diff --git a/android/generaterepo.py b/android/generaterepo.py index 899c66b..4855661 100644 --- a/android/generaterepo.py +++ b/android/generaterepo.py @@ -1,167 +1,247 @@ import json import yaml import os import re import requests import zipfile import multiprocessing import subprocess import xml.etree.ElementTree as ET import xdg.DesktopEntry import tempfile -dest = "repo" -# if dest directory doesn't exist, run fdroid init -# https://f-droid.org/en/docs/Setup_an_F-Droid_App_Repo/ - -#aaptCall = "docker-kde-android.sh /opt/android-sdk/build-tools/21.1.2/aapt dump badging /pwd/%s" -aaptCall = "aapt dump badging %s" -#fdroid = "docker run --rm -u $(id -u):$(id -g) -v $(pwd):/repo registry.gitlab.com/fdroid/docker-executable-fdroidserver:latest" -fdroid = "fdroid" - -def readAppName(apkpath): - manifest = subprocess.check_output(aaptCall % (apkpath), shell=True).decode('utf-8') - result = re.search(' name=\'([^\']*)\'', manifest) - return result.group(1) - -def readText(elem, found): - lang = elem.get('{http://www.w3.org/XML/1998/namespace}lang') - - if not lang in found: - found[lang] = "" - - if elem.tag == 'li': - found[lang] += "· " - if elem.text: - found[lang] += elem.text - for child in elem: - readText(child, found) - - if elem.tag == 'li' or elem.tag == 'p': - found[lang] += "\n" - -conv = { +# Constants used in this script +languageMap = { None: "en-US", "ca": "ca-ES" } -def createFastlaneFile(appname, filename, locales): - for lang, text in locales.items(): - path = 'metadata/' + appname + '/' + conv.get(lang, lang) - os.makedirs(path, exist_ok=True) - with open(path + '/' + filename, 'w') as f: - f.write(text) -def createYml(appname, data): - info = {} - path = 'metadata/' + appname + '.yml' - with open(path, 'r') as contents: - info = yaml.load(contents) - - info['Categories'] = data['categories'][None] + ['KDE'] - - info['Summary'] = data['summary'][None] - if 'url-homepage' in data: - info['WebSite'] = data['url-homepage'][None] - if 'url-bugtracker' in data: - info['IssueTracker'] = data['url-bugtracker'][None] - with open(path, 'w') as output: - yaml.dump(info, output, default_flow_style=False) +# Extract the internal application name from the APK given +def readApplicationName( apkPath ): + # Prepare the aapt (Android SDK command) to inspect the provided APK + commandToRun = "aapt dump badging %s" % (apkPath) + manifest = subprocess.check_output( commandToRun, shell=True ).decode('utf-8') + # Search through the aapt output for the name of the application + result = re.search(' name=\'([^\']*)\'', manifest) + return result.group(1) -def appdataContents(apkpath, name): +# Attempt to look within the APK provided for the metadata information we will need +def extractApplicationMetadata(apkpath, name): + # Open the APK file in question with zipfile.ZipFile(apkpath, 'r') as contents: - - #https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots - #https://docs.fastlane.tools/actions/supply/ - + # Prepare to start searching data = {} + + # First, let's try to look within the appdata.xml files with contents.open("assets/share/metainfo/%s.appdata.xml" % name) as appdataFile: + # Within this file we look at every entry, and where possible try to export it's content so we can use it later root = ET.fromstring(appdataFile.read()) for child in root: + # Make sure we start with a blank slate for this entry output = {} + # Grab the name of this particular attribute we're looking at + # Within the Fastlane specification, it is possible to have several items with the same name but as different types + # We therefore include this within our extracted name for the attribute to differentiate them tag = child.tag if 'type' in child.attrib: tag += '-' + child.attrib['type'] + # Have we found some information already for this particular attribute? if tag in data: output = data[tag] + # Are we dealing with category information here? + # If so, then we need to look into this items children to find out all the categories this APK belongs in if tag == 'categories': cats = [] for x in child: cats.append(x.text) output = { None: cats } + + # Otherwise this is just textual information we need to extract else: readText(child, output) + + # Save the information we've gathered! data[tag] = output + # Did we find any categories? + # Sometimes we don't find any within the Fastlane information, but without categories the F-Droid store isn't of much use + # In the event this happens, fallback to the *.desktop file for the application to see if it can provide any insight. if not 'categories' in data: with contents.open("assets/share/applications/%s.desktop" % name) as desktopFileContents: + # The Python XDG extension/wrapper requires that it be able to read the file itself + # To ensure it is able to do this, we transfer the content of the file from the APK out to a temporary file to keep it happy (fd, path) = tempfile.mkstemp(suffix=name + ".desktop") handle = open(fd, "wb") handle.write(desktopFileContents.read()) handle.close() + # Parse the XDG format *.desktop file, and extract the categories within it desktopFile = xdg.DesktopEntry.DesktopEntry(path) data['categories'] = { None: desktopFile.getCategories() } - try: - createFastlaneFile(name, "title.txt", data['name']) - createFastlaneFile(name, "short_description.txt", data['summary']) - createFastlaneFile(name, "full_description.txt", data['description']) - createYml(name, data) - except KeyError as e: - print("error: key not found", e) - - -def fetch(job): - name = job['name'] - - zipfilename = "archive-%s.zip" % name - with open(zipfilename, mode='wb') as f: - url = "https://binary-factory.kde.org/view/Android/job/%s_android/lastSuccessfulBuild/artifact/*zip*/archive.zip" % name - print("getting...", url) - - response = requests.get(url) - data = response.content - f.write(data) - print("written", len(data), zipfilename) - - with zipfile.ZipFile(zipfilename, 'r') as contents: - for entry in contents.namelist(): - if entry.endswith('-release-signed.apk'): - filename = dest + '/' + entry[entry.rfind('/')+1:] - with open(filename, mode='wb') as f: - f.write(contents.read(entry)) - print("created", filename) - - return filename - print("could not find apk for %s" % (name) ) - return None - -def decorate(filename): - if not filename: - return - print("doing...", filename) - appname = readAppName(filename) - try: - appdataContents(filename, appname) - except KeyError as e: - print("could not inspect", appname, e, filename) + # Finally, return the information we've gathered + return data + +# Android appdata.xml textual item parser +# This function handles reading standard text entries within an Android appdata.xml file +# In particular, it handles splitting out the various translations, and converts some HTML to something which F-Droid can make use of +def readText(elem, found): + # Determine the language this entry is in + lang = elem.get('{http://www.w3.org/XML/1998/namespace}lang') + + # Do we have any text for this language yet? + # If not, get everything setup + if not lang in found: + found[lang] = "" + + # Do we have a HTML List Item? + if elem.tag == 'li': + found[lang] += "· " + + # If there is text available, we'll want to extract it + # Additionally, if this element has any children, make sure we read those as well + # It isn't clear if it is possible for an item to not have text, but to have children which do have text + # The code will currently skip these if they're encountered + if elem.text: + found[lang] += elem.text + for child in elem: + readText(child, found) + + # Finally, if this element is a HTML Paragraph (p) or HTML List Item (li) make sure we add a new line for presentation purposes + if elem.tag == 'li' or elem.tag == 'p': + found[lang] += "\n" + + +# Create the various Fastlane format files per the information we've previously extracted +# These files are laid out following the Fastlane specification (links below) +# https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots +# https://docs.fastlane.tools/actions/supply/ +def createFastlaneFile( applicationName, filenameToPopulate, fileContent ): + # Go through each language and content pair we've been given + for lang, text in fileContent.items(): + # First, do we need to amend the language id, to turn the Android language ID into something more F-Droid/Fastlane friendly? + languageCode = languageMap.get(lang, lang) + + # Next we need to determine the path to the directory we're going to be writing the data into + repositoryBasePath = argument.fdroid_repository + path = os.path.join( repositoryBasePath, 'metadata', applicationName, languageCode ) + + # Make sure the directory exists + os.makedirs(path, exist_ok=True) + + # Now write out file contents! + with open(path + '/' + filenameToPopulate, 'w') as f: + f.write(text) + +# Create the summary appname.yml file used by F-Droid to summarise this particular entry in the repository +def createYml(appname, data): + # Prepare to retrieve the existing information + info = {} + + # Determine the path to the appname.yml file + # Because 'fdroid update -c' has been run already we can always assume that this file exists + # If it doesn't then something has gone very wrong and crashing out is probably the best thing to do + repositoryBasePath = argument.fdroid_repository + path = os.path.join( repositoryBasePath, 'metadata', appname + '.yml' ) + + # Read the existing content of the file in + # This ensures that any keys we don't amend will be left untouched and will continue to exist + with open(path, 'r') as contents: + info = yaml.load(contents) + + # Update the categories first + # Now is also a good time to add 'KDE' to the list of categories as well + info['Categories'] = data['categories'][None] + ['KDE'] -with open("current-jobs.json") as f: - jobs = json.load(f) + # Update the general sumamry as well + info['Summary'] = data['summary'][None] - pool = multiprocessing.Pool(8) - filenames = pool.map(fetch, jobs) + # Check to see if we have a Homepage... + if 'url-homepage' in data: + info['WebSite'] = data['url-homepage'][None] - subprocess.check_call(fdroid + " update -c", shell=True) + # What about a bug tracker? + if 'url-bugtracker' in data: + info['IssueTracker'] = data['url-bugtracker'][None] - pool.map(decorate, filenames) + # Finally, with our updates completed, we can save the updated appname.yml file back to disk + with open(path, 'w') as output: + yaml.dump(info, output, default_flow_style=False) - subprocess.check_call(fdroid + " update", shell=True) - subprocess.check_call(fdroid + " server update -v", shell=True) +# Main function for extracting metadata from APK files +def processApkFile( apkFilepath ): + # Log the file that we are going to be working with + print("doing...", apkFilepath) + # First, determine the name of the application we have here + # This is needed in order to locate the metadata files within the APK that have the information we need + applicationName = readApplicationName( apkFilepath ) + # Now that we know the application name, try to extract the metadata we need + # In some cases it is possible this will fail - if that happens then it should not stop the whole process so we just log it and continue + # Once we have gathered the information we need, try to write out the various metadata files + try: + # First extract the information + applicationData = extractApplicationMetadata( apkFilepath, applicationName ) + # Now try to create the F-Droid metadata files (which reuses the Fastlane format) + createFastlaneFile( applicationName, "title.txt", data['name'] ) + createFastlaneFile( applicationName, "short_description.txt", data['summary'] ) + createFastlaneFile( applicationName, "full_description.txt", data['description'] ) + createYml(name, data) + + except KeyError as e: + print("could not inspect", applicationName, e, apkFilename) + +### Script Commences + +# Parse the command line arguments we've been given +parser = argparse.ArgumentParser(description='Utility to update an F-Droid repository using metadata contained within APKs') +parser.add_argument('--fdroid-repository', type=str, required=True) +arguments = parser.parse_args() + +# First off, we should make sure the repository has been setup +# The path we're given should be to the folder which contains the repo/ folder +if not os.path.exists( argument.fdroid_repository ) or not os.path.exists( argument.fdroid_repository + '/repo/' ): + print("The specified F-Droid repository does not exist") + print("Please run 'fdroid init' to setup the repository then try again") + sys.exit(1) + +# Now that we know we have a valid F-Droid repository, we need to get F-Droid to prepare the metadata skeletons for us +# We'll then fill in these templates with the information from the APK files (a list of which we'll be gathering next) +subprocess.check_call("fdroid update -c", shell=True) + +# With the skeletons now available, it's time to get our list of APK files to work on ready +# For this, we go over the repo/ subdirectory mentioned above, looking at all *-release-signed.apk files +# We ignore all other APK files because they shouldn't be there +knownApks = [] +for entry in os.scandir( argument.fdroid_repository + '/repo/' ): + # First, we should ensure the item we've got is an actual file + # If it is anything else, we can skip it + if not entry.is_file(follow_symlinks=False): + continue + + # Now we need to make sure the filename is what we expect it to be + # If the file in question does not end with *-release-signed.apk then we can ignore it + if not entry.filename.endswith('-release-signed.apk'): + continue + + # Now we know we have a file that we are interested in + knownApks.append( argument.fdroid_repository + '/repo/' + entry.filename ) + +# With the list of APK files known to us now, we can go ahead and start extracting the metadata from those APK files +# This metadata will be written into the appropriate places in the F-Droid repository +# To ensure this process completes quickly, even with a large number of potential APKs we use worker threads to do this +workerPool = multiprocessing.Pool(8) +workerPool.map(processApkFile, knownApks) + +# Finally we ask F-Droid to do a full update pass +# This syncs the metadata we just updated/prepared into the actual F-Droid repository which will be used by F-Droid clients +subprocess.check_call("fdroid update", shell=True) + +# Last but not least, we publish the repository to production for clients to use +subprocess.check_call("fdroid server update -v", shell=True)