diff --git a/android/generaterepo.py b/android/generaterepo.py index 2355838..1619d4e 100644 --- a/android/generaterepo.py +++ b/android/generaterepo.py @@ -1,248 +1,248 @@ import json import yaml import os import re import argparse import requests import zipfile import multiprocessing import subprocess import xml.etree.ElementTree as ET import xdg.DesktopEntry import tempfile # Constants used in this script languageMap = { None: "en-US", "ca": "ca-ES" } # Extract the internal application name from the APK given def readApplicationName( apkPath ): # Prepare the aapt (Android SDK command) to inspect the provided APK commandToRun = "aapt dump badging %s" % (apkPath) manifest = subprocess.check_output( commandToRun, shell=True ).decode('utf-8') # Search through the aapt output for the name of the application result = re.search(' name=\'([^\']*)\'', manifest) return result.group(1) # Attempt to look within the APK provided for the metadata information we will need def extractApplicationMetadata(apkpath, name): # Open the APK file in question with zipfile.ZipFile(apkpath, 'r') as contents: # Prepare to start searching data = {} # First, let's try to look within the appdata.xml files with contents.open("assets/share/metainfo/%s.appdata.xml" % name) as appdataFile: # Within this file we look at every entry, and where possible try to export it's content so we can use it later root = ET.fromstring(appdataFile.read()) for child in root: # Make sure we start with a blank slate for this entry output = {} # Grab the name of this particular attribute we're looking at # Within the Fastlane specification, it is possible to have several items with the same name but as different types # We therefore include this within our extracted name for the attribute to differentiate them tag = child.tag if 'type' in child.attrib: tag += '-' + child.attrib['type'] # Have we found some information already for this particular attribute? if tag in data: output = data[tag] # Are we dealing with category information here? # If so, then we need to look into this items children to find out all the categories this APK belongs in if tag == 'categories': cats = [] for x in child: cats.append(x.text) output = { None: cats } # Otherwise this is just textual information we need to extract else: readText(child, output) # Save the information we've gathered! data[tag] = output # Did we find any categories? # Sometimes we don't find any within the Fastlane information, but without categories the F-Droid store isn't of much use # In the event this happens, fallback to the *.desktop file for the application to see if it can provide any insight. if not 'categories' in data: with contents.open("assets/share/applications/%s.desktop" % name) as desktopFileContents: # The Python XDG extension/wrapper requires that it be able to read the file itself # To ensure it is able to do this, we transfer the content of the file from the APK out to a temporary file to keep it happy (fd, path) = tempfile.mkstemp(suffix=name + ".desktop") handle = open(fd, "wb") handle.write(desktopFileContents.read()) handle.close() # Parse the XDG format *.desktop file, and extract the categories within it desktopFile = xdg.DesktopEntry.DesktopEntry(path) data['categories'] = { None: desktopFile.getCategories() } # Finally, return the information we've gathered return data # Android appdata.xml textual item parser # This function handles reading standard text entries within an Android appdata.xml file # In particular, it handles splitting out the various translations, and converts some HTML to something which F-Droid can make use of def readText(elem, found): # Determine the language this entry is in lang = elem.get('{http://www.w3.org/XML/1998/namespace}lang') # Do we have any text for this language yet? # If not, get everything setup if not lang in found: found[lang] = "" # Do we have a HTML List Item? if elem.tag == 'li': found[lang] += "ยท " # If there is text available, we'll want to extract it # Additionally, if this element has any children, make sure we read those as well # It isn't clear if it is possible for an item to not have text, but to have children which do have text # The code will currently skip these if they're encountered if elem.text: found[lang] += elem.text for child in elem: readText(child, found) # Finally, if this element is a HTML Paragraph (p) or HTML List Item (li) make sure we add a new line for presentation purposes if elem.tag == 'li' or elem.tag == 'p': found[lang] += "\n" # Create the various Fastlane format files per the information we've previously extracted # These files are laid out following the Fastlane specification (links below) # https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots # https://docs.fastlane.tools/actions/supply/ def createFastlaneFile( applicationName, filenameToPopulate, fileContent ): # Go through each language and content pair we've been given for lang, text in fileContent.items(): # First, do we need to amend the language id, to turn the Android language ID into something more F-Droid/Fastlane friendly? languageCode = languageMap.get(lang, lang) # Next we need to determine the path to the directory we're going to be writing the data into repositoryBasePath = arguments.fdroid_repository path = os.path.join( repositoryBasePath, 'metadata', applicationName, languageCode ) # Make sure the directory exists os.makedirs(path, exist_ok=True) # Now write out file contents! with open(path + '/' + filenameToPopulate, 'w') as f: f.write(text) # Create the summary appname.yml file used by F-Droid to summarise this particular entry in the repository def createYml(appname, data): # Prepare to retrieve the existing information info = {} # Determine the path to the appname.yml file # Because 'fdroid update -c' has been run already we can always assume that this file exists # If it doesn't then something has gone very wrong and crashing out is probably the best thing to do repositoryBasePath = arguments.fdroid_repository path = os.path.join( repositoryBasePath, 'metadata', appname + '.yml' ) # Read the existing content of the file in # This ensures that any keys we don't amend will be left untouched and will continue to exist with open(path, 'r') as contents: info = yaml.load(contents) # Update the categories first # Now is also a good time to add 'KDE' to the list of categories as well info['Categories'] = data['categories'][None] + ['KDE'] # Update the general sumamry as well info['Summary'] = data['summary'][None] # Check to see if we have a Homepage... if 'url-homepage' in data: info['WebSite'] = data['url-homepage'][None] # What about a bug tracker? if 'url-bugtracker' in data: info['IssueTracker'] = data['url-bugtracker'][None] # Finally, with our updates completed, we can save the updated appname.yml file back to disk with open(path, 'w') as output: yaml.dump(info, output, default_flow_style=False) # Main function for extracting metadata from APK files def processApkFile( apkFilepath ): # Log the file that we are going to be working with print("doing...", apkFilepath) # First, determine the name of the application we have here # This is needed in order to locate the metadata files within the APK that have the information we need applicationName = readApplicationName( apkFilepath ) # Now that we know the application name, try to extract the metadata we need # In some cases it is possible this will fail - if that happens then it should not stop the whole process so we just log it and continue # Once we have gathered the information we need, try to write out the various metadata files try: # First extract the information applicationData = extractApplicationMetadata( apkFilepath, applicationName ) # Now try to create the F-Droid metadata files (which reuses the Fastlane format) createFastlaneFile( applicationName, "title.txt", data['name'] ) createFastlaneFile( applicationName, "short_description.txt", data['summary'] ) createFastlaneFile( applicationName, "full_description.txt", data['description'] ) createYml(name, data) except KeyError as e: print("could not inspect", applicationName, e, apkFilename) ### Script Commences # Parse the command line arguments we've been given parser = argparse.ArgumentParser(description='Utility to update an F-Droid repository using metadata contained within APKs') parser.add_argument('--fdroid-repository', type=str, required=True) arguments = parser.parse_args() # First off, we should make sure the repository has been setup # The path we're given should be to the folder which contains the repo/ folder if not os.path.exists( arguments.fdroid_repository ) or not os.path.exists( arguments.fdroid_repository + '/repo/' ): print("The specified F-Droid repository does not exist") print("Please run 'fdroid init' to setup the repository then try again") sys.exit(1) # Now that we know we have a valid F-Droid repository, we need to get F-Droid to prepare the metadata skeletons for us # We'll then fill in these templates with the information from the APK files (a list of which we'll be gathering next) subprocess.check_call("fdroid update -c", shell=True, cwd=arguments.fdroid_repository) # With the skeletons now available, it's time to get our list of APK files to work on ready # For this, we go over the repo/ subdirectory mentioned above, looking at all *-release-signed.apk files # We ignore all other APK files because they shouldn't be there knownApks = [] for entry in os.scandir( arguments.fdroid_repository + '/repo/' ): # First, we should ensure the item we've got is an actual file # If it is anything else, we can skip it if not entry.is_file(follow_symlinks=False): continue # Now we need to make sure the filename is what we expect it to be # If the file in question does not end with *-release-signed.apk then we can ignore it - if not entry.filename.endswith('-release-signed.apk'): + if not entry.name.endswith('-release-signed.apk'): continue # Now we know we have a file that we are interested in - knownApks.append( arguments.fdroid_repository + '/repo/' + entry.filename ) + knownApks.append( entry.path ) # With the list of APK files known to us now, we can go ahead and start extracting the metadata from those APK files # This metadata will be written into the appropriate places in the F-Droid repository # To ensure this process completes quickly, even with a large number of potential APKs we use worker threads to do this workerPool = multiprocessing.Pool(8) workerPool.map(processApkFile, knownApks) # Finally we ask F-Droid to do a full update pass # This syncs the metadata we just updated/prepared into the actual F-Droid repository which will be used by F-Droid clients subprocess.check_call("fdroid update", shell=True, cwd=arguments.fdroid_repository) # Last but not least, we publish the repository to production for clients to use subprocess.check_call("fdroid server update -v", shell=True, cwd=arguments.fdroid_repository)