diff --git a/CHANGELOG b/CHANGELOG --- a/CHANGELOG +++ b/CHANGELOG @@ -8,6 +8,7 @@ *Correction: getNetWorth (used to compute PFS) is now computed by using all accounts *Correction: Remove color of hyperlinks in dashboard for a better rendering in dark theme *Correction: Remove broken quotes sources (BitcoinAverage, BitcoinCharts) + *Correction: Better handling of the mode and comment field using the aqbanking import backend. *Feature: New REGEXPCAPTURE operator in "Search & Process" to capture a value by regular expression *Feature: Import backend aqbanking allows to import accounts without an IBAN. (See https://phabricator.kde.org/D20875) diff --git a/plugins/import/skrooge_import_backend/backends/org.kde.skrooge-import-backend-aqbanking.desktop b/plugins/import/skrooge_import_backend/backends/org.kde.skrooge-import-backend-aqbanking.desktop --- a/plugins/import/skrooge_import_backend/backends/org.kde.skrooge-import-backend-aqbanking.desktop +++ b/plugins/import/skrooge_import_backend/backends/org.kde.skrooge-import-backend-aqbanking.desktop @@ -18,7 +18,7 @@ Name[tr]=AqBanking Arka Ucu Name[uk]=Модуль AqBanking Name[x-test]=xxAqBanking backendxx -Comment=An import backend for Skrooge using AqBanking.\n You must install AqBanking (aqbanking-cli and aqhbci-tool4) and setup all accounts manually before using this backend.\n\nAt least AqBanking version 5.6.10 or later is required.\n\nThis backend starts an AqBanking user-interactive session in the default terminal emulator.\nIf you experience issues please switch to the "xterm" terminal emulator by setting the parameters to:\naqbanking(--terminal-emulator '"xterm -e"') [Note: The single and double quotes are important!] +Comment=An import backend for Skrooge using AqBanking.\n You must install AqBanking (aqbanking-cli) and setup all accounts manually before using this backend.\n\nAt least AqBanking version 5.6.10 or later is required.\n\nThis backend starts an AqBanking user-interactive session in the default terminal emulator.\nIf you experience issues please switch to the "xterm" terminal emulator by setting the parameters to:\naqbanking(--terminal-emulator '"xterm -e"') [Note: The single and double quotes are important!]\n\nFor a list of all parameters please run "skrooge-sabb.py bulkdownload --help". Comment[ca]=Un dorsal d'importació per a l'Skrooge que usa AqBanking.\n Cal instal·lar AqBanking (aqbanking-cli i aqhbci-tool4) i configurar manualment tots els comptes abans d'usar aquest dorsal.\n\nEs requereix com a mínim la versió 5.6.10 o superior d'AqBanking.\n\nAquest dorsal inicia una sessió d'usuari interactiva de l'AqBanking a l'emulador de terminal predeterminat.\nSi experimenteu problemes commuteu a l'emulador de terminal «xterm» establint els paràmetres a:\naqbanking(--terminal-emulator '"xterm -e"') [Nota: Les cometes senzilles i les dobles són importants!] Comment[ca@valencia]=Un dorsal d'importació per a l'Skrooge que usa AqBanking.\n Cal instal·lar AqBanking (aqbanking-cli i aqhbci-tool4) i configurar manualment tots els comptes abans d'usar aquest dorsal.\n\nEs requereix com a mínim la versió 5.6.10 o superior d'AqBanking.\n\nAquest dorsal inicia una sessió d'usuari interactiva de l'AqBanking a l'emulador de terminal predeterminat.\nSi experimenteu problemes commuteu a l'emulador de terminal «xterm» establint els paràmetres a:\naqbanking(--terminal-emulator '"xterm -e"') [Nota: Les cometes senzilles i les dobles són importants!] Comment[en_GB]=An import backend for Skrooge using AqBanking.\n You must install AqBanking (aqbanking-cli and aqhbci-tool4) and setup all accounts manually before using this backend.\n\nAt least AqBanking version 5.6.10 or later is required.\n\nThis backend starts an AqBanking user-interactive session in the default terminal emulator.\nIf you experience issues please switch to the "xterm" terminal emulator by setting the parameters to:\naqbanking(--terminal-emulator '"xterm -e"') [Note: The single and double quotes are important!] @@ -40,7 +40,7 @@ X-KDE-PluginInfo-Author=Bernhard Scheirle X-KDE-PluginInfo-Email=bernhard@scheirle.de X-KDE-PluginInfo-Name=aqbanking -X-KDE-PluginInfo-Version=1.2.0 +X-KDE-PluginInfo-Version=2.0.0 X-KDE-PluginInfo-Website=http://skrooge.org/ X-KDE-PluginInfo-Category=Plugins X-KDE-PluginInfo-License=GPL diff --git a/plugins/import/skrooge_import_backend/skrooge-sabb.py b/plugins/import/skrooge_import_backend/skrooge-sabb.py --- a/plugins/import/skrooge_import_backend/skrooge-sabb.py +++ b/plugins/import/skrooge_import_backend/skrooge-sabb.py @@ -25,6 +25,12 @@ Changelog: +2.0.0 - 2019.05.09 + * Added auto repair for certain banks (Sprada, Netbank, Comdirect). + * Added --disable-auto-repair command line option + * Added --prefer-valutadate command line option + * Removed --balance command line option + 1.2.0 - 2019.04.28 * Allow processing of accounts without an IBAN (e.g credit card accounts). @@ -52,7 +58,7 @@ import tempfile from distutils.version import LooseVersion -__VERSION__ = "1.2.0" +__VERSION__ = "2.0.0" class Account(object): @@ -177,16 +183,89 @@ yield context_file_path +class RepairMan(object): + def repair_row(self, row): + pass + + +class RepairManSpardaNetBank(RepairMan): + """ + Sparda / Netbank only: + If the payees name exceeds 27 characters the overflowing characters + of the name gets stored at the beginning of the purpose field. + + This is the case, when one of the strings listed in Keywords is part + of the purpose fields but does not start at the beginning. + In this case, the part leading up to the keyword is to be treated as the + tail of the payee. + """ + + Keywords = ['SEPA-BASISLASTSCHRIFT', + 'SEPA-ÜBERWEISUNG', + 'SEPA LOHN/GEHALT'] + + def repair_row(self, row): + comment = row['comment'] + + for key in self.Keywords: + offset = comment.find(key) + if offset >= 0: + if offset > 0: + row['payee'] = row['payee'] + comment[:offset] + keyEnd = offset + len(key) + row['mode'] = comment[offset:keyEnd] + row['comment'] = comment[keyEnd:] + break + return row + + +class RepairManComdirect(RepairMan): + Keywords = ['WERTPAPIERE', + 'LASTSCHRIFT / BELASTUNG', + 'ÜBERTRAG / ÜBERWEISUNG', + 'KONTOÜBERTRAG', + 'KUPON', + 'SUMME MONATSABRECHNUNG VISA', + 'KONTOABSCHLUSSABSCHLUSS ZINSEN'] + + def repair_row(self, row): + comment = row['comment'].strip() + + for key in self.Keywords: + if comment.startswith(key): + row['comment'] = comment[len(key):] + row['mode'] = key + break + return row + + +class RepairManStrip(RepairMan): + Keywords = ['date', 'mode', 'comment', 'payee', 'amount', 'unit'] + + def repair_row(self, row): + for key in self.Keywords: + row[key] = row[key].strip() + return row + + class SABB(object): # Tools AqBanking = 'aqbanking-cli' - AqHBCI = 'aqhbci-tool4' ReturnValue_NormalExit = 0 ReturnValue_InvalidVersion = 1 def __init__(self): self.accounts = Accounts() + self.repair_mans = [RepairManSpardaNetBank(), + RepairManComdirect(), + RepairManStrip()] + + self.output_folder = None + self.balance = None + self.terminal_emulator = None + self.prefer_valutadate = None + self.disable_auto_repair = None def build_command(self, executable, args): com = [executable] @@ -266,58 +345,60 @@ print(output.getvalue().strip()) return self.ReturnValue_NormalExit - def write_balance_file(self, output_folder_path, context_file_path): - process_result = subprocess.run( - self.build_command(self.AqBanking, - ['listbal', - '-c', context_file_path]), - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL - ) - fieldnames_input = ['ignore', 'bank_number', 'account_number', 'bank_name', 'account_name', - 'booked_date', 'booked_time', 'booked_value', 'booked_currency', - 'noted_date', 'noted_time', 'noted_value', 'noted_currency'] - fieldnames_output = ['id', 'date', 'balance', 'unit'] - reader = self.get_csv_reader(process_result, fieldnames_input) - output, writer = self.get_csv_writer(fieldnames_output) - for record in reader: - row = {} - row['id'] = self.get_iban(record['bank_number'], record['account_number']) - try: - row['balance'] = record['booked_value'] - row['unit'] = record['booked_currency'] - row['date'] = datetime.datetime.strptime(record['booked_date'], "%d.%m.%Y").strftime("%Y-%m-%d") - except: - try: - row['balance'] = record['noted_value'] - row['unit'] = record['noted_currency'] - row['date'] = datetime.datetime.strptime(record['noted_date'], "%d.%m.%Y").strftime("%Y-%m-%d") - except: - continue - writer.writerow(row) - output_file_path = os.path.join(output_folder_path, "balance.csv") - with open(output_file_path, 'w') as f: - f.write(output.getvalue()) - - def convert_transactions(self, aqbanking_output, generateHeader=True): + def convert_transactions(self, aqbanking_output, generateHeader): reader = csv.DictReader(aqbanking_output.splitlines(), dialect=AqDialectTransfairs) fieldnames_output = ['date', 'mode', 'comment', 'payee', 'amount', 'unit'] output, writer = self.get_csv_writer(fieldnames_output, generateHeader) for record in sorted(reader, key=lambda row: row['date']): row = {} - row['date'] = record['date'] - row['mode'] = record['purpose'] - comment = '' + if self.prefer_valutadate: + row['date'] = record['valutadate'] + else: + row['date'] = record['date'] + row['mode'] = "" + comment = record['purpose'] for i in range(1, 12): - comment = comment + record['purpose' + str(i)] + " " - row['comment'] = comment.strip() - row['payee'] = (record['remoteName'] + " " + record['remoteName1']).strip() + comment = comment + record['purpose' + str(i)] + row['comment'] = comment + row['payee'] = record['remoteName'] + record['remoteName1'] row['amount'] = record['value_value'] row['unit'] = record['value_currency'] + if not self.disable_auto_repair: + for repair_man in self.repair_mans: + row = repair_man.repair_row(row) writer.writerow(row) return output.getvalue() - def download(self, output_folder_path, balance, terminal_emulator): + def process_context_file(self, context_file_path): + self.output_folder = os.path.abspath(self.output_folder) + if not os.path.exists(self.output_folder): + os.makedirs(self.output_folder) + + files = {} + for account in self.accounts.get_accounts(): + process_result = subprocess.run( + self.build_command(self.AqBanking, + ['listtrans', + '--bank=' + account.bank_number, + '--account=' + account.account_number, + '-c', + context_file_path + ]), + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL + ) + transactions = process_result.stdout.decode("utf-8") + output_file_path = os.path.join(self.output_folder, self.format_iban(account.iban.upper()) + ".csv") + if output_file_path in files: + files[output_file_path] = files[output_file_path] + '\n' + self.convert_transactions(transactions, False) + else: + files[output_file_path] = self.convert_transactions(transactions, True) + + for path, content in files.items(): + with open(path, 'w') as f: + f.write(content) + + def download(self): if not self.check_version(): return self.ReturnValue_InvalidVersion with TemporaryContextFile() as context_file_path: @@ -327,42 +408,13 @@ '-c', context_file_path ] - if balance: - args.append('--balance') - command = str.split(terminal_emulator) + command = str.split(self.terminal_emulator) command.extend(self.build_command(self.AqBanking, args)) subprocess.run(command) - output_folder_path = os.path.abspath(output_folder_path) - if not os.path.exists(output_folder_path): - os.makedirs(output_folder_path) - files = {} - for account in self.accounts.get_accounts(): - process_result = subprocess.run( - self.build_command(self.AqBanking, - ['listtrans', - '--bank=' + account.bank_number, - '--account=' + account.account_number, - '-c', - context_file_path - ]), - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL - ) - transactions = process_result.stdout.decode("utf-8") - output_file_path = os.path.join(output_folder_path, self.format_iban(account.iban.upper()) + ".csv") - if output_file_path in files: - files[output_file_path] = files[output_file_path] + '\n' + self.convert_transactions(transactions, False) - else: - files[output_file_path] = self.convert_transactions(transactions, True) - - for path, content in files.items(): - with open(path, 'w') as f: - f.write(content) - - if balance: - self.write_balance_file(output_folder_path, context_file_path) + self.process_context_file(context_file_path) + return self.ReturnValue_NormalExit @@ -379,22 +431,32 @@ # Command: bulkdownload parser_download = subparsers.add_parser('bulkdownload', help='Downloads all transactions into the given output folder') parser_download.add_argument('--output_folder', required=True, help='The folder to store the csv files.') - parser_download.add_argument('--balance', required=False, action='store_true', - help='Additionally also download the current balance of all accounts and stores it in a "balance.csv" file in the output folder.') parser_download.add_argument('--terminal-emulator', required=False, default="x-terminal-emulator -e", help='The terminal emulator command string that gets used to run the aqbanking user-interactive session. ' 'Use an empty value »""« to not start a new terminal, but reuse the terminal running this command. ' 'Example: "xterm -e". ' '(Default: "x-terminal-emulator -e")' ) + parser_download.add_argument('--prefer-valutadate', required=False, action='store_true', + help='Uses the valuta date instead of the normal one.' + ) + parser_download.add_argument('--disable-auto-repair', required=False, action='store_true', + help='Disables bank specific repair steps.' + ) args = parser.parse_args() + sabb = SABB() if (args.command == "listaccounts"): - return SABB().get_accounts() + return sabb.get_accounts() elif (args.command == "bulkdownload"): - return SABB().download(args.output_folder, args.balance, args.terminal_emulator) + sabb.output_folder = args.output_folder + sabb.terminal_emulator = args.terminal_emulator + sabb.prefer_valutadate = args.prefer_valutadate + sabb.disable_auto_repair = args.disable_auto_repair + return sabb.download() + #return sabb.process_context_file("") else: parser.print_help() return 1