diff --git a/kmymoney/dialogs/transactionmatcher.cpp b/kmymoney/dialogs/transactionmatcher.cpp --- a/kmymoney/dialogs/transactionmatcher.cpp +++ b/kmymoney/dialogs/transactionmatcher.cpp @@ -61,12 +61,15 @@ void TransactionMatcher::match(MyMoneyTransaction tm, MyMoneySplit sm, MyMoneyTransaction ti, MyMoneySplit si, bool allowImportedTransactions) { Q_D(TransactionMatcher); - auto sec = MyMoneyFile::instance()->security(d->m_account.currencyId()); + const auto &file = MyMoneyFile::instance(); + auto sec = file->security(d->m_account.currencyId()); // Now match the transactions. // - // 'Matching' the transactions entails DELETING the end transaction, - // and MODIFYING the start transaction as needed. + // 'Matching' the transactions entails CREATING new transaction which + // has parameters of matched transactions. The matched transactions + // change their origin attribute and shouldn't be visible + // by default anywhere. Only the new transaction will be visible. // // There are a variety of ways that a transaction can conflict. // Post date, splits, amount are the ones that seem to matter. @@ -105,6 +108,41 @@ throw MYMONEYEXCEPTION(QString::fromLatin1("Splits for %1 have conflicting values (%2,%3)").arg(d->m_account.name(), MyMoneyUtils::formatMoney(sm.shares(), d->m_account, sec), MyMoneyUtils::formatMoney(si.shares(), d->m_account, sec))); } + if (ti.isMatched()) + throw MYMONEYEXCEPTION_CSTRING("Second transaction is a product of match."); + + // if next match is comming, then we won't hide this transaction but update it + const auto isTransactionAlreadyMatched = tm.isMatched(); + if (!isTransactionAlreadyMatched) { + // hide start transaction (input for matching) + tm.setOrigin(static_cast(tm.origin() | eMyMoney::Transaction::Origin::MatchingInput)); + file->modifyTransaction(tm); + // from now on tm is matched transaction (output of matching) + tm.setOrigin(eMyMoney::Transaction::Origin::MatchingOutput); + } + + // next match has matched matched transaction and it's id is meaningless in that case + // so search for an ID of already matched transaction + QString startTransactionID; + if (isTransactionAlreadyMatched) { + for (const auto &split : tm.splits()) { + if (split.isMatched()) { + startTransactionID = split.MyMoneyKeyValueContainer::value("kmm-match-transaction").split(';').first(); + break; + } + } + } else { + startTransactionID = tm.id(); + } + + // add details about ma + sm.setValue("kmm-match-transaction", QString::fromLatin1("%1;%2").arg(startTransactionID, ti.id())); + sm.setValue("kmm-match-split", QString::fromLatin1("%1;%2").arg(sm.id(), si.id())); + + // hide end transaction (input for matching) + ti.setOrigin(static_cast(ti.origin() | eMyMoney::Transaction::Origin::MatchingInput)); + file->modifyTransaction(ti); + // ipwizard: I took over the code to keep the bank id found in the endMatchTransaction // This might not work for QIF imports as they don't setup this information. It sure // makes sense for OFX and HBCI. @@ -130,92 +168,91 @@ // if we don't have a payee assigned to the manually entered transaction // we use the one we found in the imported transaction - if (sm.payeeId().isEmpty() && !si.payeeId().isEmpty()) { - sm.setValue("kmm-orig-payee", sm.payeeId()); + if (sm.payeeId().isEmpty() && !si.payeeId().isEmpty()) sm.setPayeeId(si.payeeId()); - } // We use the imported postdate and keep the previous one for unmatch - if (tm.postDate() != ti.postDate()) { - sm.setValue("kmm-orig-postdate", tm.postDate().toString(Qt::ISODate)); + if (tm.postDate() != ti.postDate()) tm.setPostDate(ti.postDate()); - } // combine the two memos into one QString memo = sm.memo(); if (!si.memo().isEmpty() && si.memo() != memo) { - sm.setValue("kmm-orig-memo", memo); if (!memo.isEmpty()) memo += '\n'; memo += si.memo(); } sm.setMemo(memo); - // remember the split we matched - sm.setValue("kmm-match-split", si.id()); - - sm.addMatch(ti); + sm.addMatch(); tm.modifySplit(sm); - ti.modifySplit(si);/// - MyMoneyFile::instance()->modifyTransaction(tm); - // Delete the end transaction if it was stored in the engine - if (!ti.id().isEmpty()) - MyMoneyFile::instance()->removeTransaction(ti); + ti.modifySplit(si); + + // if matched transaction already exists then update it + // if not then create it + if (isTransactionAlreadyMatched) { + file->modifyTransaction(tm); + } else { + tm.clearId(); + file->addTransaction(tm); + } } void TransactionMatcher::unmatch(const MyMoneyTransaction& _t, const MyMoneySplit& _s) { - if (_s.isMatched()) { - MyMoneyTransaction tm(_t); - MyMoneySplit sm(_s); - MyMoneyTransaction ti(sm.matchedTransaction()); - MyMoneySplit si; - // if we don't have a split, then we don't have a memo - try { - si = ti.splitById(sm.value("kmm-match-split")); - } catch (const MyMoneyException &) { - } - sm.removeMatch(); - - // restore the postdate if modified - if (!sm.value("kmm-orig-postdate").isEmpty()) { - tm.setPostDate(QDate::fromString(sm.value("kmm-orig-postdate"), Qt::ISODate)); - } + if (!_s.isMatched()) + return; - // restore payee if modified - if (!sm.value("kmm-orig-payee").isEmpty()) { - sm.setPayeeId(sm.value("kmm-orig-payee")); - } + const auto &file = MyMoneyFile::instance(); + MyMoneyTransaction tm(_t); + MyMoneySplit sm(_s); + for (const auto &transactionID : sm.matchedTransactionIDs()) { + auto inputTransaction = file->transaction(transactionID); + // unhide start and end transaction + inputTransaction.setOrigin(static_cast(inputTransaction.origin() ^ eMyMoney::Transaction::Origin::MatchingInput)); + file->modifyTransaction(inputTransaction); + } - // restore memo if modified - if (!sm.value("kmm-orig-memo").isEmpty()) { - sm.setMemo(sm.value("kmm-orig-memo")); - } + // effectively remove matching information from a split + sm.MyMoneyKeyValueContainer::deletePair("kmm-match-transaction"); + sm.MyMoneyKeyValueContainer::deletePair("kmm-match-split"); + sm.removeMatch(); - sm.deletePair("kmm-orig-postdate"); - sm.deletePair("kmm-orig-payee"); - sm.deletePair("kmm-orig-memo"); - sm.deletePair("kmm-match-split"); - tm.modifySplit(sm); + tm.modifySplit(sm); + file->modifyTransaction(tm); - MyMoneyFile::instance()->modifyTransaction(tm); - MyMoneyFile::instance()->addTransaction(ti); + // if there is no another matched split in a transaction + // then this transaction can be removed + auto isAnyMatchedSplitLeft = false; + for (const auto &split : tm.splits()) { + if (split.isMatched()) { + isAnyMatchedSplitLeft = true; + break; + } } + + if (!isAnyMatchedSplitLeft) + file->removeTransaction(tm); } void TransactionMatcher::accept(const MyMoneyTransaction& _t, const MyMoneySplit& _s) { - if (_s.isMatched()) { - MyMoneyTransaction tm(_t); - MyMoneySplit sm(_s); - sm.removeMatch(); - sm.deletePair("kmm-orig-postdate"); - sm.deletePair("kmm-orig-payee"); - sm.deletePair("kmm-orig-memo"); - sm.deletePair("kmm-match-split"); - tm.modifySplit(sm); - - MyMoneyFile::instance()->modifyTransaction(tm); + if (!_s.isMatched()) + return; + MyMoneyTransaction tm(_t); + MyMoneySplit sm(_s); + + const auto &file = MyMoneyFile::instance(); + for (const auto &transactionID : sm.matchedTransactionIDs()) { + auto inputTransaction = file->transaction(transactionID); + file->removeTransaction(inputTransaction); } + + sm.MyMoneyKeyValueContainer::deletePair("kmm-match-transaction"); + sm.MyMoneyKeyValueContainer::deletePair("kmm-match-split"); + sm.removeMatch(); + tm.modifySplit(sm); + + file->modifyTransaction(tm); } diff --git a/kmymoney/mymoney/mymoneyenums.h b/kmymoney/mymoney/mymoneyenums.h --- a/kmymoney/mymoney/mymoneyenums.h +++ b/kmymoney/mymoney/mymoneyenums.h @@ -403,6 +403,14 @@ }; inline uint qHash(const Action key, uint seed) { return ::qHash(static_cast(key), seed); } + + enum Origin : int { + None = 0x0, + Typed = 0x1, + Imported = 0x2, + MatchingOutput = 0x4, + MatchingInput = 0x8, + }; } namespace Money { diff --git a/kmymoney/mymoney/mymoneyfile.cpp b/kmymoney/mymoney/mymoneyfile.cpp --- a/kmymoney/mymoney/mymoneyfile.cpp +++ b/kmymoney/mymoney/mymoneyfile.cpp @@ -677,6 +677,10 @@ throw MYMONEYEXCEPTION(QString::fromLatin1("Cannot remove transaction that references a closed account.")); d->addCacheNotification(split.accountId(), tr.postDate()); //FIXME-ALEX Do I need to add d->addCacheNotification(split.tagList()); ?? + + if (split.isMatched()) + for (const auto &transactionID : split.matchedTransactionIDs()) + removeTransaction(MyMoneyFile::transaction(transactionID)); } d->m_storage->removeTransaction(transaction); diff --git a/kmymoney/mymoney/mymoneyreport.cpp b/kmymoney/mymoney/mymoneyreport.cpp --- a/kmymoney/mymoney/mymoneyreport.cpp +++ b/kmymoney/mymoney/mymoneyreport.cpp @@ -810,6 +810,7 @@ d->m_accountGroups.clear(); MyMoneyTransactionFilter::clear(); + removeOrigin(eMyMoney::Transaction::Origin::MatchingInput); } void MyMoneyReport::assignFilter(const MyMoneyTransactionFilter& filter) diff --git a/kmymoney/mymoney/mymoneysplit.h b/kmymoney/mymoney/mymoneysplit.h --- a/kmymoney/mymoney/mymoneysplit.h +++ b/kmymoney/mymoney/mymoneysplit.h @@ -191,7 +191,7 @@ * added transaction will be overridden. @p t.isImported() must return true, * otherwise the transaction is not stored. */ - void addMatch(const MyMoneyTransaction& t); + void addMatch(); /** * remove the data of the imported transaction added with addMatch(). @@ -199,10 +199,12 @@ void removeMatch(); /** - * Return the matching imported transaction. If no such transaction - * is available (isMatched() returns false) an empty transaction is returned. + * Return ID of the matching imported transaction. If no such transaction + * is available (isMatched() returns false) an empty string is returned. */ - MyMoneyTransaction matchedTransaction() const; + QString matchedTransaction() const; + + QStringList matchedTransactionIDs() const; /** * This method replaces all occurrences of id @a oldId with diff --git a/kmymoney/mymoney/mymoneysplit.cpp b/kmymoney/mymoney/mymoneysplit.cpp --- a/kmymoney/mymoney/mymoneysplit.cpp +++ b/kmymoney/mymoney/mymoneysplit.cpp @@ -347,9 +347,9 @@ { Q_D(const MyMoneySplit); auto rc = false; - if (isMatched()) { - rc = matchedTransaction().hasReferenceTo(id); - } + if (isMatched()) + rc = matchedTransactionIDs().contains(id); + for (int i = 0; i < d->m_tagList.size(); i++) if (id == d->m_tagList[i]) return true; @@ -362,29 +362,33 @@ return d->m_isMatched; } -void MyMoneySplit::addMatch(const MyMoneyTransaction& _t) +void MyMoneySplit::addMatch() { Q_D(MyMoneySplit); - // now we allow matching of two manual transactions - d->m_matchedTransaction = _t; - d->m_matchedTransaction.clearId(); d->m_isMatched = true; } void MyMoneySplit::removeMatch() { Q_D(MyMoneySplit); - d->m_matchedTransaction = MyMoneyTransaction(); d->m_isMatched = false; } -MyMoneyTransaction MyMoneySplit::matchedTransaction() const +QString MyMoneySplit::matchedTransaction() const { Q_D(const MyMoneySplit); if (d->m_isMatched) - return d->m_matchedTransaction; + return MyMoneyKeyValueContainer::value("kmm-match-transaction").split(';').last(); + + return QString(); +} - return MyMoneyTransaction(); +QStringList MyMoneySplit::matchedTransactionIDs() const +{ + Q_D(const MyMoneySplit); + if (d->m_isMatched) + return MyMoneyKeyValueContainer::value("kmm-match-transaction").split(';'); + return QStringList(); } bool MyMoneySplit::replaceId(const QString& newId, const QString& oldId) @@ -404,10 +408,12 @@ } if (isMatched()) { - MyMoneyTransaction t = matchedTransaction(); - if (t.replaceId(newId, oldId)) { - removeMatch(); - addMatch(t); + auto oldList = MyMoneyKeyValueContainer::value("kmm-match-transaction").split(';'); + auto newList = oldList; + for (auto &item : newList) + item = item.replace(oldId, newId); + if (newList != oldList) { + MyMoneyKeyValueContainer::setValue("kmm-match-transaction", newList.join(';')); changed = true; } } diff --git a/kmymoney/mymoney/mymoneytransaction.h b/kmymoney/mymoney/mymoneytransaction.h --- a/kmymoney/mymoney/mymoneytransaction.h +++ b/kmymoney/mymoney/mymoneytransaction.h @@ -45,6 +45,8 @@ template class QList; +namespace eMyMoney { namespace Transaction { enum Origin : int; } } + /** * This class represents a transaction within the MyMoneyEngine. A transaction * contains none, one or more splits of type MyMoneySplit. They are stored in @@ -98,6 +100,9 @@ QString bankID() const; void setBankID(const QString& bankID); + eMyMoney::Transaction::Origin origin() const; + void setOrigin(eMyMoney::Transaction::Origin origin); + bool operator == (const MyMoneyTransaction& right) const; bool operator != (const MyMoneyTransaction& r) const; bool operator < (const MyMoneyTransaction& r) const; @@ -231,6 +236,12 @@ */ void setImported(bool state = true); + /** + * returns @a true if this is a transaction matched against an imported + * transaction. + */ + bool isMatched() const; + /** * This static method returns the id which will be assigned to the * first split added to a transaction. This ID can be used to figure diff --git a/kmymoney/mymoney/mymoneytransaction.cpp b/kmymoney/mymoney/mymoneytransaction.cpp --- a/kmymoney/mymoney/mymoneytransaction.cpp +++ b/kmymoney/mymoney/mymoneytransaction.cpp @@ -161,6 +161,18 @@ d->m_bankID = bankID; } +eMyMoney::Transaction::Origin MyMoneyTransaction::origin() const +{ + Q_D(const MyMoneyTransaction); + return d->m_origin; +} + +void MyMoneyTransaction::setOrigin(eMyMoney::Transaction::Origin origin) +{ + Q_D(MyMoneyTransaction); + d->m_origin = origin; +} + bool MyMoneyTransaction::operator == (const MyMoneyTransaction& right) const { Q_D(const MyMoneyTransaction); @@ -384,15 +396,23 @@ bool MyMoneyTransaction::isImported() const { - return value("Imported").toLower() == QString("true"); + Q_D(const MyMoneyTransaction); + return d->m_origin & eMyMoney::Transaction::Origin::Imported; } void MyMoneyTransaction::setImported(bool state) { + Q_D(MyMoneyTransaction); if (state) - setValue("Imported", "true"); + d->m_origin = static_cast(d->m_origin | eMyMoney::Transaction::Origin::Imported); else - deletePair("Imported"); + d->m_origin = static_cast(d->m_origin ^ eMyMoney::Transaction::Origin::Imported); +} + +bool MyMoneyTransaction::isMatched() const +{ + Q_D(const MyMoneyTransaction); + return d->m_origin & eMyMoney::Transaction::Origin::MatchingOutput; } bool MyMoneyTransaction::hasReferenceTo(const QString& id) const diff --git a/kmymoney/mymoney/mymoneytransaction_p.h b/kmymoney/mymoney/mymoneytransaction_p.h --- a/kmymoney/mymoney/mymoneytransaction_p.h +++ b/kmymoney/mymoney/mymoneytransaction_p.h @@ -36,12 +36,14 @@ #include "mymoneyobject_p.h" #include "mymoneysplit.h" +#include "mymoneyenums.h" using namespace eMyMoney; class MyMoneyTransactionPrivate : public MyMoneyObjectPrivate { public: + MyMoneyTransactionPrivate(); /** * This method returns the next id to be used for a split */ @@ -99,6 +101,14 @@ */ QString m_bankID; + eMyMoney::Transaction::Origin m_origin; }; +MyMoneyTransactionPrivate::MyMoneyTransactionPrivate() : + MyMoneyObjectPrivate(), + m_origin(eMyMoney::Transaction::Origin::Typed) +{ +} + + #endif diff --git a/kmymoney/mymoney/mymoneytransactionfilter.h b/kmymoney/mymoney/mymoneytransactionfilter.h --- a/kmymoney/mymoney/mymoneytransactionfilter.h +++ b/kmymoney/mymoney/mymoneytransactionfilter.h @@ -46,6 +46,7 @@ namespace eMyMoney { namespace TransactionFilter { enum class Date; enum class Validity; } } +namespace eMyMoney { namespace Transaction { enum Origin : int; } } /** * @author Thomas Baumgart * @author Łukasz Wojniłowicz @@ -75,6 +76,7 @@ unsigned typeFilter : 1; unsigned stateFilter : 1; unsigned validityFilter : 1; + unsigned originFilter : 1; } singleFilter; } FilterSet; @@ -216,6 +218,10 @@ */ void addValidity(const int type); + /** + */ + void removeOrigin(eMyMoney::Transaction::Origin origin); + /** */ void addState(const int state); diff --git a/kmymoney/mymoney/mymoneytransactionfilter.cpp b/kmymoney/mymoney/mymoneytransactionfilter.cpp --- a/kmymoney/mymoney/mymoneytransactionfilter.cpp +++ b/kmymoney/mymoney/mymoneytransactionfilter.cpp @@ -51,8 +51,10 @@ , m_matchOnly(false) , m_matchingSplitsCount(0) , m_invertText(false) + , m_origin({eMyMoney::Transaction::Origin::MatchingInput}) // most of the time we don't want transactions that served as an input for creating matched transaction { m_filterSet.allFilter = 0; + m_filterSet.singleFilter.originFilter = 1; // because of m_origin being set } MyMoneyTransactionFilter::FilterSet m_filterSet; @@ -71,6 +73,7 @@ QHash m_states; QHash m_types; QHash m_validity; + QSet m_origin; QString m_fromNr, m_toNr; QDate m_fromDate, m_toDate; MyMoneyMoney m_fromAmount, m_toAmount; @@ -113,6 +116,7 @@ d->m_types.clear(); d->m_states.clear(); d->m_validity.clear(); + d->m_origin.clear(); d->m_fromDate = QDate(); d->m_toDate = QDate(); } @@ -251,6 +255,13 @@ d->m_validity.insert(type, QString()); } +void MyMoneyTransactionFilter::removeOrigin(eMyMoney::Transaction::Origin origin) +{ + Q_D(MyMoneyTransactionFilter); + d->m_origin.insert(origin); + d->m_filterSet.singleFilter.originFilter = 1; +} + void MyMoneyTransactionFilter::setNumberFilter(const QString& from, const QString& to) { Q_D(MyMoneyTransactionFilter); @@ -302,6 +313,17 @@ // perform checks on the MyMoneyTransaction object first + if (filter.originFilter) { + const auto transactionOrigin = transaction.origin(); + for (const auto &filterOrigin : d->m_origin) { + if (filterOrigin & transactionOrigin) { + d->m_matchingSplitsCount = 0; + matchingSplits.clear(); + return matchingSplits; + } + } + } + // check the date range if (filter.dateFilter) { if ((d->m_fromDate != QDate() && @@ -329,9 +351,10 @@ auto isMatchingSplitsEmpty = true; auto extendedFilter = d->m_filterSet; - extendedFilter.singleFilter.dateFilter = - extendedFilter.singleFilter.accountFilter = - extendedFilter.singleFilter.categoryFilter = 0; + extendedFilter.singleFilter.dateFilter = + extendedFilter.singleFilter.accountFilter = + extendedFilter.singleFilter.categoryFilter = + extendedFilter.singleFilter.originFilter = 0; if (filter.accountFilter || filter.categoryFilter || diff --git a/kmymoney/mymoney/storage/mymoneystoragemgr.cpp b/kmymoney/mymoney/storage/mymoneystoragemgr.cpp --- a/kmymoney/mymoney/storage/mymoneystoragemgr.cpp +++ b/kmymoney/mymoney/storage/mymoneystoragemgr.cpp @@ -668,15 +668,18 @@ if (it_t == d->m_transactionList.end()) throw MYMONEYEXCEPTION_CSTRING("invalid transaction key"); - foreach (const auto split, (*it_t).splits()) { - auto acc = d->m_accountList[split.accountId()]; - // we only need to adjust non-investment accounts here - // as for investment accounts the balance will be recalculated - // after the transaction has been added. - if (!acc.isInvest()) { - d->adjustBalance(acc, split, true); - acc.touch(); - d->m_accountList.modify(acc.id(), acc); + // don't count transactions that served as input for matching + if (!((*it_t).origin() & eMyMoney::Transaction::Origin::MatchingInput)) { + foreach (const auto split, (*it_t).splits()) { + auto acc = d->m_accountList[split.accountId()]; + // we only need to adjust non-investment accounts here + // as for investment accounts the balance will be recalculated + // after the transaction has been added. + if (!acc.isInvest()) { + d->adjustBalance(acc, split, true); + acc.touch(); + d->m_accountList.modify(acc.id(), acc); + } } } @@ -688,12 +691,15 @@ d->m_transactionList.insert(newKey, transaction); d->m_transactionKeys.modify(transaction.id(), newKey); - // adjust account balances - foreach (const auto split, transaction.splits()) { - auto acc = d->m_accountList[split.accountId()]; - d->adjustBalance(acc, split, false); - acc.touch(); - d->m_accountList.modify(acc.id(), acc); + // don't count transactions that served as input for matching + if (!(transaction.origin() & eMyMoney::Transaction::Origin::MatchingInput)) { + // adjust account balances + foreach (const auto split, transaction.splits()) { + auto acc = d->m_accountList[split.accountId()]; + d->adjustBalance(acc, split, false); + acc.touch(); + d->m_accountList.modify(acc.id(), acc); + } } } @@ -1771,6 +1777,8 @@ // now scan over all transactions and all splits and setup the balances foreach (const auto transaction, d->m_transactionList) { + if (transaction.origin() & eMyMoney::Transaction::Origin::MatchingInput) + continue; foreach (const auto split, transaction.splits()) { if (!split.shares().isZero()) { const QString& id = split.accountId(); diff --git a/kmymoney/mymoney/storage/mymoneystoragemgr_p.h b/kmymoney/mymoney/storage/mymoneystoragemgr_p.h --- a/kmymoney/mymoney/storage/mymoneystoragemgr_p.h +++ b/kmymoney/mymoney/storage/mymoneystoragemgr_p.h @@ -95,7 +95,7 @@ m_dirty(false), m_creationDate(QDate::currentDate()), // initialize for file fixes (see kmymoneyview.cpp) - m_currentFixVersion(4), + m_currentFixVersion(5), m_fileFixVersion(0), // default value if no fix-version in file m_transactionListFull(false) { diff --git a/kmymoney/plugins/sql/mymoneydbdef.cpp b/kmymoney/plugins/sql/mymoneydbdef.cpp --- a/kmymoney/plugins/sql/mymoneydbdef.cpp +++ b/kmymoney/plugins/sql/mymoneydbdef.cpp @@ -36,7 +36,7 @@ #include //***************** THE CURRENT VERSION OF THE DATABASE LAYOUT **************** -unsigned int MyMoneyDbDef::m_currentVersion = 12; +unsigned int MyMoneyDbDef::m_currentVersion = 13; // ************************* Build table descriptions **************************** MyMoneyDbDef::MyMoneyDbDef() @@ -241,6 +241,7 @@ appendField(MyMoneyDbDatetimeColumn("entryDate")); appendField(MyMoneyDbColumn("currencyId", "char(3)")); appendField(MyMoneyDbTextColumn("bankId")); + appendField(MyMoneyDbIntColumn("origin", MyMoneyDbIntColumn::TINY, UNSIGNED, false, false, 13)); MyMoneyDbTable t("kmmTransactions", fields); t.buildSQLStrings(); m_tables[t.name()] = t; diff --git a/kmymoney/plugins/sql/mymoneystoragesql.cpp b/kmymoney/plugins/sql/mymoneystoragesql.cpp --- a/kmymoney/plugins/sql/mymoneystoragesql.cpp +++ b/kmymoney/plugins/sql/mymoneystoragesql.cpp @@ -1824,6 +1824,7 @@ int entryDateCol = t.fieldNumber("entryDate"); int currencyIdCol = t.fieldNumber("currencyId"); int bankIdCol = t.fieldNumber("bankId"); + auto originIdCol = t.fieldNumber("origin"); while (query.next()) { MyMoneyTransaction tx; @@ -1833,6 +1834,7 @@ tx.setEntryDate(GETDATE_D(entryDateCol)); tx.setCommodity(GETSTRING(currencyIdCol)); tx.setBankID(GETSTRING(bankIdCol)); + tx.setOrigin(static_cast(GETINT(originIdCol))); // skip all splits while the transaction id of the split is less than // the transaction id of the current transaction. Don't forget to check diff --git a/kmymoney/plugins/sql/mymoneystoragesql_p.h b/kmymoney/plugins/sql/mymoneystoragesql_p.h --- a/kmymoney/plugins/sql/mymoneystoragesql_p.h +++ b/kmymoney/plugins/sql/mymoneystoragesql_p.h @@ -45,6 +45,7 @@ #include #include #include +#include // ---------------------------------------------------------------------------- // KDE Includes @@ -516,6 +517,7 @@ while (query.next()) dbList.append(query.value(0).toString()); MyMoneyTransactionFilter filter; + filter.clear(); // to get rid of default origin filter filter.setReportAllSplits(false); QList list; m_storage->transactionList(list, filter); @@ -1109,6 +1111,7 @@ query.bindValue(":entryDate", tx.entryDate().toString(Qt::ISODate)); query.bindValue(":currencyId", tx.commodity()); query.bindValue(":bankId", tx.bankID()); + query.bindValue(":origin", tx.origin()); if (!query.exec()) // krazy:exclude=crashy throw MYMONEYEXCEPTIONSQL("writing Transaction"); // krazy:exclude=crashy @@ -1685,6 +1688,19 @@ //s.setPostDate(GETDATETIME(postDateCol)); // FIXME - when Tom puts date into split object s.setBankID(GETSTRING(bankIdCol)); + QStringList kvpIdList {QString::fromLatin1("%1%2").arg(GETSTRING(transactionIdCol), + GETSTRING(splitIdCol))}; + + // get the kvps + auto kvpMap = readKeyValuePairs("SPLIT", kvpIdList); + s.setPairs(kvpMap.value(kvpIdList.first()).pairs()); + const auto hasMatchTransaction = !s.MyMoneyKeyValueContainer::value("kmm-match-transaction").isEmpty(); + const auto hasMatchSplit = !s.MyMoneyKeyValueContainer::value("kmm-match-split").isEmpty(); + if (hasMatchTransaction && hasMatchSplit) + s.addMatch(); + else + s.removeMatch(); // MyMoneyStorageSql::fetchTransactions enter this method multiple times with old split argument, so we must remove match explicitly + return; } @@ -2133,6 +2149,10 @@ if ((rc = upgradeToV12()) != 0) return (1); ++m_dbVersion; break; + case 12: + if ((rc = upgradeToV13()) != 0) return (1); + ++m_dbVersion; + break; default: qWarning("Unknown version number in database - %d", m_dbVersion); } @@ -2497,6 +2517,390 @@ return 0; } + /** + * @brief upgradeToV13 + * Changes (2018-08-19): + * 1) make two separate transactions out of two matched transactions + * 2) create a new tramsaction as a result of matching + * 3) add origin (imported, typed, matched) of transaction + * 4) add kmm-match-transaction containing two matched transaction IDs to every matched split + * 5) add kmm-match-split containing two matched split IDs to every matched split + * @return 0 if successful, 1 if not successful + */ + int upgradeToV13() + { + Q_Q(MyMoneyStorageSql); + MyMoneyDbTransaction dbtrans(*q, Q_FUNC_INFO); + + switch(haveColumnInTable(QLatin1String("kmmTransactions"), QLatin1String("origin"))) { + case -1: + return 1; + case 1: // column exists, nothing to do + break; + case 0: // need update of kmmTransactions + if (!alterTable(m_db.m_tables["kmmTransactions"], m_dbVersion)) + return 1; + break; + } + + QSqlQuery selectQuery(*q); + QSqlQuery updateQuery(*q); + QSqlQuery deleteQuery(*q); + QSqlQuery insertQuery(*q); + + // we need to know the highest unoccupied ID to know what IDs can we assign to new (reorganized) transactions + selectQuery.prepare(QString::fromLatin1("SELECT id FROM kmmTransactions ORDER BY id DESC")); + if (!selectQuery.exec()) { + buildError(selectQuery, Q_FUNC_INFO, QString("Error retrieving highest transaction id.")); + return 1; + } + + // no transactions = no problem + if (!selectQuery.next()) + return 0; + auto transactionRecord = selectQuery.record(); + auto lastTransactionID = transactionRecord.value(0).toString().mid(1).toULong(); + + // set default transaction origin for every transaction + updateQuery.prepare(QString::fromLatin1("UPDATE kmmTransactions SET origin='%1'").arg( + QString::number(static_cast(eMyMoney::Transaction::Origin::Typed)))); + if (!updateQuery.exec()) { + buildError(updateQuery, Q_FUNC_INFO, QString("Error updating transaction origins.")); + return 1; + } + + // correct transaction origin if it has been imported + selectQuery.prepare("SELECT * FROM kmmKeyValuePairs WHERE kvpType='TRANSACTION' AND kvpKey='Imported'"); + if (!selectQuery.exec()) { + buildError(selectQuery, Q_FUNC_INFO, QString("Error retrieving key value pairs about imported transactions.")); + return 1; + } + + while (selectQuery.next()) { + auto record = selectQuery.record(); + auto transactionID = record.value("kvpID").toString(); + + updateQuery.prepare(QString::fromLatin1("UPDATE kmmTransactions SET origin='%1' WHERE id='%2'").arg( + QString::number(static_cast(eMyMoney::Transaction::Origin::Imported)), + transactionID)); + if (!updateQuery.exec()) { + buildError(updateQuery, Q_FUNC_INFO, QString("Error assigning transaction's origin to imported.")); + return 1; + } + } + + deleteQuery.prepare("DELETE FROM kmmKeyValuePairs WHERE kvpKey='Imported'"); + if (!deleteQuery.exec()) { + buildError(deleteQuery, Q_FUNC_INFO, QString("Error deleting key value pairs about imported transactions.")); + return 1; + } + + // search for embedded tranactions + QSqlQuery queryMatchedTX(*q); + queryMatchedTX.prepare("SELECT * FROM kmmKeyValuePairs WHERE kvpType='SPLIT' AND kvpKey='kmm-matched-tx'"); + if (!queryMatchedTX.exec()) { + buildError(queryMatchedTX, Q_FUNC_INFO, QString("Error retrieving key value pairs about matched transactions.")); + return 1; + } + + while (queryMatchedTX.next()) { + auto record = queryMatchedTX.record(); + auto transactionIDAndSplitID = record.value("kvpID").toString(); + auto transactionID = transactionIDAndSplitID; + transactionID = transactionID.left(19); // transaction id size + + // unembedd imported transaction + auto embeddedTransactionXML = record.value("kvpData").toString(); + embeddedTransactionXML.replace(QLatin1String("<"), QLatin1String("<")); + + QXmlStreamReader reader(embeddedTransactionXML); + QXmlStreamAttributes embeddedTransactionAttributes; + QString embeddedTransactionID; + + auto isEmbeddedTransactionImported = false; + // start parsing transaction in XML + reader.readNextStartElement(); // this reads container + while (reader.readNextStartElement()) { + if (reader.name() == "TRANSACTION") { + embeddedTransactionAttributes = reader.attributes(); + embeddedTransactionID = QString::fromLatin1("T%1").arg(QString::number(++lastTransactionID).rightJustified(18 , '0')); + + while (reader.readNextStartElement()) { + if (reader.name() == "SPLITS") { + while (reader.readNextStartElement()) { + if (reader.name() == "SPLIT") { + auto embeddedTransactionSplitAttributes = reader.attributes(); + auto embeddedSplitIDConvertedToSQL = QString::number(embeddedTransactionSplitAttributes.value("id").toString().mid(1).toInt() - 1); + + // SPLIT node has only key-value pairs node as subnode, so read it before all + while (reader.readNextStartElement()) { + if (reader.name() == "KEYVALUEPAIRS") { + while (reader.readNextStartElement()) { + // inserting all key-value pairs of embedded transaction without filtering anything out might be dangerous + if (reader.name() == "PAIR") { + auto embeddedTransactionSplitKVPAttributes = reader.attributes(); + const QStringList columnValuesInKVP = { + "SPLIT", + QString::fromLatin1("%1%2").arg(embeddedTransactionID, embeddedSplitIDConvertedToSQL), + embeddedTransactionSplitKVPAttributes.value("key").toString(), + embeddedTransactionSplitKVPAttributes.value("value").toString() + }; + insertQuery.prepare(QString::fromLatin1("INSERT INTO kmmKeyValuePairs VALUES ('%1')").arg(columnValuesInKVP.join("','"))); + if (!insertQuery.exec()) { + buildError(insertQuery, Q_FUNC_INFO, QString("Error inserting key-value pairs of splits of embedded transaction.")); + return 1; + } + } + reader.skipCurrentElement(); + } + } else { + reader.skipCurrentElement(); + } + } + + // we use highest precision for formatted values because we don't know precise precision yet + const auto valueFormatted = MyMoneyMoney(embeddedTransactionSplitAttributes.value("value").toString()).formatMoney(QString(), -1, false).replace(QChar(','), QChar('.')); + const auto priceFormatted = MyMoneyMoney(embeddedTransactionSplitAttributes.value("price").toString()).formatMoney(QString(), 8, false).replace(QChar(','), QChar('.')); + const auto sharesFormatted = MyMoneyMoney(embeddedTransactionSplitAttributes.value("shares").toString()).formatMoney(QString(), 8, false).replace(QChar(','), QChar('.')); + + // we unembedd split 1:1 except its ID an transaction ID + const QStringList columnValuesInSplit = { + embeddedTransactionID, + "N", + embeddedSplitIDConvertedToSQL, + embeddedTransactionSplitAttributes.value("payee").toString(), + embeddedTransactionSplitAttributes.value("reconciledate").toString(), + embeddedTransactionSplitAttributes.value("action").toString(), + embeddedTransactionSplitAttributes.value("reconcileflag").toString(), + embeddedTransactionSplitAttributes.value("value").toString(), + valueFormatted, + embeddedTransactionSplitAttributes.value("shares").toString(), + sharesFormatted, + embeddedTransactionSplitAttributes.value("price").toString(), + priceFormatted, + embeddedTransactionSplitAttributes.value("memo").toString(), + embeddedTransactionSplitAttributes.value("account").toString(), + QString(), + QString(), + embeddedTransactionAttributes.value("postdate").toString(), + embeddedTransactionAttributes.value("bankid").toString() + }; + + insertQuery.prepare(QString::fromLatin1("INSERT INTO kmmSplits VALUES ('%1')").arg(columnValuesInSplit.join("','"))); + if (!insertQuery.exec()) { + buildError(insertQuery, Q_FUNC_INFO, QString("Error inserting splits of embedded transaction.")); + return 1; + } + } else { + reader.skipCurrentElement(); + } + } + + // key-value pairs node as a subnode of transaction + } else if (reader.name() == "KEYVALUEPAIRS") { + while (reader.readNextStartElement()) { + if (reader.name() == "PAIR") { + auto embeddedTransactionKVPAttributes = reader.attributes(); + const QStringList columnValuesInKVP = { + "TRANSACTION", + embeddedTransactionID, + embeddedTransactionKVPAttributes.value("key").toString(), + embeddedTransactionKVPAttributes.value("value").toString() + }; + + // we don't want Imported key to be unembedded because it's replaced by origin attribute + if (embeddedTransactionKVPAttributes.value("key") == "Imported") { + isEmbeddedTransactionImported = true; + } else { + insertQuery.prepare(QString::fromLatin1("INSERT INTO kmmKeyValuePairs VALUES ('%1')").arg(columnValuesInKVP.join("','"))); + if (!insertQuery.exec()) { + buildError(insertQuery, Q_FUNC_INFO, QString("Error inserting key-value pairs of embedded transaction.")); + return 1; + } + } + } else { + reader.skipCurrentElement(); + } + + } + } else { + reader.skipCurrentElement(); + } + } + } else { + reader.skipCurrentElement(); + } + } + + // inserting embedded transaction as 1:1 except its ID + QStringList columnValuesInTransaction = { + embeddedTransactionID, + "N", + embeddedTransactionAttributes.value("postdate").toString(), + embeddedTransactionAttributes.value("memo").toString(), + embeddedTransactionAttributes.value("entrydate").toString(), + embeddedTransactionAttributes.value("commodity").toString(), + QString(), + isEmbeddedTransactionImported ? + QString::number(static_cast(eMyMoney::Transaction::Origin::Imported | eMyMoney::Transaction::Origin::MatchingInput)) : + QString::number(static_cast(eMyMoney::Transaction::Origin::Typed | eMyMoney::Transaction::Origin::MatchingInput)) + }; + + insertQuery.prepare(QString::fromLatin1("INSERT INTO kmmTransactions VALUES ('%1')").arg(columnValuesInTransaction.join("','"))); + + if (!insertQuery.exec()) { + buildError(insertQuery, Q_FUNC_INFO, QString("Error inserting embedded transaction.")); + return 1; + } + + // we already unembedded end transaction that served as input for matching + // and now we'll restore original stand of start transaction that also served as input for matching + // from matched transaction that is output from matching + selectQuery.prepare(QString::fromLatin1("SELECT * FROM kmmTransactions WHERE id='%1'").arg(transactionID)); + + if (!selectQuery.exec() || !selectQuery.next()) { + buildError(selectQuery, Q_FUNC_INFO, QString("Error retrieving matched transaction.")); + return 1; + } + auto sourceTransactionRecord = selectQuery.record(); + auto sourceTransactionID = QString::fromLatin1("T%1").arg(QString::number(++lastTransactionID).rightJustified(18 , '0')); + sourceTransactionRecord.setValue("id", QVariant(sourceTransactionID)); + auto sourceTransactionOrigin = sourceTransactionRecord.value("origin").toInt(); + sourceTransactionOrigin = sourceTransactionOrigin | eMyMoney::Transaction::Origin::MatchingInput; + sourceTransactionRecord.setValue("origin", QVariant(sourceTransactionOrigin)); + + // in matched transaction splits there are backup values of start transaction, so iterate over them + selectQuery.prepare(QString::fromLatin1("SELECT * FROM kmmSplits WHERE transactionId='%1'").arg(transactionID)); + if (!selectQuery.exec()) { + buildError(selectQuery, Q_FUNC_INFO, QString("Error retrieving splits.")); + return 1; + } + + while (selectQuery.next()) { + auto sourceTransactionSplitRecord = selectQuery.record(); + auto transactionSplitID = sourceTransactionSplitRecord.value("splitId").toInt(); + sourceTransactionSplitRecord.setValue("transactionId", QVariant(sourceTransactionID)); + + // backup values of start transaction are stored in key-value pairs of split + QSqlQuery kvpQuery(*q); + kvpQuery.prepare(QString::fromLatin1("SELECT * FROM kmmKeyValuePairs WHERE kvpType='SPLIT' AND kvpId='%1%2'").arg(transactionID, QString::number(transactionSplitID))); + if (!kvpQuery.exec()) { + buildError(kvpQuery, Q_FUNC_INFO, QString("Error retrieving key-value pairs about matched transaction.")); + return 1; + } + + // store key-value pairs because we will deltete them from database now and will be filtering them out later on + QMap keyValueMap; + while (kvpQuery.next()) { + auto kvpKey = kvpQuery.value("kvpKey").toString(); + auto kvpData = kvpQuery.value("kvpData"); + keyValueMap.insert(kvpKey, kvpData); + } + + deleteQuery.prepare(QString::fromLatin1("DELETE FROM kmmKeyValuePairs WHERE kvpType='SPLIT' AND kvpId='%1%2'").arg(transactionID, QString::number(transactionSplitID))); + if (!deleteQuery.exec()) { + buildError(deleteQuery, Q_FUNC_INFO, QString("Error deleting key-value pairs.")); + return 1; + } + + // split containg kmm-match-split means that it was matched + // otherwise it's a generic split and will be written only with new transaction ID + if (keyValueMap.contains("kmm-match-split")) { + + // restore all original values from backup for source transaction and its split + if (keyValueMap.contains("kmm-orig-postdate")) { + sourceTransactionRecord.setValue("postDate", keyValueMap.value("kmm-orig-postdate")); + keyValueMap.remove("kmm-orig-postdate"); + } + + auto sourceTransactionSplitID = keyValueMap.value("kmm-match-split").toString().mid(1).toInt() - 1; + if (transactionSplitID == sourceTransactionSplitID) { + if (keyValueMap.contains("kmm-orig-memo")) { + sourceTransactionSplitRecord.setValue("memo", keyValueMap.value("kmm-orig-memo")); + keyValueMap.remove("kmm-orig-memo"); + } + if (keyValueMap.contains("kmm-orig-payee")) { + sourceTransactionSplitRecord.setValue("payee", keyValueMap.value("kmm-orig-payee")); + keyValueMap.remove("kmm-orig-payee"); + } + } + + // add key-value pairs according to new way of storing information about matched split + QStringList columnValuesInKVP = { + "SPLIT", + QString::fromLatin1("%1%2").arg(transactionID, QString::number(transactionSplitID)), + "kmm-match-transaction", + QString::fromLatin1("%1;%2").arg(sourceTransactionID, embeddedTransactionID) + }; + + + insertQuery.prepare(QString::fromLatin1("INSERT INTO kmmKeyValuePairs VALUES ('%1')").arg(columnValuesInKVP.join("','"))); + if (!insertQuery.exec()) { + buildError(insertQuery, Q_FUNC_INFO, QString("Error inserting kmm-match-transaction information.")); + return 1; + } + + columnValuesInKVP = QStringList { + "SPLIT", + QString::fromLatin1("%1%2").arg(transactionID, QString::number(transactionSplitID)), + "kmm-match-split", + QString::fromLatin1("%1;%2").arg(QString::number(transactionSplitID), QString::number(sourceTransactionSplitID)) + }; + + insertQuery.prepare(QString::fromLatin1("INSERT INTO kmmKeyValuePairs VALUES ('%1')").arg(columnValuesInKVP.join("','"))); + if (!insertQuery.exec()) { + buildError(insertQuery, Q_FUNC_INFO, QString("Error inserting kmm-match-split information.")); + return 1; + } + + } + + QStringList columnValuesInSplit; + for (auto i = 0; i < sourceTransactionSplitRecord.count(); ++i) + columnValuesInSplit.append(sourceTransactionSplitRecord.value(i).toString()); + + insertQuery.prepare(QString::fromLatin1("INSERT INTO kmmSplits VALUES ('%1')").arg(columnValuesInSplit.join("','"))); + if (!insertQuery.exec()) { + buildError(insertQuery, Q_FUNC_INFO, QString("Error inserting split.")); + return 1; + } + + // all key-value pairs were deleted from the database, so write only the usefull ones back + QStringList columnValuesInKVP { + "SPLIT", + QString::fromLatin1("%1%2").arg(sourceTransactionID, QString::number(transactionSplitID)), + QString(), + QString() + }; + + for (auto it = keyValueMap.cbegin(); it != keyValueMap.cend(); ++it) { + if (it.key() == "kmm-matched-tx" || it.key() == "kmm-match-split") + continue; + columnValuesInKVP.replace(2, it.key()); + columnValuesInKVP.replace(3, it.value().toString()); + insertQuery.prepare(QString::fromLatin1("INSERT INTO kmmKeyValuePairs VALUES ('%1')").arg(columnValuesInKVP.join("','"))); + if (!insertQuery.exec()) { + buildError(insertQuery, Q_FUNC_INFO, QString("Error inserting key-value pair.")); + return 1; + } + } + } + + // inserting source transaction as it was just before matching + columnValuesInTransaction = QStringList(); + for (auto i = 0; i < sourceTransactionRecord.count(); ++i) + columnValuesInTransaction.append(sourceTransactionRecord.value(i).toString()); + + insertQuery.prepare(QString::fromLatin1("INSERT INTO kmmTransactions VALUES ('%1')").arg(columnValuesInTransaction.join("','"))); + if (!insertQuery.exec()) { + buildError(insertQuery, Q_FUNC_INFO, QString("Error inserting source transaction.")); + return 1; + } + } + return 0; + } + + int createTables() { Q_Q(MyMoneyStorageSql); diff --git a/kmymoney/plugins/xml/CMakeLists.txt b/kmymoney/plugins/xml/CMakeLists.txt --- a/kmymoney/plugins/xml/CMakeLists.txt +++ b/kmymoney/plugins/xml/CMakeLists.txt @@ -4,6 +4,7 @@ set(xmlstorage_SOURCES xmlstorage.cpp + xmlstorageupdater.cpp mymoneystoragexml.cpp mymoneystoragenames.cpp mymoneystorageanon.cpp diff --git a/kmymoney/plugins/xml/mymoneystorageanon.cpp b/kmymoney/plugins/xml/mymoneystorageanon.cpp --- a/kmymoney/plugins/xml/mymoneystorageanon.cpp +++ b/kmymoney/plugins/xml/mymoneystorageanon.cpp @@ -176,14 +176,6 @@ s.setShares((s.shares() * m_factor)); } s.setNumber(hideString(s.number())); - - // obfuscate a possibly matched transaction as well - if (s.isMatched()) { - MyMoneyTransaction t = s.matchedTransaction(); - fakeTransaction(t); - s.removeMatch(); - s.addMatch(t); - } tn.modifySplit(s); } tx = tn; diff --git a/kmymoney/plugins/xml/mymoneystoragenames.h b/kmymoney/plugins/xml/mymoneystoragenames.h --- a/kmymoney/plugins/xml/mymoneystoragenames.h +++ b/kmymoney/plugins/xml/mymoneystoragenames.h @@ -149,6 +149,7 @@ EntryDate, Commodity, BankID, + Origin, // insert new entries above this line LastAttribute }; diff --git a/kmymoney/plugins/xml/mymoneystoragenames.cpp b/kmymoney/plugins/xml/mymoneystoragenames.cpp --- a/kmymoney/plugins/xml/mymoneystoragenames.cpp +++ b/kmymoney/plugins/xml/mymoneystoragenames.cpp @@ -151,6 +151,7 @@ {Attribute::Transaction::EntryDate, QStringLiteral("entrydate")}, {Attribute::Transaction::Commodity, QStringLiteral("commodity")}, {Attribute::Transaction::BankID, QStringLiteral("bankid")}, + {Attribute::Transaction::Origin, QStringLiteral("origin")}, }; return attributeNames.value(attributeID); } diff --git a/kmymoney/plugins/xml/mymoneystoragexml.cpp b/kmymoney/plugins/xml/mymoneystoragexml.cpp --- a/kmymoney/plugins/xml/mymoneystoragexml.cpp +++ b/kmymoney/plugins/xml/mymoneystoragexml.cpp @@ -326,10 +326,7 @@ try { if (s == nodeName(Node::Transaction)) { auto t0 = readTransaction(m_baseNode); - if (!t0.id().isEmpty()) { - MyMoneyTransaction t1(m_reader->d->nextTransactionID(), t0); - m_reader->d->tList[t1.uniqueSortKey()] = t1; - } + m_reader->d->tList[t0.uniqueSortKey()] = t0; m_reader->signalProgress(++m_elementCount, 0); } else if (s == nodeName(Node::Account)) { auto a = readAccount(m_baseNode); @@ -547,6 +544,7 @@ transaction.setBankID(node.attribute(attributeName(Attribute::Transaction::BankID))); transaction.setMemo(node.attribute(attributeName(Attribute::Transaction::Memo))); transaction.setCommodity(node.attribute(attributeName(Attribute::Transaction::Commodity))); + transaction.setOrigin(static_cast(node.attribute(attributeName(Attribute::Transaction::Origin)).toInt())); QDomNode child = node.firstChild(); auto transactionID = transaction.id(); @@ -589,6 +587,7 @@ el.setAttribute(attributeName(Attribute::Transaction::Memo), transaction.memo()); el.setAttribute(attributeName(Attribute::Transaction::EntryDate), transaction.entryDate().toString(Qt::ISODate)); el.setAttribute(attributeName(Attribute::Transaction::Commodity), transaction.commodity()); + el.setAttribute(attributeName(Attribute::Transaction::Origin), transaction.origin()); auto splitsElement = document.createElement(elementName(Element::Transaction::Splits)); @@ -630,16 +629,20 @@ split.setNumber(node.attribute(attributeName(Attribute::Split::Number))); split.setBankID(node.attribute(attributeName(Attribute::Split::BankID))); - auto xml = split.value(attributeName(Attribute::Split::KMMatchedTx)); - if (!xml.isEmpty()) { - xml.replace(QLatin1String("<"), QLatin1String("<")); - QDomDocument docMatchedTransaction; - QDomElement nodeMatchedTransaction; - docMatchedTransaction.setContent(xml); - nodeMatchedTransaction = docMatchedTransaction.documentElement().firstChild().toElement(); - auto t = MyMoneyXmlContentHandler::readTransaction(nodeMatchedTransaction); - split.addMatch(t); - } + auto matchedTransactionID = split.MyMoneyKeyValueContainer::value("kmm-match-transaction"); + auto matchedSplitID = split.MyMoneyKeyValueContainer::value("kmm-match-split"); + if (!matchedTransactionID.isEmpty() && !matchedSplitID.isEmpty()) + split.addMatch(); +// auto xml = split.value(attributeName(Attribute::Split::KMMatchedTx)); +// if (!xml.isEmpty()) { +// xml.replace(QLatin1String("<"), QLatin1String("<")); +// QDomDocument docMatchedTransaction; +// QDomElement nodeMatchedTransaction; +// docMatchedTransaction.setContent(xml); +// nodeMatchedTransaction = docMatchedTransaction.documentElement().firstChild().toElement(); +// auto t = MyMoneyXmlContentHandler::readTransaction(nodeMatchedTransaction); +// split.addMatch(t); +// } return split; } @@ -675,17 +678,17 @@ el.appendChild(sel); } - if (split.isMatched()) { - QDomDocument docMatchedTransaction(elementName(Element::Split::Match)); - QDomElement elMatchedTransaction = docMatchedTransaction.createElement(elementName(Element::Split::Container)); - docMatchedTransaction.appendChild(elMatchedTransaction); - writeTransaction(split.matchedTransaction(), docMatchedTransaction, elMatchedTransaction); - auto xml = docMatchedTransaction.toString(); - xml.replace(QLatin1String("<"), QLatin1String("<")); - split.setValue(attributeName(Attribute::Split::KMMatchedTx), xml); - } else { - split.deletePair(attributeName(Attribute::Split::KMMatchedTx)); - } +// if (split.isMatched()) { +// QDomDocument docMatchedTransaction(elementName(Element::Split::Match)); +// QDomElement elMatchedTransaction = docMatchedTransaction.createElement(elementName(Element::Split::Container)); +// docMatchedTransaction.appendChild(elMatchedTransaction); +// writeTransaction(split.matchedTransaction(), docMatchedTransaction, elMatchedTransaction); +// auto xml = docMatchedTransaction.toString(); +// xml.replace(QLatin1String("<"), QLatin1String("<")); +// split.setValue(attributeName(Attribute::Split::KMMatchedTx), xml); +// } else { +// split.deletePair(attributeName(Attribute::Split::KMMatchedTx)); +// } writeKeyValueContainer(split, document, el); @@ -1760,6 +1763,7 @@ void MyMoneyStorageXML::writeTransactions(QDomElement& transactions) { MyMoneyTransactionFilter filter; + filter.clear(); // to get rid of default origin filter filter.setReportAllSplits(false); const auto list = m_storage->transactionList(filter); transactions.setAttribute(attributeName(Attribute::General::Count), list.count()); diff --git a/kmymoney/plugins/xml/xmlstorage.cpp b/kmymoney/plugins/xml/xmlstorage.cpp --- a/kmymoney/plugins/xml/xmlstorage.cpp +++ b/kmymoney/plugins/xml/xmlstorage.cpp @@ -27,6 +27,8 @@ #include #include #include +#include +#include // ---------------------------------------------------------------------------- // KDE Includes @@ -55,6 +57,7 @@ #include "kmymoneyutils.h" #include "kgpgfile.h" #include "kgpgkeyselectiondlg.h" +#include "xmlstorageupdater.h" #include "kmymoneyenums.h" using namespace Icons; @@ -185,20 +188,62 @@ // we know. For now, we support our own XML format and // GNUCash XML format. If the file is smaller, then it // contains no valid data and we reject it anyway. - qbaFileHeader.resize(70); - if (qfile->read(qbaFileHeader.data(), 70) != 70) + qbaFileHeader.resize(220); + if (qfile->read(qbaFileHeader.data(), 220) != 220) throw MYMONEYEXCEPTION(sFileToShort); if (haveAt) qfile->seek(0); else - ungetString(qfile, qbaFileHeader.data(), 70); + ungetString(qfile, qbaFileHeader.data(), 220); - QRegExp kmyexp(""); - QByteArray txt(qbaFileHeader, 70); - if (kmyexp.indexIn(txt) == -1) + auto re = QRegularExpression(QStringLiteral("")); + auto match = re.match(qbaFileHeader); + if (!match.hasMatch()) return nullptr; + auto fileVersion = 0; + auto fileFixVersion = 0; + re = QRegularExpression(QStringLiteral("VERSION id=\"(?\\d+)\"")); + match = re.match(qbaFileHeader); + if (match.hasMatch()) + fileVersion = match.captured(1).toInt(); + + re = QRegularExpression(QStringLiteral("FIXVERSION id=\"(?\\d+)\"")); + match = re.match(qbaFileHeader); + if (match.hasMatch()) + fileFixVersion = match.captured(1).toInt(); + + QByteArray XMLByteArray; + if (fileVersion <= 1 && fileFixVersion < 5) { + KMessageBox::information(nullptr, i18n("Your storage version is %1 and fix version is %2.\n" + "It will be upgraded to version %3 and fix version %4.\n" + "That means you won't be able to use it in previous versions of KMyMoney.\n" + "This operation won't modify your storage until you save it.", fileVersion, fileFixVersion, 1, 5), + i18n("Storage version upgrade")); + XMLByteArray = qfile->readAll(); + qfile->close(); + delete qfile; + qfile = new QBuffer(&XMLByteArray); + qfile->open(QIODevice::ReadWrite); + + if (!XMLStorageUpdater::updateFile(qfile)) { + qfile->close(); + delete qfile; + return nullptr; + } + qfile->seek(0); + } else if (fileVersion >= 1 && fileFixVersion > 5) { + KMessageBox::information(nullptr, i18n("Your storage version is %1 and fix version is %2.\n" + "The highest supported version is %3 and fix version %4.\n" + "That means your storage has been saved in newer KMyMoney version.\n" + "It's incompativle with with this version of KMyMoney.", fileVersion, fileFixVersion, 1, 5), + i18n("Incopatible storage version")); + qfile->close(); + delete qfile; + return nullptr; + } + // attach the storage before reading the file, since the online // onlineJobAdministration object queries the engine during // loading. diff --git a/kmymoney/plugins/xml/xmlstorageupdater.h b/kmymoney/plugins/xml/xmlstorageupdater.h new file mode 100644 --- /dev/null +++ b/kmymoney/plugins/xml/xmlstorageupdater.h @@ -0,0 +1,39 @@ +/* + * Copyright 2018 Łukasz Wojniłowicz + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef XMLSTORAGEUPDATER_H +#define XMLSTORAGEUPDATER_H + +// ---------------------------------------------------------------------------- +// QT Includes + +// ---------------------------------------------------------------------------- +// KDE Includes + +// ---------------------------------------------------------------------------- +// Project Includes + +class QIODevice; +class QDomDocument; + +namespace XMLStorageUpdater +{ + bool upgradeVersion1Fix5(QDomDocument &doc); + bool updateFile(QIODevice* pXMLDevice); +} + +#endif diff --git a/kmymoney/plugins/xml/xmlstorageupdater.cpp b/kmymoney/plugins/xml/xmlstorageupdater.cpp new file mode 100644 --- /dev/null +++ b/kmymoney/plugins/xml/xmlstorageupdater.cpp @@ -0,0 +1,248 @@ +/* + * Copyright 2018 Łukasz Wojniłowicz + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "xmlstorageupdater.h" + +// ---------------------------------------------------------------------------- +// QT Includes + +#include +#include +#include +#include + +// ---------------------------------------------------------------------------- +// KDE Includes + +// ---------------------------------------------------------------------------- +// Project Includes + +#include "mymoneyenums.h" + +namespace XMLStorageUpdater +{ + + /** + * @brief updateVersion1Fix5 + * @param doc as parsed document + * Changes (2018-08-19): + * 1) make two separate transactions out of two matched transactions + * 2) create a new tramsaction as a result of matching + * 3) add origin (imported, typed, matched) of transaction + * 4) add kmm-match-transaction containing two matched transaction IDs to every matched split + * 5) add kmm-match-split containing two matched split IDs to every matched split + * @return true if upgrade is successful + */ + +bool upgradeVersion1Fix5(QDomDocument &doc) +{ + auto transactionsNodes = doc.elementsByTagName("TRANSACTIONS"); + + // no transactions = no problem + if (transactionsNodes.isEmpty()) + return true; + + auto transactionsNode = transactionsNodes.item(0); + auto transactionNodes = transactionsNode.childNodes(); + if (transactionNodes.isEmpty()) + return true; + + // we need to know the highest unoccupied ID to know what IDs can we assign to new (reorganized) transactions + QStringList transactionIDs; + ulong lastTransactionID = 0; + for (auto i = 0; i < transactionNodes.length(); ++i) { + auto transationElement = transactionNodes.item(i).toElement(); + transactionIDs.append(transationElement.attribute("id")); + } + std::sort(transactionIDs.begin(), transactionIDs.end()); + lastTransactionID = transactionIDs.last().mid(1).toUInt(); + + for (auto i = 0; i < transactionNodes.length(); ++i) { + auto transactionElement = transactionNodes.item(i).toElement(); + + // search if the transaction is imported in order to set its origin properly + auto transactionOriginAttribute = static_cast(transactionElement.attribute("origin").toInt()); + auto transactionKVPElement = transactionNodes.item(i).namedItem("KEYVALUEPAIRS").toElement(); + auto transactionPairNodes = transactionKVPElement.elementsByTagName("PAIR"); + for (auto k = 0; k < transactionPairNodes.length() ; ++k) { + auto pairNodeElement = transactionPairNodes.item(k).toElement(); + if (pairNodeElement.attribute("key").compare("Imported", Qt::CaseInsensitive) == 0) { + transactionKVPElement.removeChild(pairNodeElement); + transactionOriginAttribute = static_cast(transactionOriginAttribute | eMyMoney::Transaction::Origin::Imported); + break; + } + } + + // if transaction isn't imported then it has been manually typed + if (!(transactionOriginAttribute & eMyMoney::Transaction::Origin::Imported)) + transactionOriginAttribute = static_cast(transactionOriginAttribute | eMyMoney::Transaction::Origin::Typed); + + auto splitsNode = transactionElement.elementsByTagName("SPLITS").item(0); + auto splitNodes = splitsNode.childNodes(); + for (auto j = 0; j < splitNodes.length(); ++j) { + auto splitElement = splitNodes.item(j).toElement(); + auto splitKVPElement = splitElement.elementsByTagName("KEYVALUEPAIRS").item(0).toElement(); + auto splitPairNodes = splitKVPElement.elementsByTagName("PAIR"); + + // get a map of all key-value pairs out of a split for convenient usage + QMap keyValueMap; + for (auto k = 0; k < splitPairNodes.length() ; ++k) { + auto pairNodeElement = splitPairNodes.item(k).toElement(); + keyValueMap.insert(pairNodeElement.attribute("key"), pairNodeElement.attribute("value")); + } + + // act on splits that embedded imported transaction + if (keyValueMap.contains("kmm-matched-tx")) { + // unembedd imported transaction + auto embeddedTransactionXML = keyValueMap.value("kmm-matched-tx"); + embeddedTransactionXML.replace(QLatin1String("<"), QLatin1String("<")); + QDomDocument embeddedTransactionDoc; + QDomNode embeddedTransactionNode; + embeddedTransactionDoc.setContent(embeddedTransactionXML); + embeddedTransactionNode = embeddedTransactionDoc.documentElement().firstChild().cloneNode(); + auto embeddedTransactionElement = embeddedTransactionNode.toElement(); + + // assign free transaction id + auto embeddedTransactionID = QString::fromLatin1("T%1").arg(QString::number(++lastTransactionID).rightJustified(18 , '0')); + embeddedTransactionElement.setAttribute("id", embeddedTransactionID); + + // at this point we only know that it's an input for matching + auto embeddedTransactionOrigin = eMyMoney::Transaction::Origin::None; + embeddedTransactionOrigin = static_cast(embeddedTransactionOrigin | eMyMoney::Transaction::Origin::MatchingInput); + embeddedTransactionElement.setAttribute("origin", embeddedTransactionOrigin); + + // we don't need those attributes anymore and their values have been stored earlier + const QVector backupValuesToBeRemoved {"kmm-orig-postdate", + "kmm-orig-payee", + "kmm-orig-memo", + "kmm-matched-tx", + "kmm-match-split"}; + for (auto k = splitPairNodes.length() - 1; k >= 0; --k) { + auto pairNodeElement = splitPairNodes.item(k).toElement(); + if (backupValuesToBeRemoved.contains(pairNodeElement.attribute("key"))) + splitKVPElement.removeChild(splitPairNodes.item(k)); + } + + // we had two tranactions: matched and imported, + // and now we have three: matched (result of matching) , imported (input for matching), typed (input for matching) + auto sourceTransactionNode = transactionElement.cloneNode(); + + // assign free transaction id + auto sourceTransactionElement = sourceTransactionNode.toElement(); + auto sourceTransactionID = QString::fromLatin1("T%1").arg(QString::number(++lastTransactionID).rightJustified(18 , '0')); + sourceTransactionElement.setAttribute("id", sourceTransactionID); + + // at this point we only know that it's an input for matching + auto sourceTransactionOrigin = eMyMoney::Transaction::Origin::None; + sourceTransactionOrigin = static_cast(sourceTransactionOrigin | eMyMoney::Transaction::Origin::MatchingInput); + sourceTransactionElement.setAttribute("origin", sourceTransactionOrigin); + + // previous matching changed input transaction a little bit, so restore its values to the original ones + if (keyValueMap.contains("kmm-orig-postdate")) + sourceTransactionElement.setAttribute("postdate", keyValueMap.value("kmm-orig-postdate")); + + auto sourceTransactionSplitsNode = sourceTransactionElement.elementsByTagName("SPLITS").item(0); + auto sourceTransactionSplitNodes = sourceTransactionSplitsNode.childNodes(); + for (auto k = 0; k < sourceTransactionSplitNodes.length(); ++k) { + auto sourceTransactionSplitElement = sourceTransactionSplitNodes.item(k).toElement(); + auto sourceTransactionSplitID = sourceTransactionSplitElement.attribute("id"); + if (sourceTransactionSplitID == keyValueMap.value("kmm-match-split")) { + if (keyValueMap.contains("kmm-orig-payee")) + sourceTransactionSplitElement.setAttribute("payee", keyValueMap.value("kmm-orig-payee")); + if (keyValueMap.contains("kmm-orig-memo")) + sourceTransactionSplitElement.setAttribute("memo", keyValueMap.value("kmm-orig-memo")); + break; + } + } + + // matched transaction has information about transactions (their IDs) that were input for matching + // example "T000000000000000001;T000000000000000002" + auto matchedTransactionKeyValuePair = doc.createElement("PAIR"); + matchedTransactionKeyValuePair.setAttribute("key", "kmm-match-transaction"); + matchedTransactionKeyValuePair.setAttribute("value", QString::fromLatin1("%1;%2").arg(sourceTransactionID, embeddedTransactionID)); + splitKVPElement.appendChild(matchedTransactionKeyValuePair); + + // matched transaction has information about splits (their IDs) that were matched during matching + // example "S0001;S0001" + auto matchedSplitKeyValuePair = doc.createElement("PAIR"); + matchedSplitKeyValuePair.setAttribute("key", "kmm-match-split"); + matchedSplitKeyValuePair.setAttribute("value", QString::fromLatin1("%1;%2").arg(splitElement.attribute("id"), keyValueMap.value("kmm-match-split"))); + splitKVPElement.appendChild(matchedSplitKeyValuePair); + + // append new transactions now so that they will get picked by this loop + // and their origin could be established at the beggining of this loop + transactionsNode.appendChild(embeddedTransactionNode); + transactionsNode.appendChild(sourceTransactionElement); + + // we know that it's and output of matching + transactionOriginAttribute = static_cast(transactionOriginAttribute | eMyMoney::Transaction::Origin::MatchingOutput); + } + + transactionElement.setAttribute("origin", transactionOriginAttribute); + } + } + return true; +} + +bool updateFile(QIODevice* pXMLDevice) +{ + QDomDocument doc; + auto ret = doc.setContent(pXMLDevice); + if (!ret) + return false; + + auto fixVersionElement = doc.elementsByTagName("FIXVERSION").item(0).toElement(); + const auto fixVersion = fixVersionElement.attribute("id").toUInt(); + auto versionElement = doc.elementsByTagName("VERSION").item(0).toElement(); + const auto version = versionElement.attribute("id").toUInt(); + + auto isUpdateSuccessful = true; + + switch (version) { + case 1: + switch (fixVersion) { + case 1: + case 2: + case 3: + case 4: + isUpdateSuccessful |= upgradeVersion1Fix5(doc); + // intentional fall through + case 5: + break; + default: + break; + } + // intentional fall through + default: + break; + } + + if (!(version == 1 && fixVersion < 4)) { + versionElement.setAttribute("id", "1"); + fixVersionElement.setAttribute("id", "5"); + } + + if (isUpdateSuccessful) { + pXMLDevice->close(); + pXMLDevice->open(QIODevice::ReadWrite | QIODevice::Truncate); + pXMLDevice->write(doc.toByteArray()); + } + + return isUpdateSuccessful; +} + +} diff --git a/kmymoney/views/kcategoriesview.cpp b/kmymoney/views/kcategoriesview.cpp --- a/kmymoney/views/kcategoriesview.cpp +++ b/kmymoney/views/kcategoriesview.cpp @@ -332,8 +332,12 @@ and replace that with categoryId. */ // get the list of all transactions that reference the old account - MyMoneyTransactionFilter filter(d->m_currentCategory.id()); + MyMoneyTransactionFilter filter; + filter.clear(); // to get rid of default origin filter + filter.setConsiderCategory(false); filter.setReportAllSplits(false); + filter.addAccount(d->m_currentCategory.id()); + QList tlist; QList::iterator it_t; file->transactionList(tlist, filter); diff --git a/kmymoney/widgets/ktransactionfilter.cpp b/kmymoney/widgets/ktransactionfilter.cpp --- a/kmymoney/widgets/ktransactionfilter.cpp +++ b/kmymoney/widgets/ktransactionfilter.cpp @@ -266,6 +266,7 @@ { Q_D(KTransactionFilter); d->m_filter.clear(); + d->m_filter.removeOrigin(eMyMoney::Transaction::Origin::MatchingInput); // Text tab if (!d->ui->m_textEdit->text().isEmpty()) { diff --git a/kmymoney/widgets/stdtransactionmatched.cpp b/kmymoney/widgets/stdtransactionmatched.cpp --- a/kmymoney/widgets/stdtransactionmatched.cpp +++ b/kmymoney/widgets/stdtransactionmatched.cpp @@ -98,8 +98,12 @@ font.setItalic(true); painter->setFont(font); } + const auto& file = MyMoneyFile::instance(); + const auto matchedTransactionID = d->m_split.matchedTransaction(); + auto matchedTransaction = matchedTransactionID.isEmpty() ? + MyMoneyTransaction() : + file->transaction(matchedTransactionID); - MyMoneyTransaction matchedTransaction = d->m_split.matchedTransaction(); MyMoneySplit matchedSplit; try { matchedSplit = matchedTransaction.splitById(d->m_split.value("kmm-match-split"));