Differential D18664 Diff 53838 src/engine/postingdb.cpp

Changeset View

Standalone View

View Options

src/engine/postingdb.cpp

Show All 12 Lines
13 * Lesser General Public License for more details. 13​ * Lesser General Public License for more details.
14 * 14​ *
15 * You should have received a copy of the GNU Lesser General Public 15​ * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software 16​ * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17​ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 * 18​ *
19 */ 19​ */
20 20​
21​#include "enginedebug.h"
21#include "postingdb.h" 22​#include "postingdb.h"
22#include "orpostingiterator.h" 23​#include "orpostingiterator.h"
23#include "postingcodec.h" 24​#include "postingcodec.h"
24 25​
25#include <QDebug> ​
26 ​
27using namespace Baloo; 26​using namespace Baloo;
28 27​
29PostingDB::PostingDB(MDB_dbi dbi, MDB_txn* txn) 28​PostingDB::PostingDB(MDB_dbi dbi, MDB_txn* txn)
30 : m_txn(txn) 29​ : m_txn(txn)
31 , m_dbi(dbi) 30​ , m_dbi(dbi)
32{ 31​{
33 Q_ASSERT(txn != nullptr); 32​ Q_ASSERT(txn != nullptr);
34 Q_ASSERT(dbi != 0); 33​ Q_ASSERT(dbi != 0);
35} 34​}
36 35​
37PostingDB::~PostingDB() 36​PostingDB::~PostingDB()
38{ 37​{
39} 38​}
40 39​
41MDB_dbi PostingDB::create(MDB_txn* txn) 40​MDB_dbi PostingDB::create(MDB_txn* txn)
42{ 41​{
43 MDB_dbi dbi; 42​ MDB_dbi dbi = 0;
44 int rc = mdb_dbi_open(txn, "postingdb", MDB_CREATE, &dbi); 43​ int rc = mdb_dbi_open(txn, "postingdb", MDB_CREATE, &dbi);
45 Q_ASSERT_X(rc == 0, "PostingDB::create", mdb_strerror(rc)); 44​ if (rc) {
45​ qCWarning(ENGINE) << "PostingDB::create" << mdb_strerror(rc);
46​ return 0;
47​ }
46 48​
47 return dbi; 49​ return dbi;
48} 50​}
49 51​
50MDB_dbi PostingDB::open(MDB_txn* txn) 52​MDB_dbi PostingDB::open(MDB_txn* txn)
51{ 53​{
52 MDB_dbi dbi; 54​ MDB_dbi dbi = 0;
53 int rc = mdb_dbi_open(txn, "postingdb", 0, &dbi); 55​ int rc = mdb_dbi_open(txn, "postingdb", 0, &dbi);
54 if (rc == MDB_NOTFOUND) { 56​ if (rc) {
57​ qCWarning(ENGINE) << "PostingDB::open" << mdb_strerror(rc);
55 return 0; 58​ return 0;
56 } 59​ }
57 Q_ASSERT_X(rc == 0, "PostingDB::open", mdb_strerror(rc)); ​
58 60​
59 return dbi; 61​ return dbi;
60} 62​}
61 63​
62void PostingDB::put(const QByteArray& term, const PostingList& list) 64​void PostingDB::put(const QByteArray& term, const PostingList& list)
63{ 65​{
64 Q_ASSERT(!term.isEmpty()); 66​ Q_ASSERT(!term.isEmpty());
65 Q_ASSERT(!list.isEmpty()); 67​ Q_ASSERT(!list.isEmpty());
66 68​
67 MDB_val key; 69​ MDB_val key;
68 key.mv_size = term.size(); 70​ key.mv_size = term.size();
69 key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); 71​ key.mv_data = static_cast<void*>(const_cast<char*>(term.constData()));
70 72​
71 PostingCodec codec; 73​ PostingCodec codec;
72 QByteArray arr = codec.encode(list); 74​ QByteArray arr = codec.encode(list);
73 75​
74 MDB_val val; 76​ MDB_val val;
75 val.mv_size = arr.size(); 77​ val.mv_size = arr.size();
76 val.mv_data = static_cast<void*>(arr.data()); 78​ val.mv_data = static_cast<void*>(arr.data());
77 79​
78 int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); 80​ int rc = mdb_put(m_txn, m_dbi, &key, &val, 0);
79 Q_ASSERT_X(rc == 0, "PostingDB::put", mdb_strerror(rc)); 81​ if (rc) {
82​ qCWarning(ENGINE) << "PostingDB::put" << mdb_strerror(rc);
83​ }
80} 84​}
81 85​
82PostingList PostingDB::get(const QByteArray& term) 86​PostingList PostingDB::get(const QByteArray& term)
83{ 87​{
84 Q_ASSERT(!term.isEmpty()); 88​ Q_ASSERT(!term.isEmpty());
85 89​
86 MDB_val key; 90​ MDB_val key;
87 key.mv_size = term.size(); 91​ key.mv_size = term.size();
88 key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); 92​ key.mv_data = static_cast<void*>(const_cast<char*>(term.constData()));
89 93​
90 MDB_val val; 94​ MDB_val val{0, nullptr};
91 int rc = mdb_get(m_txn, m_dbi, &key, &val); 95​ int rc = mdb_get(m_txn, m_dbi, &key, &val);
92 if (rc == MDB_NOTFOUND) { 96​ if (rc) {
97​ if (rc != MDB_NOTFOUND) {
98​ qCDebug(ENGINE) << "PostingDB::get" << term << mdb_strerror(rc);
99​ }
93 return PostingList(); 100​ return PostingList();
94 } 101​ }
95 Q_ASSERT_X(rc == 0, "PostingDB::get", mdb_strerror(rc)); ​
96 102​
97 QByteArray arr = QByteArray::fromRawData(static_cast<char*>(val.mv_data), val.mv_size); 103​ QByteArray arr = QByteArray::fromRawData(static_cast<char*>(val.mv_data), val.mv_size);
98 104​
99 PostingCodec codec; 105​ PostingCodec codec;
100 return codec.decode(arr); 106​ return codec.decode(arr);
101} 107​}
102 108​
103void PostingDB::del(const QByteArray& term) 109​void PostingDB::del(const QByteArray& term)
104{ 110​{
105 Q_ASSERT(!term.isEmpty()); 111​ Q_ASSERT(!term.isEmpty());
106 112​
107 MDB_val key; 113​ MDB_val key;
108 key.mv_size = term.size(); 114​ key.mv_size = term.size();
109 key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); 115​ key.mv_data = static_cast<void*>(const_cast<char*>(term.constData()));
110 116​
111 int rc = mdb_del(m_txn, m_dbi, &key, nullptr); 117​ int rc = mdb_del(m_txn, m_dbi, &key, nullptr);
112 if (rc == MDB_NOTFOUND) { 118​ if (rc != 0 && rc != MDB_NOTFOUND) {
113 return; 119​ qCDebug(ENGINE) << "PostingDB::del" << term << mdb_strerror(rc);
114 } 120​ }
115 Q_ASSERT_X(rc == 0, "PostingDB::del", mdb_strerror(rc)); ​
116} 121​}
117 122​
118QVector< QByteArray > PostingDB::fetchTermsStartingWith(const QByteArray& term) 123​QVector< QByteArray > PostingDB::fetchTermsStartingWith(const QByteArray& term)
119{ 124​{
120 MDB_val key; 125​ MDB_val key;
121 key.mv_size = term.size(); 126​ key.mv_size = term.size();
122 key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); 127​ key.mv_data = static_cast<void*>(const_cast<char*>(term.constData()));
123 128​
124 MDB_cursor* cursor; 129​ MDB_cursor* cursor;
125 mdb_cursor_open(m_txn, m_dbi, &cursor); 130​ mdb_cursor_open(m_txn, m_dbi, &cursor);
126 131​
127 QVector<QByteArray> terms; 132​ QVector<QByteArray> terms;
128 int rc = mdb_cursor_get(cursor, &key, nullptr, MDB_SET_RANGE); 133​ int rc = mdb_cursor_get(cursor, &key, nullptr, MDB_SET_RANGE);
129 while (rc != MDB_NOTFOUND) { 134​ while (rc == 0) {
poboikoUnsubmitted
Done

This assert is not necessary here then?

poboiko: This assert is not necessary here then?
130 Q_ASSERT_X(rc == 0, "PostingDB::fetchTermsStartingWith", mdb_strerror(rc)); ​
131 ​
132 const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size); 135​ const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size);
133 if (!arr.startsWith(term)) { 136​ if (!arr.startsWith(term)) {
134 break; 137​ break;
135 } 138​ }
136 terms << arr; 139​ terms << arr;
137 rc = mdb_cursor_get(cursor, &key, nullptr, MDB_NEXT); 140​ rc = mdb_cursor_get(cursor, &key, nullptr, MDB_NEXT);
138 } 141​ }
139 Q_ASSERT_X(rc == 0, "PostingDB::fetchTermsStartingWith", mdb_strerror(rc)); 142​ if (rc != MDB_NOTFOUND) {
143​ qCDebug(ENGINE) << "PostingDB::fetchTermsStartingWith" << mdb_strerror(rc);
144​ }
brunsUnsubmitted
Done

if (rc != MDB_NOTFOUND) ...

bruns: if (rc != MDB_NOTFOUND) ...
140 145​
141 mdb_cursor_close(cursor); 146​ mdb_cursor_close(cursor);
142 return terms; 147​ return terms;
143} 148​}
144 149​
145class DBPostingIterator : public PostingIterator { 150​class DBPostingIterator : public PostingIterator {
146public: 151​public:
147 DBPostingIterator(void* data, uint size); 152​ DBPostingIterator(void* data, uint size);
148 quint64 docId() const override; 153​ quint64 docId() const override;
149 quint64 next() override; 154​ quint64 next() override;
150 155​
151private: 156​private:
152 const QVector<quint64> m_vec; 157​ const QVector<quint64> m_vec;
153 int m_pos; 158​ int m_pos;
154}; 159​};
155 160​
156PostingIterator* PostingDB::iter(const QByteArray& term) 161​PostingIterator* PostingDB::iter(const QByteArray& term)
157{ 162​{
158 MDB_val key; 163​ MDB_val key;
159 key.mv_size = term.size(); 164​ key.mv_size = term.size();
160 key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); 165​ key.mv_data = static_cast<void*>(const_cast<char*>(term.constData()));
161 166​
162 MDB_val val; 167​ MDB_val val;
163 int rc = mdb_get(m_txn, m_dbi, &key, &val); 168​ int rc = mdb_get(m_txn, m_dbi, &key, &val);
164 if (rc == MDB_NOTFOUND) { 169​ if (rc) {
170​ qCDebug(ENGINE) << "PostingDB::iter" << term << mdb_strerror(rc);
165 return nullptr; 171​ return nullptr;
166 } 172​ }
167 Q_ASSERT_X(rc == 0, "PostingDB::iter", mdb_strerror(rc)); ​
168 173​
169 return new DBPostingIterator(val.mv_data, val.mv_size); 174​ return new DBPostingIterator(val.mv_data, val.mv_size);
170} 175​}
171 176​
172// 177​//
173// Posting Iterator 178​// Posting Iterator
174// 179​//
175DBPostingIterator::DBPostingIterator(void* data, uint size) 180​DBPostingIterator::DBPostingIterator(void* data, uint size)
Show All 33 Lines208{
209 214​
210 MDB_cursor* cursor; 215​ MDB_cursor* cursor;
211 mdb_cursor_open(m_txn, m_dbi, &cursor); 216​ mdb_cursor_open(m_txn, m_dbi, &cursor);
212 217​
213 QVector<PostingIterator*> termIterators; 218​ QVector<PostingIterator*> termIterators;
214 219​
215 MDB_val val; 220​ MDB_val val;
216 int rc = mdb_cursor_get(cursor, &key, &val, MDB_SET_RANGE); 221​ int rc = mdb_cursor_get(cursor, &key, &val, MDB_SET_RANGE);
217 while (rc != MDB_NOTFOUND) { 222​ while (rc == 0) {
218 Q_ASSERT_X(rc == 0, "PostingDB::regexpIter", mdb_strerror(rc)); ​
219 ​
220 const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size); 223​ const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size);
221 if (!arr.startsWith(prefix)) { 224​ if (!arr.startsWith(prefix)) {
222 break; 225​ break;
223 } 226​ }
224 if (validate(arr)) { 227​ if (validate(arr)) {
225 termIterators << new DBPostingIterator(val.mv_data, val.mv_size); 228​ termIterators << new DBPostingIterator(val.mv_data, val.mv_size);
226 } 229​ }
227 rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); 230​ rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT);
228 } 231​ }
229 if (rc != MDB_NOTFOUND) { 232​
230 Q_ASSERT_X(rc == 0, "PostingDB::regexpIter", mdb_strerror(rc)); 233​ if (rc != 0 && rc != MDB_NOTFOUND) {
234​ qCWarning(ENGINE) << "PostingDB::regexpIter" << mdb_strerror(rc);
231 } 235​ }
brunsUnsubmitted
Done

critical - You are leaking the cursor now

bruns: //critical - You are leaking the cursor now//
brunsUnsubmitted
Done

not done ... you have to always mdb_cursor_close

Also, MDB_NOTFOUND is not an error here.

bruns: not done ... you have to always `mdb_cursor_close` Also, MDB_NOTFOUND is not an error here.
brunsUnsubmitted
Done

For the debug message, level warning is sufficient ;-)

bruns: For the debug message, level warning is sufficient ;-)
232 236​
233 mdb_cursor_close(cursor); 237​ mdb_cursor_close(cursor);
234 if (termIterators.isEmpty()) { 238​ if (termIterators.isEmpty()) {
poboikoUnsubmitted
Done

Can it happen that rc == MDB_NOTFOUND just after some iterations of MDB_NEXT operation (i.e. we've reached the end of DB), but keys which popped up previously are perfectly valid, and we're just ignoring them here?
(shouldn't it read rc != 0 && rc != MDB_NOTFOUND, just like above?)

poboiko: Can it happen that `rc == MDB_NOTFOUND` just after some iterations of `MDB_NEXT` operation (i.e.
brunsUnsubmitted
Done

Yes, this is indeed wrong - it only works if the loop is left via 'if (!arr.startsWith(prefix)) break`, but not when all entries in the key range have a matching prefix.

If an additional rc check here is correct is hard to say. We are in an error condition when rc != {NOTFOUND, SUCCESS}, but all the values in termIterators are valid.

I vote for removing the rc check here again.

bruns: Yes, this is indeed wrong - it only works if the loop is left via 'if (!arr.startsWith(prefix))…
235 return nullptr; 239​ return nullptr;
236 } 240​ }
237 return new OrPostingIterator(termIterators); 241​ return new OrPostingIterator(termIterators);
238} 242​}
239 243​
240PostingIterator* PostingDB::prefixIter(const QByteArray& prefix) 244​PostingIterator* PostingDB::prefixIter(const QByteArray& prefix)
241{ 245​{
242 auto validate = [] (const QByteArray& arr) { 246​ auto validate = [] (const QByteArray& arr) {
Show All 31 Lines276{
274 mdb_cursor_open(m_txn, m_dbi, &cursor); 278​ mdb_cursor_open(m_txn, m_dbi, &cursor);
275 279​
276 MDB_val key = {0, nullptr}; 280​ MDB_val key = {0, nullptr};
277 MDB_val val; 281​ MDB_val val;
278 282​
279 QMap<QByteArray, PostingList> map; 283​ QMap<QByteArray, PostingList> map;
280 while (1) { 284​ while (1) {
281 int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); 285​ int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT);
282 if (rc == MDB_NOTFOUND) { 286​ if (rc) {
287​ qCDebug(ENGINE) << "PostingDB::toTestMap" << mdb_strerror(rc);
283 break; 288​ break;
284 } 289​ }
285 Q_ASSERT_X(rc == 0, "PostingDB::toTestMap", mdb_strerror(rc)); ​
286 290​
287 const QByteArray ba(static_cast<char*>(key.mv_data), key.mv_size); 291​ const QByteArray ba(static_cast<char*>(key.mv_data), key.mv_size);
288 const PostingList plist = PostingCodec().decode(QByteArray(static_cast<char*>(val.mv_data), val.mv_size)); 292​ const PostingList plist = PostingCodec().decode(QByteArray(static_cast<char*>(val.mv_data), val.mv_size));
289 map.insert(ba, plist); 293​ map.insert(ba, plist);
290 } 294​ }
291 295​
292 mdb_cursor_close(cursor); 296​ mdb_cursor_close(cursor);
293 return map; 297​ return map;
294} 298​}