Differential D18664 Diff 51252 src/engine/postingdb.cpp

Changeset View

Standalone View

View Options

src/engine/postingdb.cpp

Show All 12 Lines
13 * Lesser General Public License for more details. 13​ * Lesser General Public License for more details.
14 * 14​ *
15 * You should have received a copy of the GNU Lesser General Public 15​ * You should have received a copy of the GNU Lesser General Public
16 * License along with this library; if not, write to the Free Software 16​ * License along with this library; if not, write to the Free Software
17 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 17​ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 * 18​ *
19 */ 19​ */
20 20​
21​#include "enginedebug.h"
21#include "postingdb.h" 22​#include "postingdb.h"
22#include "orpostingiterator.h" 23​#include "orpostingiterator.h"
23#include "postingcodec.h" 24​#include "postingcodec.h"
24 25​
25#include <QDebug> ​
26 ​
27using namespace Baloo; 26​using namespace Baloo;
28 27​
29PostingDB::PostingDB(MDB_dbi dbi, MDB_txn* txn) 28​PostingDB::PostingDB(MDB_dbi dbi, MDB_txn* txn)
30 : m_txn(txn) 29​ : m_txn(txn)
31 , m_dbi(dbi) 30​ , m_dbi(dbi)
32{ 31​{
33 Q_ASSERT(txn != nullptr); 32​ Q_ASSERT(txn != nullptr);
34 Q_ASSERT(dbi != 0); 33​ Q_ASSERT(dbi != 0);
35} 34​}
36 35​
37PostingDB::~PostingDB() 36​PostingDB::~PostingDB()
38{ 37​{
39} 38​}
40 39​
41MDB_dbi PostingDB::create(MDB_txn* txn) 40​MDB_dbi PostingDB::create(MDB_txn* txn)
42{ 41​{
43 MDB_dbi dbi; 42​ MDB_dbi dbi = 0;
44 int rc = mdb_dbi_open(txn, "postingdb", MDB_CREATE, &dbi); 43​ int rc = mdb_dbi_open(txn, "postingdb", MDB_CREATE, &dbi);
45 Q_ASSERT_X(rc == 0, "PostingDB::create", mdb_strerror(rc)); 44​ if (rc) {
45​ qCDebug(ENGINE) << "PostingDB::create" << mdb_strerror(rc);
46​ return 0;
47​ }
46 48​
47 return dbi; 49​ return dbi;
48} 50​}
49 51​
50MDB_dbi PostingDB::open(MDB_txn* txn) 52​MDB_dbi PostingDB::open(MDB_txn* txn)
51{ 53​{
52 MDB_dbi dbi; 54​ MDB_dbi dbi = 0;
53 int rc = mdb_dbi_open(txn, "postingdb", 0, &dbi); 55​ int rc = mdb_dbi_open(txn, "postingdb", 0, &dbi);
54 if (rc == MDB_NOTFOUND) { 56​ if (rc) {
57​ qCDebug(ENGINE) << "PostingDB::open" << mdb_strerror(rc);
55 return 0; 58​ return 0;
56 } 59​ }
57 Q_ASSERT_X(rc == 0, "PostingDB::open", mdb_strerror(rc)); ​
58 60​
59 return dbi; 61​ return dbi;
60} 62​}
61 63​
62void PostingDB::put(const QByteArray& term, const PostingList& list) 64​void PostingDB::put(const QByteArray& term, const PostingList& list)
63{ 65​{
64 Q_ASSERT(!term.isEmpty()); 66​ Q_ASSERT(!term.isEmpty());
65 Q_ASSERT(!list.isEmpty()); 67​ Q_ASSERT(!list.isEmpty());
66 68​
67 MDB_val key; 69​ MDB_val key;
68 key.mv_size = term.size(); 70​ key.mv_size = term.size();
69 key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); 71​ key.mv_data = static_cast<void*>(const_cast<char*>(term.constData()));
70 72​
71 PostingCodec codec; 73​ PostingCodec codec;
72 QByteArray arr = codec.encode(list); 74​ QByteArray arr = codec.encode(list);
73 75​
74 MDB_val val; 76​ MDB_val val;
75 val.mv_size = arr.size(); 77​ val.mv_size = arr.size();
76 val.mv_data = static_cast<void*>(arr.data()); 78​ val.mv_data = static_cast<void*>(arr.data());
77 79​
78 int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); 80​ int rc = mdb_put(m_txn, m_dbi, &key, &val, 0);
79 Q_ASSERT_X(rc == 0, "PostingDB::put", mdb_strerror(rc)); 81​ if (rc) {
82​ qCDebug(ENGINE) << "PostingDB::put" << mdb_strerror(rc);
83​ }
80} 84​}
81 85​
82PostingList PostingDB::get(const QByteArray& term) 86​PostingList PostingDB::get(const QByteArray& term)
83{ 87​{
84 Q_ASSERT(!term.isEmpty()); 88​ Q_ASSERT(!term.isEmpty());
85 89​
86 MDB_val key; 90​ MDB_val key;
87 key.mv_size = term.size(); 91​ key.mv_size = term.size();
88 key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); 92​ key.mv_data = static_cast<void*>(const_cast<char*>(term.constData()));
89 93​
90 MDB_val val; 94​ MDB_val val{0, nullptr};
91 int rc = mdb_get(m_txn, m_dbi, &key, &val); 95​ int rc = mdb_get(m_txn, m_dbi, &key, &val);
92 if (rc == MDB_NOTFOUND) { 96​ if (rc) {
97​ qCDebug(ENGINE) << "PostingDB::get" << term << mdb_strerror(rc);
93 return PostingList(); 98​ return PostingList();
94 } 99​ }
95 Q_ASSERT_X(rc == 0, "PostingDB::get", mdb_strerror(rc)); ​
96 100​
97 QByteArray arr = QByteArray::fromRawData(static_cast<char*>(val.mv_data), val.mv_size); 101​ QByteArray arr = QByteArray::fromRawData(static_cast<char*>(val.mv_data), val.mv_size);
98 102​
99 PostingCodec codec; 103​ PostingCodec codec;
100 return codec.decode(arr); 104​ return codec.decode(arr);
101} 105​}
102 106​
103void PostingDB::del(const QByteArray& term) 107​void PostingDB::del(const QByteArray& term)
104{ 108​{
105 Q_ASSERT(!term.isEmpty()); 109​ Q_ASSERT(!term.isEmpty());
106 110​
107 MDB_val key; 111​ MDB_val key;
108 key.mv_size = term.size(); 112​ key.mv_size = term.size();
109 key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); 113​ key.mv_data = static_cast<void*>(const_cast<char*>(term.constData()));
110 114​
111 int rc = mdb_del(m_txn, m_dbi, &key, nullptr); 115​ int rc = mdb_del(m_txn, m_dbi, &key, nullptr);
112 if (rc == MDB_NOTFOUND) { 116​ if (rc) {
117​ qCDebug(ENGINE) << "PostingDB::del" << term << mdb_strerror(rc);
113 return; 118​ return;
114 } 119​ }
115 Q_ASSERT_X(rc == 0, "PostingDB::del", mdb_strerror(rc)); ​
116} 120​}
117 121​
118QVector< QByteArray > PostingDB::fetchTermsStartingWith(const QByteArray& term) 122​QVector< QByteArray > PostingDB::fetchTermsStartingWith(const QByteArray& term)
119{ 123​{
120 MDB_val key; 124​ MDB_val key;
121 key.mv_size = term.size(); 125​ key.mv_size = term.size();
122 key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); 126​ key.mv_data = static_cast<void*>(const_cast<char*>(term.constData()));
123 127​
124 MDB_cursor* cursor; 128​ MDB_cursor* cursor;
125 mdb_cursor_open(m_txn, m_dbi, &cursor); 129​ mdb_cursor_open(m_txn, m_dbi, &cursor);
126 130​
127 QVector<QByteArray> terms; 131​ QVector<QByteArray> terms;
128 int rc = mdb_cursor_get(cursor, &key, nullptr, MDB_SET_RANGE); 132​ int rc = mdb_cursor_get(cursor, &key, nullptr, MDB_SET_RANGE);
129 while (rc != MDB_NOTFOUND) { 133​ while (rc == 0) {
poboikoUnsubmitted
Done

This assert is not necessary here then?

poboiko: This assert is not necessary here then?
130 Q_ASSERT_X(rc == 0, "PostingDB::fetchTermsStartingWith", mdb_strerror(rc)); ​
131 ​
132 const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size); 134​ const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size);
133 if (!arr.startsWith(term)) { 135​ if (!arr.startsWith(term)) {
134 break; 136​ break;
135 } 137​ }
136 terms << arr; 138​ terms << arr;
137 rc = mdb_cursor_get(cursor, &key, nullptr, MDB_NEXT); 139​ rc = mdb_cursor_get(cursor, &key, nullptr, MDB_NEXT);
138 } 140​ }
139 Q_ASSERT_X(rc == 0, "PostingDB::fetchTermsStartingWith", mdb_strerror(rc)); 141​ qCDebug(ENGINE) << "PostingDB::fetchTermsStartingWith" << mdb_strerror(rc);
brunsUnsubmitted
Done

if (rc != MDB_NOTFOUND) ...

bruns: if (rc != MDB_NOTFOUND) ...
140 142​
141 mdb_cursor_close(cursor); 143​ mdb_cursor_close(cursor);
142 return terms; 144​ return terms;
143} 145​}
144 146​
145class DBPostingIterator : public PostingIterator { 147​class DBPostingIterator : public PostingIterator {
146public: 148​public:
147 DBPostingIterator(void* data, uint size); 149​ DBPostingIterator(void* data, uint size);
148 quint64 docId() const override; 150​ quint64 docId() const override;
149 quint64 next() override; 151​ quint64 next() override;
150 152​
151private: 153​private:
152 const QVector<quint64> m_vec; 154​ const QVector<quint64> m_vec;
153 int m_pos; 155​ int m_pos;
154}; 156​};
155 157​
156PostingIterator* PostingDB::iter(const QByteArray& term) 158​PostingIterator* PostingDB::iter(const QByteArray& term)
157{ 159​{
158 MDB_val key; 160​ MDB_val key;
159 key.mv_size = term.size(); 161​ key.mv_size = term.size();
160 key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); 162​ key.mv_data = static_cast<void*>(const_cast<char*>(term.constData()));
161 163​
162 MDB_val val; 164​ MDB_val val;
163 int rc = mdb_get(m_txn, m_dbi, &key, &val); 165​ int rc = mdb_get(m_txn, m_dbi, &key, &val);
164 if (rc == MDB_NOTFOUND) { 166​ if (rc) {
167​ qCDebug(ENGINE) << "PostingDB::iter" << term << mdb_strerror(rc);
165 return nullptr; 168​ return nullptr;
166 } 169​ }
167 Q_ASSERT_X(rc == 0, "PostingDB::iter", mdb_strerror(rc)); ​
168 170​
169 return new DBPostingIterator(val.mv_data, val.mv_size); 171​ return new DBPostingIterator(val.mv_data, val.mv_size);
170} 172​}
171 173​
172// 174​//
173// Posting Iterator 175​// Posting Iterator
174// 176​//
175DBPostingIterator::DBPostingIterator(void* data, uint size) 177​DBPostingIterator::DBPostingIterator(void* data, uint size)
Show All 33 Lines205{
209 211​
210 MDB_cursor* cursor; 212​ MDB_cursor* cursor;
211 mdb_cursor_open(m_txn, m_dbi, &cursor); 213​ mdb_cursor_open(m_txn, m_dbi, &cursor);
212 214​
213 QVector<PostingIterator*> termIterators; 215​ QVector<PostingIterator*> termIterators;
214 216​
215 MDB_val val; 217​ MDB_val val;
216 int rc = mdb_cursor_get(cursor, &key, &val, MDB_SET_RANGE); 218​ int rc = mdb_cursor_get(cursor, &key, &val, MDB_SET_RANGE);
217 while (rc != MDB_NOTFOUND) { 219​ while (rc == 0) {
218 Q_ASSERT_X(rc == 0, "PostingDB::regexpIter", mdb_strerror(rc)); ​
219 ​
220 const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size); 220​ const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size);
221 if (!arr.startsWith(prefix)) { 221​ if (!arr.startsWith(prefix)) {
222 break; 222​ break;
223 } 223​ }
224 if (validate(arr)) { 224​ if (validate(arr)) {
225 termIterators << new DBPostingIterator(val.mv_data, val.mv_size); 225​ termIterators << new DBPostingIterator(val.mv_data, val.mv_size);
226 } 226​ }
227 rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); 227​ rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT);
228 } 228​ }
229 if (rc != MDB_NOTFOUND) { 229​
230 Q_ASSERT_X(rc == 0, "PostingDB::regexpIter", mdb_strerror(rc)); 230​ if (rc) {
231​ qCDebug(ENGINE) << "PostingDB::regexpIter" << mdb_strerror(rc);
232​ return nullptr;
brunsUnsubmitted
Done

critical - You are leaking the cursor now

bruns: //critical - You are leaking the cursor now//
brunsUnsubmitted
Done

not done ... you have to always mdb_cursor_close

Also, MDB_NOTFOUND is not an error here.

bruns: not done ... you have to always `mdb_cursor_close` Also, MDB_NOTFOUND is not an error here.
brunsUnsubmitted
Done

For the debug message, level warning is sufficient ;-)

bruns: For the debug message, level warning is sufficient ;-)
231 } 233​ }
232 234​
233 mdb_cursor_close(cursor); 235​ mdb_cursor_close(cursor);
234 if (termIterators.isEmpty()) { 236​ if (termIterators.isEmpty()) {
poboikoUnsubmitted
Done

Can it happen that rc == MDB_NOTFOUND just after some iterations of MDB_NEXT operation (i.e. we've reached the end of DB), but keys which popped up previously are perfectly valid, and we're just ignoring them here?
(shouldn't it read rc != 0 && rc != MDB_NOTFOUND, just like above?)

poboiko: Can it happen that `rc == MDB_NOTFOUND` just after some iterations of `MDB_NEXT` operation (i.e.
brunsUnsubmitted
Done

Yes, this is indeed wrong - it only works if the loop is left via 'if (!arr.startsWith(prefix)) break`, but not when all entries in the key range have a matching prefix.

If an additional rc check here is correct is hard to say. We are in an error condition when rc != {NOTFOUND, SUCCESS}, but all the values in termIterators are valid.

I vote for removing the rc check here again.

bruns: Yes, this is indeed wrong - it only works if the loop is left via 'if (!arr.startsWith(prefix))…
235 return nullptr; 237​ return nullptr;
236 } 238​ }
237 return new OrPostingIterator(termIterators); 239​ return new OrPostingIterator(termIterators);
238} 240​}
239 241​
240PostingIterator* PostingDB::prefixIter(const QByteArray& prefix) 242​PostingIterator* PostingDB::prefixIter(const QByteArray& prefix)
241{ 243​{
242 auto validate = [] (const QByteArray& arr) { 244​ auto validate = [] (const QByteArray& arr) {
Show All 31 Lines274{
274 mdb_cursor_open(m_txn, m_dbi, &cursor); 276​ mdb_cursor_open(m_txn, m_dbi, &cursor);
275 277​
276 MDB_val key = {0, nullptr}; 278​ MDB_val key = {0, nullptr};
277 MDB_val val; 279​ MDB_val val;
278 280​
279 QMap<QByteArray, PostingList> map; 281​ QMap<QByteArray, PostingList> map;
280 while (1) { 282​ while (1) {
281 int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); 283​ int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT);
282 if (rc == MDB_NOTFOUND) { 284​ if (rc) {
285​ qCDebug(ENGINE) << "PostingDB::toTestMap" << mdb_strerror(rc);
283 break; 286​ break;
284 } 287​ }
285 Q_ASSERT_X(rc == 0, "PostingDB::toTestMap", mdb_strerror(rc)); ​
286 288​
287 const QByteArray ba(static_cast<char*>(key.mv_data), key.mv_size); 289​ const QByteArray ba(static_cast<char*>(key.mv_data), key.mv_size);
288 const PostingList plist = PostingCodec().decode(QByteArray(static_cast<char*>(val.mv_data), val.mv_size)); 290​ const PostingList plist = PostingCodec().decode(QByteArray(static_cast<char*>(val.mv_data), val.mv_size));
289 map.insert(ba, plist); 291​ map.insert(ba, plist);
290 } 292​ }
291 293​
292 mdb_cursor_close(cursor); 294​ mdb_cursor_close(cursor);
293 return map; 295​ return map;
294} 296​}