Changeset View
Changeset View
Standalone View
Standalone View
src/engine/postingdb.cpp
Show All 12 Lines | |||||
13 | * Lesser General Public License for more details. | 13 | * Lesser General Public License for more details. | ||
14 | * | 14 | * | ||
15 | * You should have received a copy of the GNU Lesser General Public | 15 | * You should have received a copy of the GNU Lesser General Public | ||
16 | * License along with this library; if not, write to the Free Software | 16 | * License along with this library; if not, write to the Free Software | ||
17 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | 17 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
18 | * | 18 | * | ||
19 | */ | 19 | */ | ||
20 | 20 | | |||
21 | #include "enginedebug.h" | ||||
21 | #include "postingdb.h" | 22 | #include "postingdb.h" | ||
22 | #include "orpostingiterator.h" | 23 | #include "orpostingiterator.h" | ||
23 | #include "postingcodec.h" | 24 | #include "postingcodec.h" | ||
24 | 25 | | |||
25 | #include <QDebug> | | |||
26 | | ||||
27 | using namespace Baloo; | 26 | using namespace Baloo; | ||
28 | 27 | | |||
29 | PostingDB::PostingDB(MDB_dbi dbi, MDB_txn* txn) | 28 | PostingDB::PostingDB(MDB_dbi dbi, MDB_txn* txn) | ||
30 | : m_txn(txn) | 29 | : m_txn(txn) | ||
31 | , m_dbi(dbi) | 30 | , m_dbi(dbi) | ||
32 | { | 31 | { | ||
33 | Q_ASSERT(txn != nullptr); | 32 | Q_ASSERT(txn != nullptr); | ||
34 | Q_ASSERT(dbi != 0); | 33 | Q_ASSERT(dbi != 0); | ||
35 | } | 34 | } | ||
36 | 35 | | |||
37 | PostingDB::~PostingDB() | 36 | PostingDB::~PostingDB() | ||
38 | { | 37 | { | ||
39 | } | 38 | } | ||
40 | 39 | | |||
41 | MDB_dbi PostingDB::create(MDB_txn* txn) | 40 | MDB_dbi PostingDB::create(MDB_txn* txn) | ||
42 | { | 41 | { | ||
43 | MDB_dbi dbi; | 42 | MDB_dbi dbi = 0; | ||
44 | int rc = mdb_dbi_open(txn, "postingdb", MDB_CREATE, &dbi); | 43 | int rc = mdb_dbi_open(txn, "postingdb", MDB_CREATE, &dbi); | ||
45 | Q_ASSERT_X(rc == 0, "PostingDB::create", mdb_strerror(rc)); | 44 | if (rc) { | ||
45 | qCWarning(ENGINE) << "PostingDB::create" << mdb_strerror(rc); | ||||
46 | return 0; | ||||
47 | } | ||||
46 | 48 | | |||
47 | return dbi; | 49 | return dbi; | ||
48 | } | 50 | } | ||
49 | 51 | | |||
50 | MDB_dbi PostingDB::open(MDB_txn* txn) | 52 | MDB_dbi PostingDB::open(MDB_txn* txn) | ||
51 | { | 53 | { | ||
52 | MDB_dbi dbi; | 54 | MDB_dbi dbi = 0; | ||
53 | int rc = mdb_dbi_open(txn, "postingdb", 0, &dbi); | 55 | int rc = mdb_dbi_open(txn, "postingdb", 0, &dbi); | ||
54 | if (rc == MDB_NOTFOUND) { | 56 | if (rc) { | ||
57 | qCWarning(ENGINE) << "PostingDB::open" << mdb_strerror(rc); | ||||
55 | return 0; | 58 | return 0; | ||
56 | } | 59 | } | ||
57 | Q_ASSERT_X(rc == 0, "PostingDB::open", mdb_strerror(rc)); | | |||
58 | 60 | | |||
59 | return dbi; | 61 | return dbi; | ||
60 | } | 62 | } | ||
61 | 63 | | |||
62 | void PostingDB::put(const QByteArray& term, const PostingList& list) | 64 | void PostingDB::put(const QByteArray& term, const PostingList& list) | ||
63 | { | 65 | { | ||
64 | Q_ASSERT(!term.isEmpty()); | 66 | Q_ASSERT(!term.isEmpty()); | ||
65 | Q_ASSERT(!list.isEmpty()); | 67 | Q_ASSERT(!list.isEmpty()); | ||
66 | 68 | | |||
67 | MDB_val key; | 69 | MDB_val key; | ||
68 | key.mv_size = term.size(); | 70 | key.mv_size = term.size(); | ||
69 | key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); | 71 | key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); | ||
70 | 72 | | |||
71 | PostingCodec codec; | 73 | PostingCodec codec; | ||
72 | QByteArray arr = codec.encode(list); | 74 | QByteArray arr = codec.encode(list); | ||
73 | 75 | | |||
74 | MDB_val val; | 76 | MDB_val val; | ||
75 | val.mv_size = arr.size(); | 77 | val.mv_size = arr.size(); | ||
76 | val.mv_data = static_cast<void*>(arr.data()); | 78 | val.mv_data = static_cast<void*>(arr.data()); | ||
77 | 79 | | |||
78 | int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); | 80 | int rc = mdb_put(m_txn, m_dbi, &key, &val, 0); | ||
79 | Q_ASSERT_X(rc == 0, "PostingDB::put", mdb_strerror(rc)); | 81 | if (rc) { | ||
82 | qCWarning(ENGINE) << "PostingDB::put" << mdb_strerror(rc); | ||||
83 | } | ||||
80 | } | 84 | } | ||
81 | 85 | | |||
82 | PostingList PostingDB::get(const QByteArray& term) | 86 | PostingList PostingDB::get(const QByteArray& term) | ||
83 | { | 87 | { | ||
84 | Q_ASSERT(!term.isEmpty()); | 88 | Q_ASSERT(!term.isEmpty()); | ||
85 | 89 | | |||
86 | MDB_val key; | 90 | MDB_val key; | ||
87 | key.mv_size = term.size(); | 91 | key.mv_size = term.size(); | ||
88 | key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); | 92 | key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); | ||
89 | 93 | | |||
90 | MDB_val val; | 94 | MDB_val val{0, nullptr}; | ||
91 | int rc = mdb_get(m_txn, m_dbi, &key, &val); | 95 | int rc = mdb_get(m_txn, m_dbi, &key, &val); | ||
92 | if (rc == MDB_NOTFOUND) { | 96 | if (rc) { | ||
97 | if (rc != MDB_NOTFOUND) { | ||||
98 | qCDebug(ENGINE) << "PostingDB::get" << term << mdb_strerror(rc); | ||||
99 | } | ||||
93 | return PostingList(); | 100 | return PostingList(); | ||
94 | } | 101 | } | ||
95 | Q_ASSERT_X(rc == 0, "PostingDB::get", mdb_strerror(rc)); | | |||
96 | 102 | | |||
97 | QByteArray arr = QByteArray::fromRawData(static_cast<char*>(val.mv_data), val.mv_size); | 103 | QByteArray arr = QByteArray::fromRawData(static_cast<char*>(val.mv_data), val.mv_size); | ||
98 | 104 | | |||
99 | PostingCodec codec; | 105 | PostingCodec codec; | ||
100 | return codec.decode(arr); | 106 | return codec.decode(arr); | ||
101 | } | 107 | } | ||
102 | 108 | | |||
103 | void PostingDB::del(const QByteArray& term) | 109 | void PostingDB::del(const QByteArray& term) | ||
104 | { | 110 | { | ||
105 | Q_ASSERT(!term.isEmpty()); | 111 | Q_ASSERT(!term.isEmpty()); | ||
106 | 112 | | |||
107 | MDB_val key; | 113 | MDB_val key; | ||
108 | key.mv_size = term.size(); | 114 | key.mv_size = term.size(); | ||
109 | key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); | 115 | key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); | ||
110 | 116 | | |||
111 | int rc = mdb_del(m_txn, m_dbi, &key, nullptr); | 117 | int rc = mdb_del(m_txn, m_dbi, &key, nullptr); | ||
112 | if (rc == MDB_NOTFOUND) { | 118 | if (rc != 0 && rc != MDB_NOTFOUND) { | ||
113 | return; | 119 | qCDebug(ENGINE) << "PostingDB::del" << term << mdb_strerror(rc); | ||
114 | } | 120 | } | ||
115 | Q_ASSERT_X(rc == 0, "PostingDB::del", mdb_strerror(rc)); | | |||
116 | } | 121 | } | ||
117 | 122 | | |||
118 | QVector< QByteArray > PostingDB::fetchTermsStartingWith(const QByteArray& term) | 123 | QVector< QByteArray > PostingDB::fetchTermsStartingWith(const QByteArray& term) | ||
119 | { | 124 | { | ||
120 | MDB_val key; | 125 | MDB_val key; | ||
121 | key.mv_size = term.size(); | 126 | key.mv_size = term.size(); | ||
122 | key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); | 127 | key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); | ||
123 | 128 | | |||
124 | MDB_cursor* cursor; | 129 | MDB_cursor* cursor; | ||
125 | mdb_cursor_open(m_txn, m_dbi, &cursor); | 130 | mdb_cursor_open(m_txn, m_dbi, &cursor); | ||
126 | 131 | | |||
127 | QVector<QByteArray> terms; | 132 | QVector<QByteArray> terms; | ||
128 | int rc = mdb_cursor_get(cursor, &key, nullptr, MDB_SET_RANGE); | 133 | int rc = mdb_cursor_get(cursor, &key, nullptr, MDB_SET_RANGE); | ||
129 | while (rc != MDB_NOTFOUND) { | 134 | while (rc == 0) { | ||
poboiko: This assert is not necessary here then? | |||||
130 | Q_ASSERT_X(rc == 0, "PostingDB::fetchTermsStartingWith", mdb_strerror(rc)); | | |||
131 | | ||||
132 | const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size); | 135 | const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size); | ||
133 | if (!arr.startsWith(term)) { | 136 | if (!arr.startsWith(term)) { | ||
134 | break; | 137 | break; | ||
135 | } | 138 | } | ||
136 | terms << arr; | 139 | terms << arr; | ||
137 | rc = mdb_cursor_get(cursor, &key, nullptr, MDB_NEXT); | 140 | rc = mdb_cursor_get(cursor, &key, nullptr, MDB_NEXT); | ||
138 | } | 141 | } | ||
139 | Q_ASSERT_X(rc == 0, "PostingDB::fetchTermsStartingWith", mdb_strerror(rc)); | 142 | if (rc != MDB_NOTFOUND) { | ||
143 | qCDebug(ENGINE) << "PostingDB::fetchTermsStartingWith" << mdb_strerror(rc); | ||||
144 | } | ||||
bruns: if (rc != MDB_NOTFOUND) ... | |||||
140 | 145 | | |||
141 | mdb_cursor_close(cursor); | 146 | mdb_cursor_close(cursor); | ||
142 | return terms; | 147 | return terms; | ||
143 | } | 148 | } | ||
144 | 149 | | |||
145 | class DBPostingIterator : public PostingIterator { | 150 | class DBPostingIterator : public PostingIterator { | ||
146 | public: | 151 | public: | ||
147 | DBPostingIterator(void* data, uint size); | 152 | DBPostingIterator(void* data, uint size); | ||
148 | quint64 docId() const override; | 153 | quint64 docId() const override; | ||
149 | quint64 next() override; | 154 | quint64 next() override; | ||
150 | 155 | | |||
151 | private: | 156 | private: | ||
152 | const QVector<quint64> m_vec; | 157 | const QVector<quint64> m_vec; | ||
153 | int m_pos; | 158 | int m_pos; | ||
154 | }; | 159 | }; | ||
155 | 160 | | |||
156 | PostingIterator* PostingDB::iter(const QByteArray& term) | 161 | PostingIterator* PostingDB::iter(const QByteArray& term) | ||
157 | { | 162 | { | ||
158 | MDB_val key; | 163 | MDB_val key; | ||
159 | key.mv_size = term.size(); | 164 | key.mv_size = term.size(); | ||
160 | key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); | 165 | key.mv_data = static_cast<void*>(const_cast<char*>(term.constData())); | ||
161 | 166 | | |||
162 | MDB_val val; | 167 | MDB_val val; | ||
163 | int rc = mdb_get(m_txn, m_dbi, &key, &val); | 168 | int rc = mdb_get(m_txn, m_dbi, &key, &val); | ||
164 | if (rc == MDB_NOTFOUND) { | 169 | if (rc) { | ||
170 | qCDebug(ENGINE) << "PostingDB::iter" << term << mdb_strerror(rc); | ||||
165 | return nullptr; | 171 | return nullptr; | ||
166 | } | 172 | } | ||
167 | Q_ASSERT_X(rc == 0, "PostingDB::iter", mdb_strerror(rc)); | | |||
168 | 173 | | |||
169 | return new DBPostingIterator(val.mv_data, val.mv_size); | 174 | return new DBPostingIterator(val.mv_data, val.mv_size); | ||
170 | } | 175 | } | ||
171 | 176 | | |||
172 | // | 177 | // | ||
173 | // Posting Iterator | 178 | // Posting Iterator | ||
174 | // | 179 | // | ||
175 | DBPostingIterator::DBPostingIterator(void* data, uint size) | 180 | DBPostingIterator::DBPostingIterator(void* data, uint size) | ||
Show All 33 Lines | 208 | { | |||
209 | 214 | | |||
210 | MDB_cursor* cursor; | 215 | MDB_cursor* cursor; | ||
211 | mdb_cursor_open(m_txn, m_dbi, &cursor); | 216 | mdb_cursor_open(m_txn, m_dbi, &cursor); | ||
212 | 217 | | |||
213 | QVector<PostingIterator*> termIterators; | 218 | QVector<PostingIterator*> termIterators; | ||
214 | 219 | | |||
215 | MDB_val val; | 220 | MDB_val val; | ||
216 | int rc = mdb_cursor_get(cursor, &key, &val, MDB_SET_RANGE); | 221 | int rc = mdb_cursor_get(cursor, &key, &val, MDB_SET_RANGE); | ||
217 | while (rc != MDB_NOTFOUND) { | 222 | while (rc == 0) { | ||
218 | Q_ASSERT_X(rc == 0, "PostingDB::regexpIter", mdb_strerror(rc)); | | |||
219 | | ||||
220 | const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size); | 223 | const QByteArray arr(static_cast<char*>(key.mv_data), key.mv_size); | ||
221 | if (!arr.startsWith(prefix)) { | 224 | if (!arr.startsWith(prefix)) { | ||
222 | break; | 225 | break; | ||
223 | } | 226 | } | ||
224 | if (validate(arr)) { | 227 | if (validate(arr)) { | ||
225 | termIterators << new DBPostingIterator(val.mv_data, val.mv_size); | 228 | termIterators << new DBPostingIterator(val.mv_data, val.mv_size); | ||
226 | } | 229 | } | ||
227 | rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); | 230 | rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); | ||
228 | } | 231 | } | ||
229 | if (rc != MDB_NOTFOUND) { | 232 | | ||
230 | Q_ASSERT_X(rc == 0, "PostingDB::regexpIter", mdb_strerror(rc)); | 233 | if (rc != 0 && rc != MDB_NOTFOUND) { | ||
234 | qCWarning(ENGINE) << "PostingDB::regexpIter" << mdb_strerror(rc); | ||||
231 | } | 235 | } | ||
bruns: //critical - You are leaking the cursor now//
| |||||
not done ... you have to always mdb_cursor_close Also, MDB_NOTFOUND is not an error here. bruns: not done ... you have to always `mdb_cursor_close`
Also, MDB_NOTFOUND is not an error here. | |||||
bruns: For the debug message, level warning is sufficient ;-) | |||||
232 | 236 | | |||
233 | mdb_cursor_close(cursor); | 237 | mdb_cursor_close(cursor); | ||
234 | if (termIterators.isEmpty()) { | 238 | if (rc || termIterators.isEmpty()) { | ||
Can it happen that rc == MDB_NOTFOUND just after some iterations of MDB_NEXT operation (i.e. we've reached the end of DB), but keys which popped up previously are perfectly valid, and we're just ignoring them here? poboiko: Can it happen that `rc == MDB_NOTFOUND` just after some iterations of `MDB_NEXT` operation (i.e. | |||||
Yes, this is indeed wrong - it only works if the loop is left via 'if (!arr.startsWith(prefix)) break`, but not when all entries in the key range have a matching prefix. If an additional rc check here is correct is hard to say. We are in an error condition when rc != {NOTFOUND, SUCCESS}, but all the values in termIterators are valid. I vote for removing the rc check here again. bruns: Yes, this is indeed wrong - it only works if the loop is left via 'if (!arr.startsWith(prefix))… | |||||
235 | return nullptr; | 239 | return nullptr; | ||
236 | } | 240 | } | ||
237 | return new OrPostingIterator(termIterators); | 241 | return new OrPostingIterator(termIterators); | ||
238 | } | 242 | } | ||
239 | 243 | | |||
240 | PostingIterator* PostingDB::prefixIter(const QByteArray& prefix) | 244 | PostingIterator* PostingDB::prefixIter(const QByteArray& prefix) | ||
241 | { | 245 | { | ||
242 | auto validate = [] (const QByteArray& arr) { | 246 | auto validate = [] (const QByteArray& arr) { | ||
Show All 31 Lines | 276 | { | |||
274 | mdb_cursor_open(m_txn, m_dbi, &cursor); | 278 | mdb_cursor_open(m_txn, m_dbi, &cursor); | ||
275 | 279 | | |||
276 | MDB_val key = {0, nullptr}; | 280 | MDB_val key = {0, nullptr}; | ||
277 | MDB_val val; | 281 | MDB_val val; | ||
278 | 282 | | |||
279 | QMap<QByteArray, PostingList> map; | 283 | QMap<QByteArray, PostingList> map; | ||
280 | while (1) { | 284 | while (1) { | ||
281 | int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); | 285 | int rc = mdb_cursor_get(cursor, &key, &val, MDB_NEXT); | ||
282 | if (rc == MDB_NOTFOUND) { | 286 | if (rc) { | ||
287 | qCDebug(ENGINE) << "PostingDB::toTestMap" << mdb_strerror(rc); | ||||
283 | break; | 288 | break; | ||
284 | } | 289 | } | ||
285 | Q_ASSERT_X(rc == 0, "PostingDB::toTestMap", mdb_strerror(rc)); | | |||
286 | 290 | | |||
287 | const QByteArray ba(static_cast<char*>(key.mv_data), key.mv_size); | 291 | const QByteArray ba(static_cast<char*>(key.mv_data), key.mv_size); | ||
288 | const PostingList plist = PostingCodec().decode(QByteArray(static_cast<char*>(val.mv_data), val.mv_size)); | 292 | const PostingList plist = PostingCodec().decode(QByteArray(static_cast<char*>(val.mv_data), val.mv_size)); | ||
289 | map.insert(ba, plist); | 293 | map.insert(ba, plist); | ||
290 | } | 294 | } | ||
291 | 295 | | |||
292 | mdb_cursor_close(cursor); | 296 | mdb_cursor_close(cursor); | ||
293 | return map; | 297 | return map; | ||
294 | } | 298 | } |
This assert is not necessary here then?