Changeset View
Changeset View
Standalone View
Standalone View
common/typeindex.cpp
Show All 15 Lines | 1 | /* | |||
---|---|---|---|---|---|
16 | Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA | 16 | Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA | ||
17 | 02110-1301, USA. | 17 | 02110-1301, USA. | ||
18 | */ | 18 | */ | ||
19 | #include "typeindex.h" | 19 | #include "typeindex.h" | ||
20 | 20 | | |||
21 | #include "log.h" | 21 | #include "log.h" | ||
22 | #include "index.h" | 22 | #include "index.h" | ||
23 | #include "fulltextindex.h" | 23 | #include "fulltextindex.h" | ||
24 | | ||||
24 | #include <QDateTime> | 25 | #include <QDateTime> | ||
25 | #include <QDataStream> | 26 | #include <QDataStream> | ||
26 | 27 | | |||
28 | #include <cmath> | ||||
29 | | ||||
27 | using namespace Sink; | 30 | using namespace Sink; | ||
28 | 31 | | |||
29 | static QByteArray getByteArray(const QVariant &value) | 32 | static QByteArray getByteArray(const QVariant &value) | ||
30 | { | 33 | { | ||
31 | if (value.type() == QVariant::DateTime) { | 34 | if (value.type() == QVariant::DateTime) { | ||
32 | QByteArray result; | 35 | QByteArray result; | ||
33 | QDataStream ds(&result, QIODevice::WriteOnly); | 36 | QDataStream ds(&result, QIODevice::WriteOnly); | ||
34 | ds << value.toDateTime(); | 37 | ds << value.toDateTime(); | ||
Show All 10 Lines | |||||
45 | } | 48 | } | ||
46 | if (value.isValid() && !value.toByteArray().isEmpty()) { | 49 | if (value.isValid() && !value.toByteArray().isEmpty()) { | ||
47 | return value.toByteArray(); | 50 | return value.toByteArray(); | ||
48 | } | 51 | } | ||
49 | // LMDB can't handle empty keys, so use something different | 52 | // LMDB can't handle empty keys, so use something different | ||
50 | return "toplevel"; | 53 | return "toplevel"; | ||
51 | } | 54 | } | ||
52 | 55 | | |||
53 | static QByteArray toSortableByteArray(const QDateTime &date) | 56 | | ||
57 | static QByteArray toSortableByteArrayImpl(const QDateTime &date) | ||||
54 | { | 58 | { | ||
55 | // Sort invalid last | 59 | // Sort invalid last | ||
56 | if (!date.isValid()) { | 60 | if (!date.isValid()) { | ||
57 | return QByteArray::number(std::numeric_limits<unsigned int>::max()); | 61 | return QByteArray::number(std::numeric_limits<unsigned int>::max()); | ||
58 | } | 62 | } | ||
59 | return QByteArray::number(std::numeric_limits<unsigned int>::max() - date.toTime_t()); | 63 | static unsigned int uint_num_digits = std::log10(std::numeric_limits<unsigned int>::max()) + 1; | ||
64 | return QByteArray::number(std::numeric_limits<unsigned int>::max() - date.toTime_t()).rightJustified(uint_num_digits, '0'); | ||||
65 | } | ||||
66 | | ||||
67 | static QByteArray toSortableByteArray(const QVariant &value) | ||||
68 | { | ||||
69 | if (!value.isValid()) { | ||||
70 | // FIXME: we don't know the type, so we don't know what to return | ||||
71 | // This mean we're fixing every sorted index keys to use unsigned int | ||||
72 | return QByteArray::number(std::numeric_limits<unsigned int>::max()); | ||||
60 | } | 73 | } | ||
61 | 74 | | |||
75 | switch (value.type()) { | ||||
76 | case QMetaType::QDateTime: | ||||
77 | return toSortableByteArrayImpl(value.toDateTime()); | ||||
78 | default: | ||||
79 | SinkWarning() << "Not knowing how to convert a" << value.typeName() | ||||
80 | << "to a sortable key, falling back to default conversion"; | ||||
81 | return getByteArray(value); | ||||
82 | } | ||||
83 | } | ||||
62 | 84 | | |||
63 | TypeIndex::TypeIndex(const QByteArray &type, const Sink::Log::Context &ctx) : mLogCtx(ctx), mType(type) | 85 | TypeIndex::TypeIndex(const QByteArray &type, const Sink::Log::Context &ctx) : mLogCtx(ctx), mType(type) | ||
64 | { | 86 | { | ||
65 | } | 87 | } | ||
66 | 88 | | |||
67 | QByteArray TypeIndex::indexName(const QByteArray &property, const QByteArray &sortProperty) const | 89 | QByteArray TypeIndex::indexName(const QByteArray &property, const QByteArray &sortProperty) const | ||
68 | { | 90 | { | ||
69 | if (sortProperty.isEmpty()) { | 91 | if (sortProperty.isEmpty()) { | ||
70 | return mType + ".index." + property; | 92 | return mType + ".index." + property; | ||
71 | } | 93 | } | ||
72 | return mType + ".index." + property + ".sort." + sortProperty; | 94 | return mType + ".index." + property + ".sort." + sortProperty; | ||
73 | } | 95 | } | ||
74 | 96 | | |||
97 | QByteArray TypeIndex::sortedIndexName(const QByteArray &property) const | ||||
98 | { | ||||
99 | return mType + ".index." + property + ".sorted"; | ||||
100 | } | ||||
101 | | ||||
75 | template <> | 102 | template <> | ||
76 | void TypeIndex::addProperty<QByteArray>(const QByteArray &property) | 103 | void TypeIndex::addProperty<QByteArray>(const QByteArray &property) | ||
77 | { | 104 | { | ||
78 | auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { | 105 | auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { | ||
79 | // SinkTraceCtx(mLogCtx) << "Indexing " << mType + ".index." + property << value.toByteArray(); | 106 | // SinkTraceCtx(mLogCtx) << "Indexing " << mType + ".index." + property << value.toByteArray(); | ||
80 | if (add) { | 107 | if (add) { | ||
81 | Index(indexName(property), transaction).add(getByteArray(value), identifier); | 108 | Index(indexName(property), transaction).add(getByteArray(value), identifier); | ||
82 | } else { | 109 | } else { | ||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Line(s) | |||||
133 | 160 | | |||
134 | template <> | 161 | template <> | ||
135 | void TypeIndex::addProperty<ApplicationDomain::Reference>(const QByteArray &property) | 162 | void TypeIndex::addProperty<ApplicationDomain::Reference>(const QByteArray &property) | ||
136 | { | 163 | { | ||
137 | addProperty<QByteArray>(property); | 164 | addProperty<QByteArray>(property); | ||
138 | } | 165 | } | ||
139 | 166 | | |||
140 | template <> | 167 | template <> | ||
168 | void TypeIndex::addSortedProperty<QDateTime>(const QByteArray &property) | ||||
169 | { | ||||
170 | auto indexer = [this, property](bool add, const QByteArray &identifier, const QVariant &value, | ||||
171 | Sink::Storage::DataStore::Transaction &transaction) { | ||||
172 | const auto sortableDate = toSortableByteArray(value); | ||||
173 | if (add) { | ||||
174 | Index(sortedIndexName(property), transaction).add(sortableDate, identifier); | ||||
175 | } else { | ||||
176 | Index(sortedIndexName(property), transaction).remove(sortableDate, identifier); | ||||
177 | } | ||||
178 | }; | ||||
179 | mSortIndexer.insert(property, indexer); | ||||
180 | mSortedProperties << property; | ||||
181 | } | ||||
182 | | ||||
183 | template <> | ||||
141 | void TypeIndex::addPropertyWithSorting<QByteArray, QDateTime>(const QByteArray &property, const QByteArray &sortProperty) | 184 | void TypeIndex::addPropertyWithSorting<QByteArray, QDateTime>(const QByteArray &property, const QByteArray &sortProperty) | ||
142 | { | 185 | { | ||
143 | auto indexer = [=](bool add, const QByteArray &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction) { | 186 | auto indexer = [=](bool add, const QByteArray &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction) { | ||
144 | const auto date = sortValue.toDateTime(); | 187 | const auto date = sortValue.toDateTime(); | ||
145 | const auto propertyValue = getByteArray(value); | 188 | const auto propertyValue = getByteArray(value); | ||
146 | if (add) { | 189 | if (add) { | ||
147 | Index(indexName(property, sortProperty), transaction).add(propertyValue + toSortableByteArray(date), identifier); | 190 | Index(indexName(property, sortProperty), transaction).add(propertyValue + toSortableByteArray(date), identifier); | ||
148 | } else { | 191 | } else { | ||
149 | Index(indexName(property, sortProperty), transaction).remove(propertyValue + toSortableByteArray(date), identifier); | 192 | Index(indexName(property, sortProperty), transaction).remove(propertyValue + toSortableByteArray(date), identifier); | ||
150 | } | 193 | } | ||
151 | }; | 194 | }; | ||
152 | mSortIndexer.insert(property + sortProperty, indexer); | 195 | mGroupedSortIndexer.insert(property + sortProperty, indexer); | ||
153 | mSortedProperties.insert(property, sortProperty); | 196 | mGroupedSortedProperties.insert(property, sortProperty); | ||
154 | } | 197 | } | ||
155 | 198 | | |||
156 | template <> | 199 | template <> | ||
157 | void TypeIndex::addPropertyWithSorting<ApplicationDomain::Reference, QDateTime>(const QByteArray &property, const QByteArray &sortProperty) | 200 | void TypeIndex::addPropertyWithSorting<ApplicationDomain::Reference, QDateTime>(const QByteArray &property, const QByteArray &sortProperty) | ||
158 | { | 201 | { | ||
159 | addPropertyWithSorting<QByteArray, QDateTime>(property, sortProperty); | 202 | addPropertyWithSorting<QByteArray, QDateTime>(property, sortProperty); | ||
160 | } | 203 | } | ||
161 | 204 | | |||
162 | void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) | 205 | void TypeIndex::updateIndex(bool add, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) | ||
163 | { | 206 | { | ||
164 | for (const auto &property : mProperties) { | 207 | for (const auto &property : mProperties) { | ||
165 | const auto value = entity.getProperty(property); | 208 | const auto value = entity.getProperty(property); | ||
166 | auto indexer = mIndexer.value(property); | 209 | auto indexer = mIndexer.value(property); | ||
167 | indexer(add, identifier, value, transaction); | 210 | indexer(add, identifier, value, transaction); | ||
168 | } | 211 | } | ||
169 | for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { | 212 | for (const auto &property : mSortedProperties) { | ||
213 | const auto value = entity.getProperty(property); | ||||
214 | auto indexer = mSortIndexer.value(property); | ||||
215 | indexer(add, identifier, value, transaction); | ||||
216 | } | ||||
217 | for (auto it = mGroupedSortedProperties.constBegin(); it != mGroupedSortedProperties.constEnd(); it++) { | ||||
170 | const auto value = entity.getProperty(it.key()); | 218 | const auto value = entity.getProperty(it.key()); | ||
171 | const auto sortValue = entity.getProperty(it.value()); | 219 | const auto sortValue = entity.getProperty(it.value()); | ||
172 | auto indexer = mSortIndexer.value(it.key() + it.value()); | 220 | auto indexer = mGroupedSortIndexer.value(it.key() + it.value()); | ||
173 | indexer(add, identifier, value, sortValue, transaction); | 221 | indexer(add, identifier, value, sortValue, transaction); | ||
174 | } | 222 | } | ||
175 | for (const auto &indexer : mCustomIndexer) { | 223 | for (const auto &indexer : mCustomIndexer) { | ||
176 | indexer->setup(this, &transaction, resourceInstanceId); | 224 | indexer->setup(this, &transaction, resourceInstanceId); | ||
177 | if (add) { | 225 | if (add) { | ||
178 | indexer->add(entity); | 226 | indexer->add(entity); | ||
179 | } else { | 227 | } else { | ||
180 | indexer->remove(entity); | 228 | indexer->remove(entity); | ||
Show All 21 Lines | 249 | { | |||
202 | updateIndex(true, identifier, entity, transaction, resourceInstanceId); | 250 | updateIndex(true, identifier, entity, transaction, resourceInstanceId); | ||
203 | } | 251 | } | ||
204 | 252 | | |||
205 | void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) | 253 | void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) | ||
206 | { | 254 | { | ||
207 | updateIndex(false, identifier, entity, transaction, resourceInstanceId); | 255 | updateIndex(false, identifier, entity, transaction, resourceInstanceId); | ||
208 | } | 256 | } | ||
209 | 257 | | |||
210 | static QVector<QByteArray> indexLookup(Index &index, QueryBase::Comparator filter) | 258 | static QVector<QByteArray> indexLookup(Index &index, QueryBase::Comparator filter, | ||
259 | std::function<QByteArray(const QVariant &)> valueToKey = getByteArray) | ||||
211 | { | 260 | { | ||
212 | QVector<QByteArray> keys; | 261 | QVector<QByteArray> keys; | ||
213 | QByteArrayList lookupKeys; | 262 | QByteArrayList lookupKeys; | ||
214 | if (filter.comparator == Query::Comparator::Equals) { | 263 | if (filter.comparator == Query::Comparator::Equals) { | ||
215 | lookupKeys << getByteArray(filter.value); | 264 | lookupKeys << valueToKey(filter.value); | ||
216 | } else if (filter.comparator == Query::Comparator::In) { | 265 | } else if (filter.comparator == Query::Comparator::In) { | ||
217 | lookupKeys = filter.value.value<QByteArrayList>(); | 266 | for(const QVariant &value : filter.value.value<QVariantList>()) { | ||
267 | lookupKeys << valueToKey(value); | ||||
268 | } | ||||
218 | } else { | 269 | } else { | ||
219 | Q_ASSERT(false); | 270 | Q_ASSERT(false); | ||
220 | } | 271 | } | ||
221 | 272 | | |||
222 | for (const auto &lookupKey : lookupKeys) { | 273 | for (const auto &lookupKey : lookupKeys) { | ||
223 | index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; }, | 274 | index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; }, | ||
224 | [lookupKey](const Index::Error &error) { SinkWarning() << "Lookup error in index: " << error.message << lookupKey; }, true); | 275 | [lookupKey](const Index::Error &error) { | ||
276 | SinkWarning() << "Lookup error in index: " << error.message << lookupKey; | ||||
277 | }, | ||||
278 | true); | ||||
279 | } | ||||
280 | return keys; | ||||
281 | } | ||||
282 | | ||||
283 | static QVector<QByteArray> sortedIndexLookup(Index &index, QueryBase::Comparator filter) | ||||
284 | { | ||||
285 | if (filter.comparator == Query::Comparator::In || filter.comparator == Query::Comparator::Contains) { | ||||
286 | SinkWarning() << "In and Contains comparison not supported on sorted indexes"; | ||||
287 | } | ||||
288 | | ||||
289 | if (filter.comparator != Query::Comparator::Within) { | ||||
290 | return indexLookup(index, filter, toSortableByteArray); | ||||
291 | } | ||||
292 | | ||||
293 | QVector<QByteArray> keys; | ||||
294 | | ||||
295 | QByteArray lowerBound, upperBound; | ||||
296 | auto bounds = filter.value.value<QVariantList>(); | ||||
297 | if (bounds[0].canConvert<QDateTime>()) { | ||||
298 | // Inverse the bounds because dates are stored newest first | ||||
299 | upperBound = toSortableByteArray(bounds[0].toDateTime()); | ||||
300 | lowerBound = toSortableByteArray(bounds[1].toDateTime()); | ||||
301 | } else { | ||||
302 | lowerBound = bounds[0].toByteArray(); | ||||
303 | upperBound = bounds[1].toByteArray(); | ||||
225 | } | 304 | } | ||
305 | | ||||
306 | index.rangeLookup(lowerBound, upperBound, [&](const QByteArray &value) { keys << value; }, | ||||
307 | [bounds](const Index::Error &error) { | ||||
308 | SinkWarning() << "Lookup error in index:" << error.message | ||||
309 | << "with bounds:" << bounds[0] << bounds[1]; | ||||
310 | }); | ||||
311 | | ||||
226 | return keys; | 312 | return keys; | ||
227 | } | 313 | } | ||
228 | 314 | | |||
229 | QVector<QByteArray> TypeIndex::query(const Sink::QueryBase &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) | 315 | QVector<QByteArray> TypeIndex::query(const Sink::QueryBase &query, QSet<QByteArray> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) | ||
230 | { | 316 | { | ||
231 | const auto baseFilters = query.getBaseFilters(); | 317 | const auto baseFilters = query.getBaseFilters(); | ||
232 | for (auto it = baseFilters.constBegin(); it != baseFilters.constEnd(); it++) { | 318 | for (auto it = baseFilters.constBegin(); it != baseFilters.constEnd(); it++) { | ||
233 | if (it.value().comparator == QueryBase::Comparator::Fulltext) { | 319 | if (it.value().comparator == QueryBase::Comparator::Fulltext) { | ||
234 | FulltextIndex fulltextIndex{resourceInstanceId}; | 320 | FulltextIndex fulltextIndex{resourceInstanceId}; | ||
235 | const auto keys = fulltextIndex.lookup(it.value().value.toString()); | 321 | const auto keys = fulltextIndex.lookup(it.value().value.toString()); | ||
236 | appliedFilters << it.key(); | 322 | appliedFilters << it.key(); | ||
237 | SinkTraceCtx(mLogCtx) << "Fulltext index lookup found " << keys.size() << " keys."; | 323 | SinkTraceCtx(mLogCtx) << "Fulltext index lookup found " << keys.size() << " keys."; | ||
238 | return keys; | 324 | return keys; | ||
239 | } | 325 | } | ||
240 | } | 326 | } | ||
241 | 327 | | |||
242 | for (auto it = mSortedProperties.constBegin(); it != mSortedProperties.constEnd(); it++) { | 328 | for (auto it = mGroupedSortedProperties.constBegin(); it != mGroupedSortedProperties.constEnd(); it++) { | ||
243 | if (query.hasFilter(it.key()) && query.sortProperty() == it.value()) { | 329 | if (query.hasFilter(it.key()) && query.sortProperty() == it.value()) { | ||
244 | Index index(indexName(it.key(), it.value()), transaction); | 330 | Index index(indexName(it.key(), it.value()), transaction); | ||
245 | const auto keys = indexLookup(index, query.getFilter(it.key())); | 331 | const auto keys = indexLookup(index, query.getFilter(it.key())); | ||
246 | appliedFilters << it.key(); | 332 | appliedFilters << it.key(); | ||
247 | appliedSorting = it.value(); | 333 | appliedSorting = it.value(); | ||
248 | SinkTraceCtx(mLogCtx) << "Sorted index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys."; | 334 | SinkTraceCtx(mLogCtx) << "Grouped sorted index lookup on " << it.key() << it.value() << " found " << keys.size() << " keys."; | ||
249 | return keys; | 335 | return keys; | ||
250 | } | 336 | } | ||
251 | } | 337 | } | ||
338 | | ||||
339 | for (const auto &property : mSortedProperties) { | ||||
340 | if (query.hasFilter(property)) { | ||||
341 | Index index(sortedIndexName(property), transaction); | ||||
342 | const auto keys = sortedIndexLookup(index, query.getFilter(property)); | ||||
343 | appliedFilters << property; | ||||
344 | SinkTraceCtx(mLogCtx) << "Sorted index lookup on " << property << " found " << keys.size() << " keys."; | ||||
345 | return keys; | ||||
346 | } | ||||
347 | } | ||||
348 | | ||||
252 | for (const auto &property : mProperties) { | 349 | for (const auto &property : mProperties) { | ||
253 | if (query.hasFilter(property)) { | 350 | if (query.hasFilter(property)) { | ||
254 | Index index(indexName(property), transaction); | 351 | Index index(indexName(property), transaction); | ||
255 | const auto keys = indexLookup(index, query.getFilter(property)); | 352 | const auto keys = indexLookup(index, query.getFilter(property)); | ||
256 | appliedFilters << property; | 353 | appliedFilters << property; | ||
257 | SinkTraceCtx(mLogCtx) << "Index lookup on " << property << " found " << keys.size() << " keys."; | 354 | SinkTraceCtx(mLogCtx) << "Index lookup on " << property << " found " << keys.size() << " keys."; | ||
258 | return keys; | 355 | return keys; | ||
259 | } | 356 | } | ||
▲ Show 20 Lines • Show All 84 Lines • Show Last 20 Lines |