1 #include <mbgl/storage/offline_database.hpp>
2 #include <mbgl/storage/response.hpp>
3 #include <mbgl/util/compression.hpp>
4 #include <mbgl/util/io.hpp>
5 #include <mbgl/util/string.hpp>
6 #include <mbgl/util/chrono.hpp>
7 #include <mbgl/util/logging.hpp>
8
9 #include "offline_schema.hpp"
10
11 #include "sqlite3.hpp"
12
13 namespace mbgl {
14
OfflineDatabase(std::string path_,uint64_t maximumCacheSize_)15 OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_)
16 : path(std::move(path_)),
17 maximumCacheSize(maximumCacheSize_) {
18 ensureSchema();
19 }
20
~OfflineDatabase()21 OfflineDatabase::~OfflineDatabase() {
22 // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
23 // can't throw anything.
24 try {
25 statements.clear();
26 db.reset();
27 } catch (mapbox::sqlite::Exception& ex) {
28 Log::Error(Event::Database, (int)ex.code, ex.what());
29 }
30 }
31
ensureSchema()32 void OfflineDatabase::ensureSchema() {
33 auto result = mapbox::sqlite::Database::tryOpen(path, mapbox::sqlite::ReadWriteCreate);
34 if (result.is<mapbox::sqlite::Exception>()) {
35 const auto& ex = result.get<mapbox::sqlite::Exception>();
36 if (ex.code == mapbox::sqlite::ResultCode::NotADB) {
37 // Corrupted; blow it away.
38 removeExisting();
39 result = mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate);
40 } else {
41 Log::Error(Event::Database, "Unexpected error connecting to database: %s", ex.what());
42 throw ex;
43 }
44 }
45
46 try {
47 assert(result.is<mapbox::sqlite::Database>());
48 db = std::make_unique<mapbox::sqlite::Database>(std::move(result.get<mapbox::sqlite::Database>()));
49 db->setBusyTimeout(Milliseconds::max());
50 db->exec("PRAGMA foreign_keys = ON");
51
52 switch (userVersion()) {
53 case 0:
54 case 1:
55 // Newly created database, or old cache-only database; remove old table if it exists.
56 removeOldCacheTable();
57 break;
58 case 2:
59 migrateToVersion3();
60 // fall through
61 case 3:
62 case 4:
63 migrateToVersion5();
64 // fall through
65 case 5:
66 migrateToVersion6();
67 // fall through
68 case 6:
69 // happy path; we're done
70 return;
71 default:
72 // downgrade, delete the database
73 removeExisting();
74 break;
75 }
76 } catch (const mapbox::sqlite::Exception& ex) {
77 // Unfortunately, SQLITE_NOTADB is not always reported upon opening the database.
78 // Apparently sometimes it is delayed until the first read operation.
79 if (ex.code == mapbox::sqlite::ResultCode::NotADB) {
80 removeExisting();
81 } else {
82 throw;
83 }
84 }
85
86 try {
87 // When downgrading the database, or when the database is corrupt, we've deleted the old database handle,
88 // so we need to reopen it.
89 if (!db) {
90 db = std::make_unique<mapbox::sqlite::Database>(mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate));
91 db->setBusyTimeout(Milliseconds::max());
92 db->exec("PRAGMA foreign_keys = ON");
93 }
94
95 db->exec("PRAGMA auto_vacuum = INCREMENTAL");
96 db->exec("PRAGMA journal_mode = DELETE");
97 db->exec("PRAGMA synchronous = FULL");
98 db->exec(offlineDatabaseSchema);
99 db->exec("PRAGMA user_version = 6");
100 } catch (...) {
101 Log::Error(Event::Database, "Unexpected error creating database schema: %s", util::toString(std::current_exception()).c_str());
102 throw;
103 }
104 }
105
userVersion()106 int OfflineDatabase::userVersion() {
107 return static_cast<int>(getPragma<int64_t>("PRAGMA user_version"));
108 }
109
removeExisting()110 void OfflineDatabase::removeExisting() {
111 Log::Warning(Event::Database, "Removing existing incompatible offline database");
112
113 statements.clear();
114 db.reset();
115
116 try {
117 util::deleteFile(path);
118 } catch (util::IOException& ex) {
119 Log::Error(Event::Database, ex.code, ex.what());
120 }
121 }
122
removeOldCacheTable()123 void OfflineDatabase::removeOldCacheTable() {
124 db->exec("DROP TABLE IF EXISTS http_cache");
125 db->exec("VACUUM");
126 }
127
migrateToVersion3()128 void OfflineDatabase::migrateToVersion3() {
129 db->exec("PRAGMA auto_vacuum = INCREMENTAL");
130 db->exec("VACUUM");
131 db->exec("PRAGMA user_version = 3");
132 }
133
134 // Schema version 4 was WAL journal + NORMAL sync. It was reverted during pre-
135 // release development and the migration was removed entirely to avoid potential
136 // conflicts from quickly (and needlessly) switching journal and sync modes.
137 //
138 // See: https://github.com/mapbox/mapbox-gl-native/pull/6320
139
migrateToVersion5()140 void OfflineDatabase::migrateToVersion5() {
141 db->exec("PRAGMA journal_mode = DELETE");
142 db->exec("PRAGMA synchronous = FULL");
143 db->exec("PRAGMA user_version = 5");
144 }
145
migrateToVersion6()146 void OfflineDatabase::migrateToVersion6() {
147 mapbox::sqlite::Transaction transaction(*db);
148 db->exec("ALTER TABLE resources ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0");
149 db->exec("ALTER TABLE tiles ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0");
150 db->exec("PRAGMA user_version = 6");
151 transaction.commit();
152 }
153
getStatement(const char * sql)154 mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) {
155 auto it = statements.find(sql);
156 if (it == statements.end()) {
157 it = statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(*db, sql)).first;
158 }
159 return *it->second;
160 }
161
get(const Resource & resource)162 optional<Response> OfflineDatabase::get(const Resource& resource) {
163 auto result = getInternal(resource);
164 return result ? result->first : optional<Response>();
165 }
166
getInternal(const Resource & resource)167 optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) {
168 if (resource.kind == Resource::Kind::Tile) {
169 assert(resource.tileData);
170 return getTile(*resource.tileData);
171 } else {
172 return getResource(resource);
173 }
174 }
175
hasInternal(const Resource & resource)176 optional<int64_t> OfflineDatabase::hasInternal(const Resource& resource) {
177 if (resource.kind == Resource::Kind::Tile) {
178 assert(resource.tileData);
179 return hasTile(*resource.tileData);
180 } else {
181 return hasResource(resource);
182 }
183 }
184
put(const Resource & resource,const Response & response)185 std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) {
186 mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate);
187 auto result = putInternal(resource, response, true);
188 transaction.commit();
189 return result;
190 }
191
putInternal(const Resource & resource,const Response & response,bool evict_)192 std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, const Response& response, bool evict_) {
193 if (response.error) {
194 return { false, 0 };
195 }
196
197 std::string compressedData;
198 bool compressed = false;
199 uint64_t size = 0;
200
201 if (response.data) {
202 compressedData = util::compress(*response.data);
203 compressed = compressedData.size() < response.data->size();
204 size = compressed ? compressedData.size() : response.data->size();
205 }
206
207 if (evict_ && !evict(size)) {
208 Log::Info(Event::Database, "Unable to make space for entry");
209 return { false, 0 };
210 }
211
212 bool inserted;
213
214 if (resource.kind == Resource::Kind::Tile) {
215 assert(resource.tileData);
216 inserted = putTile(*resource.tileData, response,
217 compressed ? compressedData : response.data ? *response.data : "",
218 compressed);
219 } else {
220 inserted = putResource(resource, response,
221 compressed ? compressedData : response.data ? *response.data : "",
222 compressed);
223 }
224
225 return { inserted, size };
226 }
227
getResource(const Resource & resource)228 optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resource& resource) {
229 // Update accessed timestamp used for LRU eviction.
230 {
231 mapbox::sqlite::Query accessedQuery{ getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2") };
232 accessedQuery.bind(1, util::now());
233 accessedQuery.bind(2, resource.url);
234 accessedQuery.run();
235 }
236
237 // clang-format off
238 mapbox::sqlite::Query query{ getStatement(
239 // 0 1 2 3 4 5
240 "SELECT etag, expires, must_revalidate, modified, data, compressed "
241 "FROM resources "
242 "WHERE url = ?") };
243 // clang-format on
244
245 query.bind(1, resource.url);
246
247 if (!query.run()) {
248 return {};
249 }
250
251 Response response;
252 uint64_t size = 0;
253
254 response.etag = query.get<optional<std::string>>(0);
255 response.expires = query.get<optional<Timestamp>>(1);
256 response.mustRevalidate = query.get<bool>(2);
257 response.modified = query.get<optional<Timestamp>>(3);
258
259 auto data = query.get<optional<std::string>>(4);
260 if (!data) {
261 response.noContent = true;
262 } else if (query.get<bool>(5)) {
263 response.data = std::make_shared<std::string>(util::decompress(*data));
264 size = data->length();
265 } else {
266 response.data = std::make_shared<std::string>(*data);
267 size = data->length();
268 }
269
270 return std::make_pair(response, size);
271 }
272
hasResource(const Resource & resource)273 optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) {
274 mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") };
275 query.bind(1, resource.url);
276 if (!query.run()) {
277 return {};
278 }
279
280 return query.get<optional<int64_t>>(0);
281 }
282
putResource(const Resource & resource,const Response & response,const std::string & data,bool compressed)283 bool OfflineDatabase::putResource(const Resource& resource,
284 const Response& response,
285 const std::string& data,
286 bool compressed) {
287 if (response.notModified) {
288 // clang-format off
289 mapbox::sqlite::Query notModifiedQuery{ getStatement(
290 "UPDATE resources "
291 "SET accessed = ?1, "
292 " expires = ?2, "
293 " must_revalidate = ?3 "
294 "WHERE url = ?4 ") };
295 // clang-format on
296
297 notModifiedQuery.bind(1, util::now());
298 notModifiedQuery.bind(2, response.expires);
299 notModifiedQuery.bind(3, response.mustRevalidate);
300 notModifiedQuery.bind(4, resource.url);
301 notModifiedQuery.run();
302 return false;
303 }
304
305 // We can't use REPLACE because it would change the id value.
306 // clang-format off
307 mapbox::sqlite::Query updateQuery{ getStatement(
308 "UPDATE resources "
309 "SET kind = ?1, "
310 " etag = ?2, "
311 " expires = ?3, "
312 " must_revalidate = ?4, "
313 " modified = ?5, "
314 " accessed = ?6, "
315 " data = ?7, "
316 " compressed = ?8 "
317 "WHERE url = ?9 ") };
318 // clang-format on
319
320 updateQuery.bind(1, int(resource.kind));
321 updateQuery.bind(2, response.etag);
322 updateQuery.bind(3, response.expires);
323 updateQuery.bind(4, response.mustRevalidate);
324 updateQuery.bind(5, response.modified);
325 updateQuery.bind(6, util::now());
326 updateQuery.bind(9, resource.url);
327
328 if (response.noContent) {
329 updateQuery.bind(7, nullptr);
330 updateQuery.bind(8, false);
331 } else {
332 updateQuery.bindBlob(7, data.data(), data.size(), false);
333 updateQuery.bind(8, compressed);
334 }
335
336 updateQuery.run();
337 if (updateQuery.changes() != 0) {
338 return false;
339 }
340
341 // clang-format off
342 mapbox::sqlite::Query insertQuery{ getStatement(
343 "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) "
344 "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") };
345 // clang-format on
346
347 insertQuery.bind(1, resource.url);
348 insertQuery.bind(2, int(resource.kind));
349 insertQuery.bind(3, response.etag);
350 insertQuery.bind(4, response.expires);
351 insertQuery.bind(5, response.mustRevalidate);
352 insertQuery.bind(6, response.modified);
353 insertQuery.bind(7, util::now());
354
355 if (response.noContent) {
356 insertQuery.bind(8, nullptr);
357 insertQuery.bind(9, false);
358 } else {
359 insertQuery.bindBlob(8, data.data(), data.size(), false);
360 insertQuery.bind(9, compressed);
361 }
362
363 insertQuery.run();
364
365 return true;
366 }
367
getTile(const Resource::TileData & tile)368 optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) {
369 {
370 // clang-format off
371 mapbox::sqlite::Query accessedQuery{ getStatement(
372 "UPDATE tiles "
373 "SET accessed = ?1 "
374 "WHERE url_template = ?2 "
375 " AND pixel_ratio = ?3 "
376 " AND x = ?4 "
377 " AND y = ?5 "
378 " AND z = ?6 ") };
379 // clang-format on
380
381 accessedQuery.bind(1, util::now());
382 accessedQuery.bind(2, tile.urlTemplate);
383 accessedQuery.bind(3, tile.pixelRatio);
384 accessedQuery.bind(4, tile.x);
385 accessedQuery.bind(5, tile.y);
386 accessedQuery.bind(6, tile.z);
387 accessedQuery.run();
388 }
389
390 // clang-format off
391 mapbox::sqlite::Query query{ getStatement(
392 // 0 1 2, 3, 4, 5
393 "SELECT etag, expires, must_revalidate, modified, data, compressed "
394 "FROM tiles "
395 "WHERE url_template = ?1 "
396 " AND pixel_ratio = ?2 "
397 " AND x = ?3 "
398 " AND y = ?4 "
399 " AND z = ?5 ") };
400 // clang-format on
401
402 query.bind(1, tile.urlTemplate);
403 query.bind(2, tile.pixelRatio);
404 query.bind(3, tile.x);
405 query.bind(4, tile.y);
406 query.bind(5, tile.z);
407
408 if (!query.run()) {
409 return {};
410 }
411
412 Response response;
413 uint64_t size = 0;
414
415 response.etag = query.get<optional<std::string>>(0);
416 response.expires = query.get<optional<Timestamp>>(1);
417 response.mustRevalidate = query.get<bool>(2);
418 response.modified = query.get<optional<Timestamp>>(3);
419
420 optional<std::string> data = query.get<optional<std::string>>(4);
421 if (!data) {
422 response.noContent = true;
423 } else if (query.get<bool>(5)) {
424 response.data = std::make_shared<std::string>(util::decompress(*data));
425 size = data->length();
426 } else {
427 response.data = std::make_shared<std::string>(*data);
428 size = data->length();
429 }
430
431 return std::make_pair(response, size);
432 }
433
hasTile(const Resource::TileData & tile)434 optional<int64_t> OfflineDatabase::hasTile(const Resource::TileData& tile) {
435 // clang-format off
436 mapbox::sqlite::Query size{ getStatement(
437 "SELECT length(data) "
438 "FROM tiles "
439 "WHERE url_template = ?1 "
440 " AND pixel_ratio = ?2 "
441 " AND x = ?3 "
442 " AND y = ?4 "
443 " AND z = ?5 ") };
444 // clang-format on
445
446 size.bind(1, tile.urlTemplate);
447 size.bind(2, tile.pixelRatio);
448 size.bind(3, tile.x);
449 size.bind(4, tile.y);
450 size.bind(5, tile.z);
451
452 if (!size.run()) {
453 return {};
454 }
455
456 return size.get<optional<int64_t>>(0);
457 }
458
putTile(const Resource::TileData & tile,const Response & response,const std::string & data,bool compressed)459 bool OfflineDatabase::putTile(const Resource::TileData& tile,
460 const Response& response,
461 const std::string& data,
462 bool compressed) {
463 if (response.notModified) {
464 // clang-format off
465 mapbox::sqlite::Query notModifiedQuery{ getStatement(
466 "UPDATE tiles "
467 "SET accessed = ?1, "
468 " expires = ?2, "
469 " must_revalidate = ?3 "
470 "WHERE url_template = ?4 "
471 " AND pixel_ratio = ?5 "
472 " AND x = ?6 "
473 " AND y = ?7 "
474 " AND z = ?8 ") };
475 // clang-format on
476
477 notModifiedQuery.bind(1, util::now());
478 notModifiedQuery.bind(2, response.expires);
479 notModifiedQuery.bind(3, response.mustRevalidate);
480 notModifiedQuery.bind(4, tile.urlTemplate);
481 notModifiedQuery.bind(5, tile.pixelRatio);
482 notModifiedQuery.bind(6, tile.x);
483 notModifiedQuery.bind(7, tile.y);
484 notModifiedQuery.bind(8, tile.z);
485 notModifiedQuery.run();
486 return false;
487 }
488
489 // We can't use REPLACE because it would change the id value.
490
491 // clang-format off
492 mapbox::sqlite::Query updateQuery{ getStatement(
493 "UPDATE tiles "
494 "SET modified = ?1, "
495 " etag = ?2, "
496 " expires = ?3, "
497 " must_revalidate = ?4, "
498 " accessed = ?5, "
499 " data = ?6, "
500 " compressed = ?7 "
501 "WHERE url_template = ?8 "
502 " AND pixel_ratio = ?9 "
503 " AND x = ?10 "
504 " AND y = ?11 "
505 " AND z = ?12 ") };
506 // clang-format on
507
508 updateQuery.bind(1, response.modified);
509 updateQuery.bind(2, response.etag);
510 updateQuery.bind(3, response.expires);
511 updateQuery.bind(4, response.mustRevalidate);
512 updateQuery.bind(5, util::now());
513 updateQuery.bind(8, tile.urlTemplate);
514 updateQuery.bind(9, tile.pixelRatio);
515 updateQuery.bind(10, tile.x);
516 updateQuery.bind(11, tile.y);
517 updateQuery.bind(12, tile.z);
518
519 if (response.noContent) {
520 updateQuery.bind(6, nullptr);
521 updateQuery.bind(7, false);
522 } else {
523 updateQuery.bindBlob(6, data.data(), data.size(), false);
524 updateQuery.bind(7, compressed);
525 }
526
527 updateQuery.run();
528 if (updateQuery.changes() != 0) {
529 return false;
530 }
531
532 // clang-format off
533 mapbox::sqlite::Query insertQuery{ getStatement(
534 "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) "
535 "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") };
536 // clang-format on
537
538 insertQuery.bind(1, tile.urlTemplate);
539 insertQuery.bind(2, tile.pixelRatio);
540 insertQuery.bind(3, tile.x);
541 insertQuery.bind(4, tile.y);
542 insertQuery.bind(5, tile.z);
543 insertQuery.bind(6, response.modified);
544 insertQuery.bind(7, response.mustRevalidate);
545 insertQuery.bind(8, response.etag);
546 insertQuery.bind(9, response.expires);
547 insertQuery.bind(10, util::now());
548
549 if (response.noContent) {
550 insertQuery.bind(11, nullptr);
551 insertQuery.bind(12, false);
552 } else {
553 insertQuery.bindBlob(11, data.data(), data.size(), false);
554 insertQuery.bind(12, compressed);
555 }
556
557 insertQuery.run();
558
559 return true;
560 }
561
listRegions()562 std::vector<OfflineRegion> OfflineDatabase::listRegions() {
563 mapbox::sqlite::Query query{ getStatement("SELECT id, definition, description FROM regions") };
564
565 std::vector<OfflineRegion> result;
566
567 while (query.run()) {
568 result.push_back(OfflineRegion(
569 query.get<int64_t>(0),
570 decodeOfflineRegionDefinition(query.get<std::string>(1)),
571 query.get<std::vector<uint8_t>>(2)));
572 }
573
574 return result;
575 }
576
createRegion(const OfflineRegionDefinition & definition,const OfflineRegionMetadata & metadata)577 OfflineRegion OfflineDatabase::createRegion(const OfflineRegionDefinition& definition,
578 const OfflineRegionMetadata& metadata) {
579 // clang-format off
580 mapbox::sqlite::Query query{ getStatement(
581 "INSERT INTO regions (definition, description) "
582 "VALUES (?1, ?2) ") };
583 // clang-format on
584
585 query.bind(1, encodeOfflineRegionDefinition(definition));
586 query.bindBlob(2, metadata);
587 query.run();
588
589 return OfflineRegion(query.lastInsertRowId(), definition, metadata);
590 }
591
updateMetadata(const int64_t regionID,const OfflineRegionMetadata & metadata)592 OfflineRegionMetadata OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) {
593 // clang-format off
594 mapbox::sqlite::Query query{ getStatement(
595 "UPDATE regions SET description = ?1 "
596 "WHERE id = ?2") };
597 // clang-format on
598 query.bindBlob(1, metadata);
599 query.bind(2, regionID);
600 query.run();
601
602 return metadata;
603 }
604
deleteRegion(OfflineRegion && region)605 void OfflineDatabase::deleteRegion(OfflineRegion&& region) {
606 {
607 mapbox::sqlite::Query query{ getStatement("DELETE FROM regions WHERE id = ?") };
608 query.bind(1, region.getID());
609 query.run();
610 }
611
612 evict(0);
613 db->exec("PRAGMA incremental_vacuum");
614
615 // Ensure that the cached offlineTileCount value is recalculated.
616 offlineMapboxTileCount = {};
617 }
618
getRegionResource(int64_t regionID,const Resource & resource)619 optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) {
620 auto response = getInternal(resource);
621
622 if (response) {
623 markUsed(regionID, resource);
624 }
625
626 return response;
627 }
628
hasRegionResource(int64_t regionID,const Resource & resource)629 optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) {
630 auto response = hasInternal(resource);
631
632 if (response) {
633 markUsed(regionID, resource);
634 }
635
636 return response;
637 }
638
putRegionResource(int64_t regionID,const Resource & resource,const Response & response)639 uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) {
640 mapbox::sqlite::Transaction transaction(*db);
641 auto size = putRegionResourceInternal(regionID, resource, response);
642 transaction.commit();
643 return size;
644 }
645
putRegionResources(int64_t regionID,const std::list<std::tuple<Resource,Response>> & resources,OfflineRegionStatus & status)646 void OfflineDatabase::putRegionResources(int64_t regionID, const std::list<std::tuple<Resource, Response>>& resources, OfflineRegionStatus& status) {
647 mapbox::sqlite::Transaction transaction(*db);
648
649 for (const auto& elem : resources) {
650 const auto& resource = std::get<0>(elem);
651 const auto& response = std::get<1>(elem);
652
653 try {
654 uint64_t resourceSize = putRegionResourceInternal(regionID, resource, response);
655 status.completedResourceCount++;
656 status.completedResourceSize += resourceSize;
657 if (resource.kind == Resource::Kind::Tile) {
658 status.completedTileCount += 1;
659 status.completedTileSize += resourceSize;
660 }
661 } catch (const MapboxTileLimitExceededException&) {
662 // Commit the rest of the batch and retrow
663 transaction.commit();
664 throw;
665 }
666 }
667
668 // Commit the completed batch
669 transaction.commit();
670 }
671
putRegionResourceInternal(int64_t regionID,const Resource & resource,const Response & response)672 uint64_t OfflineDatabase::putRegionResourceInternal(int64_t regionID, const Resource& resource, const Response& response) {
673 if (exceedsOfflineMapboxTileCountLimit(resource)) {
674 throw MapboxTileLimitExceededException();
675 }
676
677 uint64_t size = putInternal(resource, response, false).second;
678 bool previouslyUnused = markUsed(regionID, resource);
679
680 if (offlineMapboxTileCount
681 && resource.kind == Resource::Kind::Tile
682 && util::mapbox::isMapboxURL(resource.url)
683 && previouslyUnused) {
684 *offlineMapboxTileCount += 1;
685 }
686
687 return size;
688 }
689
markUsed(int64_t regionID,const Resource & resource)690 bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
691 if (resource.kind == Resource::Kind::Tile) {
692 // clang-format off
693 mapbox::sqlite::Query insertQuery{ getStatement(
694 "INSERT OR IGNORE INTO region_tiles (region_id, tile_id) "
695 "SELECT ?1, tiles.id "
696 "FROM tiles "
697 "WHERE url_template = ?2 "
698 " AND pixel_ratio = ?3 "
699 " AND x = ?4 "
700 " AND y = ?5 "
701 " AND z = ?6 ") };
702 // clang-format on
703
704 const Resource::TileData& tile = *resource.tileData;
705 insertQuery.bind(1, regionID);
706 insertQuery.bind(2, tile.urlTemplate);
707 insertQuery.bind(3, tile.pixelRatio);
708 insertQuery.bind(4, tile.x);
709 insertQuery.bind(5, tile.y);
710 insertQuery.bind(6, tile.z);
711 insertQuery.run();
712
713 if (insertQuery.changes() == 0) {
714 return false;
715 }
716
717 // clang-format off
718 mapbox::sqlite::Query selectQuery{ getStatement(
719 "SELECT region_id "
720 "FROM region_tiles, tiles "
721 "WHERE region_id != ?1 "
722 " AND url_template = ?2 "
723 " AND pixel_ratio = ?3 "
724 " AND x = ?4 "
725 " AND y = ?5 "
726 " AND z = ?6 "
727 "LIMIT 1 ") };
728 // clang-format on
729
730 selectQuery.bind(1, regionID);
731 selectQuery.bind(2, tile.urlTemplate);
732 selectQuery.bind(3, tile.pixelRatio);
733 selectQuery.bind(4, tile.x);
734 selectQuery.bind(5, tile.y);
735 selectQuery.bind(6, tile.z);
736 return !selectQuery.run();
737 } else {
738 // clang-format off
739 mapbox::sqlite::Query insertQuery{ getStatement(
740 "INSERT OR IGNORE INTO region_resources (region_id, resource_id) "
741 "SELECT ?1, resources.id "
742 "FROM resources "
743 "WHERE resources.url = ?2 ") };
744 // clang-format on
745
746 insertQuery.bind(1, regionID);
747 insertQuery.bind(2, resource.url);
748 insertQuery.run();
749
750 if (insertQuery.changes() == 0) {
751 return false;
752 }
753
754 // clang-format off
755 mapbox::sqlite::Query selectQuery{ getStatement(
756 "SELECT region_id "
757 "FROM region_resources, resources "
758 "WHERE region_id != ?1 "
759 " AND resources.url = ?2 "
760 "LIMIT 1 ") };
761 // clang-format on
762
763 selectQuery.bind(1, regionID);
764 selectQuery.bind(2, resource.url);
765 return !selectQuery.run();
766 }
767 }
768
getRegionDefinition(int64_t regionID)769 OfflineRegionDefinition OfflineDatabase::getRegionDefinition(int64_t regionID) {
770 mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") };
771 query.bind(1, regionID);
772 query.run();
773
774 return decodeOfflineRegionDefinition(query.get<std::string>(0));
775 }
776
getRegionCompletedStatus(int64_t regionID)777 OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) {
778 OfflineRegionStatus result;
779
780 std::tie(result.completedResourceCount, result.completedResourceSize)
781 = getCompletedResourceCountAndSize(regionID);
782 std::tie(result.completedTileCount, result.completedTileSize)
783 = getCompletedTileCountAndSize(regionID);
784
785 result.completedResourceCount += result.completedTileCount;
786 result.completedResourceSize += result.completedTileSize;
787
788 return result;
789 }
790
getCompletedResourceCountAndSize(int64_t regionID)791 std::pair<int64_t, int64_t> OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) {
792 // clang-format off
793 mapbox::sqlite::Query query{ getStatement(
794 "SELECT COUNT(*), SUM(LENGTH(data)) "
795 "FROM region_resources, resources "
796 "WHERE region_id = ?1 "
797 "AND resource_id = resources.id ") };
798 // clang-format on
799 query.bind(1, regionID);
800 query.run();
801 return { query.get<int64_t>(0), query.get<int64_t>(1) };
802 }
803
getCompletedTileCountAndSize(int64_t regionID)804 std::pair<int64_t, int64_t> OfflineDatabase::getCompletedTileCountAndSize(int64_t regionID) {
805 // clang-format off
806 mapbox::sqlite::Query query{ getStatement(
807 "SELECT COUNT(*), SUM(LENGTH(data)) "
808 "FROM region_tiles, tiles "
809 "WHERE region_id = ?1 "
810 "AND tile_id = tiles.id ") };
811 // clang-format on
812 query.bind(1, regionID);
813 query.run();
814 return { query.get<int64_t>(0), query.get<int64_t>(1) };
815 }
816
817 template <class T>
getPragma(const char * sql)818 T OfflineDatabase::getPragma(const char* sql) {
819 mapbox::sqlite::Query query{ getStatement(sql) };
820 query.run();
821 return query.get<T>(0);
822 }
823
824 // Remove least-recently used resources and tiles until the used database size,
825 // as calculated by multiplying the number of in-use pages by the page size, is
826 // less than the maximum cache size. Returns false if this condition cannot be
827 // satisfied.
828 //
829 // SQLite database never shrinks in size unless we call VACCUM. We here
830 // are monitoring the soft limit (i.e. number of free pages in the file)
831 // and as it approaches to the hard limit (i.e. the actual file size) we
832 // delete an arbitrary number of old cache entries. The free pages approach saves
833 // us from calling VACCUM or keeping a running total, which can be costly.
evict(uint64_t neededFreeSize)834 bool OfflineDatabase::evict(uint64_t neededFreeSize) {
835 uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size");
836 uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count");
837
838 auto usedSize = [&] {
839 return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count"));
840 };
841
842 // The addition of pageSize is a fudge factor to account for non `data` column
843 // size, and because pages can get fragmented on the database.
844 while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) {
845 // clang-format off
846 mapbox::sqlite::Query accessedQuery{ getStatement(
847 "SELECT max(accessed) "
848 "FROM ( "
849 " SELECT accessed "
850 " FROM resources "
851 " LEFT JOIN region_resources "
852 " ON resource_id = resources.id "
853 " WHERE resource_id IS NULL "
854 " UNION ALL "
855 " SELECT accessed "
856 " FROM tiles "
857 " LEFT JOIN region_tiles "
858 " ON tile_id = tiles.id "
859 " WHERE tile_id IS NULL "
860 " ORDER BY accessed ASC LIMIT ?1 "
861 ") "
862 ) };
863 accessedQuery.bind(1, 50);
864 // clang-format on
865 if (!accessedQuery.run()) {
866 return false;
867 }
868 Timestamp accessed = accessedQuery.get<Timestamp>(0);
869
870 // clang-format off
871 mapbox::sqlite::Query resourceQuery{ getStatement(
872 "DELETE FROM resources "
873 "WHERE id IN ( "
874 " SELECT id FROM resources "
875 " LEFT JOIN region_resources "
876 " ON resource_id = resources.id "
877 " WHERE resource_id IS NULL "
878 " AND accessed <= ?1 "
879 ") ") };
880 // clang-format on
881 resourceQuery.bind(1, accessed);
882 resourceQuery.run();
883 const uint64_t resourceChanges = resourceQuery.changes();
884
885 // clang-format off
886 mapbox::sqlite::Query tileQuery{ getStatement(
887 "DELETE FROM tiles "
888 "WHERE id IN ( "
889 " SELECT id FROM tiles "
890 " LEFT JOIN region_tiles "
891 " ON tile_id = tiles.id "
892 " WHERE tile_id IS NULL "
893 " AND accessed <= ?1 "
894 ") ") };
895 // clang-format on
896 tileQuery.bind(1, accessed);
897 tileQuery.run();
898 const uint64_t tileChanges = tileQuery.changes();
899
900 // The cached value of offlineTileCount does not need to be updated
901 // here because only non-offline tiles can be removed by eviction.
902
903 if (resourceChanges == 0 && tileChanges == 0) {
904 return false;
905 }
906 }
907
908 return true;
909 }
910
setOfflineMapboxTileCountLimit(uint64_t limit)911 void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) {
912 offlineMapboxTileCountLimit = limit;
913 }
914
getOfflineMapboxTileCountLimit()915 uint64_t OfflineDatabase::getOfflineMapboxTileCountLimit() {
916 return offlineMapboxTileCountLimit;
917 }
918
offlineMapboxTileCountLimitExceeded()919 bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() {
920 return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit;
921 }
922
getOfflineMapboxTileCount()923 uint64_t OfflineDatabase::getOfflineMapboxTileCount() {
924 // Calculating this on every call would be much simpler than caching and
925 // manually updating the value, but it would make offline downloads an O(n²)
926 // operation, because the database query below involves an index scan of
927 // region_tiles.
928
929 if (offlineMapboxTileCount) {
930 return *offlineMapboxTileCount;
931 }
932
933 // clang-format off
934 mapbox::sqlite::Query query{ getStatement(
935 "SELECT COUNT(DISTINCT id) "
936 "FROM region_tiles, tiles "
937 "WHERE tile_id = tiles.id "
938 "AND url_template LIKE 'mapbox://%' ") };
939 // clang-format on
940
941 query.run();
942
943 offlineMapboxTileCount = query.get<int64_t>(0);
944 return *offlineMapboxTileCount;
945 }
946
exceedsOfflineMapboxTileCountLimit(const Resource & resource)947 bool OfflineDatabase::exceedsOfflineMapboxTileCountLimit(const Resource& resource) {
948 return resource.kind == Resource::Kind::Tile
949 && util::mapbox::isMapboxURL(resource.url)
950 && offlineMapboxTileCountLimitExceeded();
951 }
952
953 } // namespace mbgl
954