1 #include <mbgl/storage/file_source.hpp>
2 #include <mbgl/storage/offline_database.hpp>
3 #include <mbgl/storage/offline_download.hpp>
4 #include <mbgl/storage/resource.hpp>
5 #include <mbgl/storage/response.hpp>
6 #include <mbgl/storage/http_file_source.hpp>
7 #include <mbgl/style/parser.hpp>
8 #include <mbgl/style/sources/vector_source.hpp>
9 #include <mbgl/style/sources/raster_source.hpp>
10 #include <mbgl/style/sources/raster_dem_source.hpp>
11 #include <mbgl/style/sources/geojson_source.hpp>
12 #include <mbgl/style/sources/image_source.hpp>
13 #include <mbgl/style/conversion/json.hpp>
14 #include <mbgl/style/conversion/tileset.hpp>
15 #include <mbgl/text/glyph.hpp>
16 #include <mbgl/util/mapbox.hpp>
17 #include <mbgl/util/run_loop.hpp>
18 #include <mbgl/util/tile_cover.hpp>
19 #include <mbgl/util/tileset.hpp>
20
21 #include <set>
22
23 namespace mbgl {
24
25 using namespace style;
26
OfflineDownload(int64_t id_,OfflineRegionDefinition && definition_,OfflineDatabase & offlineDatabase_,FileSource & onlineFileSource_)27 OfflineDownload::OfflineDownload(int64_t id_,
28 OfflineRegionDefinition&& definition_,
29 OfflineDatabase& offlineDatabase_,
30 FileSource& onlineFileSource_)
31 : id(id_),
32 definition(definition_),
33 offlineDatabase(offlineDatabase_),
34 onlineFileSource(onlineFileSource_) {
35 setObserver(nullptr);
36 }
37
38 OfflineDownload::~OfflineDownload() = default;
39
setObserver(std::unique_ptr<OfflineRegionObserver> observer_)40 void OfflineDownload::setObserver(std::unique_ptr<OfflineRegionObserver> observer_) {
41 observer = observer_ ? std::move(observer_) : std::make_unique<OfflineRegionObserver>();
42 }
43
setState(OfflineRegionDownloadState state)44 void OfflineDownload::setState(OfflineRegionDownloadState state) {
45 if (status.downloadState == state) {
46 return;
47 }
48
49 status.downloadState = state;
50
51 if (status.downloadState == OfflineRegionDownloadState::Active) {
52 activateDownload();
53 } else {
54 deactivateDownload();
55 }
56
57 observer->statusChanged(status);
58 }
59
getStatus() const60 OfflineRegionStatus OfflineDownload::getStatus() const {
61 if (status.downloadState == OfflineRegionDownloadState::Active) {
62 return status;
63 }
64
65 OfflineRegionStatus result = offlineDatabase.getRegionCompletedStatus(id);
66
67 result.requiredResourceCount++;
68 optional<Response> styleResponse = offlineDatabase.get(Resource::style(definition.styleURL));
69 if (!styleResponse) {
70 return result;
71 }
72
73 style::Parser parser;
74 parser.parse(*styleResponse->data);
75
76 result.requiredResourceCountIsPrecise = true;
77
78 for (const auto& source : parser.sources) {
79 SourceType type = source->getType();
80
81 auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
82 if (urlOrTileset.is<Tileset>()) {
83 result.requiredResourceCount +=
84 definition.tileCount(type, tileSize, urlOrTileset.get<Tileset>().zoomRange);
85 } else {
86 result.requiredResourceCount += 1;
87 const auto& url = urlOrTileset.get<std::string>();
88 optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url));
89 if (sourceResponse) {
90 style::conversion::Error error;
91 optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data, error);
92 if (tileset) {
93 result.requiredResourceCount +=
94 definition.tileCount(type, tileSize, (*tileset).zoomRange);
95 }
96 } else {
97 result.requiredResourceCountIsPrecise = false;
98 }
99 }
100 };
101
102 switch (type) {
103 case SourceType::Vector: {
104 const auto& vectorSource = *source->as<VectorSource>();
105 handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize);
106 break;
107 }
108
109 case SourceType::Raster: {
110 const auto& rasterSource = *source->as<RasterSource>();
111 handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
112 break;
113 }
114
115 case SourceType::RasterDEM: {
116 const auto& rasterDEMSource = *source->as<RasterDEMSource>();
117 handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
118 break;
119 }
120
121 case SourceType::GeoJSON: {
122 const auto& geojsonSource = *source->as<GeoJSONSource>();
123 if (geojsonSource.getURL()) {
124 result.requiredResourceCount += 1;
125 }
126 break;
127 }
128
129 case SourceType::Image: {
130 const auto& imageSource = *source->as<ImageSource>();
131 if (imageSource.getURL()) {
132 result.requiredResourceCount += 1;
133 }
134 break;
135 }
136
137 case SourceType::Video:
138 case SourceType::Annotations:
139 case SourceType::CustomVector:
140 break;
141 }
142 }
143
144 if (!parser.glyphURL.empty()) {
145 result.requiredResourceCount += parser.fontStacks().size() * GLYPH_RANGES_PER_FONT_STACK;
146 }
147
148 if (!parser.spriteURL.empty()) {
149 result.requiredResourceCount += 2;
150 }
151
152 return result;
153 }
154
activateDownload()155 void OfflineDownload::activateDownload() {
156 status = OfflineRegionStatus();
157 status.downloadState = OfflineRegionDownloadState::Active;
158 status.requiredResourceCount++;
159 ensureResource(Resource::style(definition.styleURL), [&](Response styleResponse) {
160 status.requiredResourceCountIsPrecise = true;
161
162 style::Parser parser;
163 parser.parse(*styleResponse.data);
164
165 for (const auto& source : parser.sources) {
166 SourceType type = source->getType();
167
168 auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
169 if (urlOrTileset.is<Tileset>()) {
170 queueTiles(type, tileSize, urlOrTileset.get<Tileset>());
171 } else {
172 const auto& url = urlOrTileset.get<std::string>();
173 status.requiredResourceCountIsPrecise = false;
174 status.requiredResourceCount++;
175 requiredSourceURLs.insert(url);
176
177 ensureResource(Resource::source(url), [=](Response sourceResponse) {
178 style::conversion::Error error;
179 optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data, error);
180 if (tileset) {
181 util::mapbox::canonicalizeTileset(*tileset, url, type, tileSize);
182 queueTiles(type, tileSize, *tileset);
183
184 requiredSourceURLs.erase(url);
185 if (requiredSourceURLs.empty()) {
186 status.requiredResourceCountIsPrecise = true;
187 }
188 }
189 });
190 }
191 };
192
193 switch (type) {
194 case SourceType::Vector: {
195 const auto& vectorSource = *source->as<VectorSource>();
196 handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize);
197 break;
198 }
199
200 case SourceType::Raster: {
201 const auto& rasterSource = *source->as<RasterSource>();
202 handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
203 break;
204 }
205
206 case SourceType::RasterDEM: {
207 const auto& rasterDEMSource = *source->as<RasterDEMSource>();
208 handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
209 break;
210 }
211
212 case SourceType::GeoJSON: {
213 const auto& geojsonSource = *source->as<GeoJSONSource>();
214 if (geojsonSource.getURL()) {
215 queueResource(Resource::source(*geojsonSource.getURL()));
216 }
217 break;
218 }
219
220 case SourceType::Image: {
221 const auto& imageSource = *source->as<ImageSource>();
222 auto imageUrl = imageSource.getURL();
223 if (imageUrl && !imageUrl->empty()) {
224 queueResource(Resource::image(*imageUrl));
225 }
226 break;
227 }
228
229 case SourceType::Video:
230 case SourceType::Annotations:
231 case SourceType::CustomVector:
232 break;
233 }
234 }
235
236 if (!parser.glyphURL.empty()) {
237 for (const auto& fontStack : parser.fontStacks()) {
238 for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
239 queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE)));
240 }
241 }
242 }
243
244 if (!parser.spriteURL.empty()) {
245 queueResource(Resource::spriteImage(parser.spriteURL, definition.pixelRatio));
246 queueResource(Resource::spriteJSON(parser.spriteURL, definition.pixelRatio));
247 }
248
249 continueDownload();
250 });
251 }
252
253 /*
254 Fill up our own request queue by requesting the next few resources. This is called
255 when activating the download, or when a request completes successfully.
256
257 Note "successfully"; it's not called when a requests receives an error. A request
258 that errors will be retried after some delay. So in that sense it's still "active"
259 and consuming resources, notably the request object, its timer, and network resources
260 when the timer fires.
261
262 We could try to squeeze in subsequent requests while we wait for the errored request
263 to retry. But that risks overloading the upstream request queue -- defeating our own
264 metering -- if there are a lot of errored requests that all come up for retry at the
265 same time. And many times, the cause of a request error will apply to many requests
266 of the same type. For instance if a server is unreachable, all the requests to that
267 host are going to error. In that case, continuing to try subsequent resources after
268 the first few errors is fruitless anyway.
269 */
continueDownload()270 void OfflineDownload::continueDownload() {
271 if (resourcesRemaining.empty() && status.complete()) {
272 setState(OfflineRegionDownloadState::Inactive);
273 return;
274 }
275
276 while (!resourcesRemaining.empty() && requests.size() < HTTPFileSource::maximumConcurrentRequests()) {
277 ensureResource(resourcesRemaining.front());
278 resourcesRemaining.pop_front();
279 }
280 }
281
deactivateDownload()282 void OfflineDownload::deactivateDownload() {
283 requiredSourceURLs.clear();
284 resourcesRemaining.clear();
285 requests.clear();
286 }
287
queueResource(Resource resource)288 void OfflineDownload::queueResource(Resource resource) {
289 status.requiredResourceCount++;
290 resourcesRemaining.push_front(std::move(resource));
291 }
292
queueTiles(SourceType type,uint16_t tileSize,const Tileset & tileset)293 void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tileset& tileset) {
294 for (const auto& tile : definition.tileCover(type, tileSize, tileset.zoomRange)) {
295 status.requiredResourceCount++;
296 resourcesRemaining.push_back(
297 Resource::tile(tileset.tiles[0], definition.pixelRatio, tile.x, tile.y, tile.z, tileset.scheme));
298 }
299 }
300
ensureResource(const Resource & resource,std::function<void (Response)> callback)301 void OfflineDownload::ensureResource(const Resource& resource,
302 std::function<void(Response)> callback) {
303 auto workRequestsIt = requests.insert(requests.begin(), nullptr);
304 *workRequestsIt = util::RunLoop::Get()->invokeCancellable([=]() {
305 requests.erase(workRequestsIt);
306
307 auto getResourceSizeInDatabase = [&] () -> optional<int64_t> {
308 if (!callback) {
309 return offlineDatabase.hasRegionResource(id, resource);
310 }
311 optional<std::pair<Response, uint64_t>> response = offlineDatabase.getRegionResource(id, resource);
312 if (!response) {
313 return {};
314 }
315 callback(response->first);
316 return response->second;
317 };
318
319 optional<int64_t> offlineResponse = getResourceSizeInDatabase();
320 if (offlineResponse) {
321 status.completedResourceCount++;
322 status.completedResourceSize += *offlineResponse;
323 if (resource.kind == Resource::Kind::Tile) {
324 status.completedTileCount += 1;
325 status.completedTileSize += *offlineResponse;
326 }
327
328 observer->statusChanged(status);
329 continueDownload();
330 return;
331 }
332
333 if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
334 onMapboxTileCountLimitExceeded();
335 return;
336 }
337
338 auto fileRequestsIt = requests.insert(requests.begin(), nullptr);
339 *fileRequestsIt = onlineFileSource.request(resource, [=](Response onlineResponse) {
340 if (onlineResponse.error) {
341 observer->responseError(*onlineResponse.error);
342 return;
343 }
344
345 requests.erase(fileRequestsIt);
346
347 if (callback) {
348 callback(onlineResponse);
349 }
350
351 // Queue up for batched insertion
352 buffer.emplace_back(resource, onlineResponse);
353
354 // Flush buffer periodically
355 if (buffer.size() == 64 || resourcesRemaining.size() == 0) {
356 try {
357 offlineDatabase.putRegionResources(id, buffer, status);
358 } catch (const MapboxTileLimitExceededException&) {
359 onMapboxTileCountLimitExceeded();
360 return;
361 }
362
363 buffer.clear();
364 observer->statusChanged(status);
365 }
366
367 if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
368 onMapboxTileCountLimitExceeded();
369 return;
370 }
371
372 continueDownload();
373 });
374 });
375 }
376
onMapboxTileCountLimitExceeded()377 void OfflineDownload::onMapboxTileCountLimitExceeded() {
378 observer->mapboxTileCountLimitExceeded(offlineDatabase.getOfflineMapboxTileCountLimit());
379 setState(OfflineRegionDownloadState::Inactive);
380 }
381
382 } // namespace mbgl
383