Coverage for gws-app/gws/base/grabber/core.py: 97%
188 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-05 13:35 +0200
1"""Base raster grabber."""
3import math
4import os
6import gws
7import gws.lib.extent
8import gws.lib.gdalx
9import gws.lib.grid
10import gws.lib.crs
11import gws.lib.image
12import gws.lib.mime
13import gws.gis.cache
15_BlobImagePair = tuple[bytes | None, gws.Image | None]
17MAX_LEVEL = 30
18"""Serving stopper: levels beyond this are never served or precomputed."""
20EPHEMERAL_MAX_AGE = 60
21"""Lifetime (seconds) of tiles in the ephemeral store."""
23BLOCK_LOCK_TIMEOUT = 60
24"""Seconds to wait for a block being composed by another process."""
27class Options(gws.Data):
28 """Options for creating a grabber."""
30 crs: gws.Crs
31 """Target CRS."""
32 cache: gws.MapCache
33 """Cache settings, with the final cache name including the SRID."""
34 extent: gws.Extent
35 """Layer extent in the target CRS."""
36 imageFormat: gws.ImageFormat
37 """Format tiles are stored and returned in."""
40class Object(gws.Grabber):
41 """Base raster grabber.
43 Implements the ``gws.Grabber`` API on top of the composition methods
44 (``compose_*``), which subclasses implement for their kind of source.
45 """
47 defaultEphemeralStore: gws.TileStore
48 """Short-lived store for static tiles outside the cache settings, so that blocks are composed once."""
49 maxLevel: int
50 """Finest served level."""
51 mimeType: str
52 """Mime type of ``imageFormat``."""
53 minLevel: int
54 """Coarsest served level."""
55 mtrByLevel: dict[int, gws.MapTileRange]
56 """Tile range covered by ``extent``, per served level."""
57 requestTiles: int
58 """Tiles per side composed in one block; 1 means no meta-tiling."""
60 def __init__(self, opts: Options):
61 """Create a grabber.
63 Args:
64 opts: Grabber options.
66 Raises:
67 ``gws.ConfigurationError``: If the extent covers no tiles at some level.
68 """
69 self.targetCrs = opts.crs
70 self.grid = gws.lib.grid.for_crs(self.targetCrs)
72 self.imageFormat = opts.imageFormat
73 self.mimeType = self.imageFormat.mimeTypes[0]
75 self.cache = opts.cache
76 self.requestTiles = 1
77 self.store = gws.gis.cache.store.Object(
78 f'{gws.c.MAP_CACHE_DIR}/{self.cache.name}',
79 max_age=self.cache.maxAge,
80 extension=gws.lib.mime.extension_for(self.mimeType),
81 )
82 self.defaultEphemeralStore = self._ephemeral_store('')
84 self.extent = opts.extent
85 self.minLevel = 0
86 self.maxLevel = MAX_LEVEL
88 self.mtrByLevel = {}
89 for z in range(self.minLevel, self.maxLevel + 1):
90 mtr = gws.lib.grid.range_for_extent(self.grid, self.extent, z)
91 if not mtr:
92 raise gws.ConfigurationError(f'grabber {self.cache.name!r}: empty tile range for level {z}')
93 self.mtrByLevel[z] = mtr
95 ##
97 def get_tile_as_bytes(self, mt, params=None):
98 return self.pair_to_bytes(self._get_tile_as_pair(mt, params))
100 def get_tile_as_image(self, mt, params=None):
101 return self.pair_to_image(self._get_tile_as_pair(mt, params))
103 def get_tiles_as_bytes_dict(self, mtr, params=None):
104 return {t: self.get_tile_as_bytes(t, params) for t in self._valid_tiles_in_range(mtr)}
106 def get_tiles_as_image_dict(self, mtr, params=None):
107 return {t: self.get_tile_as_image(t, params) for t in self._valid_tiles_in_range(mtr)}
109 def get_box_as_bytes(self, extent, w, h, params=None):
110 return self.pair_to_bytes(self._get_box_as_pair(extent, w, h, params))
112 def get_box_as_image(self, extent, w, h, params=None):
113 return self.pair_to_image(self._get_box_as_pair(extent, w, h, params))
115 def levels(self):
116 return list(self.mtrByLevel)
118 def tile_range_for_level(self, z):
119 return self.mtrByLevel[z]
121 ##
123 def compose_tile_block_as_image_dict(self, mt: gws.MapTile, params: dict | None = None) -> dict[gws.MapTile, gws.Image]:
124 """Compose the block of ``requestTiles`` x ``requestTiles`` tiles containing a tile.
126 The base implementation supports only ``requestTiles`` 1 and composes the tile alone.
128 Args:
129 mt: A tile in the block.
130 params: Dynamic request parameters.
132 Returns:
133 Images of the tiles of the block within the layer's tile range.
135 Raises:
136 ``NotImplementedError``: If ``requestTiles`` is not 1.
137 """
139 if self.requestTiles != 1:
140 raise NotImplementedError(f'compose_tile_block_as_image_dict not implemented in {self!r}')
141 return {mt: self.compose_tile_as_image(mt, params)}
143 def compose_tile_as_image(self, mt: gws.MapTile, params: dict | None = None) -> gws.Image:
144 """Compose a single tile over its own extent.
146 Args:
147 mt: The tile.
148 params: Dynamic request parameters.
150 Returns:
151 The tile image.
152 """
154 return self.compose_box_as_image(
155 gws.lib.grid.extent_for_tile(self.grid, mt),
156 self.grid.tileSize,
157 self.grid.tileSize,
158 params,
159 )
161 def compose_box_as_image(self, extent: gws.Extent, w: int, h: int, params: dict | None = None) -> gws.Image:
162 """Compose an image for an arbitrary extent and pixel size from the source.
164 Subclasses must implement this.
166 Args:
167 extent: Extent in the target CRS.
168 w: Width in pixels.
169 h: Height in pixels.
170 params: Dynamic request parameters.
172 Returns:
173 The image.
175 Raises:
176 ``NotImplementedError``: In the base class.
177 """
179 raise NotImplementedError(f'compose_box_as_image not implemented in {self!r}')
181 ##
183 def pair_to_bytes(self, bi: _BlobImagePair) -> bytes:
184 """Return the bytes form of a pair, encoding the image if needed.
186 Args:
187 bi: A pair of encoded bytes and image, one of which is set.
189 Returns:
190 The encoded image.
192 Raises:
193 ``gws.Error``: If neither is set.
194 """
196 blob, img = bi
197 if blob is not None:
198 return blob
199 if img is not None:
200 return self.to_bytes(img)
201 raise gws.Error('unexpected state')
203 def pair_to_image(self, bi: _BlobImagePair) -> gws.Image:
204 """Return the image form of a pair, decoding the bytes if needed.
206 Args:
207 bi: A pair of encoded bytes and image, one of which is set.
209 Returns:
210 The image.
212 Raises:
213 ``gws.Error``: If neither is set.
214 """
216 blob, img = bi
217 if img is not None:
218 return img
219 if blob is not None:
220 return self.to_image(blob)
221 raise gws.Error('unexpected state')
223 def to_bytes(self, img: gws.Image) -> bytes:
224 """Encode an image in the grabber's image format.
226 Args:
227 img: The image.
229 Returns:
230 The encoded image.
231 """
233 return img.to_bytes(self.mimeType, self.imageFormat.options)
235 def to_image(self, blob: bytes) -> gws.Image:
236 """Decode an encoded image.
238 Args:
239 blob: The encoded image.
241 Returns:
242 The image.
243 """
245 return gws.lib.image.from_bytes(blob)
247 def normalize_image(self, img: gws.Image, w: int, h: int) -> gws.Image:
248 """Ensure a source image is RGBA and has the requested size.
250 Args:
251 img: Image from the source.
252 w: Expected width.
253 h: Expected height.
255 Returns:
256 The image converted to RGBA.
258 Raises:
259 ``gws.ExternalServiceError``: If the image size differs from the requested size.
260 """
262 if img.size() != (w, h):
263 raise gws.ExternalServiceError(f'grabber {self.cache.name!r}: unexpected image size {img.size()!r}')
264 return img.convert('RGBA')
266 def block_start_tile(self, mt: gws.MapTile) -> gws.MapTile:
267 """Return the first tile of the block containing a tile.
269 Args:
270 mt: The tile.
272 Returns:
273 The top-left tile of the block.
274 """
276 x, y, z = mt
277 n = self.requestTiles
278 return (x // n) * n, (y // n) * n, z
280 def extent_to_source_crs(self, extent: gws.Extent) -> gws.Extent | None:
281 """Transform a target extent into the source CRS, clipped to the source area of use.
283 Args:
284 extent: Extent in the target CRS.
286 Returns:
287 The extent in the source CRS, or ``None`` if it lies outside the source area of use.
288 """
290 wgs_extent = gws.lib.extent.transform_to_wgs(extent, self.targetCrs)
291 wgs_extent = self.sourceCrs.clip_wgs_extent(wgs_extent)
292 if not wgs_extent:
293 return
294 src_extent = gws.lib.extent.transform_from_wgs(wgs_extent, self.sourceCrs)
295 if not gws.lib.extent.is_valid(src_extent):
296 return
297 return src_extent
299 def warp_image(self, img: gws.Image, src_bounds: gws.Bounds, target_bounds: gws.Bounds, w: int, h: int) -> gws.Image:
300 """Warp an image covering the source bounds onto the target bounds and pixel size.
302 Uses nearest neighbour resampling when the CRS and the resolution are the same,
303 and bilinear resampling otherwise.
305 Args:
306 img: Source image.
307 src_bounds: Bounds covered by the source image.
308 target_bounds: Target bounds.
309 w: Target width in pixels.
310 h: Target height in pixels.
312 Returns:
313 The warped image.
314 """
316 same_crs = src_bounds.crs == target_bounds.crs
317 src_res = gws.lib.extent.w(src_bounds.extent) / img.size()[0]
318 target_res = gws.lib.extent.w(target_bounds.extent) / w
320 # Pixels copied 1:1 within the same CRS use 'nearest', so that a sub-pixel offset
321 # between the source and target grids does not blur the image.
322 resample_alg = 'bilinear'
323 if same_crs and math.isclose(src_res, target_res, rel_tol=gws.lib.grid.RESOLUTION_TOLERANCE):
324 resample_alg = 'nearest'
326 # GDAL widens the bilinear kernel by the ratio of the source window to the destination chunk.
327 # Cross-CRS, with a source covering e.g. the whole mercator world, the window estimate can fail
328 # and fall back to the whole image, and the kernel then averages a strip of source pixels (diagonal smear).
329 # Pinning the scale keeps a 2x2 kernel. Same-CRS the estimate is correct, so leave it alone.
330 warp_options = [] if same_crs else ['XSCALE=1', 'YSCALE=1']
332 with gws.lib.gdalx.open_from_image(img, src_bounds) as ds:
333 return ds.warp_to_image(
334 dict(
335 dstSRS=target_bounds.crs.epsg,
336 outputBounds=target_bounds.extent,
337 outputBoundsSRS=target_bounds.crs.epsg,
338 width=w,
339 height=h,
340 resampleAlg=resample_alg,
341 warpOptions=warp_options,
342 )
343 )
345 def empty_image(self, w: int, h: int) -> gws.Image:
346 """Return a transparent image of the given size.
348 Args:
349 w: Width in pixels.
350 h: Height in pixels.
352 Returns:
353 The image.
354 """
356 return gws.lib.image.from_size((w, h))
358 def empty_box(self, w: int, h: int) -> bytes:
359 """Return a transparent image of the given size, as encoded bytes.
361 Args:
362 w: Width in pixels.
363 h: Height in pixels.
365 Returns:
366 The encoded image.
367 """
369 return self.to_bytes(self.empty_image(w, h))
371 def empty_tile(self) -> bytes:
372 """Return the transparent tile, encoded once and then reused.
374 Returns:
375 The encoded tile.
376 """
378 if not hasattr(self, '_emptyTile'):
379 self._emptyTile = self.empty_box(self.grid.tileSize, self.grid.tileSize)
380 return self._emptyTile
382 def is_serving(self, z: int) -> bool:
383 """Check if a level is within the serving range.
385 Args:
386 z: Level.
388 Returns:
389 ``True`` if the level is served.
390 """
392 return self.minLevel <= z <= self.maxLevel
394 def is_storing(self, z: int) -> bool:
395 """Check if tiles of a level go to the persistent store.
397 Args:
398 z: Level.
400 Returns:
401 ``True`` if the cache has a positive ``maxAge`` and the level is not beyond ``cache.maxLevel``.
402 """
404 return self.cache.maxAge > 0 and z <= self.cache.maxLevel
406 ##
408 def _get_tile_as_pair(self, mt: gws.MapTile, params: dict | None) -> _BlobImagePair:
409 """Return a tile from the store, composing and storing its block on a miss."""
411 z = mt[-1]
412 if not self.is_serving(z):
413 return self.empty_tile(), None
415 if not gws.lib.grid.in_range(mt, self.tile_range_for_level(z)):
416 return self.empty_tile(), None
418 store_key = gws.u.sha256(params)[:12] if params else ''
419 store = self._store_for(z, store_key)
421 blob = store.read(mt)
422 if blob is not None:
423 return blob, None
425 try:
426 bx, by, _ = self.block_start_tile(mt)
427 with gws.u.server_lock(f'grabber_{self.cache.name}_{store_key}_{z}_{bx}_{by}', BLOCK_LOCK_TIMEOUT):
428 # the tile might be written by another render
429 blob = store.read(mt)
430 if blob is not None:
431 return blob, None
432 block_images = self._compose_and_store_tile_block(mt, params, store)
433 return None, block_images[mt]
434 except gws.LockBusyError:
435 gws.log.warning(f'grabber {self.cache.name!r}: block lock busy for {mt!r}')
436 return self.empty_tile(), None
438 def _compose_and_store_tile_block(self, mt: gws.MapTile, params: dict | None, store: gws.TileStore) -> dict[gws.MapTile, gws.Image]:
439 """Compose the block containing a tile and write all its tiles to the store."""
441 block_images = self.compose_tile_block_as_image_dict(mt, params)
442 for t, img in block_images.items():
443 blob = self.to_bytes(img)
444 store.write(t, blob)
445 if store is not self.store:
446 gws.u.ephemeral_cleanup()
447 return block_images
449 def _get_box_as_pair(self, extent: gws.Extent, w: float, h: float, params: dict | None) -> _BlobImagePair:
450 """Return a box, mosaicked from stored tiles at storing levels, composed directly otherwise."""
452 w = gws.u.to_rounded_int(w)
453 h = gws.u.to_rounded_int(h)
455 z = gws.lib.grid.level_for_resolution(self.grid, gws.lib.extent.w(extent) / w)
456 if params or not self.is_storing(z):
457 return None, self.compose_box_as_image(extent, w, h, params)
459 mtr = gws.lib.grid.range_for_extent(self.grid, extent, z)
460 if not mtr:
461 return self.empty_box(w, h), None
463 x0, y0, x1, y1, _ = mtr
464 ts = self.grid.tileSize
465 mosaic = gws.lib.image.from_size(((x1 - x0 + 1) * ts, (y1 - y0 + 1) * ts))
466 for (tx, ty, _), img in self.get_tiles_as_image_dict(mtr).items():
467 mosaic.paste(img, ((tx - x0) * ts, (ty - y0) * ts))
469 mosaic_extent = gws.lib.grid.extent_for_range(self.grid, mtr)
470 return None, self.warp_image(
471 mosaic,
472 gws.Bounds(crs=self.targetCrs, extent=mosaic_extent),
473 gws.Bounds(crs=self.targetCrs, extent=extent),
474 w,
475 h,
476 )
478 def _valid_tiles_in_range(self, mtr: gws.MapTileRange) -> list[gws.MapTile]:
479 """Return the tiles of a range that lie within the served levels and extent."""
481 z = mtr[-1]
482 if not self.is_serving(z):
483 return []
484 level_mtr = self.tile_range_for_level(z)
485 return [t for t in gws.lib.grid.enum_tiles(mtr) if gws.lib.grid.in_range(t, level_mtr)]
487 def _store_for(self, z: int, key: str = '') -> gws.TileStore:
488 """Return the store for a level and params key."""
490 if key:
491 return self._ephemeral_store(key)
492 if self.is_storing(z):
493 return self.store
494 return self.defaultEphemeralStore
496 def _ephemeral_store(self, key: str) -> gws.TileStore:
497 """Create an ephemeral store for a params key."""
499 return gws.gis.cache.store.Object(
500 gws.u.ephemeral_dir(f'tiles_{self.cache.name}_{key}'),
501 max_age=EPHEMERAL_MAX_AGE,
502 extension=gws.lib.mime.extension_for(self.mimeType),
503 )