Coverage for gws-app/gws/gis/cache/core.py: 96%
166 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-05 13:35 +0200
1"""Tile cache inventory, filtering and maintenance."""
3import os
4from typing import Optional
6import gws
7import gws.lib.grid
8import gws.lib.osx as osx
11class LayerConfig(gws.Config):
12 """Tile cache settings of a layer."""
14 name: str = ''
15 """Cache directory name. (added in 8.5)"""
16 maxAge: gws.Duration = '7d'
17 """Max. age of cached tiles."""
18 maxLevel: int = 18
19 """Finest zoom level to cache. (changed in 8.5)"""
20 requestBuffer: int = 64
21 """Pixel buffer around source requests, to keep labels consistent across tiles. (changed in 8.5)"""
22 requestTiles: int = 4
23 """Tiles per side of the block rendered in one source request. (changed in 8.5)"""
24 crs: Optional[list[gws.CrsName]]
25 """CRS to cache tiles in. (added in 8.5)"""
28class GlobalConfig(gws.Config):
29 """Global tile cache options."""
31 seedingMaxTime: gws.Duration = '10m'
32 """Time limit for a cache seeding run."""
33 seedingConcurrency: int = 1
34 """Number of parallel threads for cache seeding."""
37class Level(gws.Data):
38 """A zoom level of a cache, with statistics."""
40 z: int
41 """Zoom level."""
42 gridRange: gws.MapTileRange
43 """Tile range of the level, possibly restricted by a filter."""
44 gridSize: gws.Size
45 """Number of tile columns and rows in ``gridRange``."""
46 resolution: float
47 """Resolution of the level in map units per pixel."""
48 seedTime: float
49 """Seconds spent seeding this level."""
50 cachedTiles: int
51 """Number of tiles already in the store."""
52 failedTiles: int
53 """Number of tiles that failed during seeding."""
54 fetchedTiles: int
55 """Number of tiles fetched during seeding."""
56 totalTiles: int
57 """Number of tiles in ``gridRange``."""
58 percentCached: int
59 """Percentage of cached tiles."""
60 cachedRange: Optional[gws.MapTileRange]
61 """Bounding tile range of the stored tiles."""
62 fileSize: int
63 """Total size of the stored tiles in bytes."""
66class Cache(gws.Data):
67 """A tile cache, shared by all layers with the same cache name."""
69 name: str
70 """Cache name, also the name of the cache directory."""
71 dir: str
72 """Path to the cache directory, empty if it does not exist."""
73 grabber: gws.Grabber
74 """Grabber of the first layer that uses this cache."""
75 layers: list[gws.Layer]
76 """Layers that use this cache."""
77 layerTitle: str
78 """Title of the first layer."""
79 layerType: str
80 """Type of the first layer."""
81 levels: list[Level]
82 """Zoom levels of the cache, up to the cache max. level."""
83 seedStatus: str
84 """Status of the last seeding run, e.g. ``timeout``, empty if it completed."""
85 cachedTiles: int
86 """Number of stored tiles."""
87 fileSize: int
88 """Total size of the stored tiles in bytes."""
91class Inventory(gws.Data):
92 """All tile caches of a configuration."""
94 caches: list[Cache]
95 """Configured caches."""
96 orphanDirs: list[str]
97 """Directories in the cache directory that belong to no configured cache."""
100class Filter(gws.Data):
101 """Selects caches, levels and tiles from an inventory. Empty fields select everything."""
103 layerUids: list[str]
104 """Select caches used by any of these layers."""
105 cacheNames: list[str]
106 """Select caches whose names start with any of these prefixes."""
107 srids: list[int]
108 """Select caches in these CRS."""
109 levels: list[int]
110 """Select these zoom levels."""
111 bbox: Optional[gws.Bounds]
112 """Select tiles in this area, which must be in the CRS of the caches."""
115class SeedOptions(gws.Data):
116 """Options for a seeding run."""
118 filter: Filter
119 """Selects caches, levels and tiles to seed."""
120 maxTime: int
121 """Time limit in seconds."""
122 concurrency: int
123 """Number of worker threads."""
124 maxAge: Optional[int]
125 """Refetch tiles older than this (seconds), capped by the cache max. age."""
128class SeedResult(gws.Data):
129 """Result of a seeding run."""
131 caches: list[Cache]
132 """Seeded caches, with per-level statistics."""
133 seedTime: float
134 """Total run time in seconds."""
135 seedStatus: str
136 """Status of the run: empty if completed, ``timeout``, ``interrupted`` or ``locked``."""
139def inventory(root: gws.Root) -> Inventory:
140 """Collect all tile caches of a configuration.
142 Caches are taken from the grabbers of all layers, skipping grabbers with a zero max. age.
143 Each cache gets its levels up to the cache max. level and the existing cache directory.
144 Directories in the cache directory that belong to no cache are listed as orphans.
145 Statistics are not computed here, see ``add_stats``.
147 Args:
148 root: Configuration root.
150 Returns:
151 The inventory, caches sorted by layer type, layer title and name.
152 """
154 inv = Inventory(caches=[], orphanDirs=[])
155 cmap = {}
157 for la in root.find_all(gws.ext.object.layer):
158 for gr in getattr(la, 'grabbers', {}).values():
159 if gr.cache.maxAge <= 0:
160 continue
161 if gr.cache.name not in cmap:
162 cmap[gr.cache.name] = Cache(
163 name=gr.cache.name,
164 grabber=gr,
165 dir='',
166 layers=[],
167 layerTitle=getattr(la, 'title', ''),
168 layerType=la.extType,
169 levels=[],
170 seedStatus='',
171 cachedTiles=0,
172 fileSize=0,
173 )
174 cmap[gr.cache.name].layers.append(la)
176 inv.caches = list(cmap.values())
177 inv.caches.sort(key=lambda c: (c.layerType, c.layerTitle, c.name))
179 for c in inv.caches:
180 for z in c.grabber.levels():
181 if z > c.grabber.cache.maxLevel:
182 break
183 mtr = c.grabber.tile_range_for_level(z)
184 nx = mtr[2] - mtr[0] + 1
185 ny = mtr[3] - mtr[1] + 1
186 c.levels.append(
187 Level(
188 z=z,
189 gridRange=mtr,
190 gridSize=(nx, ny),
191 resolution=gws.lib.grid.resolution_for_level(c.grabber.grid, z),
192 seedTime=0,
193 cachedTiles=0,
194 failedTiles=0,
195 fetchedTiles=0,
196 totalTiles=nx * ny,
197 percentCached=0,
198 fileSize=0,
199 )
200 )
202 for de in osx.find_entries(gws.c.MAP_CACHE_DIR, deep=False):
203 if not de.is_dir():
204 continue
205 c = cmap.get(de.name)
206 if not c:
207 inv.orphanDirs.append(de.path)
208 continue
209 c.dir = de.path
211 return inv
214def apply_filter(inv: Inventory, flt: Filter):
215 """Filter the inventory in place.
217 Selects caches by layer, cache name prefix and CRS, then restricts their levels to ``flt.levels``
218 and to the tile ranges covering ``flt.bbox``, which must be in the CRS of the caches.
219 Caches without levels are removed.
221 Args:
222 inv: Inventory to filter.
223 flt: Filter.
224 """
226 caches = []
228 for c in inv.caches:
229 b1 = not flt.layerUids or any(la.uid in flt.layerUids for la in c.layers)
230 b2 = not flt.cacheNames or any(c.name.startswith(cn) for cn in flt.cacheNames)
231 b3 = not flt.srids or c.grabber.targetCrs.srid in flt.srids
232 if not (b1 and b2 and b3):
233 continue
235 levels = []
236 for lv in c.levels:
237 if flt.levels and lv.z not in flt.levels:
238 continue
239 if flt.bbox:
240 mtr = gws.lib.grid.range_for_extent(c.grabber.grid, flt.bbox.extent, lv.z)
241 mtr = gws.lib.grid.intersect_ranges(lv.gridRange, mtr) if mtr else None
242 if not mtr:
243 continue
244 lv.gridRange = mtr
245 lv.gridSize = (mtr[2] - mtr[0] + 1, mtr[3] - mtr[1] + 1)
246 lv.totalTiles = lv.gridSize[0] * lv.gridSize[1]
247 levels.append(lv)
249 if levels:
250 c.levels = levels
251 caches.append(c)
253 inv.caches = caches
256def add_stats(inv: Inventory):
257 """Read store statistics into the inventory.
259 Sets the number and size of stored tiles and the percentage cached for each level and cache.
261 Args:
262 inv: Inventory to update.
263 """
265 for c in inv.caches:
266 for lv in c.levels:
267 s = c.grabber.store.stats_for_level(lv.z)
268 lv.cachedTiles = s.count
269 lv.fileSize = s.size
270 lv.cachedRange = s.range
271 lv.percentCached = percent_cached(lv)
272 c.fileSize = sum(lv.fileSize for lv in c.levels)
273 c.cachedTiles = sum(lv.cachedTiles for lv in c.levels)
276def percent_cached(lv: Level) -> int:
277 """Compute the percentage of cached and fetched tiles of a level.
279 Args:
280 lv: Level.
282 Returns:
283 A percentage from 0 to 100, at least 1 if any tile is present.
284 """
286 n = lv.cachedTiles + lv.fetchedTiles
287 if not n or not lv.totalTiles:
288 return 0
289 return min(100, max(1, int(n * 100 / lv.totalTiles)))
292def percentage_by_level(c: Cache) -> list[int]:
293 """Compute the cached percentages of all levels of a cache.
295 Args:
296 c: Cache.
298 Returns:
299 A list of percentages indexed by level, up to the cache max. level.
300 Levels not in the cache are 0.
301 """
303 ps = [0] * (c.grabber.cache.maxLevel + 1)
304 for lv in c.levels:
305 ps[lv.z] = percent_cached(lv)
306 return ps
309def cleanup(root: gws.Root):
310 """Remove orphan cache directories.
312 Args:
313 root: Configuration root.
314 """
316 inv = inventory(root)
317 for d in inv.orphanDirs:
318 gws.log.info(f'cleanup: removing orphan cache directory {d}')
319 osx.rmdir(d)
322def drop(root: gws.Root, flt: Optional[Filter] = None):
323 """Remove cached tiles.
325 If the filter has neither levels nor a bbox, the whole store of each selected cache is removed.
326 Otherwise, the tile ranges covering the bbox, or the selected levels, are removed.
328 Args:
329 root: Configuration root.
330 flt: Filter, selects all caches if omitted.
331 """
333 flt = flt or Filter()
334 inv = inventory(root)
335 apply_filter(inv, flt)
337 for c in inv.caches:
338 if not flt.bbox and not flt.levels:
339 if c.dir:
340 gws.log.info(f'drop: removing cache directory {c.dir}')
341 c.grabber.store.drop()
342 continue
343 for lv in c.levels:
344 if flt.bbox:
345 gws.log.info(f'drop: {c.name}: removing tiles {lv.gridRange}')
346 c.grabber.store.drop_range(lv.gridRange)
347 else:
348 gws.log.info(f'drop: {c.name}: removing level {lv.z}')
349 c.grabber.store.drop_level(lv.z)
352def store_in_web_cache(url: str, img: bytes):
353 """Store an image in the web cache.
355 Writes the image to ``FASTCACHE_DIR`` under the URL path. Write errors are logged and ignored.
357 Args:
358 url: URL path, used as the file path in the web cache.
359 img: Image data.
360 """
361 path = gws.c.FASTCACHE_DIR + url
362 try:
363 os.makedirs(os.path.dirname(path), 0o755, exist_ok=True)
364 gws.u.write_file_b(path, img)
365 except OSError:
366 gws.log.warning(f'store_in_web_cache FAILED path={path!r}')