Coverage for gws-app/gws/gis/cache/cli.py: 48%

1"""Command-line cache commands.""" 

2 

3from typing import Optional 

4 

5import re 

6 

7import gws 

8import gws.lib.shape 

9import gws.lib.crs 

10import gws.lib.datetimex 

11import gws.lib.jsonx 

12import gws.lib.cli as cli 

13 

14from . import core, seed 

15 

16 

17class FilterParams(gws.CliParams): 

18 """Parameters that select caches.""" 

19 

20 layerUids: Optional[list[str]] 

21 """List of layer IDs.""" 

22 cacheNames: Optional[list[str]] 

23 """List of cache names or prefixes.""" 

24 crs: Optional[list[int]] 

25 """List of CRS codes.""" 

26 

27 

28class StatusParams(FilterParams): 

29 """Parameters of the ``cache status`` command.""" 

30 

31 details: bool = False 

32 """Print detailed status.""" 

33 json: str = '' 

34 """Write json report to path.""" 

35 

36 

37class FilterParamsWithGeom(FilterParams): 

38 """Parameters that select caches, levels and an area.""" 

39 

40 bbox: Optional[list[float]] 

41 """Bounding box [minx, miny, maxx, maxy].""" 

42 wkt: str = '' 

43 """WKT geometry.""" 

44 levels: str = '' 

45 """Zoom levels (1,2,3 or 1-3).""" 

46 

47 

48class DropParams(FilterParamsWithGeom): 

49 """Parameters of the ``cache drop`` command.""" 

50 

51 pass 

52 

53 

54class SeedParams(FilterParamsWithGeom): 

55 """Parameters of the ``cache seed`` command.""" 

56 

57 maxTime: Optional[int] 

58 """Max. seeding time in seconds.""" 

59 concurrency: Optional[int] 

60 """Number of concurrent seeding threads.""" 

61 maxAge: str = '' 

62 """Reseed tiles older than this duration (c.g. 1d, 0 to reseed all).""" 

63 json: str = '' 

64 """Write json report to path.""" 

65 

66 

67@gws.ext.object.cli('cache') 

68class Object(gws.Node): 

69 """Command line commands for the tile cache.""" 

70 

71 @gws.ext.command.cli('cacheStatus') 

72 def do_status(self, p: StatusParams): 

73 """Display the cache status.""" 

74 

75 root = gws.load_root() 

76 inv = core.inventory(root) 

77 core.apply_filter(inv, _filter(p)) 

78 core.add_stats(inv) 

79 if p.json: 

80 gws.lib.jsonx.to_path(p.json, _status_to_json(inv)) 

81 elif p.details: 

82 _display_status_details(inv) 

83 else: 

84 _display_status_brief(inv) 

85 

86 @gws.ext.command.cli('cacheCleanup') 

87 def do_cleanup(self, p: gws.CliParams): 

88 """Remove orphan cache directories.""" 

89 

90 root = gws.load_root() 

91 core.cleanup(root) 

92 

93 @gws.ext.command.cli('cacheDrop') 

94 def do_drop(self, p: DropParams): 

95 """Remove cached tiles.""" 

96 

97 root = gws.load_root() 

98 core.drop(root, _filter_with_geom(p)) 

99 

100 @gws.ext.command.cli('cacheSeed') 

101 def do_seed(self, p: SeedParams): 

102 """Seed the selected caches.""" 

103 

104 root = gws.load_root() 

105 

106 opts = core.SeedOptions( 

107 filter=_filter_with_geom(p), 

108 maxTime=p.maxTime or root.app.cfg('cache.seedingMaxTime'), 

109 concurrency=p.concurrency or root.app.cfg('cache.seedingConcurrency'), 

110 maxAge=gws.lib.datetimex.parse_duration(p.maxAge) if p.maxAge else None, 

111 ) 

112 res = seed.seed(root, opts) 

113 if p.json: 

114 gws.lib.jsonx.to_path(p.json, _seed_to_json(res)) 

115 else: 

116 _display_seed(res) 

117 

118 

119## 

120 

121 

122def _filter(p: FilterParams) -> core.Filter: 

123 return core.Filter( 

124 layerUids=gws.u.to_list(p.layerUids), 

125 cacheNames=gws.u.to_list(p.cacheNames), 

126 srids=[int(s) for s in gws.u.to_list(p.crs)], 

127 ) 

128 

129 

130def _levels(s: str) -> list[int]: 

131 if not s: 

132 return [] 

133 m = re.match(r'^(\d+)-(\d+)$', s) 

134 if m: 

135 return list(range(int(m.group(1)), int(m.group(2)) + 1)) 

136 return [int(x) for x in gws.u.to_list(s)] 

137 

138 

139def _filter_with_geom(p: FilterParamsWithGeom) -> core.Filter: 

140 flt = _filter(p) 

141 flt.levels = _levels(p.levels) 

142 

143 bbox = gws.u.to_list(p.bbox) 

144 wkt = p.wkt or '' 

145 if not bbox and not wkt: 

146 return flt 

147 

148 if bbox and wkt: 

149 raise gws.Error('only one of bbox and wkt can be given') 

150 if len(flt.srids) != 1: 

151 raise gws.Error('exactly one crs is required with bbox or wkt') 

152 

153 crs = gws.lib.crs.require(flt.srids[0]) 

154 

155 if bbox: 

156 if len(bbox) != 4: 

157 raise gws.Error(f'invalid bbox {p.bbox!r}') 

158 flt.bbox = gws.Bounds(crs=crs, extent=tuple(float(v) for v in bbox)) 

159 else: 

160 flt.bbox = gws.lib.shape.from_wkt(wkt, crs).bounds() 

161 

162 return flt 

163 

164 

165def _status_to_json(inv: core.Inventory) -> dict: 

166 caches = [] 

167 

168 for c in inv.caches: 

169 caches.append( 

170 { 

171 'name': c.name, 

172 'dir': c.dir, 

173 'crs': c.grabber.targetCrs.srid, 

174 'layers': [ 

175 { 

176 'uid': la.uid, 

177 'type': la.extType, 

178 'title': gws.u.get(la, 'title', ''), 

179 } 

180 for la in c.layers 

181 ], 

182 'levels': [ 

183 { 

184 'level': lv.z, 

185 'gridSize': list(lv.gridSize), 

186 'gridRange': list(lv.gridRange), 

187 'totalTiles': lv.totalTiles, 

188 'cachedTiles': lv.cachedTiles, 

189 'fileSize': lv.fileSize, 

190 'percentCached': lv.percentCached, 

191 } 

192 for lv in c.levels 

193 ], 

194 } 

195 ) 

196 

197 return { 

198 'caches': caches, 

199 'orphanDirs': list(inv.orphanDirs), 

200 } 

201 

202 

203def _display_status_brief(inv: core.Inventory): 

204 cli.info(f'{len(inv.caches)} CACHES') 

205 if inv.orphanDirs: 

206 cli.info(f'{len(inv.orphanDirs)} ORPHAN DIRECTORIES') 

207 cli.info('') 

208 

209 table = [] 

210 for c in inv.caches: 

211 ls = c.layerType + ' "' + c.layerTitle[:30] + '"' 

212 if len(c.layers) > 1: 

213 ls += f' (+{len(c.layers) - 1})' 

214 percents = ' '.join(f'{z}:{p}' for z, p in enumerate(core.percentage_by_level(c))) 

215 table.append( 

216 { 

217 'cache': c.name, 

218 'layer': ls, 

219 'levels': max(lv.z for lv in c.levels), 

220 'files': c.cachedTiles, 

221 'size': _format_file_size(c.fileSize), 

222 '%%': percents, 

223 } 

224 ) 

225 

226 cli.info(cli.text_table(table, header='auto')) 

227 

228 

229def _display_status_details(inv: core.Inventory): 

230 cli.info(f'{len(inv.caches)} CACHES') 

231 if inv.orphanDirs: 

232 cli.info(f'{len(inv.orphanDirs)} ORPHAN DIRECTORIES') 

233 cli.info('') 

234 

235 for c in inv.caches: 

236 ls = c.layerType + ' "' + c.layerTitle[:30] + '"' 

237 if len(c.layers) > 1: 

238 ls += f' (+{len(c.layers) - 1})' 

239 percents = ' '.join(f'{z}:{p}' for z, p in enumerate(core.percentage_by_level(c))) 

240 

241 cli.info('') 

242 cli.info('=' * 80) 

243 cli.info('') 

244 cli.info(f'CACHE {c.name}') 

245 cli.info(f'LAYER {ls}') 

246 cli.info(f'FILES {c.cachedTiles}, {_format_file_size(c.fileSize)}') 

247 cli.info(f'%% {percents}') 

248 

249 if not c.levels: 

250 continue 

251 

252 table = [] 

253 

254 for lv in c.levels: 

255 table.append( 

256 { 

257 'level': lv.z, 

258 'range': f'{lv.gridRange[0]},{lv.gridRange[1]} - {lv.gridRange[2]},{lv.gridRange[3]}', 

259 'grid': f'{lv.gridSize[0]} x {lv.gridSize[1]}', 

260 'total': lv.totalTiles, 

261 'cached': lv.cachedTiles, 

262 '%%': lv.percentCached, 

263 } 

264 ) 

265 cli.info('') 

266 cli.info(cli.text_table(table, header='auto')) 

267 

268 if inv.orphanDirs: 

269 cli.info('') 

270 cli.info('=' * 80) 

271 cli.info(f'{len(inv.orphanDirs)} ORPHAN DIRECTORIES ("gws cache cleanup" to remove):') 

272 for d in inv.orphanDirs: 

273 cli.info(f' {d}') 

274 

275 

276def _seed_to_json(res: core.SeedResult) -> dict: 

277 caches = [] 

278 

279 for c in res.caches: 

280 caches.append( 

281 { 

282 'name': c.name, 

283 'dir': c.dir, 

284 'layers': [ 

285 { 

286 'uid': la.uid, 

287 'type': la.extType, 

288 'title': gws.u.get(la, 'title', ''), 

289 } 

290 for la in c.layers 

291 ], 

292 'levels': [ 

293 { 

294 'level': lv.z, 

295 'totalTiles': lv.totalTiles, 

296 'cachedTiles': lv.cachedTiles, 

297 'fetchedTiles': lv.fetchedTiles, 

298 'failedTiles': lv.failedTiles, 

299 'seedTime': round(lv.seedTime, 1), 

300 } 

301 for lv in c.levels 

302 ], 

303 'seedStatus': c.seedStatus or '', 

304 } 

305 ) 

306 

307 return { 

308 'caches': caches, 

309 'seedTime': round(res.seedTime, 1), 

310 'seedStatus': res.seedStatus, 

311 } 

312 

313 

314def _display_seed(res: core.SeedResult): 

315 table = [] 

316 

317 for c in res.caches: 

318 for lv in c.levels: 

319 if not lv.fetchedTiles and not lv.failedTiles: 

320 continue 

321 table.append( 

322 { 

323 'cache': c.name, 

324 'level': lv.z, 

325 'total': lv.totalTiles, 

326 'fetched': lv.fetchedTiles, 

327 'failed': lv.failedTiles, 

328 '%%': core.percent_cached(lv), 

329 'time': round(lv.seedTime, 1), 

330 'tps': int(lv.fetchedTiles / lv.seedTime) if lv.seedTime else 0, 

331 } 

332 ) 

333 

334 cli.info('') 

335 cli.info(cli.text_table(table, header='auto')) 

336 

337 

338def _format_file_size(size: int) -> str: 

339 if size == 0: 

340 return '0' 

341 s = float(size) 

342 for unit in ['B', 'KB', 'MB', 'GB', 'TB']: 

343 if s < 1024: 

344 return f'{s:.1f} {unit}' 

345 s /= 1024 

346 return str(size) + '??'