Coverage for gws-app/gws/plugin/qgis/provider.py: 64%

1"""QGIS project provider.""" 

2 

3from typing import Optional, cast 

4 

5import gws 

6import gws.base.database 

7import gws.base.ows.client 

8import gws.config.util 

9import gws.plugin.postgres.provider 

10import gws.base.metadata 

11import gws.lib.net 

12import gws.lib.mime 

13import gws.lib.osx 

14import gws.lib.crs 

15import gws.lib.bounds 

16import gws.gis.source 

17import gws.lib.extent 

18import gws.lib.net 

19 

20from . import caps as caps_module, project 

21 

22 

23class Config(gws.Config): 

24 """QGIS project, served by QGIS Server.""" 

25 

26 path: Optional[gws.FilePath] 

27 """QGIS project file.""" 

28 dbUid: Optional[str] 

29 """Database provider UID for projects stored in a database.""" 

30 schema: Optional[str] 

31 """Database schema for projects stored in a database.""" 

32 projectName: Optional[str] 

33 """Project name for projects stored in a database.""" 

34 defaultLegendOptions: Optional[dict] 

35 """Legend options applied to all layers of this project.""" 

36 directRender: Optional[list[str]] 

37 """Layer sources to load directly, not through QGIS Server.""" 

38 directSearch: Optional[list[str]] 

39 """Layer sources to search directly, not through QGIS Server.""" 

40 forceCrs: Optional[gws.CrsName] 

41 """CRS for QGIS Server requests.""" 

42 extentBuffer: Optional[int] 

43 """Buffer around the extent computed from layer data.""" 

44 useCanvasExtent: Optional[bool] 

45 """Use the map canvas extent when the project has no WMS extent.""" 

46 withWatch: Optional[bool] 

47 """Reload the application when the QGIS project changes.""" 

48 watchFrequency: Optional[gws.Duration] 

49 """Interval between checks for project changes.""" 

50 

51 

52class Object(gws.OwsServiceProvider): 

53 """Provider for a QGIS project, served by QGIS Server. 

54 

55 The provider loads and parses the project, computes the project bounds 

56 and sends requests to QGIS Server. 

57 """ 

58 

59 store: project.Store 

60 """Location of the project.""" 

61 printTemplates: list[caps_module.PrintTemplate] 

62 """Print layouts of the project.""" 

63 

64 directRender: set[str] 

65 """Data source providers rendered directly, not through QGIS Server (``wms``, ``wmts``, ``xyz``).""" 

66 directSearch: set[str] 

67 """Data source providers searched directly, not through QGIS Server (``wms``, ``wfs``, ``postgres``).""" 

68 

69 defaultLegendOptions: dict 

70 """Legend options applied to all layers of the project.""" 

71 

72 caps: caps_module.Caps 

73 """Parsed project capabilities.""" 

74 sourceHash: str 

75 """Hash of the project XML when watching is enabled, otherwise empty.""" 

76 

77 def configure(self): 

78 self.configure_store() 

79 

80 self.url = 'http://{}:{}'.format( 

81 self.root.app.cfg('server.qgis.host'), 

82 self.root.app.cfg('server.qgis.port')) 

83 

84 self.caps = self.qgis_project().caps() 

85 

86 self.metadata = self.caps.metadata 

87 self.printTemplates = self.caps.printTemplates 

88 self.sourceLayers = self.caps.sourceLayers 

89 self.version = self.caps.version 

90 

91 self.forceCrs = gws.lib.crs.get(self.cfg('forceCrs')) or self.caps.projectCrs 

92 self.alwaysXY = False 

93 

94 self.bounds = self._project_bounds() 

95 self.wgsExtent = gws.lib.bounds.wgs_extent(self.bounds) 

96 

97 self.directRender = self._direct_formats('directRender', {'wms', 'wmts', 'xyz'}) 

98 self.directSearch = self._direct_formats('directSearch', {'wms', 'wfs', 'postgres'}) 

99 

100 self.defaultLegendOptions = self.cfg('defaultLegendOptions', default={}) 

101 

102 self.sourceHash = '' 

103 if self.cfg('withWatch'): 

104 self.sourceHash = self.qgis_project().sourceHash 

105 self.root.app.monitor.register_periodic_task(self, frequency=self.cfg('watchFrequency', default=0)) 

106 gws.log.info(f'QGIS: monitoring: enabled: {self.server_project_path()!r}') 

107 

108 def cache_hash(self): 

109 """Compute a hash of the provider settings that affect rendering. 

110 

111 Returns: 

112 Hash string, built from the store and the request CRS. 

113 """ 

114 return gws.u.sha256([ 

115 vars(self.store), 

116 self.forceCrs.srid, 

117 ]) 

118 

119 def periodic_task(self): 

120 h = self.qgis_project().sourceHash 

121 if h != self.sourceHash: 

122 gws.log.info(f'QGIS: monitoring: CHANGED: {self.server_project_path()!r}') 

123 self.sourceHash = h 

124 self.root.app.monitor.schedule_reload(with_reconfigure=True) 

125 

126 def _project_bounds(self): 

127 # explicit WMS extent? 

128 if self.caps.projectBounds: 

129 return gws.lib.bounds.transform(self.caps.projectBounds, self.forceCrs) 

130 

131 # canvas extent? 

132 if self.cfg('useCanvasExtent') and self.caps.projectCanvasBounds: 

133 return gws.lib.bounds.transform(self.caps.projectCanvasBounds, self.forceCrs) 

134 

135 # combined data extents + buffer 

136 b = gws.gis.source.combined_bounds(self.sourceLayers, self.forceCrs) 

137 if b: 

138 return gws.lib.bounds.buffer(b, self.cfg('extentBuffer') or 0) 

139 

140 return self.forceCrs.bounds 

141 

142 def _direct_formats(self, opt, allowed): 

143 p = self.cfg(opt) 

144 if not p: 

145 return set() 

146 

147 res = set() 

148 

149 for s in p: 

150 s = s.lower() 

151 if s not in allowed: 

152 raise gws.ConfigurationError(f'{opt} not supported for {s!r}') 

153 res.add(s) 

154 

155 return res 

156 

157 def configure_store(self): 

158 """Set the project store from ``path`` or ``projectName``. 

159 

160 Raises: 

161 ``gws.ConfigurationError``: If neither ``path`` nor ``projectName`` is configured. 

162 """ 

163 p = self.cfg('path') 

164 if p: 

165 pp = gws.lib.osx.parse_path(p) 

166 self.store = project.Store( 

167 type=project.StoreType.file, 

168 projectName=pp.stem, 

169 path=p, 

170 ) 

171 return 

172 p = self.cfg('projectName') 

173 if p: 

174 self.store = project.Store( 

175 type=project.StoreType.postgres, 

176 projectName=p, 

177 dbUid=self.cfg('dbUid'), 

178 schema=self.cfg('schema') or 'public', 

179 ) 

180 return 

181 # @TODO gpkg, etc 

182 raise gws.ConfigurationError('cannot load qgis project ("path" or "projectName" must be specified)') 

183 

184 ## 

185 

186 def qgis_project(self) -> project.Object: 

187 """Load the project from its store. 

188 

189 Returns: 

190 A freshly loaded project. 

191 

192 Raises: 

193 ``project.Error``: If the project cannot be loaded. 

194 """ 

195 return project.from_store(self.root, self.store) 

196 

197 def server_project_path(self): 

198 """Return the project address for the QGIS Server ``MAP`` parameter. 

199 

200 For a file store this is the file path. For a Postgres store this is 

201 the database connection URL with ``schema`` and ``project`` parameters; 

202 when watching is enabled, the project hash is appended to invalidate 

203 the QGIS Server cache. 

204 

205 Returns: 

206 Project path or URL. 

207 """ 

208 if self.store.type == project.StoreType.file: 

209 return self.store.path 

210 if self.store.type == project.StoreType.postgres: 

211 prov = self.root.app.databaseMgr.find_provider(ext_type='postgres', uid=self.store.dbUid) 

212 p = {'schema': self.store.schema, 'project': self.store.projectName} 

213 if self.sourceHash: 

214 # NB the hash is to invalidate QGIS Server internal cache 

215 p['h'] = self.sourceHash 

216 return gws.lib.net.add_params(prov.url(), p) 

217 

218 def server_params(self, params: dict) -> dict: 

219 """Add default parameters to a QGIS Server request. 

220 

221 Args: 

222 params: Request parameters; keys are converted to upper case. 

223 

224 Returns: 

225 Parameters with ``MAP``, ``SERVICE`` and ``VERSION`` defaults. 

226 """ 

227 defaults = dict( 

228 MAP=self.server_project_path(), 

229 SERVICE=gws.OwsProtocol.WMS, 

230 VERSION='1.3.0', 

231 ) 

232 return gws.u.merge(defaults, gws.u.to_upper_dict(params)) 

233 

234 def call_server(self, params: dict) -> gws.lib.net.HTTPResponse: 

235 """Send a request to QGIS Server. 

236 

237 Args: 

238 params: Request parameters, completed by ``server_params``. 

239 

240 Returns: 

241 HTTP response. 

242 

243 Raises: 

244 ``gws.lib.net.Error``: If the request fails. 

245 """ 

246 params = self.server_params(params) 

247 res = gws.lib.net.http_request(self.url, params=params, timeout=1000) 

248 res.raise_if_failed() 

249 return res 

250 

251 ## 

252 

253 def get_map(self, bounds: gws.Bounds, width: float, height: float, params: dict, mime: gws.MimeType = '') -> bytes: 

254 """Render a map image with a GetMap request. 

255 

256 Args: 

257 bounds: Box to render. 

258 width: Image width in pixels. 

259 height: Image height in pixels. 

260 params: Extra request parameters, e.g. ``LAYERS``; they override the defaults. 

261 mime: MIME type to request, PNG if empty. 

262 

263 Returns: 

264 Image bytes. 

265 

266 Raises: 

267 ``gws.Error``: If QGIS Server returns a non-image response. 

268 """ 

269 bbox = bounds.extent 

270 if bounds.crs.isYX and not self.alwaysXY: 

271 bbox = gws.lib.extent.swap_xy(bbox) 

272 

273 defaults = dict( 

274 REQUEST=gws.OwsVerb.GetMap, 

275 BBOX=bbox, 

276 WIDTH=gws.u.to_rounded_int(width), 

277 HEIGHT=gws.u.to_rounded_int(height), 

278 CRS=bounds.crs.epsg, 

279 FORMAT=mime or gws.lib.mime.PNG, 

280 TRANSPARENT='true', 

281 STYLES='', 

282 ) 

283 

284 params = gws.u.merge(defaults, params) 

285 

286 res = self.call_server(params) 

287 if res.content_type.startswith('image/'): 

288 return res.content 

289 raise gws.Error(res.text) 

290 

291 def get_features(self, search, source_layers, mime=''): 

292 shape = search.shape 

293 if not shape or shape.type != gws.GeometryType.point: 

294 return [] 

295 

296 request_crs = self.forceCrs 

297 

298 box_size_m = 500 

299 box_size_deg = 1 

300 box_size_px = 500 

301 

302 size = None 

303 

304 if shape.crs.uom == gws.Uom.m: 

305 size = box_size_px * search.resolution 

306 if shape.crs.uom == gws.Uom.deg: 

307 # @TODO use search.resolution here as well 

308 size = box_size_deg 

309 if not size: 

310 gws.log.debug(f'cannot request crs {shape.crs!r}, unsupported unit') 

311 return [] 

312 

313 bbox = ( 

314 shape.x - (size / 2), 

315 shape.y - (size / 2), 

316 shape.x + (size / 2), 

317 shape.y + (size / 2), 

318 ) 

319 

320 bbox = gws.lib.extent.transform(bbox, shape.crs, request_crs) 

321 

322 layer_names = [sl.name for sl in source_layers] 

323 

324 params = { 

325 'BBOX': bbox, 

326 'CRS': request_crs.to_string(gws.CrsFormat.epsg), 

327 'WIDTH': box_size_px, 

328 'HEIGHT': box_size_px, 

329 'I': box_size_px >> 1, 

330 'J': box_size_px >> 1, 

331 'LAYERS': layer_names, 

332 'QUERY_LAYERS': layer_names, 

333 'STYLES': [''] * len(layer_names), 

334 'FEATURE_COUNT': search.limit or 100, 

335 'INFO_FORMAT': mime or gws.lib.mime.XML, 

336 'REQUEST': gws.OwsVerb.GetFeatureInfo, 

337 'WITH_GEOMETRY': 'true', 

338 } 

339 

340 if search.extraParams: 

341 params = gws.u.merge(params, gws.u.to_upper_dict(search.extraParams)) 

342 

343 res = self.call_server(params) 

344 

345 try: 

346 records = gws.base.ows.client.featureinfo.parse(res.text, default_crs=request_crs, always_xy=self.alwaysXY) 

347 except gws.Error as exc: 

348 gws.log.error(f'get_features: parse error: {exc!r}') 

349 return [] 

350 

351 gws.log.debug(f'get_features: FOUND={len(records)} params={params!r}') 

352 

353 for rec in records: 

354 if rec.shape: 

355 rec.shape = rec.shape.transformed_to(shape.crs) 

356 

357 return records 

358 

359 ## 

360 

361 def create_leaf_layer_config(self, source_layers): 

362 """Create the configuration of a leaf layer for the ``qgis`` tree layer. 

363 

364 By default, this is a ``qgisflat`` layer. For a single source layer 

365 whose data source provider is listed in ``directRender`` or 

366 ``directSearch``, the layer is rendered directly (``wmsflat``, 

367 ``wmts`` or ``tile``) and gets its own finders and models. 

368 

369 Args: 

370 source_layers: Source layers of the leaf. 

371 

372 Returns: 

373 Layer configuration dict. 

374 """ 

375 simple_cfg = { 

376 'type': 'qgisflat', 

377 '_defaultProvider': self, 

378 '_defaultSourceLayers': source_layers, 

379 # opacity is handled by QGIS, so we set it to 1 here. 

380 'opacity': 1, 

381 } 

382 

383 if len(source_layers) > 1 or source_layers[0].isGroup: 

384 return simple_cfg 

385 

386 ds = source_layers[0].dataSource 

387 if not ds or not ds.get('provider'): 

388 return simple_cfg 

389 

390 render = self._leaf_render_config(ds) 

391 search = self._leaf_search_config(ds) 

392 

393 cfg = {} 

394 cfg.update(render or {}) 

395 cfg.update(search or {}) 

396 

397 if not cfg.get('type'): 

398 cfg.update(simple_cfg) 

399 

400 return cfg 

401 

402 def _leaf_render_config(self, ds): 

403 """Create a direct render layer configuration for a data source.""" 

404 prov = ds.get('provider') 

405 if prov not in self.directRender: 

406 return 

407 

408 url = self._leaf_service_url(ds.get('url'), ds.get('params')) 

409 if not url: 

410 return 

411 

412 if prov == 'wms': 

413 layers = ds.get('layers') 

414 if not layers: 

415 return 

416 return { 

417 'type': 'wmsflat', 

418 'sourceLayers': {'names': layers}, 

419 'display': 'tile', 

420 'provider': {'url': url}, 

421 } 

422 

423 if prov == 'wmts': 

424 layers = ds.get('layers') 

425 if not layers: 

426 return 

427 cfg = { 

428 'type': 'wmts', 

429 'sourceLayers': {'names': layers}, 

430 'display': 'tile', 

431 'provider': {'url': url}, 

432 } 

433 p = ds.get('styles') 

434 if p: 

435 cfg['style'] = p[0] 

436 return cfg 

437 

438 if prov == 'xyz': 

439 return { 

440 'type': 'tile', 

441 'provider': {'url': url}, 

442 } 

443 

444 def _leaf_search_config(self, ds): 

445 """Create direct finder and model configurations for a data source.""" 

446 prov = ds.get('provider') 

447 if prov not in self.directSearch: 

448 return 

449 

450 if prov == 'wms': 

451 url = self._leaf_service_url(ds.get('url'), ds.get('params')) 

452 layers = ds.get('layers') 

453 if not url or not layers: 

454 return 

455 finder = { 

456 'type': 'wms', 

457 'provider': {'url': url}, 

458 'sourceLayers': {'names': layers}, 

459 } 

460 return {'finders': [finder]} 

461 

462 if prov == 'wfs': 

463 url = self._leaf_service_url(ds.get('url'), ds.get('params')) 

464 if not url: 

465 return 

466 finder = { 

467 'type': 'wfs', 

468 'provider': {'url': url}, 

469 } 

470 p = ds.get('typename') 

471 if p: 

472 finder['sourceLayers'] = {'names': [p]} 

473 p = ds.get('srsname') 

474 if p: 

475 finder['forceCrs'] = p 

476 p = ds.get('ignoreaxisorientation') 

477 if p == '1': 

478 finder['alwaysXY'] = True 

479 p = ds.get('invertaxisorientation') 

480 if p == '1': 

481 # NB assuming this might be only '1' for lat-lon projections 

482 finder['alwaysXY'] = True 

483 

484 return {'finders': [finder]} 

485 

486 if prov == 'postgres': 

487 table_name = ds.get('table') 

488 

489 # 'table' can also be a select statement, in which case it might be enclosed in parens 

490 if not table_name or table_name.startswith('(') or table_name.upper().startswith('SELECT '): 

491 return 

492 

493 db = self.postgres_provider_from_datasource(ds) 

494 

495 model = { 

496 'type': 'postgres', 

497 'tableName': table_name, 

498 'sqlFilter': ds.get('sql'), 

499 '_defaultDb': db 

500 } 

501 finder = { 

502 'type': 'postgres', 

503 'tableName': table_name, 

504 'sqlFilter': ds.get('sql'), 

505 '_defaultDb': db 

506 } 

507 return {'models': [model], 'finders': [finder]} 

508 

509 def postgres_provider_from_datasource(self, ds: dict) -> gws.plugin.postgres.provider.Object: 

510 """Find or create a Postgres provider for a QGIS data source. 

511 

512 An existing provider with the same connection URL is reused, 

513 otherwise a new provider is created. 

514 

515 Args: 

516 ds: Parsed Postgres data source. 

517 

518 Returns: 

519 Postgres provider. 

520 """ 

521 cfg = gws.Config( 

522 host=ds.get('host'), 

523 port=ds.get('port'), 

524 database=ds.get('dbname'), 

525 username=ds.get('user'), 

526 password=ds.get('password'), 

527 serviceName=ds.get('service'), 

528 options=ds.get('options'), 

529 ) 

530 url = gws.plugin.postgres.provider.connection_url(cfg) 

531 mgr = self.root.app.databaseMgr 

532 

533 for p in mgr.providers: 

534 if p.extType == 'postgres' and p.url() == url: 

535 return cast(gws.plugin.postgres.provider.Object, p) 

536 

537 gws.log.debug(f'creating an ad-hoc postgres provider for qgis {url=}') 

538 p = mgr.create_provider(cfg, type='postgres') 

539 return cast(gws.plugin.postgres.provider.Object, p) 

540 

541 _std_ows_params = { 

542 'bbox', 

543 'bgcolor', 

544 'crs', 

545 'exceptions', 

546 'format', 

547 'height', 

548 'layers', 

549 'request', 

550 'service', 

551 'sld', 

552 'sld_body', 

553 'srs', 

554 'styles', 

555 'time', 

556 'transparent', 

557 'version', 

558 'width', 

559 } 

560 

561 def _leaf_service_url(self, url, params): 

562 """Add the non-standard OWS parameters of a data source to its URL.""" 

563 if not url: 

564 return 

565 if not params: 

566 return url 

567 

568 # a wms url can be like "server?service=WMS....&bbox=.... &some-non-std-param=... 

569 # we need to keep non-std params for caps requests 

570 

571 p = {k: v for k, v in params.items() if k.lower() not in self._std_ows_params} 

572 return gws.lib.net.add_params(url, p)