Coverage for gws-app/gws/plugin/qgis/provider.py: 42%

1"""QGIS project provider."""

3from typing import Optional, cast

5import gws

6import gws.base.database

7import gws.base.ows.client

8import gws.config.util

9import gws.plugin.postgres.provider

10import gws.base.metadata

11import gws.lib.net

12import gws.lib.mime

13import gws.lib.osx

14import gws.lib.crs

15import gws.lib.bounds

16import gws.gis.source

17import gws.lib.extent

18import gws.lib.net

20from . import caps as caps_module, project

23class Config(gws.Config):

24 """QGIS project, served by QGIS Server."""

26 path: Optional[gws.FilePath]

27 """QGIS project file."""

28 dbUid: Optional[str]

29 """Database provider UID for projects stored in a database."""

30 schema: Optional[str]

31 """Database schema for projects stored in a database."""

32 projectName: Optional[str]

33 """Project name for projects stored in a database."""

34 defaultLegendOptions: Optional[dict]

35 """Legend options applied to all layers of this project."""

36 directRender: Optional[list[str]]

37 """Layer sources to load directly, not through QGIS Server."""

38 directSearch: Optional[list[str]]

39 """Layer sources to search directly, not through QGIS Server."""

40 forceCrs: Optional[gws.CrsName]

41 """CRS for QGIS Server requests."""

42 extentBuffer: Optional[int]

43 """Buffer around the extent computed from layer data."""

44 useCanvasExtent: Optional[bool]

45 """Use the map canvas extent when the project has no WMS extent."""

46 withWatch: Optional[bool]

47 """Reload the application when the QGIS project changes."""

48 watchFrequency: Optional[gws.Duration]

49 """Interval between checks for project changes."""

52class Object(gws.OwsServiceProvider):

53 """Provider for a QGIS project, served by QGIS Server.

55 The provider loads and parses the project, computes the project bounds

56 and sends requests to QGIS Server.

57 """

59 store: project.Store

60 """Location of the project."""

61 printTemplates: list[caps_module.PrintTemplate]

62 """Print layouts of the project."""

64 directRender: set[str]

65 """Data source providers rendered directly, not through QGIS Server (``wms``, ``wmts``, ``xyz``)."""

66 directSearch: set[str]

67 """Data source providers searched directly, not through QGIS Server (``wms``, ``wfs``, ``postgres``)."""

69 defaultLegendOptions: dict

70 """Legend options applied to all layers of the project."""

72 caps: caps_module.Caps

73 """Parsed project capabilities."""

74 sourceHash: str

75 """Hash of the project XML when watching is enabled, otherwise empty."""

77 def configure(self):

78 self.configure_store()

80 self.url = 'http://{}:{}'.format(

81 self.root.app.cfg('server.qgis.host'),

82 self.root.app.cfg('server.qgis.port'))

84 self.caps = self.qgis_project().caps()

86 self.metadata = self.caps.metadata

87 self.printTemplates = self.caps.printTemplates

88 self.sourceLayers = self.caps.sourceLayers

89 self.version = self.caps.version

91 self.forceCrs = gws.lib.crs.get(self.cfg('forceCrs')) or self.caps.projectCrs

92 self.alwaysXY = False

94 self.bounds = self._project_bounds()

95 self.wgsExtent = gws.lib.bounds.wgs_extent(self.bounds)

97 self.directRender = self._direct_formats('directRender', {'wms', 'wmts', 'xyz'})

98 self.directSearch = self._direct_formats('directSearch', {'wms', 'wfs', 'postgres'})

100 self.defaultLegendOptions = self.cfg('defaultLegendOptions', default={})

101

102 self.sourceHash = ''

103 if self.cfg('withWatch'):

104 self.sourceHash = self.qgis_project().sourceHash

105 self.root.app.monitor.register_periodic_task(self, frequency=self.cfg('watchFrequency', default=0))

106 gws.log.info(f'QGIS: monitoring: enabled: {self.server_project_path()!r}')

107

108 def cache_hash(self):

109 """Compute a hash of the provider settings that affect rendering.

110

111 Returns:

112 Hash string, built from the store and the request CRS.

113 """

114 return gws.u.sha256([

115 vars(self.store),

116 self.forceCrs.srid,

117 ])

118

119 def periodic_task(self):

120 h = self.qgis_project().sourceHash

121 if h != self.sourceHash:

122 gws.log.info(f'QGIS: monitoring: CHANGED: {self.server_project_path()!r}')

123 self.sourceHash = h

124 self.root.app.monitor.schedule_reload(with_reconfigure=True)

125

126 def _project_bounds(self):

127 # explicit WMS extent?

128 if self.caps.projectBounds:

129 return gws.lib.bounds.transform(self.caps.projectBounds, self.forceCrs)

130

131 # canvas extent?

132 if self.cfg('useCanvasExtent') and self.caps.projectCanvasBounds:

133 return gws.lib.bounds.transform(self.caps.projectCanvasBounds, self.forceCrs)

134

135 # combined data extents + buffer

136 b = gws.gis.source.combined_bounds(self.sourceLayers, self.forceCrs)

137 if b:

138 return gws.lib.bounds.buffer(b, self.cfg('extentBuffer') or 0)

139

140 return self.forceCrs.bounds

141

142 def _direct_formats(self, opt, allowed):

143 p = self.cfg(opt)

144 if not p:

145 return set()

146

147 res = set()

148

149 for s in p:

150 s = s.lower()

151 if s not in allowed:

152 raise gws.ConfigurationError(f'{opt} not supported for {s!r}')

153 res.add(s)

154

155 return res

156

157 def configure_store(self):

158 """Set the project store from ``path`` or ``projectName``.

159

160 Raises:

161 ``gws.ConfigurationError``: If neither ``path`` nor ``projectName`` is configured.

162 """

163 p = self.cfg('path')

164 if p:

165 pp = gws.lib.osx.parse_path(p)

166 self.store = project.Store(

167 type=project.StoreType.file,

168 projectName=pp.stem,

169 path=p,

170 )

171 return

172 p = self.cfg('projectName')

173 if p:

174 self.store = project.Store(

175 type=project.StoreType.postgres,

176 projectName=p,

177 dbUid=self.cfg('dbUid'),

178 schema=self.cfg('schema') or 'public',

179 )

180 return

181 # @TODO gpkg, etc

182 raise gws.ConfigurationError('cannot load qgis project ("path" or "projectName" must be specified)')

183

184 ##

185

186 def qgis_project(self) -> project.Object:

187 """Load the project from its store.

188

189 Returns:

190 A freshly loaded project.

191

192 Raises:

193 ``project.Error``: If the project cannot be loaded.

194 """

195 return project.from_store(self.root, self.store)

196

197 def server_project_path(self):

198 """Return the project address for the QGIS Server ``MAP`` parameter.

199

200 For a file store this is the file path. For a Postgres store this is

201 the database connection URL with ``schema`` and ``project`` parameters;

202 when watching is enabled, the project hash is appended to invalidate

203 the QGIS Server cache.

204

205 Returns:

206 Project path or URL.

207 """

208 if self.store.type == project.StoreType.file:

209 return self.store.path

210 if self.store.type == project.StoreType.postgres:

211 prov = self.root.app.databaseMgr.find_provider(ext_type='postgres', uid=self.store.dbUid)

212 p = {'schema': self.store.schema, 'project': self.store.projectName}

213 if self.sourceHash:

214 # NB the hash is to invalidate QGIS Server internal cache

215 p['h'] = self.sourceHash

216 return gws.lib.net.add_params(prov.url(), p)

217

218 def server_params(self, params: dict) -> dict:

219 """Add default parameters to a QGIS Server request.

220

221 Args:

222 params: Request parameters; keys are converted to upper case.

223

224 Returns:

225 Parameters with ``MAP``, ``SERVICE`` and ``VERSION`` defaults.

226 """

227 defaults = dict(

228 MAP=self.server_project_path(),

229 SERVICE=gws.OwsProtocol.WMS,

230 VERSION='1.3.0',

231 )

232 return gws.u.merge(defaults, gws.u.to_upper_dict(params))

233

234 def call_server(self, params: dict) -> gws.lib.net.HTTPResponse:

235 """Send a request to QGIS Server.

236

237 Args:

238 params: Request parameters, completed by ``server_params``.

239

240 Returns:

241 HTTP response.

242

243 Raises:

244 ``gws.lib.net.Error``: If the request fails.

245 """

246 params = self.server_params(params)

247 res = gws.lib.net.http_request(self.url, params=params, timeout=1000)

248 res.raise_if_failed()

249 return res

250

251 ##

252

253 def get_map(self, bounds: gws.Bounds, width: float, height: float, params: dict) -> bytes:

254 """Render a map image with a GetMap request.

255

256 Args:

257 bounds: Box to render.

258 width: Image width in pixels.

259 height: Image height in pixels.

260 params: Extra request parameters, e.g. ``LAYERS``; they override the defaults (transparent PNG).

261

262 Returns:

263 Image bytes.

264

265 Raises:

266 ``gws.Error``: If QGIS Server returns a non-image response.

267 """

268 bbox = bounds.extent

269 if bounds.crs.isYX and not self.alwaysXY:

270 bbox = gws.lib.extent.swap_xy(bbox)

271

272 defaults = dict(

273 REQUEST=gws.OwsVerb.GetMap,

274 BBOX=bbox,

275 WIDTH=gws.u.to_rounded_int(width),

276 HEIGHT=gws.u.to_rounded_int(height),

277 CRS=bounds.crs.epsg,

278 FORMAT=gws.lib.mime.PNG,

279 TRANSPARENT='true',

280 STYLES='',

281 )

282

283 params = gws.u.merge(defaults, params)

284

285 res = self.call_server(params)

286 if res.content_type.startswith('image/'):

287 return res.content

288 raise gws.Error(res.text)

289

290 def get_features(self, search, source_layers):

291 shape = search.shape

292 if not shape or shape.type != gws.GeometryType.point:

293 return []

294

295 request_crs = self.forceCrs

296 if not request_crs:

297 request_crs = gws.lib.crs.best_match(

298 shape.crs,

299 gws.gis.source.combined_crs_list(source_layers))

300

301 box_size_m = 500

302 box_size_deg = 1

303 box_size_px = 500

304

305 size = None

306

307 if shape.crs.uom == gws.Uom.m:

308 size = box_size_px * search.resolution

309 if shape.crs.uom == gws.Uom.deg:

310 # @TODO use search.resolution here as well

311 size = box_size_deg

312 if not size:

313 gws.log.debug('cannot request crs {crs!r}, unsupported unit')

314 return []

315

316 bbox = (

317 shape.x - (size / 2),

318 shape.y - (size / 2),

319 shape.x + (size / 2),

320 shape.y + (size / 2),

321 )

322

323 bbox = gws.lib.extent.transform(bbox, shape.crs, request_crs)

324

325 layer_names = [sl.name for sl in source_layers]

326

327 params = {

328 'BBOX': bbox,

329 'CRS': request_crs.to_string(gws.CrsFormat.epsg),

330 'WIDTH': box_size_px,

331 'HEIGHT': box_size_px,

332 'I': box_size_px >> 1,

333 'J': box_size_px >> 1,

334 'LAYERS': layer_names,

335 'QUERY_LAYERS': layer_names,

336 'STYLES': [''] * len(layer_names),

337 'FEATURE_COUNT': search.limit or 100,

338 'INFO_FORMAT': 'text/xml',

339 'REQUEST': gws.OwsVerb.GetFeatureInfo,

340 'WITH_GEOMETRY': 'true',

341 }

342

343 if search.extraParams:

344 params = gws.u.merge(params, gws.u.to_upper_dict(search.extraParams))

345

346 res = self.call_server(params)

347

348 try:

349 records = gws.base.ows.client.featureinfo.parse(res.text, default_crs=request_crs, always_xy=self.alwaysXY)

350 except gws.Error as exc:

351 gws.log.error(f'get_features: parse error: {exc!r}')

352 return []

353

354 gws.log.debug(f'get_features: FOUND={len(records)} params={params!r}')

355

356 for rec in records:

357 if rec.shape:

358 rec.shape = rec.shape.transformed_to(shape.crs)

359

360 return records

361

362 ##

363

364 def create_leaf_layer_config(self, source_layers):

365 """Create the configuration of a leaf layer for the ``qgis`` tree layer.

366

367 By default, this is a ``qgisflat`` layer. For a single source layer

368 whose data source provider is listed in ``directRender`` or

369 ``directSearch``, the layer is rendered directly (``wmsflat``,

370 ``wmts`` or ``tile``) and gets its own finders and models.

371

372 Args:

373 source_layers: Source layers of the leaf.

374

375 Returns:

376 Layer configuration dict.

377 """

378 simple_cfg = {

379 'type': 'qgisflat',

380 '_defaultProvider': self,

381 '_defaultSourceLayers': source_layers,

382 }

383

384 if len(source_layers) > 1 or source_layers[0].isGroup:

385 return simple_cfg

386

387 ds = source_layers[0].dataSource

388 if not ds or not ds.get('provider'):

389 return simple_cfg

390

391 render = self._leaf_render_config(ds)

392 search = self._leaf_search_config(ds)

393

394 cfg = {}

395 cfg.update(render or {})

396 cfg.update(search or {})

397

398 if not cfg.get('type'):

399 cfg.update(simple_cfg)

400

401 return cfg

402

403 def _leaf_render_config(self, ds):

404 """Create a direct render layer configuration for a data source."""

405 prov = ds.get('provider')

406 if prov not in self.directRender:

407 return

408

409 url = self._leaf_service_url(ds.get('url'), ds.get('params'))

410 if not url:

411 return

412

413 if prov == 'wms':

414 layers = ds.get('layers')

415 if not layers:

416 return

417 return {

418 'type': 'wmsflat',

419 'sourceLayers': {'names': layers},

420 'display': 'tile',

421 'provider': {'url': url},

422 }

423

424 if prov == 'wmts':

425 layers = ds.get('layers')

426 if not layers:

427 return

428 cfg = {

429 'type': 'wmts',

430 'sourceLayers': {'names': layers},

431 'display': 'tile',

432 'provider': {'url': url},

433 }

434 p = ds.get('styles')

435 if p:

436 cfg['style'] = p[0]

437 return cfg

438

439 if prov == 'xyz':

440 return {

441 'type': 'tile',

442 'provider': {'url': url},

443 }

444

445 def _leaf_search_config(self, ds):

446 """Create direct finder and model configurations for a data source."""

447 prov = ds.get('provider')

448 if prov not in self.directSearch:

449 return

450

451 if prov == 'wms':

452 url = self._leaf_service_url(ds.get('url'), ds.get('params'))

453 layers = ds.get('layers')

454 if not url or not layers:

455 return

456 finder = {

457 'type': 'wms',

458 'provider': {'url': url},

459 'sourceLayers': {'names': layers},

460 }

461 return {'finders': [finder]}

462

463 if prov == 'wfs':

464 url = self._leaf_service_url(ds.get('url'), ds.get('params'))

465 if not url:

466 return

467 finder = {

468 'type': 'wfs',

469 'provider': {'url': url},

470 }

471 p = ds.get('typename')

472 if p:

473 finder['sourceLayers'] = {'names': [p]}

474 p = ds.get('srsname')

475 if p:

476 finder['forceCrs'] = p

477 p = ds.get('ignoreaxisorientation')

478 if p == '1':

479 finder['alwaysXY'] = True

480 p = ds.get('invertaxisorientation')

481 if p == '1':

482 # NB assuming this might be only '1' for lat-lon projections

483 finder['alwaysXY'] = True

484

485 return {'finders': [finder]}

486

487 if prov == 'postgres':

488 table_name = ds.get('table')

489

490 # 'table' can also be a select statement, in which case it might be enclosed in parens

491 if not table_name or table_name.startswith('(') or table_name.upper().startswith('SELECT '):

492 return

493

494 db = self.postgres_provider_from_datasource(ds)

495

496 model = {

497 'type': 'postgres',

498 'tableName': table_name,

499 'sqlFilter': ds.get('sql'),

500 '_defaultDb': db

501 }

502 finder = {

503 'type': 'postgres',

504 'tableName': table_name,

505 'sqlFilter': ds.get('sql'),

506 '_defaultDb': db

507 }

508 return {'models': [model], 'finders': [finder]}

509

510 def postgres_provider_from_datasource(self, ds: dict) -> gws.plugin.postgres.provider.Object:

511 """Find or create a Postgres provider for a QGIS data source.

512

513 An existing provider with the same connection URL is reused,

514 otherwise a new provider is created.

515

516 Args:

517 ds: Parsed Postgres data source.

518

519 Returns:

520 Postgres provider.

521 """

522 cfg = gws.Config(

523 host=ds.get('host'),

524 port=ds.get('port'),

525 database=ds.get('dbname'),

526 username=ds.get('user'),

527 password=ds.get('password'),

528 serviceName=ds.get('service'),

529 options=ds.get('options'),

530 )

531 url = gws.plugin.postgres.provider.connection_url(cfg)

532 mgr = self.root.app.databaseMgr

533

534 for p in mgr.providers:

535 if p.extType == 'postgres' and p.url() == url:

536 return cast(gws.plugin.postgres.provider.Object, p)

537

538 gws.log.debug(f'creating an ad-hoc postgres provider for qgis {url=}')

539 p = mgr.create_provider(cfg, type='postgres')

540 return cast(gws.plugin.postgres.provider.Object, p)

541

542 _std_ows_params = {

543 'bbox',

544 'bgcolor',

545 'crs',

546 'exceptions',

547 'format',

548 'height',

549 'layers',

550 'request',

551 'service',

552 'sld',

553 'sld_body',

554 'srs',

555 'styles',

556 'time',

557 'transparent',

558 'version',

559 'width',

560 }

561

562 def _leaf_service_url(self, url, params):

563 """Add the non-standard OWS parameters of a data source to its URL."""

564 if not url:

565 return

566 if not params:

567 return url

568

569 # a wms url can be like "server?service=WMS....&bbox=.... &some-non-std-param=...

570 # we need to keep non-std params for caps requests

571

572 p = {k: v for k, v in params.items() if k.lower() not in self._std_ows_params}

573 return gws.lib.net.add_params(url, p)