Coverage for gws-app/gws/plugin/qfieldcloud/caps.py: 99%

1"""Parser for QField-related capabilities of a QGIS project.""" 

2 

3from typing import Optional, cast 

4 

5import gws 

6import gws.lib.shape 

7import gws.gis.source 

8import gws.lib.datetimex as dtx 

9import gws.lib.jsonx 

10import gws.lib.osx 

11import gws.lib.crs 

12import gws.plugin.qgis.caps 

13 

14from . import core 

15 

16 

17class ProjectProps(gws.Data): 

18 """Custom project properties as defined by QField.""" 

19 

20 areaOfInterest: str 

21 """Area of interest as WKT.""" 

22 areaOfInterestCrs: str 

23 """CRS of the area of interest.""" 

24 baseMapLayer: str 

25 """Id of the layer used as the base map, if ``baseMapType`` is ``singleLayer``.""" 

26 baseMapTheme: str 

27 """Map theme used as the base map, if ``baseMapType`` is ``mapTheme``.""" 

28 baseMapTileSize: int 

29 """Base map tile size.""" 

30 baseMapTilesMaxZoomLevel: int 

31 """Maximum zoom level of the base map.""" 

32 baseMapTilesMinZoomLevel: int 

33 """Minimum zoom level of the base map.""" 

34 baseMapType: str 

35 """Base map type, ``mapTheme`` or ``singleLayer``.""" 

36 createBaseMap: bool 

37 """Whether a base map is created.""" 

38 digitizingLogsLayer: str 

39 """Id of the digitizing logs layer.""" 

40 forceAutoPush: bool 

41 """Whether changes are pushed automatically.""" 

42 forceAutoPushIntervalMins: int 

43 """Automatic push interval in minutes.""" 

44 forceStamping: bool 

45 """Whether photos are stamped.""" 

46 geofencingBehavior: int 

47 """Geofencing behavior.""" 

48 geofencingIsActive: bool 

49 """Whether geofencing is active.""" 

50 geofencingLayer: str 

51 """Id of the geofencing layer.""" 

52 geofencingShouldPreventDigitizing: bool 

53 """Whether geofencing prevents digitizing.""" 

54 mapThemesActiveLayers: dict 

55 """Active layers of map themes.""" 

56 maximumImageWidthHeight: int 

57 """Maximum width and height of images.""" 

58 offlineCopyOnlyAoi: bool 

59 """Whether only features in the area of interest are packaged.""" 

60 stampingDetailsTemplate: str 

61 """Template for photo stamping.""" 

62 stampingFontStyle: str 

63 """Font style for photo stamping.""" 

64 stampingHorizontalAlignment: int 

65 """Horizontal alignment for photo stamping.""" 

66 stampingImageDecoration: str 

67 """Image decoration for photo stamping.""" 

68 

69 attachmentDirs: list[str] 

70 """Attachment directories.""" 

71 dataDirs: list[str] 

72 """Data directories.""" 

73 dirsToCopy: dict 

74 """Directories to copy, as a dict ``{dirname: bool}``.""" 

75 

76 

77class LayerProps(gws.Data): 

78 """Custom layer properties as defined by QField.""" 

79 

80 action: str 

81 """QFieldSync layer action.""" 

82 attachment_naming: dict 

83 """Naming rules for attachments.""" 

84 attribute_editing_locked_expression: str 

85 """Expression that locks attribute editing.""" 

86 cloud_action: str 

87 """Cloud action: ``offline``, ``no_action`` or ``remove``.""" 

88 feature_addition_locked_expression: str 

89 """Expression that locks feature addition.""" 

90 feature_deletion_locked_expression: str 

91 """Expression that locks feature deletion.""" 

92 geometry_editing_locked_expression: str 

93 """Expression that locks geometry editing.""" 

94 is_attribute_editing_locked: bool 

95 """Whether attribute editing is locked.""" 

96 is_feature_addition_locked: bool 

97 """Whether feature addition is locked.""" 

98 is_feature_deletion_locked: bool 

99 """Whether feature deletion is locked.""" 

100 is_geometry_editing_locked: bool 

101 """Whether geometry editing is locked.""" 

102 photo_naming: dict 

103 """Naming rules for photos.""" 

104 relationship_maximum_visible: dict 

105 """Maximum number of visible related features.""" 

106 tracking_distance_requirement_minimum_meters: int 

107 """Minimum distance between tracked positions, in meters.""" 

108 tracking_erroneous_distance_safeguard_maximum_meters: int 

109 """Maximum distance of a tracked position before it is considered erroneous, in meters.""" 

110 tracking_measurement_type: int 

111 """Tracking measurement type.""" 

112 tracking_time_requirement_interval_seconds: int 

113 """Minimum time between tracked positions, in seconds.""" 

114 value_map_button_interface_threshold: int 

115 """Threshold for showing value maps as buttons.""" 

116 

117 

118class LayerAction(gws.Enum): 

119 """What the packager does with a layer.""" 

120 

121 remove = 'remove' 

122 """Remove the layer from the project.""" 

123 edit = 'edit' 

124 """Package the layer data for offline editing.""" 

125 baseMap = 'baseMap' 

126 """Render the layer into the base map.""" 

127 

128 

129class ModelEntry(gws.Data): 

130 """A model for an offline table, with its GeoPackage layer name.""" 

131 

132 gpName: str 

133 """Name of the GeoPackage file and layer.""" 

134 tableName: str 

135 """Database table name.""" 

136 model: gws.DatabaseModel 

137 """Model for the table.""" 

138 

139 

140class LayerEntry(gws.Data): 

141 """QField-related information about a QGIS layer.""" 

142 

143 action: LayerAction 

144 """What to do with the layer.""" 

145 qgisId: str 

146 """Layer id in the QGIS project.""" 

147 modelEntry: ModelEntry 

148 """Model entry, for ``edit`` layers.""" 

149 readOnly: bool 

150 """Whether all editing is locked in QFieldSync.""" 

151 sqlFilter: str 

152 """SQL filter (subset string) of the layer.""" 

153 dataSourceFileName: str 

154 """Name of the packaged data file.""" 

155 dataSource: str 

156 """Data source string in the packaged QGIS project.""" 

157 dataProvider: str 

158 """Data provider in the packaged QGIS project.""" 

159 sourceLayer: gws.SourceLayer 

160 """Source layer from the QGIS project.""" 

161 props: LayerProps 

162 """QFieldSync layer properties.""" 

163 

164 

165class Caps(gws.Data): 

166 """QField related capabilities extracted from the QGIS project and GWS config.""" 

167 

168 sourceHash: str 

169 """Hash of the QGIS project source, used to invalidate cached caps.""" 

170 qgisPath: str 

171 """Path to the QGIS project file, empty if the project is not stored in a file.""" 

172 layerMap: dict[str, LayerEntry] 

173 """Layer entries by QGIS layer id.""" 

174 modelMap: dict[str, ModelEntry] 

175 """Model entries by GeoPackage name.""" 

176 copyDirs: list[str] 

177 """Absolute paths of directories to copy into the package.""" 

178 baseMapLayerIds: list[str] 

179 """Ids of the layers rendered into the base map.""" 

180 areaOfInterest: Optional[gws.Bounds] 

181 """Area of interest.""" 

182 copyOnlyAreaOfInterest: bool 

183 """Whether only features in the area of interest are packaged.""" 

184 projectProps: ProjectProps 

185 """QFieldSync project properties.""" 

186 

187 

188class Parser: 

189 """Reads QField-related capabilities from a QGIS project. 

190 

191 ``parse`` reads the project and layer properties. ``create_models`` and 

192 ``assign_path_props`` complete the layer entries and are called separately. 

193 """ 

194 

195 project: core.QfcProject 

196 """QField project (unused, the project is stored in ``qfcProject``).""" 

197 caps: Caps 

198 """Capabilities being built.""" 

199 qgisCaps: gws.plugin.qgis.caps.Caps 

200 """Capabilities of the QGIS project.""" 

201 

202 def __init__(self, qfc_project: core.QfcProject): 

203 """Create a parser. 

204 

205 Args: 

206 qfc_project: QField project to parse. 

207 """ 

208 self.qfcProject = qfc_project 

209 

210 def parse(self) -> Caps: 

211 """Parse the QGIS project. 

212 

213 Reads the project properties, the area of interest, the directories to copy, the base map layers and the layer entries. 

214 

215 Returns: 

216 Capabilities, also stored in ``self.caps``. 

217 """ 

218 qp = self.qfcProject.qgisProvider.qgis_project() 

219 self.qgisCaps = qp.caps() 

220 

221 self.caps = Caps( 

222 qgisPath='', 

223 sourceHash=qp.sourceHash, 

224 layerMap={}, 

225 modelMap={}, 

226 copyDirs=[], 

227 baseMapLayerIds=[], 

228 areaOfInterest=None, 

229 copyOnlyAreaOfInterest=False, 

230 projectProps=self.extract_project_props(), 

231 ) 

232 

233 if self.qfcProject.qgisProvider.store.type == gws.plugin.qgis.project.StoreType.file: 

234 self.caps.qgisPath = self.qfcProject.qgisProvider.store.path 

235 

236 self.parse_area_of_interest() 

237 self.parse_copy_dirs() 

238 self.parse_base_map() 

239 

240 self.iter_layers() 

241 

242 return self.caps 

243 

244 ## 

245 

246 def parse_area_of_interest(self): 

247 """Set the area of interest from the project properties.""" 

248 aoi = self.caps.projectProps.areaOfInterest 

249 if not aoi: 

250 return 

251 crs = self.caps.projectProps.areaOfInterestCrs 

252 shape = gws.lib.shape.from_wkt(aoi, gws.lib.crs.get(crs) or self.qgisCaps.projectCrs) 

253 self.caps.areaOfInterest = shape.bounds() 

254 self.caps.copyOnlyAreaOfInterest = self.caps.projectProps.offlineCopyOnlyAoi is True 

255 

256 def parse_copy_dirs(self): 

257 """Set the directories to copy into the package. 

258 

259 Relative paths are resolved against the QGIS project file. Nested directories are dropped. 

260 """ 

261 raw_dirs = [] 

262 

263 # dirsToCopy is a dict (dirname: bool) 

264 dc = self.caps.projectProps.dirsToCopy or {} 

265 for k, v in dc.items(): 

266 if v: 

267 raw_dirs.append(k) 

268 

269 # attachmentDirs and dataDirs are lists 

270 raw_dirs.extend(self.caps.projectProps.attachmentDirs or []) 

271 raw_dirs.extend(self.caps.projectProps.dataDirs or []) 

272 

273 abs_dirs = [] 

274 

275 for p in raw_dirs: 

276 if not p.startswith('/'): 

277 if not self.caps.qgisPath: 

278 gws.log.warning(f'cannot determine an absolute path for {p!r}') 

279 continue 

280 p = gws.lib.osx.abs_path(p, self.caps.qgisPath) 

281 abs_dirs.append(p) 

282 

283 unnest_dirs = [] 

284 

285 for p in sorted(abs_dirs): 

286 if any(p.startswith(d) for d in unnest_dirs): 

287 continue 

288 unnest_dirs.append(p) 

289 

290 self.caps.copyDirs = unnest_dirs 

291 

292 def parse_base_map(self): 

293 """Set the base map layer ids from the map theme or the single base map layer.""" 

294 if not self.caps.projectProps.createBaseMap: 

295 return 

296 

297 bt = self.caps.projectProps.baseMapType 

298 

299 if bt == 'mapTheme': 

300 theme = self.caps.projectProps.baseMapTheme 

301 if not theme: 

302 gws.log.warning(f'map theme not defined') 

303 return 

304 

305 vp = self.qgisCaps.visibilityPresets.get(theme) 

306 if not vp: 

307 gws.log.warning(f'map theme {theme!r} not found') 

308 return 

309 self.caps.baseMapLayerIds = vp 

310 return 

311 

312 if bt == 'singleLayer': 

313 uid = self.caps.projectProps.baseMapLayer 

314 if uid: 

315 self.caps.baseMapLayerIds = [uid] 

316 

317 ## 

318 

319 def iter_layers(self): 

320 """Create layer entries for all non-group source layers.""" 

321 for sl in gws.gis.source.filter_layers(self.qgisCaps.sourceLayers, is_group=False): 

322 le = self.layer_entry(sl) 

323 if le: 

324 self.caps.layerMap[le.qgisId] = le 

325 

326 def layer_entry(self, sl: gws.SourceLayer) -> Optional[LayerEntry]: 

327 """Create a layer entry for a source layer. 

328 

329 Args: 

330 sl: Source layer. 

331 

332 Returns: 

333 Layer entry, or ``None`` if the layer is left unchanged in the package. 

334 """ 

335 le = self.layer_entry_2(sl) 

336 if not le: 

337 return 

338 le.qgisId = sl.sourceId 

339 le.sourceLayer = sl 

340 

341 return le 

342 

343 def layer_entry_2(self, sl: gws.SourceLayer) -> Optional[LayerEntry]: 

344 """Determine the action for a source layer. 

345 

346 Base map layers get ``baseMap``. Layers with the cloud action ``remove`` get ``remove``. 

347 Postgres layers with the cloud action ``offline`` get ``edit``, offline layers of other providers get ``remove``. 

348 

349 Args: 

350 sl: Source layer. 

351 

352 Returns: 

353 Layer entry without the id and source layer, or ``None`` for other layers. 

354 """ 

355 props = self.extract_layer_props(sl) 

356 

357 if sl.sourceId in self.caps.baseMapLayerIds: 

358 return LayerEntry(action=LayerAction.baseMap, props=props) 

359 

360 # 'offline', 'no_action' or 'remove' 

361 # for "cable", use "action" instead of "cloud_action" 

362 act = props.cloud_action 

363 

364 if act == 'remove': 

365 return LayerEntry(action=LayerAction.remove, props=props) 

366 

367 if act == 'offline': 

368 prov = sl.dataSource.get('provider') 

369 if prov == 'postgres': 

370 return self.postgres_layer_entry(sl, props) 

371 # @TODO support offline for other providers? 

372 gws.log.warning(f'layer {sl.sourceId!r}: offline editing of {prov!r} not supported') 

373 return LayerEntry(action=LayerAction.remove, props=props) 

374 

375 def postgres_layer_entry(self, sl, props: LayerProps) -> LayerEntry: 

376 """Create a layer entry for an offline Postgres layer. 

377 

378 Layers without a plain table name (e.g. SQL queries) get the ``remove`` action. 

379 

380 Args: 

381 sl: Source layer. 

382 props: QFieldSync layer properties. 

383 

384 Returns: 

385 Layer entry. 

386 """ 

387 read_only = ( 

388 props.is_attribute_editing_locked 

389 and props.is_geometry_editing_locked 

390 and props.is_feature_addition_locked 

391 and props.is_feature_deletion_locked 

392 ) 

393 

394 table_name = sl.dataSource.get('table') 

395 if not table_name or table_name.startswith('(') or table_name.upper().startswith('SELECT '): 

396 gws.log.warning(f'layer {sl.sourceId!r}: no table name') 

397 return LayerEntry(action=LayerAction.remove, props=props) 

398 

399 return LayerEntry( 

400 action=LayerAction.edit, 

401 readOnly=read_only, 

402 sqlFilter=sl.dataSource.get('sql', ''), 

403 props=props, 

404 ) 

405 

406 ## 

407 

408 def extract_project_props(self) -> ProjectProps: 

409 """Read the QFieldSync project properties. 

410 

411 Returns: 

412 Project properties. 

413 """ 

414 d = {} 

415 # there are two of them, QFieldSync and libqfieldsync 

416 d.update(self.qgisCaps.properties.get('qfieldsync', {})) 

417 d.update(self.qgisCaps.properties.get('QFieldSync', {})) 

418 

419 t = ProjectProps() 

420 _dict_to_data(d, t) 

421 return t 

422 

423 def extract_layer_props(self, sl: gws.SourceLayer) -> LayerProps: 

424 """Read the QFieldSync properties of a layer. 

425 

426 Args: 

427 sl: Source layer. 

428 

429 Returns: 

430 Layer properties. 

431 """ 

432 d = {} 

433 

434 for k, v in sl.properties.items(): 

435 if k.startswith('QFieldSync/'): 

436 d[k.split('/').pop()] = v 

437 

438 t = LayerProps() 

439 _dict_to_data(d, t) 

440 return t 

441 

442 ## 

443 

444 def assign_path_props(self): 

445 """Set the data file name, data source and provider of ``edit`` and ``baseMap`` layers in the package.""" 

446 for le in self.caps.layerMap.values(): 

447 if le.action == LayerAction.edit: 

448 name = le.modelEntry.gpName 

449 le.dataSourceFileName = f'{name}.gpkg' 

450 le.dataSource = f'./{le.dataSourceFileName}|layername={name}' 

451 if le.sqlFilter: 

452 le.dataSource += f'|subset={le.sqlFilter}' 

453 le.dataProvider = 'ogr' 

454 

455 if le.action == LayerAction.baseMap: 

456 u = gws.u.to_uid(le.qgisId) 

457 le.dataSourceFileName = f'{u}.gpkg' 

458 le.dataSource = f'./{le.dataSourceFileName}' 

459 le.dataProvider = 'gdal' 

460 

461 def create_models(self): 

462 """Create model entries for all ``edit`` layers.""" 

463 self.caps.modelMap = {} 

464 

465 for le in self.caps.layerMap.values(): 

466 self.create_model_entry_for_layer(le) 

467 

468 def create_model_entry_for_layer(self, le: LayerEntry): 

469 """Assign a model entry to an ``edit`` layer. 

470 

471 If no model is found, or the layer is editable but the model is not, the layer gets the ``remove`` action. 

472 

473 Args: 

474 le: Layer entry. 

475 """ 

476 if le.action != LayerAction.edit: 

477 return 

478 me = self.model_entry_for_source_layer(le.sourceLayer) 

479 if not me: 

480 gws.log.warning(f'layer {le.qgisId!r}: no model') 

481 le.action = LayerAction.remove 

482 return 

483 if not le.readOnly and not me.model.isEditable: 

484 gws.log.warning(f'layer {le.qgisId!r}: table {me.tableName!r} is not editable') 

485 le.action = LayerAction.remove 

486 return 

487 le.modelEntry = me 

488 

489 def model_entry_for_source_layer(self, sl: gws.SourceLayer) -> Optional[ModelEntry]: 

490 """Find or create a model entry for the table of a source layer. 

491 

492 A configured model with the same table name is used if present. 

493 Otherwise a generic Postgres model is created for the table. 

494 

495 Args: 

496 sl: Source layer. 

497 

498 Returns: 

499 Model entry, or ``None`` if the layer has no table or the table does not exist. 

500 """ 

501 table_name = sl.dataSource.get('table') 

502 if not table_name: 

503 return 

504 

505 for model in self.qfcProject.models: 

506 full_name = model.db.join_table_name('', model.tableName) 

507 if full_name == model.db.join_table_name('', table_name): 

508 gp_name = self.gp_name_for_model(full_name) 

509 if gp_name not in self.caps.modelMap: 

510 self.caps.modelMap[gp_name] = ModelEntry(gpName=gp_name, tableName=full_name, model=model) 

511 return self.caps.modelMap[gp_name] 

512 

513 db = self.qfcProject.qgisProvider.postgres_provider_from_datasource(sl.dataSource) 

514 if not db.has_table(table_name): 

515 gws.log.warning(f'layer {sl.sourceId!r}: table {table_name!r} not found') 

516 return 

517 

518 gp_name = self.gp_name_for_model(table_name) 

519 

520 if gp_name not in self.caps.modelMap: 

521 model = self.qfcProject.root.create_shared( 

522 gws.ext.object.model, 

523 gws.Config( 

524 uid=f'qfield_model_{table_name}', 

525 type='postgres', 

526 # NB: permissions are checked in the public export/import functions above 

527 permissions=gws.Config(read=gws.c.PUBLIC, edit=gws.c.PUBLIC), 

528 tableName=table_name, 

529 isEditable=True, 

530 _defaultDb=db, 

531 ), 

532 ) 

533 self.caps.modelMap[gp_name] = ModelEntry(gpName=gp_name, tableName=table_name, model=cast(gws.DatabaseModel, model)) 

534 

535 return self.caps.modelMap[gp_name] 

536 

537 def gp_name_for_model(self, table_name): 

538 """Return the GeoPackage name for a table. 

539 

540 Args: 

541 table_name: Table name, optionally with a schema (``public`` by default). 

542 

543 Returns: 

544 Name in the form ``qm_<schema>_<table>``, lowercase. 

545 """ 

546 if '.' not in table_name: 

547 table_name = 'public.' + table_name 

548 return 'qm_' + table_name.replace('.', '_').lower() 

549 

550 

551## 

552 

553 

554def _dict_to_data(d: dict, t: gws.Data): 

555 """Copy dict values into a Data object, converting them to the annotated types.""" 

556 for k, typ in t.__class__.__annotations__.items(): 

557 v = d.get(k) 

558 if v is None: 

559 continue 

560 try: 

561 if typ is bool: 

562 # QFieldSync writes flags both as `int` (1/0) and as `bool` (true/false) 

563 v = v is True or str(v).lower() in ('1', 'true') 

564 elif typ is int: 

565 v = int(v) 

566 elif typ is float: 

567 v = float(v) 

568 elif typ is dict: 

569 v = gws.lib.jsonx.from_string(v) 

570 except Exception as exc: 

571 gws.log.warning(f'invalid property value {k!r}={v!r}: {exc}') 

572 continue 

573 setattr(t, k, v) 

574 return t