Coverage for gws-app/gws/plugin/qfieldcloud/packager.py: 99%

1"""Creates QField packages from QGIS projects."""

3from typing import cast, Optional

5import gws

6import gws.lib.gdalx

7import gws.lib.jsonx

8import gws.lib.image

9import gws.lib.bounds

10import gws.lib.osx as osx

11import gws.lib.extent

12import gws.gis.render

13import gws.lib.grid

15from . import core, caps as caps_mod

17PATH_MAP_FILE = 'path_map.json'

18"""Name of the file that maps package file names to paths on disk."""

19COMPLETE_FILE = 'package_complete'

20"""Name of the marker file written when a package is complete."""

23class Args(gws.Data):

24 """Arguments for the packager."""

26 uid: str

27 """Package uid, used in log messages."""

28 qfcProject: core.QfcProject

29 """QField project."""

30 caps: caps_mod.Caps

31 """Capabilities of the QField project."""

32 project: Optional[gws.Project]

33 """GWS project context."""

34 user: gws.User

35 """User the package is created for."""

36 packageDir: str

37 """Directory to write the package into."""

38 mapCacheDir: str

39 """Directory for cached base map images."""

40 withBaseMap: bool

41 """Render base maps."""

42 withData: bool

43 """Write the data of offline layers."""

44 withMedia: bool

45 """Add media files."""

46 withQgis: bool

47 """Write the modified QGIS project."""

50class Object:

51 """Packager, writes a QField package into a directory.

53 Data files and the QGIS project are written into the package directory.

54 Base maps and media files are not copied; the package path map refers to

55 them where they are. The path map is written into ``PATH_MAP_FILE``, and

56 ``COMPLETE_FILE`` marks the package as complete.

57 """

59 uid: str

60 """Package uid."""

61 root: gws.Root

62 """Root object."""

63 qfcProject: core.QfcProject

64 """QField project."""

65 project: Optional[gws.Project]

66 """GWS project context."""

67 user: gws.User

68 """User the package is created for."""

69 args: Args

70 """Packager arguments."""

71 caps: caps_mod.Caps

72 """Capabilities of the QField project."""

74 def create_package(self, root: gws.Root, args: Args):

75 """Create a package.

77 Args:

78 root: Root object.

79 args: Packager arguments.

80 """

81 self.root = root

82 self.uid = args.uid

83 self.pathMap = {}

85 self.args = args

86 self.qfcProject = self.args.qfcProject

87 self.project = self.args.project

88 self.user = self.args.user

89 self.caps = args.caps

91 if self.args.withData:

92 self.write_data()

94 if self.args.withBaseMap:

95 self.write_base_map()

97 if self.args.withMedia:

98 self.write_media()

100 if self.args.withQgis:

101 self.write_qgis_project()

102

103 gws.lib.jsonx.to_path(

104 f'{self.args.packageDir}/{PATH_MAP_FILE}',

105 self.pathMap,

106 pretty=True,

107 )

108 gws.u.write_file(

109 f'{self.args.packageDir}/{COMPLETE_FILE}',

110 '1',

111 )

112

113 def write_data(self):

114 """Write the features of each ``edit`` layer into a GeoPackage file."""

115 for le in self.caps.layerMap.values():

116 if le.action != caps_mod.LayerAction.edit:

117 continue

118 if le.dataSourceFileName in self.pathMap:

119 continue

120 path = f'{self.args.packageDir}/{le.dataSourceFileName}'

121 self.pathMap[le.dataSourceFileName] = path

122 with gws.lib.gdalx.open_vector(path, 'w') as ds:

123 self.write_features(le, ds)

124

125 def write_base_map(self):

126 """Render all ``baseMap`` layers."""

127 # @TODO options for flattened base maps

128 for le in self.caps.layerMap.values():

129 if le.action == caps_mod.LayerAction.baseMap:

130 self.write_base_map_layer(le)

131

132 def write_media(self):

133 """Add the files from the directories to copy to the path map.

134

135 Paths in the package are relative to the QGIS project file. If the project is not stored in a file, the last directory name is used.

136 """

137 for d in self.caps.copyDirs:

138 if not gws.u.is_dir(d):

139 gws.log.warning(f'{self.uid}: media dir not found: {d!r}')

140 continue

141 if self.caps.qgisPath:

142 rel_dir = osx.rel_path(d, self.caps.qgisPath)

143 else:

144 # @TODO absolute dir with a postgres-based qgis project?

145 rel_dir = d.split('/')[-1]

146 for p in osx.find_files(d):

147 self.pathMap[rel_dir + '/' + osx.rel_path(p, d)] = p

148

149 #

150

151 def get_features_for_layer(self, le: caps_mod.LayerEntry) -> list[gws.Feature]:

152 """Read the features of an ``edit`` layer.

153

154 If only the area of interest is copied, features are limited to the area of interest, or the project bounds.

155

156 Args:

157 le: Layer entry.

158

159 Returns:

160 Features.

161 """

162 me = le.modelEntry

163 q = gws.SearchQuery()

164 if self.caps.copyOnlyAreaOfInterest:

165 q.bounds = gws.u.require(self.caps.areaOfInterest or self.qfcProject.qgisProvider.bounds)

166 mc = gws.ModelContext(user=self.user, project=self.project, op=gws.ModelOperation.read)

167 return me.model.find_features(q, mc)

168

169 _SUPPORTED_ATTRIBUTE_TYPES = {

170 gws.AttributeType.bool,

171 gws.AttributeType.date,

172 gws.AttributeType.datetime,

173 gws.AttributeType.float,

174 gws.AttributeType.int,

175 gws.AttributeType.str,

176 gws.AttributeType.time,

177 }

178

179 def write_features(self, le: caps_mod.LayerEntry, ds: gws.lib.gdalx.VectorDataSet):

180 """Write the features of a layer into a GeoPackage layer.

181

182 Only fields of simple types are written. A field named ``fid`` is written as ``fid_gws``, because GDAL uses ``fid`` internally.

183

184 Args:

185 le: Layer entry.

186 ds: GeoPackage data set.

187 """

188 features = self.get_features_for_layer(le)

189

190 me = le.modelEntry

191 gws.log.debug(f'{self.uid}: {self.qfcProject.uid}::{me.gpName!r} BEGIN write_features')

192

193 columns = {}

194 field_names = {}

195 for f in me.model.fields:

196 if f.attributeType not in self._SUPPORTED_ATTRIBUTE_TYPES:

197 continue

198 # FID field needs to be renamed, because GDAL uses it internally

199 name = f.name

200 if f.name.lower() == 'fid':

201 name = 'fid_gws'

202 columns[name] = f.attributeType

203 field_names[name] = f.name

204

205 gp_layer = ds.create_layer(

206 me.gpName,

207 columns=columns,

208 geometry_type=me.model.geometryType,

209 crs=me.model.geometryCrs,

210 overwrite=True,

211 )

212

213 records = []

214 for feat in features:

215 rec = gws.FeatureRecord(

216 attributes={name: feat.get(field_name) for name, field_name in field_names.items()},

217 shape=feat.shape(),

218 meta={},

219 )

220 records.append(rec)

221

222 with ds.transaction():

223 gp_layer.insert(records)

224

225 gws.log.debug(f'{self.uid}: {self.qfcProject.uid}::{me.gpName!r} END write_features, count={gp_layer.count()}')

226

227 ##

228

229 def write_base_map_layer(self, le: caps_mod.LayerEntry):

230 """Render a base map layer into a raster file.

231

232 The layer is rendered as a single image covering the area of interest

233 (or the project bounds), at the resolution of the maximum zoom level

234 (clamped to 3..20) in the grid of the target CRS. The file is stored in the

235 map cache directory and reused while it is younger than ``mapCacheLifeTime``.

236

237 Args:

238 le: Layer entry.

239 """

240 max_zoom = max(

241 self.caps.projectProps.baseMapTilesMinZoomLevel or 0,

242 self.caps.projectProps.baseMapTilesMaxZoomLevel or 0,

243 )

244 if max_zoom < 3:

245 gws.log.warning(f'{self.uid}: write_base_map_layer: invalid zoom level {max_zoom=}')

246 max_zoom = 3

247 if max_zoom > 20:

248 gws.log.warning(f'{self.uid}: write_base_map_layer: invalid zoom level {max_zoom=}')

249 max_zoom = 20

250

251 cache_path = f'{self.args.mapCacheDir}/{max_zoom}_{le.dataSourceFileName}'

252 age = osx.file_age(cache_path)

253 ttl = self.qfcProject.mapCacheLifeTime

254 gws.log.debug(f'{self.uid}: write_base_map_layer: {le.qgisId}: {cache_path=} {ttl=}/{age=}')

255

256 if ttl > 0 and (0 < age < ttl):

257 gws.log.debug(f'{self.uid}: write_base_map_layer: CACHED!')

258 self.pathMap[le.dataSourceFileName] = cache_path

259 return

260

261 bounds = gws.u.require(self.caps.areaOfInterest or self.qfcProject.qgisProvider.bounds)

262 bounds = gws.lib.bounds.transform(bounds, self.qfcProject.qgisProvider.forceCrs)

263

264 resolution = gws.lib.grid.resolution_for_level(gws.lib.grid.for_crs(bounds.crs), max_zoom)

265

266 w, h = gws.lib.extent.size(bounds.extent)

267 px_size = (w / resolution, h / resolution, gws.Uom.px)

268

269 gws.log.debug(f'{self.uid}: write_base_map_layer: {max_zoom=} {resolution=} {px_size=} {bounds=}')

270

271 flat_layer = cast(

272 gws.Layer,

273 self.qfcProject.root.create_temporary(

274 gws.ext.object.layer,

275 type='qgisflat',

276 _parentWgsExtent=gws.lib.bounds.wgs_extent(bounds),

277 _mapCrs=bounds.crs,

278 _parentResolutions=[1],

279 _defaultProvider=self.qfcProject.qgisProvider,

280 _defaultSourceLayers=[le.sourceLayer],

281 ),

282 )

283

284 mv = gws.gis.render.map_view_from_bbox(

285 size=px_size,

286 bbox=bounds.extent,

287 crs=bounds.crs,

288 dpi=96,

289 rotation=0,

290 )

291

292 lri = gws.LayerRenderInput(

293 type=gws.LayerRenderInputType.box,

294 targetCrs=bounds.crs,

295 user=self.user,

296 view=mv,

297 )

298

299 lro = gws.u.require(flat_layer.render(lri))

300 img = gws.lib.image.from_bytes(lro.content).convert('RGBA')

301

302 with gws.lib.gdalx.open_from_image(img, bounds) as src:

303 src.save_as(cache_path)

304

305 self.pathMap[le.dataSourceFileName] = cache_path

306

307 def write_qgis_project(self):

308 """Write the modified QGIS project into the package.

309

310 The original project is also written next to it, with the extension ``.source.qgs``.

311 """

312 fname = f'{self.qfcProject.uid}.qgs'

313 path = f'{self.args.packageDir}/{fname}'

314

315 qp = self.qfcProject.qgisProvider.qgis_project()

316 gws.u.write_file(path + '.source.qgs', qp.text)

317

318 root_el = qp.xml_root()

319 QgisXmlTransformer().run(self, root_el)

320 xml = root_el.to_string()

321 xml = self.replace_vars(xml)

322 gws.u.write_file(path, xml)

323 self.pathMap[fname] = path

324

325 def replace_vars(self, s: str) -> str:

326 """Replace user variables in a string.

327

328 Supported variables are ``{user.authToken}``, ``{user.loginName}`` and ``{user.displayName}``.

329

330 Args:

331 s: Source string.

332

333 Returns:

334 String with the variables replaced.

335 """

336 # @TODO render attributes as templates

337 s = s.replace('{user.authToken}', self.user.authToken)

338 s = s.replace('{user.loginName}', self.user.loginName)

339 s = s.replace('{user.displayName}', self.user.displayName)

340 return s

341

342

343class QgisXmlTransformer:

344 """Modifies the QGIS project XML for the package.

345

346 Points layers and references to the packaged data sources, removes layers

347 with the ``remove`` action and empty layer groups, and makes paths relative.

348 """

349

350 po: Object

351 """Packager."""

352 root: gws.XmlElement

353 """Root element of the project."""

354 toRemove: list[gws.XmlElement]

355 """Elements to remove."""

356

357 def run(self, po: Object, root_el: gws.XmlElement):

358 """Transform the project XML in place.

359

360 Args:

361 po: Packager.

362 root_el: Root element of the project.

363 """

364 self.po = po

365 self.root = root_el

366 self.toRemove = []

367

368 self.change_global_props()

369 self.update_layer_tree()

370 self.update_map_layers()

371 self.update_referenced_layers()

372 self.update_referenced_layers()

373 self.update_edit_widgets()

374

375 self.cleanup_layer_group(root_el.find('layer-tree-group'))

376 self.remove_elements(root_el, None)

377

378 def change_global_props(self):

379 """Set the project to use relative paths."""

380 # change global properties

381

382 properties = self.root.find('properties') or self.root.add('properties')

383

384 # this is added by the Sync plugin

385 # p = properties.add('OfflineEditingPlugin').add('OfflineDbPath', type='QString')

386 # p.text = f'{self.po.deviceDbPath}'

387

388 # ensure relative paths

389 p = properties.find('Paths/Absolute')

390 if not p:

391 p = properties.add('Paths').add('Absolute', type='bool')

392 p.text = 'false'

393

394 def update_layer_tree(self):

395 """Update the data source of layer tree entries, or mark them for removal."""

396 for el in self.root.findall('.//layer-tree-layer'):

397 le = self.po.caps.layerMap.get(el.get('id'))

398 if not le:

399 continue

400

401 if le.action == caps_mod.LayerAction.remove:

402 self.toRemove.append(el)

403 continue

404

405 el.set('source', le.dataSource)

406 el.set('providerKey', le.dataProvider)

407

408 def update_map_layers(self):

409 """Update the data source of map layers, or mark them for removal."""

410 for el in self.root.findall('.//maplayer'):

411 le = self.po.caps.layerMap.get(el.textof('id'))

412 if not le:

413 continue

414

415 if le.action == caps_mod.LayerAction.remove:

416 self.toRemove.append(el)

417 continue

418

419 el.require('datasource').text = le.dataSource

420 el.require('provider').text = le.dataProvider

421

422 # @TODO do we need to change properties at all?

423 #

424 # if le.action == caps_mod.LayerAction.edit:

425 # cp = el.find('customproperties')

426 # if cp:

427 # el.remove(cp)

428 # opt = el.add('customproperties').add('Option', type='Map')

429

430 # opt.add('Option', type='QString', name='QFieldSync/action', value='offline')

431 # opt.add('Option', type='QString', name='QFieldSync/attachment_naming', value='{}')

432 # opt.add('Option', type='QString', name='QFieldSync/photo_naming', value='{}')

433 # opt.add('Option', type='QString', name='QFieldSync/sourceDataPrimaryKeys', value='fid')

434

435 # if le.action == caps_mod.LayerAction.edit:

436 # if le.readOnly:

437 # opt.add('Option', type='bool', name='QFieldSync/is_geometry_locked', value='true')

438 # else:

439 # opt.add('Option', type='bool', name='isOfflineEditable', value='true')

440

441 def update_referenced_layers(self):

442 """Update the data source in relations that reference ``edit`` layers."""

443 for el in self.root.findall('.//referencedLayers/relation'):

444 """

445 <referencedLayers>

446 <relation

447 strength="Association"

448 referencingLayer="..."

449 layerId="..."

450 referencedLayer="REF_ID"

451 providerKey="<REPLACE THIS>"

452 dataSource="<REPLACE THIS>"

453 >

454 ...

455 """

456

457 ref_id = el.get('referencedLayer')

458

459 le = self.po.caps.layerMap.get(ref_id)

460 if not le or le.action != caps_mod.LayerAction.edit:

461 gws.log.warning(f'{self.po.uid}: relation: referenced layer not found: {ref_id!r}')

462 continue

463

464 if 'dataSource' in el.attrib:

465 el.set('dataSource', le.dataSource)

466 if 'providerKey' in el.attrib:

467 el.set('providerKey', le.dataProvider)

468

469 def update_edit_widgets(self):

470 """Update the data source in relation reference widgets that reference ``edit`` layers."""

471 for el in self.root.findall('.//editWidget'):

472 """

473 <editWidget type="RelationReference">

474 <config>

475 <Option type="Map">

476 ...

477 <Option value="<REF_ID>" name="ReferencedLayerId" type="QString"/>

478 <Option value="<REPLACE THIS>" name="ReferencedLayerDataSource" type="QString"/>

479 <Option value="<REPLACE THIS>" name="ReferencedLayerProviderKey" type="QString"/>

480 ...

481 """

482

483 if el.get('type') != 'RelationReference':

484 continue

485

486 ref_id = None

487 for opt in el.findall('.//Option'):

488 if opt.get('name') == 'ReferencedLayerId':

489 ref_id = opt.get('value')

490 break

491

492 if not ref_id:

493 continue

494

495 le = self.po.caps.layerMap.get(ref_id)

496 if not le or le.action != caps_mod.LayerAction.edit:

497 gws.log.warning(f'{self.po.uid}: editWidget: referenced layer not found: {ref_id!r}')

498 continue

499

500 for opt in el.findall('.//Option'):

501 if opt.get('name') == 'ReferencedLayerDataSource':

502 opt.set('value', le.dataSource)

503 if opt.get('name') == 'ReferencedLayerProviderKey':

504 opt.set('value', le.dataProvider)

505

506 def cleanup_layer_group(self, group_el):

507 """Mark empty layer groups for removal, recursively.

508

509 Args:

510 group_el: Layer tree group element.

511

512 Returns:

513 ``True`` if the group contains layers that are kept.

514 """

515 is_empty = True

516

517 for sub in group_el.children():

518 if sub.tag == 'layer-tree-group':

519 if self.cleanup_layer_group(sub):

520 is_empty = False

521 if sub.tag == 'layer-tree-layer' and sub not in self.toRemove:

522 is_empty = False

523

524 if is_empty:

525 self.toRemove.append(group_el)

526

527 return not is_empty

528

529 def remove_elements(self, el, parent_el):

530 """Remove the elements marked for removal, recursively.

531

532 Args:

533 el: Element to check.

534 parent_el: Parent element.

535 """

536 if el in self.toRemove:

537 parent_el.remove(el)

538 return

539 ns = el.children()

540 for n in ns:

541 self.remove_elements(n, el)