Coverage for gws-app/gws/plugin/qfieldcloud/packager.py: 98%
345 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-23 20:06 +0200
1"""Creates QField packages from QGIS projects."""
3from typing import cast, Optional
5import os
7import gws
8import gws.plugin.model_field.file as file_field
9import gws.lib.gdalx
10import gws.lib.jsonx
11import gws.lib.image
12import gws.lib.bounds
13import gws.lib.osx as osx
14import gws.lib.extent
15import gws.gis.render
16import gws.lib.grid
18from . import core, caps as caps_mod
20PATH_MAP_FILE = 'path_map.json'
21"""Name of the file that maps package file names to paths on disk."""
22COMPLETE_FILE = 'package_complete'
23"""Name of the marker file written when a package is complete."""
26class Args(gws.Data):
27 """Arguments for the packager."""
29 uid: str
30 """Package uid, used in log messages."""
31 qfcProject: core.QfcProject
32 """QField project."""
33 caps: caps_mod.Caps
34 """Capabilities of the QField project."""
35 project: Optional[gws.Project]
36 """GWS project context."""
37 user: gws.User
38 """User the package is created for."""
39 packageDir: str
40 """Directory to write the package into."""
41 cacheDir: str
42 """Directory for cached base maps and ``syncFiles`` content."""
43 withBaseMap: bool
44 """Render base maps."""
45 withData: bool
46 """Write the data of offline layers."""
47 withMedia: bool
48 """Add media files."""
49 withQgis: bool
50 """Write the modified QGIS project."""
53class Object:
54 """Packager, writes a QField package into a directory.
56 Data files and the QGIS project are written into the package directory.
57 Base maps and media files are not copied; the package path map refers to
58 them where they are. The path map is written into ``PATH_MAP_FILE``, and
59 ``COMPLETE_FILE`` marks the package as complete.
60 """
62 uid: str
63 """Package uid."""
64 root: gws.Root
65 """Root object."""
66 qfcProject: core.QfcProject
67 """QField project."""
68 project: Optional[gws.Project]
69 """GWS project context."""
70 user: gws.User
71 """User the package is created for."""
72 args: Args
73 """Packager arguments."""
74 caps: caps_mod.Caps
75 """Capabilities of the QField project."""
76 layerMap: dict[str, caps_mod.LayerEntry]
77 """Layer entries for the user, by QGIS layer id."""
79 def create_package(self, root: gws.Root, args: Args):
80 """Create a package.
82 Args:
83 root: Root object.
84 args: Packager arguments.
85 """
86 self.root = root
87 self.uid = args.uid
88 self.pathMap = {}
90 self.args = args
91 self.qfcProject = self.args.qfcProject
92 self.project = self.args.project
93 self.user = self.args.user
94 self.caps = args.caps
95 self.layerMap = {uid: self.layer_entry_for_user(le) for uid, le in self.caps.layerMap.items()}
97 if self.args.withData:
98 self.write_data()
100 if self.args.withBaseMap:
101 self.write_base_map()
103 if self.args.withMedia:
104 self.write_media()
106 if self.args.withQgis:
107 self.write_qgis_project()
109 gws.lib.jsonx.to_path(
110 f'{self.args.packageDir}/{PATH_MAP_FILE}',
111 self.pathMap,
112 pretty=True,
113 )
114 gws.u.write_file(
115 f'{self.args.packageDir}/{COMPLETE_FILE}',
116 '1',
117 )
119 def layer_entry_for_user(self, le: caps_mod.LayerEntry) -> caps_mod.LayerEntry:
120 """Create a copy of an ``edit`` layer entry with the read-only flag and custom properties for the user.
122 Operations the user may not perform on the model are locked in QField.
123 Geometry editing is also locked if the user may not write the geometry field.
124 Operations locked in the QGIS project, or all operations if the layer is read-only there, are locked as well.
125 Locks controlled by an expression in the QGIS project are not considered, the expressions are kept as is.
126 If all operations are locked, the layer is read-only.
128 Args:
129 le: Layer entry.
131 Returns:
132 Layer entry for the user. Other layer entries are returned unchanged.
133 """
134 if le.action != caps_mod.LayerAction.edit:
135 return le
137 model = le.modelEntry.model
138 geom_field = model.field(model.geometryName) if model.geometryName else None
140 if not model.isEditable or le.readOnly:
141 can_create = can_write = can_delete = can_geom = False
142 else:
143 can_create = self.user.can_create(model) and not caps_mod.is_locked(le.props, 'feature_addition')
144 can_write = self.user.can_write(model) and not caps_mod.is_locked(le.props, 'attribute_editing')
145 can_delete = self.user.can_delete(model) and not caps_mod.is_locked(le.props, 'feature_deletion')
146 can_geom = (
147 self.user.can_write(model)
148 and geom_field is not None
149 and self.user.can_write(geom_field)
150 and not caps_mod.is_locked(le.props, 'geometry_editing')
151 )
153 set_props = {
154 'QFieldSync/is_feature_addition_locked': not can_create,
155 'QFieldSync/is_attribute_editing_locked': not can_write,
156 'QFieldSync/is_geometry_editing_locked': not can_geom,
157 'QFieldSync/is_feature_deletion_locked': not can_delete,
158 'QFieldSync/is_geometry_locked': None,
159 'QFieldSync/is_geometry_locked_expression_active': None,
160 'QFieldSync/geometry_locked_expression': None,
161 }
163 if model.uidName:
164 set_props['QFieldSync/sourceDataPrimaryKeys'] = core.gp_field_name(model.uidName)
166 return caps_mod.LayerEntry(
167 vars(le),
168 readOnly=not (can_create or can_write or can_geom or can_delete),
169 setProps=set_props,
170 )
172 def write_data(self):
173 """Write the features of each ``edit`` layer into a GeoPackage file, and the files of ``syncFiles`` fields without ``useBlob``."""
174 for le in self.layerMap.values():
175 if le.action != caps_mod.LayerAction.edit:
176 continue
177 if le.dataSourceFileName in self.pathMap:
178 continue
179 path = f'{self.args.packageDir}/{le.dataSourceFileName}'
180 self.pathMap[le.dataSourceFileName] = path
181 with gws.lib.gdalx.open_vector(path, 'w') as ds:
182 self.write_features(le, ds)
183 self.write_files(le)
185 def write_base_map(self):
186 """Render all ``baseMap`` layers."""
187 # @TODO options for flattened base maps
188 for le in self.layerMap.values():
189 if le.action == caps_mod.LayerAction.baseMap:
190 self.write_base_map_layer(le)
192 def write_media(self):
193 """Add the files from the directories to copy to the path map.
195 Paths in the package are relative to the QGIS project file. If the project is not stored in a file, the last directory name is used.
196 """
197 for d in self.caps.copyDirs:
198 if not gws.u.is_dir(d):
199 gws.log.warning(f'{self.uid}: media dir not found: {d!r}')
200 continue
201 if self.caps.qgisPath:
202 rel_dir = osx.rel_path(d, self.caps.qgisPath)
203 else:
204 # @TODO absolute dir with a postgres-based qgis project?
205 rel_dir = d.split('/')[-1]
206 for p in osx.find_files(d):
207 self.pathMap[rel_dir + '/' + osx.rel_path(p, d)] = p
209 def get_features_for_layer(self, le: caps_mod.LayerEntry) -> list[gws.Feature]:
210 """Read the features of an ``edit`` layer.
212 If only the area of interest is copied, features are limited to the area of interest, or the project bounds.
214 Args:
215 le: Layer entry.
217 Returns:
218 Features.
219 """
220 me = le.modelEntry
221 q = gws.SearchQuery()
222 if self.caps.copyOnlyAreaOfInterest:
223 q.bounds = gws.u.require(self.caps.areaOfInterest or self.qfcProject.qgisProvider.bounds)
224 mc = gws.ModelContext(user=self.user, project=self.project, op=gws.ModelOperation.read)
225 return me.model.find_features(q, mc)
227 _SUPPORTED_ATTRIBUTE_TYPES = {
228 gws.AttributeType.bool,
229 gws.AttributeType.date,
230 gws.AttributeType.datetime,
231 gws.AttributeType.float,
232 gws.AttributeType.int,
233 gws.AttributeType.str,
234 gws.AttributeType.time,
235 }
237 def write_features(self, le: caps_mod.LayerEntry, ds: gws.lib.gdalx.VectorDataSet):
238 """Write the features of a layer into a GeoPackage layer.
240 Only fields of simple types that the user can read are written. A field named ``fid`` is written as ``fid_gws``, because GDAL uses ``fid`` internally.
241 The content of ``syncFiles`` fields with ``useBlob`` that the user can read is written into a binary column named after the ``contentColumn``.
243 Args:
244 le: Layer entry.
245 ds: GeoPackage data set.
246 """
247 features = self.get_features_for_layer(le)
249 me = le.modelEntry
250 gws.log.debug(f'{self.uid}: {self.qfcProject.uid}::{me.gpName!r} BEGIN write_features')
252 columns = {}
253 field_names = {}
254 for f in me.model.fields:
255 if f.attributeType not in self._SUPPORTED_ATTRIBUTE_TYPES:
256 continue
257 if not self.user.can_read(f):
258 continue
259 name = core.gp_field_name(f.name)
260 columns[name] = f.attributeType
261 field_names[name] = f.name
263 blob_fields = {}
264 for sf in self.qfcProject.syncFiles:
265 if sf.model != me.model or not sf.useBlob or not self.user.can_read(sf.field):
266 continue
267 name = core.gp_field_name(sf.field.contentColumnName)
268 columns[name] = gws.AttributeType.bytes
269 blob_fields[name] = sf.field
271 contents = {}
272 if blob_fields:
273 contents = self.load_file_contents(
274 me,
275 list(blob_fields.values()),
276 [fe.uid() for fe in features],
277 )
279 gp_layer = ds.create_layer(
280 me.gpName,
281 columns=columns,
282 geometry_type=me.model.geometryType,
283 crs=me.model.geometryCrs,
284 overwrite=True,
285 )
287 records = []
288 for fe in features:
289 rec = gws.FeatureRecord(
290 attributes={name: fe.get(field_name) for name, field_name in field_names.items()},
291 shape=fe.shape(),
292 meta={},
293 )
294 for name, fld in blob_fields.items():
295 fv = contents.get(fe.uid(), {}).get(fld.name)
296 rec.attributes[name] = fv.content if fv else None
297 records.append(rec)
299 with ds.transaction():
300 gp_layer.insert(records)
302 gws.log.debug(f'{self.uid}: {self.qfcProject.uid}::{me.gpName!r} END write_features, count={gp_layer.count()}')
304 def write_files(self, le: caps_mod.LayerEntry):
305 """Write the content of ``syncFiles`` fields without ``useBlob`` into the package.
307 Each file is added to the package under the path stored in the ``nameColumn``.
308 Files are stored in the map cache directory and reused while their size matches the size of the content in the database.
309 Fields the user cannot read, empty files, and absolute paths or paths with ``..`` are skipped.
311 Args:
312 le: Layer entry.
313 """
314 me = le.modelEntry
316 file_fields = {}
318 for sf in self.qfcProject.syncFiles:
319 if sf.model == me.model and self.user.can_read(sf.field) and not sf.useBlob:
320 name = core.gp_field_name(sf.field.nameColumnName)
321 file_fields[name] = sf.field
323 if not file_fields:
324 return
326 cache_dir = gws.u.ensure_dir(f'{self.args.cacheDir}/files_{me.gpName}')
327 pending = {}
328 pending_uids = set()
330 for fe in self.get_features_for_layer(le):
331 for fld in file_fields.values():
332 fv = cast(file_field.FileValue, fe.get(fld.name))
333 if not fv or not fv.name or not fv.size:
334 continue
335 if fv.name.startswith('/') or '..' in fv.name:
336 gws.log.warning(f'{self.uid}: write_files: invalid path {fv.name!r} in {me.gpName!r}')
337 continue
338 if fv.name in self.pathMap or fv.name in pending:
339 continue
340 path = f'{cache_dir}/{fv.name.replace("/", "_")}'
341 if osx.file_size(path) == fv.size:
342 self.pathMap[fv.name] = path
343 continue
344 pending[fv.name] = (fe, fld, path)
345 pending_uids.add(fe.uid())
347 if not pending:
348 return
350 contents = self.load_file_contents(
351 me,
352 list(file_fields.values()),
353 list(pending_uids),
354 )
356 for name, (fe, fld, path) in pending.items():
357 fv = contents.get(fe.uid(), {}).get(fld.name)
358 if not fv or not fv.content:
359 continue
360 gws.u.write_file_b(path, fv.content)
361 self.pathMap[name] = path
363 def load_file_contents(
364 self,
365 me: caps_mod.ModelEntry,
366 fields: list[file_field.Object],
367 uids: list[gws.FeatureUid],
368 ) -> dict[str, dict]:
369 """Read the content of file fields.
371 Args:
372 me: Model entry.
373 fields: File fields of the model.
374 uids: Feature UIDs to read the content for.
376 Returns:
377 File values with content, by field name, by feature uid as a string.
378 """
379 if not uids:
380 return {}
381 q = gws.SearchQuery(
382 uids=uids,
383 extraColumns=[me.model.column(fld.contentColumnName) for fld in fields],
384 )
385 mc = gws.ModelContext(user=self.user, project=self.project, op=gws.ModelOperation.read)
387 res = {}
388 for fe in me.model.find_features(q, mc):
389 res[fe.uid()] = {fld.name: fe.get(fld.name) for fld in fields}
391 return res
393 ##
395 def write_base_map_layer(self, le: caps_mod.LayerEntry):
396 """Render a base map layer into a raster file.
398 The layer is rendered as a single image covering the area of interest
399 (or the project bounds), at the resolution of the maximum zoom level
400 (clamped to 3..20) in the grid of the target CRS. The file is stored in the
401 map cache directory and reused while it is younger than ``mapCacheLifeTime``.
403 Args:
404 le: Layer entry.
405 """
406 max_zoom = max(
407 self.caps.projectProps.baseMapTilesMinZoomLevel or 0,
408 self.caps.projectProps.baseMapTilesMaxZoomLevel or 0,
409 )
410 if max_zoom < 3:
411 gws.log.warning(f'{self.uid}: write_base_map_layer: invalid zoom level {max_zoom=}')
412 max_zoom = 3
413 if max_zoom > 20:
414 gws.log.warning(f'{self.uid}: write_base_map_layer: invalid zoom level {max_zoom=}')
415 max_zoom = 20
417 cache_path = f'{self.args.cacheDir}/{max_zoom}_{le.dataSourceFileName}'
418 age = osx.file_age(cache_path)
419 ttl = self.qfcProject.mapCacheLifeTime
420 cache_ok = ttl > 0 and (0 < age < ttl)
421 gws.log.debug(f'{self.uid}: write_base_map_layer: {le.qgisId}: {cache_path=} {ttl=}/{age=} {cache_ok=}')
423 if cache_ok:
424 self.pathMap[le.dataSourceFileName] = cache_path
425 return
427 bounds = gws.u.require(self.caps.areaOfInterest or self.qfcProject.qgisProvider.bounds)
428 bounds = gws.lib.bounds.transform(bounds, self.qfcProject.qgisProvider.forceCrs)
430 resolution = gws.lib.grid.resolution_for_level(gws.lib.grid.for_crs(bounds.crs), max_zoom)
432 w, h = gws.lib.extent.size(bounds.extent)
433 px_size = (gws.u.to_rounded_int(w / resolution), gws.u.to_rounded_int(h / resolution), gws.Uom.px)
435 gws.log.debug(f'{self.uid}: write_base_map_layer: {max_zoom=} {resolution=} {px_size=} {bounds=}')
437 if px_size[0] * px_size[1] > gws.lib.image.MAX_PIXELS:
438 raise gws.Error(f'{self.uid}: write_base_map_layer: base map too large: {px_size[0]}x{px_size[1]} at zoom {max_zoom}')
440 flat_layer = cast(
441 gws.Layer,
442 self.qfcProject.root.create_temporary(
443 gws.ext.object.layer,
444 type='qgisflat',
445 _parentWgsExtent=gws.lib.bounds.wgs_extent(bounds),
446 _mapCrs=bounds.crs,
447 _parentResolutions=[1],
448 _defaultProvider=self.qfcProject.qgisProvider,
449 _defaultSourceLayers=[le.sourceLayer],
450 ),
451 )
453 mv = gws.gis.render.map_view_from_bbox(
454 size=px_size,
455 bbox=bounds.extent,
456 crs=bounds.crs,
457 dpi=96,
458 rotation=0,
459 )
461 lri = gws.LayerRenderInput(
462 type=gws.LayerRenderInputType.box,
463 targetCrs=bounds.crs,
464 user=self.user,
465 view=mv,
466 )
468 lro = flat_layer.render(lri)
469 if not lro:
470 raise gws.Error(f'{self.uid}: write_base_map_layer: failed to render layer {flat_layer.uid}')
472 with gws.lib.gdalx.open_from_bytes(lro.content, bounds) as src:
473 src.save_as(cache_path)
475 self.pathMap[le.dataSourceFileName] = cache_path
477 def write_qgis_project(self):
478 """Write the modified QGIS project into the package.
480 The original project is also written next to it, with the extension ``.source.qgs``.
481 """
482 fname = f'{self.qfcProject.uid}.qgs'
483 path = f'{self.args.packageDir}/{fname}'
485 qp = self.qfcProject.qgisProvider.qgis_project()
486 gws.u.write_file(path + '.source.qgs', qp.text)
488 root_el = qp.xml_root()
489 QgisXmlTransformer().run(self, root_el)
490 xml = root_el.to_string()
491 xml = self.replace_vars(xml)
492 gws.u.write_file(path, xml)
493 self.pathMap[fname] = path
495 def replace_vars(self, s: str) -> str:
496 """Replace user variables in a string.
498 Supported variables are ``{user.authToken}``, ``{user.loginName}`` and ``{user.displayName}``.
500 Args:
501 s: Source string.
503 Returns:
504 String with the variables replaced.
505 """
506 # @TODO render attributes as templates
507 s = s.replace('{user.authToken}', self.user.authToken)
508 s = s.replace('{user.loginName}', self.user.loginName)
509 s = s.replace('{user.displayName}', self.user.displayName)
510 return s
513class QgisXmlTransformer:
514 """Modifies the QGIS project XML for the package.
516 Points layers and references to the packaged data sources, removes layers
517 with the ``remove`` action and empty layer groups, and makes paths relative.
518 """
520 po: Object
521 """Packager."""
522 root: gws.XmlElement
523 """Root element of the project."""
524 toRemove: list[gws.XmlElement]
525 """Elements to remove."""
527 def run(self, po: Object, root_el: gws.XmlElement):
528 """Transform the project XML in place.
530 Args:
531 po: Packager.
532 root_el: Root element of the project.
533 """
534 self.po = po
535 self.root = root_el
536 self.toRemove = []
538 self.change_global_props()
539 self.update_layer_tree()
540 self.update_map_layers()
541 self.update_referenced_layers()
542 self.update_edit_widgets()
544 self.cleanup_layer_group(root_el.find('layer-tree-group'))
545 self.remove_elements(root_el, None)
547 def change_global_props(self):
548 """Set the project to use relative paths."""
549 # change global properties
551 properties = self.root.find('properties') or self.root.add('properties')
553 # this is added by the Sync plugin
554 # p = properties.add('OfflineEditingPlugin').add('OfflineDbPath', type='QString')
555 # p.text = f'{self.po.deviceDbPath}'
557 # ensure relative paths
558 p = properties.find('Paths/Absolute')
559 if not p:
560 p = properties.add('Paths').add('Absolute', type='bool')
561 p.text = 'false'
563 def update_layer_tree(self):
564 """Update the data source of layer tree entries, or mark them for removal."""
565 for el in self.root.findall('.//layer-tree-layer'):
566 le = self.po.layerMap.get(el.get('id'))
567 if not le:
568 continue
570 if le.action == caps_mod.LayerAction.remove:
571 self.toRemove.append(el)
572 continue
574 el.set('source', le.dataSource)
575 el.set('providerKey', le.dataProvider)
577 def update_map_layers(self):
578 """Update the data source of map layers, or mark them for removal."""
579 for el in self.root.findall('.//maplayer'):
580 le = self.po.layerMap.get(el.textof('id'))
581 if not le:
582 continue
584 if le.action == caps_mod.LayerAction.remove:
585 self.toRemove.append(el)
586 continue
588 el.require('datasource').text = le.dataSource
589 el.require('provider').text = le.dataProvider
591 if le.action == caps_mod.LayerAction.edit:
592 el.set('readOnly', '1' if le.readOnly else '0')
593 self.update_custom_properties(el, le.setProps or {})
595 def update_custom_properties(self, layer_el: gws.XmlElement, props: dict):
596 """Set or remove custom properties of a map layer.
598 Args:
599 layer_el: Map layer element.
600 props: Property values by name, ``None`` removes the property.
601 """
602 cp = layer_el.find('customproperties')
603 if cp is None:
604 cp = layer_el.add('customproperties')
605 opt_map = cp.find('Option')
606 if opt_map is None:
607 opt_map = cp.add('Option', type='Map')
609 for opt in opt_map.findall('Option'):
610 if opt.get('name') in props:
611 opt_map.remove(opt)
613 for name, val in props.items():
614 if val is None:
615 continue
616 if isinstance(val, bool):
617 opt_map.add('Option', name=name, type='bool', value='true' if val else 'false')
618 else:
619 opt_map.add('Option', name=name, type='QString', value=str(val))
621 def update_referenced_layers(self):
622 """Update the data source in relations that reference ``edit`` layers."""
623 for el in self.root.findall('.//referencedLayers/relation'):
624 """
625 <referencedLayers>
626 <relation
627 strength="Association"
628 referencingLayer="..."
629 layerId="..."
630 referencedLayer="REF_ID"
631 providerKey="<REPLACE THIS>"
632 dataSource="<REPLACE THIS>"
633 >
634 ...
635 """
637 ref_id = el.get('referencedLayer')
639 le = self.po.layerMap.get(ref_id)
640 if not le or le.action != caps_mod.LayerAction.edit:
641 gws.log.warning(f'{self.po.uid}: relation: referenced layer not found: {ref_id!r}')
642 continue
644 if 'dataSource' in el.attrib:
645 el.set('dataSource', le.dataSource)
646 if 'providerKey' in el.attrib:
647 el.set('providerKey', le.dataProvider)
649 def update_edit_widgets(self):
650 """Update the data source in relation reference widgets that reference ``edit`` layers."""
651 for el in self.root.findall('.//editWidget'):
652 """
653 <editWidget type="RelationReference">
654 <config>
655 <Option type="Map">
656 ...
657 <Option value="<REF_ID>" name="ReferencedLayerId" type="QString"/>
658 <Option value="<REPLACE THIS>" name="ReferencedLayerDataSource" type="QString"/>
659 <Option value="<REPLACE THIS>" name="ReferencedLayerProviderKey" type="QString"/>
660 ...
661 """
663 if el.get('type') != 'RelationReference':
664 continue
666 ref_id = None
667 for opt in el.findall('.//Option'):
668 if opt.get('name') == 'ReferencedLayerId':
669 ref_id = opt.get('value')
670 break
672 if not ref_id:
673 continue
675 le = self.po.layerMap.get(ref_id)
676 if not le or le.action != caps_mod.LayerAction.edit:
677 gws.log.warning(f'{self.po.uid}: editWidget: referenced layer not found: {ref_id!r}')
678 continue
680 for opt in el.findall('.//Option'):
681 if opt.get('name') == 'ReferencedLayerDataSource':
682 opt.set('value', le.dataSource)
683 if opt.get('name') == 'ReferencedLayerProviderKey':
684 opt.set('value', le.dataProvider)
686 def cleanup_layer_group(self, group_el):
687 """Mark empty layer groups for removal, recursively.
689 Args:
690 group_el: Layer tree group element.
692 Returns:
693 ``True`` if the group contains layers that are kept.
694 """
695 is_empty = True
697 for sub in group_el.children():
698 if sub.tag == 'layer-tree-group':
699 if self.cleanup_layer_group(sub):
700 is_empty = False
701 if sub.tag == 'layer-tree-layer' and sub not in self.toRemove:
702 is_empty = False
704 if is_empty:
705 self.toRemove.append(group_el)
707 return not is_empty
709 def remove_elements(self, el, parent_el):
710 """Remove the elements marked for removal, recursively.
712 Args:
713 el: Element to check.
714 parent_el: Parent element.
715 """
716 if el in self.toRemove:
717 parent_el.remove(el)
718 return
719 ns = el.children()
720 for n in ns:
721 self.remove_elements(n, el)