Coverage for gws-app/gws/plugin/qfieldcloud/packager.py: 99%

1from typing import cast 

2 

3import gws 

4import gws.lib.gdalx 

5import gws.lib.jsonx 

6import gws.lib.image 

7import gws.lib.bounds 

8import gws.lib.osx as osx 

9import gws.lib.extent 

10import gws.gis.render 

11import gws.gis.zoom 

12 

13from . import core, caps as caps_mod 

14 

15PATH_MAP_FILE = 'path_map.json' 

16COMPLETE_FILE = 'package_complete' 

17 

18 

19class Args(gws.Data): 

20 uid: str 

21 qfcProject: core.QfcProject 

22 caps: caps_mod.Caps 

23 project: gws.Project 

24 user: gws.User 

25 packageDir: str 

26 mapCacheDir: str 

27 withBaseMap: bool 

28 withData: bool 

29 withMedia: bool 

30 withQgis: bool 

31 

32 

33class Object: 

34 uid: str 

35 root: gws.Root 

36 qfcProject: core.QfcProject 

37 project: gws.Project 

38 user: gws.User 

39 args: Args 

40 caps: caps_mod.Caps 

41 

42 def create_package(self, root: gws.Root, args: Args): 

43 self.root = root 

44 self.uid = args.uid 

45 self.pathMap = {} 

46 

47 self.args = args 

48 self.qfcProject = self.args.qfcProject 

49 self.project = self.args.project 

50 self.user = self.args.user 

51 self.caps = args.caps 

52 

53 if self.args.withData: 

54 self.write_data() 

55 

56 if self.args.withBaseMap: 

57 self.write_base_map() 

58 

59 if self.args.withMedia: 

60 self.write_media() 

61 

62 if self.args.withQgis: 

63 self.write_qgis_project() 

64 

65 gws.lib.jsonx.to_path( 

66 f'{self.args.packageDir}/{PATH_MAP_FILE}', 

67 self.pathMap, 

68 pretty=True, 

69 ) 

70 gws.u.write_file( 

71 f'{self.args.packageDir}/{COMPLETE_FILE}', 

72 '1', 

73 ) 

74 

75 def write_data(self): 

76 for le in self.caps.layerMap.values(): 

77 if le.action != caps_mod.LayerAction.edit: 

78 continue 

79 if le.dataSourceFileName in self.pathMap: 

80 continue 

81 path = f'{self.args.packageDir}/{le.dataSourceFileName}' 

82 self.pathMap[le.dataSourceFileName] = path 

83 with gws.lib.gdalx.open_vector(path, 'w') as ds: 

84 self.write_features(le, ds) 

85 

86 def write_base_map(self): 

87 # @TODO options for flattened base maps 

88 for le in self.caps.layerMap.values(): 

89 if le.action == caps_mod.LayerAction.baseMap: 

90 self.write_base_map_layer(le) 

91 

92 def write_media(self): 

93 for d in self.caps.copyDirs: 

94 if not gws.u.is_dir(d): 

95 gws.log.warning(f'{self.uid}: media dir not found: {d!r}') 

96 continue 

97 if self.caps.qgisPath: 

98 rel_dir = osx.rel_path(d, self.caps.qgisPath) 

99 else: 

100 # @TODO absolute dir with a postgres-based qgis project? 

101 rel_dir = d.split('/')[-1] 

102 for p in osx.find_files(d): 

103 self.pathMap[rel_dir + '/' + osx.rel_path(p, d)] = p 

104 

105 # 

106 

107 def get_features_for_layer(self, le: caps_mod.LayerEntry) -> list[gws.Feature]: 

108 me = le.modelEntry 

109 q = gws.SearchQuery() 

110 if self.caps.copyOnlyAreaOfInterest: 

111 q.bounds = gws.u.require(self.caps.areaOfInterest or self.qfcProject.qgisProvider.bounds) 

112 mc = gws.ModelContext(user=self.user, project=self.project, op=gws.ModelOperation.read) 

113 return me.model.find_features(q, mc) 

114 

115 _SUPPORTED_ATTRIBUTE_TYPES = { 

116 gws.AttributeType.bool, 

117 gws.AttributeType.date, 

118 gws.AttributeType.datetime, 

119 gws.AttributeType.float, 

120 gws.AttributeType.int, 

121 gws.AttributeType.str, 

122 gws.AttributeType.time, 

123 } 

124 

125 def write_features(self, le: caps_mod.LayerEntry, ds: gws.lib.gdalx.VectorDataSet): 

126 features = self.get_features_for_layer(le) 

127 

128 me = le.modelEntry 

129 gws.log.debug(f'{self.uid}: {self.qfcProject.uid}::{me.gpName!r} BEGIN write_features') 

130 

131 columns = {} 

132 field_names = {} 

133 for f in me.model.fields: 

134 if f.attributeType not in self._SUPPORTED_ATTRIBUTE_TYPES: 

135 continue 

136 # FID field needs to be renamed, because GDAL uses it internally 

137 name = f.name 

138 if f.name.lower() == 'fid': 

139 name = 'fid_gws' 

140 columns[name] = f.attributeType 

141 field_names[name] = f.name 

142 

143 gp_layer = ds.create_layer( 

144 me.gpName, 

145 columns=columns, 

146 geometry_type=me.model.geometryType, 

147 crs=me.model.geometryCrs, 

148 overwrite=True, 

149 ) 

150 

151 records = [] 

152 for feat in features: 

153 rec = gws.FeatureRecord( 

154 attributes={name: feat.get(field_name) for name, field_name in field_names.items()}, 

155 shape=feat.shape(), 

156 meta={}, 

157 ) 

158 records.append(rec) 

159 

160 with ds.transaction(): 

161 gp_layer.insert(records) 

162 

163 gws.log.debug(f'{self.uid}: {self.qfcProject.uid}::{me.gpName!r} END write_features, count={gp_layer.count()}') 

164 

165 ## 

166 

167 def write_base_map_layer(self, le: caps_mod.LayerEntry): 

168 max_zoom = max( 

169 self.caps.projectProps.baseMapTilesMinZoomLevel or 0, 

170 self.caps.projectProps.baseMapTilesMaxZoomLevel or 0, 

171 ) 

172 if max_zoom < 3: 

173 gws.log.warning(f'{self.uid}: write_base_map_layer: invalid zoom level {max_zoom=}') 

174 max_zoom = 3 

175 if max_zoom > 20: 

176 gws.log.warning(f'{self.uid}: write_base_map_layer: invalid zoom level {max_zoom=}') 

177 max_zoom = 20 

178 

179 cache_path = f'{self.args.mapCacheDir}/{max_zoom}_{le.dataSourceFileName}' 

180 age = osx.file_age(cache_path) 

181 ttl = self.qfcProject.mapCacheLifeTime 

182 gws.log.debug(f'{self.uid}: write_base_map_layer: {le.qgisId}: {cache_path=} {ttl=}/{age=}') 

183 

184 if ttl > 0 and (0 < age < ttl): 

185 gws.log.debug(f'{self.uid}: write_base_map_layer: CACHED!') 

186 self.pathMap[le.dataSourceFileName] = cache_path 

187 return 

188 

189 bounds = gws.u.require(self.caps.areaOfInterest or self.qfcProject.qgisProvider.bounds) 

190 bounds = gws.lib.bounds.transform(bounds, le.sourceLayer.supportedCrs[0]) 

191 

192 ls = list(reversed(gws.gis.zoom.OSM_RESOLUTIONS)) 

193 resolution = ls[max_zoom] 

194 

195 w, h = gws.lib.extent.size(bounds.extent) 

196 px_size = (w / resolution, h / resolution, gws.Uom.px) 

197 

198 gws.log.debug(f'{self.uid}: write_base_map_layer: {max_zoom=} {resolution=} {px_size=} {bounds=}') 

199 

200 flat_layer = cast( 

201 gws.Layer, 

202 self.qfcProject.root.create_temporary( 

203 gws.ext.object.layer, 

204 type='qgisflat', 

205 _parentBounds=bounds, 

206 _parentResolutions=[1], 

207 _defaultProvider=self.qfcProject.qgisProvider, 

208 _defaultSourceLayers=[le.sourceLayer], 

209 ), 

210 ) 

211 

212 mv = gws.gis.render.map_view_from_bbox( 

213 size=px_size, 

214 bbox=bounds.extent, 

215 crs=bounds.crs, 

216 dpi=96, 

217 rotation=0, 

218 ) 

219 

220 lri = gws.LayerRenderInput( 

221 type=gws.LayerRenderInputType.box, 

222 user=self.user, 

223 view=mv, 

224 ) 

225 

226 lro = gws.u.require(flat_layer.render(lri)) 

227 img = gws.lib.image.from_bytes(lro.content) 

228 

229 with gws.lib.gdalx.open_from_image(img, bounds) as src: 

230 src.save_as(cache_path) 

231 

232 self.pathMap[le.dataSourceFileName] = cache_path 

233 

234 def write_qgis_project(self): 

235 fname = f'{self.qfcProject.uid}.qgs' 

236 path = f'{self.args.packageDir}/{fname}' 

237 

238 qp = self.qfcProject.qgisProvider.qgis_project() 

239 gws.u.write_file(path + '.source.qgs', qp.text) 

240 

241 root_el = qp.xml_root() 

242 QgisXmlTransformer().run(self, root_el) 

243 xml = root_el.to_string() 

244 xml = self.replace_vars(xml) 

245 gws.u.write_file(path, xml) 

246 self.pathMap[fname] = path 

247 

248 def replace_vars(self, s: str) -> str: 

249 # @TODO render attributes as templates 

250 s = s.replace('{user.authToken}', self.user.authToken) 

251 s = s.replace('{user.loginName}', self.user.loginName) 

252 s = s.replace('{user.displayName}', self.user.displayName) 

253 return s 

254 

255 

256class QgisXmlTransformer: 

257 po: Object 

258 root: gws.XmlElement 

259 toRemove: list[gws.XmlElement] 

260 

261 def run(self, po: Object, root_el: gws.XmlElement): 

262 self.po = po 

263 self.root = root_el 

264 self.toRemove = [] 

265 

266 self.change_global_props() 

267 self.update_layer_tree() 

268 self.update_map_layers() 

269 self.update_referenced_layers() 

270 self.update_referenced_layers() 

271 self.update_edit_widgets() 

272 

273 self.cleanup_layer_group(root_el.find('layer-tree-group')) 

274 self.remove_elements(root_el, None) 

275 

276 def change_global_props(self): 

277 # change global properties 

278 

279 properties = self.root.find('properties') or self.root.add('properties') 

280 

281 # this is added by the Sync plugin 

282 # p = properties.add('OfflineEditingPlugin').add('OfflineDbPath', type='QString') 

283 # p.text = f'{self.po.deviceDbPath}' 

284 

285 # ensure relative paths 

286 p = properties.find('Paths/Absolute') 

287 if not p: 

288 p = properties.add('Paths').add('Absolute', type='bool') 

289 p.text = 'false' 

290 

291 def update_layer_tree(self): 

292 for el in self.root.findall('.//layer-tree-layer'): 

293 le = self.po.caps.layerMap.get(el.get('id')) 

294 if not le: 

295 continue 

296 

297 if le.action == caps_mod.LayerAction.remove: 

298 self.toRemove.append(el) 

299 continue 

300 

301 el.set('source', le.dataSource) 

302 el.set('providerKey', le.dataProvider) 

303 

304 def update_map_layers(self): 

305 for el in self.root.findall('.//maplayer'): 

306 le = self.po.caps.layerMap.get(el.textof('id')) 

307 if not le: 

308 continue 

309 

310 if le.action == caps_mod.LayerAction.remove: 

311 self.toRemove.append(el) 

312 continue 

313 

314 el.require('datasource').text = le.dataSource 

315 el.require('provider').text = le.dataProvider 

316 

317 # @TODO do we need to change properties at all? 

318 # 

319 # if le.action == caps_mod.LayerAction.edit: 

320 # cp = el.find('customproperties') 

321 # if cp: 

322 # el.remove(cp) 

323 # opt = el.add('customproperties').add('Option', type='Map') 

324 

325 # opt.add('Option', type='QString', name='QFieldSync/action', value='offline') 

326 # opt.add('Option', type='QString', name='QFieldSync/attachment_naming', value='{}') 

327 # opt.add('Option', type='QString', name='QFieldSync/photo_naming', value='{}') 

328 # opt.add('Option', type='QString', name='QFieldSync/sourceDataPrimaryKeys', value='fid') 

329 

330 # if le.action == caps_mod.LayerAction.edit: 

331 # if le.readOnly: 

332 # opt.add('Option', type='bool', name='QFieldSync/is_geometry_locked', value='true') 

333 # else: 

334 # opt.add('Option', type='bool', name='isOfflineEditable', value='true') 

335 

336 def update_referenced_layers(self): 

337 for el in self.root.findall('.//referencedLayers/relation'): 

338 """ 

339 <referencedLayers> 

340 <relation  

341 strength="Association"  

342 referencingLayer="..." 

343 layerId="..." 

344 referencedLayer="REF_ID"  

345 providerKey="<REPLACE THIS>" 

346 dataSource="<REPLACE THIS>" 

347 > 

348 ... 

349 """ 

350 

351 ref_id = el.get('referencedLayer') 

352 

353 le = self.po.caps.layerMap.get(ref_id) 

354 if not le or le.action != caps_mod.LayerAction.edit: 

355 gws.log.warning(f'{self.po.uid}: relation: referenced layer not found: {ref_id!r}') 

356 continue 

357 

358 if 'dataSource' in el.attrib: 

359 el.set('dataSource', le.dataSource) 

360 if 'providerKey' in el.attrib: 

361 el.set('providerKey', le.dataProvider) 

362 

363 def update_edit_widgets(self): 

364 for el in self.root.findall('.//editWidget'): 

365 """ 

366 <editWidget type="RelationReference"> 

367 <config> 

368 <Option type="Map"> 

369 ... 

370 <Option value="<REF_ID>" name="ReferencedLayerId" type="QString"/> 

371 <Option value="<REPLACE THIS>" name="ReferencedLayerDataSource" type="QString"/> 

372 <Option value="<REPLACE THIS>" name="ReferencedLayerProviderKey" type="QString"/> 

373 ... 

374 """ 

375 

376 if el.get('type') != 'RelationReference': 

377 continue 

378 

379 ref_id = None 

380 for opt in el.findall('.//Option'): 

381 if opt.get('name') == 'ReferencedLayerId': 

382 ref_id = opt.get('value') 

383 break 

384 

385 if not ref_id: 

386 continue 

387 

388 le = self.po.caps.layerMap.get(ref_id) 

389 if not le or le.action != caps_mod.LayerAction.edit: 

390 gws.log.warning(f'{self.po.uid}: editWidget: referenced layer not found: {ref_id!r}') 

391 continue 

392 

393 for opt in el.findall('.//Option'): 

394 if opt.get('name') == 'ReferencedLayerDataSource': 

395 opt.set('value', le.dataSource) 

396 if opt.get('name') == 'ReferencedLayerProviderKey': 

397 opt.set('value', le.dataProvider) 

398 

399 def cleanup_layer_group(self, group_el): 

400 is_empty = True 

401 

402 for sub in group_el.children(): 

403 if sub.tag == 'layer-tree-group': 

404 if self.cleanup_layer_group(sub): 

405 is_empty = False 

406 if sub.tag == 'layer-tree-layer' and sub not in self.toRemove: 

407 is_empty = False 

408 

409 if is_empty: 

410 self.toRemove.append(group_el) 

411 

412 return not is_empty 

413 

414 def remove_elements(self, el, parent_el): 

415 if el in self.toRemove: 

416 parent_el.remove(el) 

417 return 

418 ns = el.children() 

419 for n in ns: 

420 self.remove_elements(n, el)