Coverage for gws-app / gws / plugin / qfieldcloud / packager.py: 0%

1from typing import cast 

2 

3import gws 

4import gws.gis.gdalx 

5import gws.lib.jsonx 

6import gws.lib.image 

7import gws.lib.bounds 

8import gws.lib.osx as osx 

9import gws.lib.extent 

10import gws.gis.render 

11import gws.gis.zoom 

12 

13from . import core, caps as caps_mod 

14 

15PATH_MAP_FILE = 'path_map.json' 

16COMPLETE_FILE = 'package_complete' 

17 

18 

19class Args(gws.Data): 

20 uid: str 

21 qfcProject: core.QfcProject 

22 caps: caps_mod.Caps 

23 project: gws.Project 

24 user: gws.User 

25 packageDir: str 

26 mapCacheDir: str 

27 withBaseMap: bool 

28 withData: bool 

29 withMedia: bool 

30 withQgis: bool 

31 

32 

33class Object: 

34 uid: str 

35 root: gws.Root 

36 qfcProject: core.QfcProject 

37 project: gws.Project 

38 user: gws.User 

39 args: Args 

40 caps: caps_mod.Caps 

41 

42 def create_package(self, root: gws.Root, args: Args): 

43 self.root = root 

44 self.uid = args.uid 

45 self.pathMap = {} 

46 

47 self.args = args 

48 self.qfcProject = self.args.qfcProject 

49 self.project = self.args.project 

50 self.user = self.args.user 

51 self.caps = args.caps 

52 

53 if self.args.withData: 

54 self.write_data() 

55 

56 if self.args.withBaseMap: 

57 self.write_base_map() 

58 

59 if self.args.withMedia: 

60 self.write_media() 

61 

62 if self.args.withQgis: 

63 self.write_qgis_project() 

64 

65 gws.lib.jsonx.to_path( 

66 f'{self.args.packageDir}/{PATH_MAP_FILE}', 

67 self.pathMap, 

68 pretty=True, 

69 ) 

70 gws.u.write_file( 

71 f'{self.args.packageDir}/{COMPLETE_FILE}', 

72 '1', 

73 ) 

74 

75 def write_data(self): 

76 for le in self.caps.layerMap.values(): 

77 if le.action != caps_mod.LayerAction.edit: 

78 continue 

79 if le.dataSourceFileName in self.pathMap: 

80 continue 

81 path = f'{self.args.packageDir}/{le.dataSourceFileName}' 

82 self.pathMap[le.dataSourceFileName] = path 

83 with gws.gis.gdalx.open_vector(path, 'w') as ds: 

84 self.write_features(le, ds) 

85 

86 def write_base_map(self): 

87 # @TODO options for flattened base maps 

88 for le in self.caps.layerMap.values(): 

89 if le.action == caps_mod.LayerAction.baseMap: 

90 self.write_base_map_layer(le) 

91 

92 def write_media(self): 

93 for d in self.caps.copyDirs: 

94 if not gws.u.is_dir(d): 

95 gws.log.warning(f'{self.uid}: media dir not found: {d!r}') 

96 continue 

97 if self.caps.qgisPath: 

98 rel_dir = osx.rel_path(d, self.caps.qgisPath) 

99 else: 

100 # @TODO absolute dir with a postgres-based qgis project? 

101 rel_dir = d.split('/')[-1] 

102 for p in osx.find_files(d): 

103 self.pathMap[rel_dir + '/' + osx.rel_path(p, d)] = p 

104 

105 # 

106 

107 def get_features_for_layer(self, le: caps_mod.LayerEntry) -> list[gws.Feature]: 

108 me = le.modelEntry 

109 q = gws.SearchQuery() 

110 if self.caps.copyOnlyAreaOfInterest: 

111 q.bounds = gws.u.require(self.caps.areaOfInterest or self.qfcProject.qgisProvider.bounds) 

112 mc = gws.ModelContext(user=self.user, project=self.project, op=gws.ModelOperation.read) 

113 return me.model.find_features(q, mc) 

114 

115 _SUPPORTED_ATTRIBUTE_TYPES = { 

116 gws.AttributeType.bool, 

117 gws.AttributeType.date, 

118 gws.AttributeType.datetime, 

119 gws.AttributeType.float, 

120 gws.AttributeType.int, 

121 gws.AttributeType.str, 

122 gws.AttributeType.time, 

123 } 

124 

125 def write_features(self, le: caps_mod.LayerEntry, ds: gws.gis.gdalx.VectorDataSet): 

126 features = self.get_features_for_layer(le) 

127 

128 me = le.modelEntry 

129 gws.log.debug(f'{self.uid}: {self.qfcProject.uid}::{me.gpName!r} BEGIN write_features') 

130 

131 columns = {} 

132 for f in me.model.fields: 

133 if f.attributeType not in self._SUPPORTED_ATTRIBUTE_TYPES: 

134 continue 

135 # FID field needs to be renamed, because GDAL uses it internally 

136 name = f.name 

137 if f.name.lower() == 'fid': 

138 name = 'fid_gws' 

139 columns[name] = f.attributeType 

140 

141 gp_layer = ds.create_layer( 

142 me.gpName, 

143 columns=columns, 

144 geometry_type=me.model.geometryType, 

145 crs=me.model.geometryCrs, 

146 overwrite=True, 

147 ) 

148 

149 records = [] 

150 for feat in features: 

151 rec = gws.FeatureRecord( 

152 attributes={name: feat.get(name) for name in columns}, 

153 shape=feat.shape(), 

154 meta={}, 

155 ) 

156 # see above 

157 if 'fid' in rec.attributes: 

158 rec.attributes['fid_gws'] = rec.attributes.pop('fid') 

159 records.append(rec) 

160 

161 with ds.transaction(): 

162 gp_layer.insert(records) 

163 

164 gws.log.debug(f'{self.uid}: {self.qfcProject.uid}::{me.gpName!r} END write_features, count={gp_layer.count()}') 

165 

166 ## 

167 

168 def write_base_map_layer(self, le: caps_mod.LayerEntry): 

169 max_zoom = max( 

170 self.caps.projectProps.baseMapTilesMinZoomLevel or 0, 

171 self.caps.projectProps.baseMapTilesMaxZoomLevel or 0, 

172 ) 

173 if max_zoom < 3: 

174 gws.log.warning(f'{self.uid}: write_base_map_layer: invalid zoom level {max_zoom=}') 

175 max_zoom = 3 

176 if max_zoom > 20: 

177 gws.log.warning(f'{self.uid}: write_base_map_layer: invalid zoom level {max_zoom=}') 

178 max_zoom = 20 

179 

180 cache_path = f'{self.args.mapCacheDir}/{max_zoom}_{le.dataSourceFileName}' 

181 age = osx.file_age(cache_path) 

182 ttl = self.qfcProject.mapCacheLifeTime 

183 gws.log.debug(f'{self.uid}: write_base_map_layer: {le.qgisId}: {cache_path=} {ttl=}/{age=}') 

184 

185 if ttl > 0 and (0 < age < ttl): 

186 gws.log.debug(f'{self.uid}: write_base_map_layer: CACHED!') 

187 self.pathMap[le.dataSourceFileName] = cache_path 

188 return 

189 

190 bounds = gws.u.require(self.caps.areaOfInterest or self.qfcProject.qgisProvider.bounds) 

191 bounds = gws.lib.bounds.transform(bounds, le.sourceLayer.supportedCrs[0]) 

192 

193 ls = list(reversed(gws.gis.zoom.OSM_RESOLUTIONS)) 

194 resolution = ls[max_zoom] 

195 

196 w, h = gws.lib.extent.size(bounds.extent) 

197 px_size = (w / resolution, h / resolution, gws.Uom.px) 

198 

199 gws.log.debug(f'{self.uid}: write_base_map_layer: {max_zoom=} {resolution=} {px_size=} {bounds=}') 

200 

201 flat_layer = cast( 

202 gws.Layer, 

203 self.qfcProject.root.create_temporary( 

204 gws.ext.object.layer, 

205 type='qgisflat', 

206 _parentBounds=bounds, 

207 _parentResolutions=[1], 

208 _defaultProvider=self.qfcProject.qgisProvider, 

209 _defaultSourceLayers=[le.sourceLayer], 

210 ), 

211 ) 

212 

213 mv = gws.gis.render.map_view_from_bbox( 

214 size=px_size, 

215 bbox=bounds.extent, 

216 crs=bounds.crs, 

217 dpi=96, 

218 rotation=0, 

219 ) 

220 

221 lri = gws.LayerRenderInput( 

222 type=gws.LayerRenderInputType.box, 

223 user=self.user, 

224 view=mv, 

225 ) 

226 

227 lro = gws.u.require(flat_layer.render(lri)) 

228 img = gws.lib.image.from_bytes(lro.content) 

229 

230 with gws.gis.gdalx.open_from_image(img, bounds) as src: 

231 src.create_copy(cache_path) 

232 

233 self.pathMap[le.dataSourceFileName] = cache_path 

234 

235 def write_qgis_project(self): 

236 fname = f'{self.qfcProject.uid}.qgs' 

237 path = f'{self.args.packageDir}/{fname}' 

238 

239 qp = self.qfcProject.qgisProvider.qgis_project() 

240 gws.u.write_file(path + '.source.qgs', qp.text) 

241 

242 root_el = qp.xml_root() 

243 QgisXmlTransformer().run(self, root_el) 

244 xml = root_el.to_string() 

245 xml = self.replace_vars(xml) 

246 gws.u.write_file(path, xml) 

247 self.pathMap[fname] = path 

248 

249 def replace_vars(self, s: str) -> str: 

250 # @TODO render attributes as templates 

251 s = s.replace('{user.authToken}', self.user.authToken) 

252 s = s.replace('{user.loginName}', self.user.loginName) 

253 s = s.replace('{user.displayName}', self.user.displayName) 

254 return s 

255 

256 

257class QgisXmlTransformer: 

258 po: Object 

259 root: gws.XmlElement 

260 toRemove: list[gws.XmlElement] 

261 

262 def run(self, po: Object, root_el: gws.XmlElement): 

263 self.po = po 

264 self.root = root_el 

265 self.toRemove = [] 

266 

267 self.change_global_props() 

268 self.update_layer_tree() 

269 self.update_map_layers() 

270 self.update_referenced_layers() 

271 self.update_referenced_layers() 

272 self.update_edit_widgets() 

273 

274 self.cleanup_layer_group(root_el.find('layer-tree-group')) 

275 self.remove_elements(root_el, None) 

276 

277 def change_global_props(self): 

278 # change global properties 

279 

280 properties = self.root.find('properties') or self.root.add('properties') 

281 

282 # this is added by the Sync plugin 

283 # p = properties.add('OfflineEditingPlugin').add('OfflineDbPath', type='QString') 

284 # p.text = f'{self.po.deviceDbPath}' 

285 

286 # ensure relative paths 

287 p = properties.find('Paths/Absolute') 

288 if not p: 

289 p = properties.add('Paths').add('Absolute', type='bool') 

290 p.text = 'false' 

291 

292 def update_layer_tree(self): 

293 for el in self.root.findall('.//layer-tree-layer'): 

294 le = self.po.caps.layerMap.get(el.get('id')) 

295 if not le: 

296 continue 

297 

298 if le.action == caps_mod.LayerAction.remove: 

299 self.toRemove.append(el) 

300 continue 

301 

302 el.set('source', le.dataSource) 

303 el.set('providerKey', le.dataProvider) 

304 

305 def update_map_layers(self): 

306 for el in self.root.findall('.//maplayer'): 

307 le = self.po.caps.layerMap.get(el.textof('id')) 

308 if not le: 

309 continue 

310 

311 if le.action == caps_mod.LayerAction.remove: 

312 self.toRemove.append(el) 

313 continue 

314 

315 el.require('datasource').text = le.dataSource 

316 el.require('provider').text = le.dataProvider 

317 

318 # @TODO do we need to change properties at all? 

319 # 

320 # if le.action == caps_mod.LayerAction.edit: 

321 # cp = el.find('customproperties') 

322 # if cp: 

323 # el.remove(cp) 

324 # opt = el.add('customproperties').add('Option', type='Map') 

325 

326 # opt.add('Option', type='QString', name='QFieldSync/action', value='offline') 

327 # opt.add('Option', type='QString', name='QFieldSync/attachment_naming', value='{}') 

328 # opt.add('Option', type='QString', name='QFieldSync/photo_naming', value='{}') 

329 # opt.add('Option', type='QString', name='QFieldSync/sourceDataPrimaryKeys', value='fid') 

330 

331 # if le.action == caps_mod.LayerAction.edit: 

332 # if le.readOnly: 

333 # opt.add('Option', type='bool', name='QFieldSync/is_geometry_locked', value='true') 

334 # else: 

335 # opt.add('Option', type='bool', name='isOfflineEditable', value='true') 

336 

337 def update_referenced_layers(self): 

338 for el in self.root.findall('.//referencedLayers/relation'): 

339 """ 

340 <referencedLayers> 

341 <relation  

342 strength="Association"  

343 referencingLayer="..." 

344 layerId="..." 

345 referencedLayer="REF_ID"  

346 providerKey="<REPLACE THIS>" 

347 dataSource="<REPLACE THIS>" 

348 > 

349 ... 

350 """ 

351 

352 ref_id = el.get('referencedLayer') 

353 

354 le = self.po.caps.layerMap.get(ref_id) 

355 if not le or le.action != caps_mod.LayerAction.edit: 

356 gws.log.warning(f'{self.po.uid}: relation: referenced layer not found: {ref_id!r}') 

357 continue 

358 

359 if 'dataSource' in el.attrib: 

360 el.set('dataSource', le.dataSource) 

361 if 'providerKey' in el.attrib: 

362 el.set('providerKey', le.dataProvider) 

363 

364 def update_edit_widgets(self): 

365 for el in self.root.findall('.//editWidget'): 

366 """ 

367 <editWidget type="RelationReference"> 

368 <config> 

369 <Option type="Map"> 

370 ... 

371 <Option value="<REF_ID>" name="ReferencedLayerId" type="QString"/> 

372 <Option value="<REPLACE THIS>" name="ReferencedLayerDataSource" type="QString"/> 

373 <Option value="<REPLACE THIS>" name="ReferencedLayerProviderKey" type="QString"/> 

374 ... 

375 """ 

376 

377 if el.get('type') != 'RelationReference': 

378 continue 

379 

380 ref_id = None 

381 for opt in el.findall('.//Option'): 

382 if opt.get('name') == 'ReferencedLayerId': 

383 ref_id = opt.get('value') 

384 break 

385 

386 if not ref_id: 

387 continue 

388 

389 le = self.po.caps.layerMap.get(ref_id) 

390 if not le or le.action != caps_mod.LayerAction.edit: 

391 gws.log.warning(f'{self.po.uid}: editWidget: referenced layer not found: {ref_id!r}') 

392 continue 

393 

394 for opt in el.findall('.//Option'): 

395 if opt.get('name') == 'ReferencedLayerDataSource': 

396 opt.set('value', le.dataSource) 

397 if opt.get('name') == 'ReferencedLayerProviderKey': 

398 opt.set('value', le.dataProvider) 

399 

400 def cleanup_layer_group(self, group_el): 

401 is_empty = True 

402 

403 for sub in group_el.children(): 

404 if sub.tag == 'layer-tree-group': 

405 if self.cleanup_layer_group(sub): 

406 is_empty = False 

407 if sub.tag == 'layer-tree-layer' and sub not in self.toRemove: 

408 is_empty = False 

409 

410 if is_empty: 

411 self.toRemove.append(group_el) 

412 

413 return not is_empty 

414 

415 def remove_elements(self, el, parent_el): 

416 if el in self.toRemove: 

417 parent_el.remove(el) 

418 return 

419 ns = el.children() 

420 for n in ns: 

421 self.remove_elements(n, el)