Coverage for gws-app/gws/plugin/qfieldcloud/action_base.py: 98%

1"""QField Cloud API action: projects, capabilities, packaging and directories.""" 

2 

3from typing import Optional, cast 

4 

5import re 

6 

7import gws 

8import gws.base.job 

9import gws.base.action 

10import gws.lib.jsonx 

11import gws.lib.datetimex as dtx 

12import gws.lib.osx as osx 

13 

14from . import core, packager, patcher, caps, auth 

15 

16 

17class WorkerPayload(gws.Data): 

18 """Job payload for the package worker.""" 

19 

20 actionUid: str 

21 """Uid of the action.""" 

22 jobType: str 

23 """Job type, ``package``.""" 

24 qfcProjectUid: str 

25 """QField project uid.""" 

26 projectUid: Optional[str] 

27 """GWS project uid.""" 

28 

29 

30class BaseAction(gws.base.action.Object): 

31 """Base of the QField Cloud API action. 

32 

33 Holds the configured QField projects and the authorization method, caches 

34 the project capabilities, creates packages and manages the project directories. 

35 Request handling is done by ``action_handler.Handler``. 

36 """ 

37 

38 qfcProjects: list[core.QfcProject] 

39 """Configured QField projects.""" 

40 capsCache: dict[str, caps.Caps] 

41 """Capabilities by QField project uid. Not serialized.""" 

42 authMethod: auth.Object 

43 """Authorization method for QField clients.""" 

44 deltaLifeTime: int 

45 """How long stored delta payloads are kept, in seconds.""" 

46 

47 def configure(self): 

48 self.deltaLifeTime = self.cfg('deltaLifeTime', default=3600 * 24 * 30) 

49 self.authMethod = cast( 

50 auth.Object, 

51 self.create_child(gws.ext.object.authMethod, self.cfg('auth'), type='qfieldcloud'), 

52 ) 

53 self.root.app.authMgr.add_method(self.authMethod) 

54 self.qfcProjects = [] 

55 for p in self.cfg('projects') or []: 

56 qp = self.create_child(core.QfcProject, p) 

57 if qp: 

58 self.qfcProjects.append(cast(core.QfcProject, qp)) 

59 

60 def __getstate__(self): 

61 """Return the state for pickling, without the caps cache.""" 

62 return gws.u.omit(vars(self), 'capsCache') 

63 

64 ## 

65 

66 def get_packager(self) -> packager.Object: 

67 """Return a new packager. Override to use a custom packager. 

68 

69 Returns: 

70 Packager. 

71 """ 

72 return packager.Object() 

73 

74 def get_patcher(self) -> patcher.Object: 

75 """Return a new patcher. Override to use a custom patcher. 

76 

77 Returns: 

78 Patcher. 

79 """ 

80 return patcher.Object() 

81 

82 ## 

83 

84 def get_caps(self, qfc_project: core.QfcProject) -> caps.Caps: 

85 """Return the capabilities of a QField project, from the cache if they are still valid. 

86 

87 Args: 

88 qfc_project: QField project. 

89 

90 Returns: 

91 Capabilities. 

92 """ 

93 if not hasattr(self, 'capsCache'): 

94 self.capsCache = {} 

95 cs = self.get_cached_caps(qfc_project) 

96 if cs: 

97 return cs 

98 

99 pa = caps.Parser(qfc_project) 

100 pa.parse() 

101 pa.create_models() 

102 pa.assign_path_props() 

103 

104 self.capsCache[qfc_project.uid] = pa.caps 

105 gws.log.debug(f'get_caps: {qfc_project.uid=}: created') 

106 return pa.caps 

107 

108 def get_db_provider(self, qfc_project: core.QfcProject) -> gws.DatabaseProvider: 

109 """Return the database provider of the models of a QField project. 

110 

111 Args: 

112 qfc_project: QField project. 

113 

114 Returns: 

115 Database provider. 

116 

117 Raises: 

118 ``gws.Error``: If the project has no models, or its models use more than one database provider. 

119 """ 

120 cs = self.get_caps(qfc_project) 

121 

122 db = None 

123 uids = set() 

124 for me in cs.modelMap.values(): 

125 db = me.model.db 

126 uids.add(me.model.db.uid) 

127 

128 if len(uids) != 1 or db is None: 

129 raise gws.Error(f'QField project {qfc_project.uid!r}: expected one database provider, found {sorted(uids)}') 

130 

131 return db 

132 

133 def get_cached_caps(self, qfc_project: core.QfcProject) -> Optional[caps.Caps]: 

134 """Return cached capabilities, if the QGIS project source has not changed. 

135 

136 Args: 

137 qfc_project: QField project. 

138 

139 Returns: 

140 Capabilities, or ``None`` if not cached or outdated. 

141 """ 

142 cs = self.capsCache.get(qfc_project.uid) 

143 if not cs: 

144 gws.log.debug(f'get_caps: {qfc_project.uid=}: not found') 

145 return 

146 

147 qp = qfc_project.qgisProvider.qgis_project() 

148 if qp.sourceHash != cs.sourceHash: 

149 gws.log.debug(f'get_caps: {qfc_project.uid=}: hash changed: {cs.sourceHash=} != {qp.sourceHash=}') 

150 self.capsCache.pop(qfc_project.uid, None) 

151 return 

152 

153 gws.log.debug(f'get_caps: {qfc_project.uid=}: CACHED!') 

154 return cs 

155 

156 ## 

157 

158 def get_qfc_projects(self, user: gws.User) -> list[core.QfcProject]: 

159 """Return the QField projects a user can use. 

160 

161 Args: 

162 user: User. 

163 

164 Returns: 

165 QField projects. 

166 """ 

167 return [p for p in self.qfcProjects if user.can_use(p)] 

168 

169 def get_qfc_project(self, qfc_project_uid: str, user: gws.User) -> Optional[core.QfcProject]: 

170 """Return a QField project if the user can use it. 

171 

172 Args: 

173 qfc_project_uid: QField project uid. 

174 user: User. 

175 

176 Returns: 

177 QField project, or ``None`` if not found. 

178 """ 

179 for qp in self.get_qfc_projects(user): 

180 if qp.uid == qfc_project_uid: 

181 return qp 

182 

183 ## 

184 

185 def create_package_from_worker(self, worker: 'PackageWorker', pa: WorkerPayload): 

186 """Create a package in a new package directory. Called by the package worker. 

187 

188 Packages older than an hour are removed first. 

189 

190 Args: 

191 worker: Package worker. 

192 pa: Job payload. 

193 """ 

194 project = worker.user.require_project(pa.projectUid) if pa.projectUid else None 

195 qfc_project = gws.u.require(self.get_qfc_project(pa.qfcProjectUid, worker.user)) 

196 

197 self.cleanup_old_packages(qfc_project) 

198 

199 uid = dtx.to_basic_string(with_ms=True) 

200 pkg_dir = self.new_package_dir(qfc_project, uid) 

201 args = packager.Args( 

202 uid=uid, 

203 qfcProject=qfc_project, 

204 caps=self.get_caps(qfc_project), 

205 project=project, 

206 user=worker.user, 

207 packageDir=pkg_dir, 

208 cacheDir=self.project_cache_dir(qfc_project), 

209 withBaseMap=True, 

210 withData=True, 

211 withMedia=True, 

212 withQgis=True, 

213 ) 

214 self.get_packager().create_package(self.root, args) 

215 

216 def create_package_from_cli(self, qfc_project_uid: str, target_dir: str, project: Optional[gws.Project], user: gws.User): 

217 """Create a package in a given directory. 

218 

219 Args: 

220 qfc_project_uid: QField project uid. 

221 target_dir: Directory to write the package into. 

222 project: GWS project context. 

223 user: User the package is created for. 

224 

225 Raises: 

226 ``gws.NotFoundError``: If the QField project is not found. 

227 """ 

228 qfc_project = self.get_qfc_project(qfc_project_uid, user) 

229 if not qfc_project: 

230 raise gws.NotFoundError(f'project {qfc_project_uid!r} not found') 

231 

232 args = packager.Args( 

233 uid='cli', 

234 qfcProject=qfc_project, 

235 caps=self.get_caps(qfc_project), 

236 project=project, 

237 user=user, 

238 packageDir=target_dir, 

239 cacheDir=self.project_cache_dir(qfc_project), 

240 withBaseMap=True, 

241 withData=True, 

242 withMedia=True, 

243 withQgis=True, 

244 ) 

245 self.get_packager().create_package(self.root, args) 

246 

247 def get_job(self, job_id: str, user: gws.User) -> Optional[gws.Job]: 

248 """Return a job of a user. 

249 

250 Args: 

251 job_id: Job uid. 

252 user: User. 

253 

254 Returns: 

255 The job, or ``None`` if not found. 

256 """ 

257 return self.root.app.jobMgr.get_job(job_id, user=user) 

258 

259 ## 

260 

261 def project_base_dir(self, qfc_project: core.QfcProject) -> str: 

262 """Return the base directory of a QField project, creating it if needed. 

263 

264 The directory is ``<VAR_DIR>/qfieldcloud/projects/<uid>``. Packages, caches 

265 and stored deltas are kept below it. 

266 

267 Args: 

268 qfc_project: QField project. 

269 

270 Returns: 

271 Directory path. 

272 """ 

273 return gws.u.ensure_dir(f'{gws.c.VAR_DIR}/qfieldcloud/projects/{qfc_project.uid}') 

274 

275 def latest_package_dir(self, qfc_project: core.QfcProject) -> Optional[str]: 

276 """Return the directory of the latest complete package. 

277 

278 Args: 

279 qfc_project: QField project. 

280 

281 Returns: 

282 Directory path, or ``None`` if there is no complete package. 

283 """ 

284 base_dir = self.project_base_dir(qfc_project) 

285 for pkg in sorted(osx.find_directories(base_dir, deep=False), reverse=True): 

286 m = re.search(r'package_(\d+)', pkg) 

287 if m and gws.u.is_file(f'{pkg}/{packager.COMPLETE_FILE}'): 

288 return pkg 

289 

290 def new_package_dir(self, qfc_project: core.QfcProject, uid: str) -> str: 

291 """Create a directory for a new package. 

292 

293 Args: 

294 qfc_project: QField project. 

295 uid: Package uid. 

296 

297 Returns: 

298 Directory path. 

299 """ 

300 base_dir = self.project_base_dir(qfc_project) 

301 pkg_dir = gws.u.ensure_dir(f'{base_dir}/package_{uid}') 

302 return pkg_dir 

303 

304 def cleanup_old_packages(self, qfc_project: core.QfcProject, keep_seconds: int = 3600): 

305 """Remove package directories older than ``keep_seconds``. 

306 

307 Args: 

308 qfc_project: QField project. 

309 keep_seconds: Maximum age in seconds. 

310 """ 

311 base_dir = self.project_base_dir(qfc_project) 

312 now = dtx.now().timestamp() 

313 for pkg in osx.find_directories(base_dir, deep=False): 

314 m = re.search(r'package_(\d+)', pkg) 

315 if not m: 

316 continue 

317 t = osx.file_mtime(pkg) 

318 if now - t > keep_seconds: 

319 gws.log.info(f'cleanup_old_packages: removing old package: {pkg=}') 

320 osx.rmdir(pkg) 

321 

322 def latest_package_path_map(self, qfc_project: core.QfcProject) -> dict[str, str]: 

323 """Return the path map of the latest package of a QField project. 

324 

325 Args: 

326 qfc_project: QField project. 

327 

328 Returns: 

329 Paths on disk by package file name, empty if there is no package. 

330 """ 

331 d = self.latest_package_dir(qfc_project) 

332 try: 

333 return gws.lib.jsonx.from_path(f'{d}/{packager.PATH_MAP_FILE}') 

334 except Exception: 

335 return {} 

336 

337 def project_cache_dir(self, qfc_project: core.QfcProject) -> str: 

338 """Return the cache directory of a QField project, creating it if needed. 

339 

340 Args: 

341 qfc_project: QField project. 

342 

343 Returns: 

344 Directory path. 

345 """ 

346 base_dir = self.project_base_dir(qfc_project) 

347 return gws.u.ensure_dir(f'{base_dir}/cache') 

348 

349 def deltas_db_path(self, qfc_project: core.QfcProject) -> str: 

350 """Return the path of the delta database of a QField project. 

351 

352 Args: 

353 qfc_project: QField project. 

354 

355 Returns: 

356 File path. 

357 """ 

358 base_dir = self.project_base_dir(qfc_project) 

359 return f'{base_dir}/deltas.sqlite' 

360 

361 

362## 

363 

364 

365class PackageWorker(gws.base.job.worker.Object): 

366 """Job worker that creates a package.""" 

367 

368 @classmethod 

369 def run(cls, root: gws.Root, job: gws.Job): 

370 """Run a packaging job. 

371 

372 Args: 

373 root: Root object. 

374 job: The job. 

375 """ 

376 w = cls(root, job.user, job) 

377 w.work() 

378 

379 def work(self): 

380 """Create the package and mark the job as complete.""" 

381 self.update_job(state=gws.JobState.running) 

382 pa = WorkerPayload(gws.u.require(self.get_job()).payload) 

383 action = cast(BaseAction, self.root.get(pa.actionUid)) 

384 action.create_package_from_worker(self, pa) 

385 self.update_job(state=gws.JobState.complete)