Coverage for gws-app/gws/config/loader.py: 20%

1"""Configuration loader: configure, save and load the root object.""" 

2 

3from typing import Optional 

4import sys 

5 

6import gws 

7import gws.spec.runtime 

8import gws.lib.osx 

9import gws.lib.dynimport 

10 

11from . import parser 

12 

13_ERROR_PREFIX = 'CONFIGURATION ERROR' 

14_WARNING_PREFIX = 'CONFIGURATION WARNING' 

15 

16_DEFAULT_CONFIG_PATHS = [ 

17 '/data/config.cx', 

18 '/data/config.json', 

19 '/data/config.yaml', 

20 '/data/config.py', 

21] 

22 

23_DEFAULT_MANIFEST_PATHS = [ 

24 '/data/MANIFEST.json', 

25] 

26 

27 

28class Object: 

29 """Configuration loader. 

30 

31 Holds the state of one configuration run: the specs, the parsed config, 

32 the root object and the collected errors and warnings. 

33 """ 

34 

35 ctx: gws.ConfigContext 

36 """Context with specs, errors, warnings and read paths.""" 

37 manifestPath: str 

38 """Path to the application manifest, or an empty string.""" 

39 configPath: str 

40 """Path to the configuration file, or an empty string.""" 

41 fallbackConfig: Optional[gws.Config] 

42 """Configuration used when the main configuration fails, if the manifest allows it.""" 

43 withSpecCache: bool 

44 """Read and write the specs cache.""" 

45 

46 def __init__( 

47 self, 

48 manifest_path='', 

49 config_path='', 

50 specs=None, 

51 raw_config=None, 

52 fallback_config=None, 

53 with_spec_cache=False, 

54 hooks=None, 

55 ): 

56 """Create a loader. 

57 

58 Args: 

59 manifest_path: Path to the application manifest. Defaults to ``GWS_MANIFEST`` or a default path. 

60 config_path: Path to the configuration file. Defaults to ``GWS_CONFIG`` or a default path. 

61 specs: Specs to use. If not given, specs are created from the manifest. 

62 raw_config: Configuration dict, used instead of the configuration file. 

63 fallback_config: Configuration used when the main configuration fails. 

64 with_spec_cache: Read and write the specs cache. 

65 hooks: List of ``(event, fn)`` pairs, see ``configure``. 

66 """ 

67 self.tm1 = _time_and_memory() 

68 

69 self.ctx = gws.ConfigContext( 

70 errors=[], 

71 warnings=[], 

72 ) 

73 

74 self.manifestPath = real_manifest_path(manifest_path) 

75 if self.manifestPath: 

76 gws.log.info(f'using manifest {self.manifestPath!r}...') 

77 

78 self.configPath = real_config_path(config_path) 

79 self.rawConfig = raw_config 

80 self.fallbackConfig = fallback_config 

81 self.withSpecCache = with_spec_cache 

82 self.hooks = hooks or [] 

83 self.specs = specs 

84 

85 self.config = None 

86 self.root = None 

87 

88 def configure(self) -> gws.ConfigResult: 

89 """Parse the configuration and create the root object. 

90 

91 Hooks are called with the loader object at the events ``preConfigure``, 

92 ``postConfigure`` (around parsing), ``preInitialize`` and ``postInitialize`` 

93 (around creating the root). Exceptions in hooks are recorded as errors. 

94 

95 If no root can be created, the fallback config is used, provided the 

96 manifest sets ``withFallbackConfig``. If there are errors and the manifest 

97 sets ``withStrictConfig``, the root is discarded. 

98 

99 Returns: 

100 The configuration result. 

101 """ 

102 if not self._init_specs(): 

103 return self._result() 

104 

105 self._run_hook('preConfigure') 

106 if not self.config: 

107 self.config = self._create_config() 

108 self._run_hook('postConfigure') 

109 

110 if self.config: 

111 self._run_hook('preInitialize') 

112 if not self.root: 

113 self.root = self._create_root(self.config) 

114 self._run_hook('postInitialize') 

115 

116 if not self.root and self.ctx.specs.manifest.withFallbackConfig and self.fallbackConfig: 

117 gws.log.warning(f'using fallback config') 

118 self.root = self._create_root(self.fallbackConfig) 

119 

120 if not self.root: 

121 return self._result() 

122 

123 if self.ctx.errors and self.ctx.specs.manifest.withStrictConfig: 

124 self.root = None 

125 return self._result() 

126 

127 self.root.configPaths = list(self.ctx.paths) 

128 return self._result() 

129 

130 def parse(self) -> gws.ConfigResult: 

131 """Parse and validate the configuration without creating objects. 

132 

133 Returns: 

134 The configuration result, without the root object. 

135 """ 

136 if not self._init_specs(): 

137 return self._result() 

138 

139 self.config = self._create_config() 

140 if not self.config: 

141 return self._result() 

142 

143 return self._result() 

144 

145 ## 

146 

147 def _init_specs(self): 

148 if self.specs: 

149 self.ctx.specs = self.specs 

150 return True 

151 

152 try: 

153 self.ctx.specs = gws.spec.runtime.create( 

154 manifest_path=self.manifestPath, 

155 read_cache=self.withSpecCache, 

156 write_cache=self.withSpecCache, 

157 ) 

158 return True 

159 except Exception as exc: 

160 gws.log.exception() 

161 self._error(exc) 

162 return False 

163 

164 def _create_config(self): 

165 if self.rawConfig: 

166 return parser.parse_app_dict(self.rawConfig, '', self.ctx) 

167 if not self.configPath: 

168 self._error(gws.ConfigurationError('no configuration file found')) 

169 return 

170 gws.log.info(f'using config {self.configPath!r}...') 

171 return parser.parse_app_from_path(self.configPath, self.ctx) 

172 

173 def _create_root(self, cfg): 

174 root = initialize(self.ctx.specs, cfg) 

175 if root: 

176 for ce in root.configErrors: 

177 self.ctx.errors.append(gws.ConfigErrorInfo(ce)) 

178 for cw in root.configWarnings: 

179 self.ctx.warnings.append(gws.ConfigErrorInfo(cw)) 

180 return root 

181 

182 def _run_hook(self, event): 

183 for evt, fn in self.hooks: 

184 if event != evt: 

185 continue 

186 try: 

187 fn(self) 

188 except Exception as exc: 

189 gws.log.exception() 

190 self._error(exc) 

191 

192 def _error(self, exc): 

193 cei = gws.ConfigErrorInfo(message=str(exc)) 

194 if exc.__cause__: 

195 cei.cause = repr(exc.__cause__) 

196 self.ctx.errors.append(cei) 

197 

198 def _result(self): 

199 return gws.ConfigResult( 

200 errors=self.ctx.errors, 

201 warnings=self.ctx.warnings, 

202 root=self.root, 

203 config=self.config, 

204 info=_info_string(self.root, self.tm1), 

205 ) 

206 

207 

208def configure( 

209 manifest_path='', 

210 config_path='', 

211 specs: Optional[gws.SpecRuntime] = None, 

212 raw_config: dict | gws.Data = None, 

213 fallback_config: dict | gws.Data = None, 

214 with_spec_cache=False, 

215 hooks: list = None, 

216) -> gws.ConfigResult: 

217 """Parse the configuration and create the root object. 

218 

219 Args: 

220 manifest_path: Path to the application manifest. 

221 config_path: Path to the configuration file. 

222 specs: Specs to use. If not given, specs are created from the manifest. 

223 raw_config: Configuration dict, used instead of the configuration file. 

224 fallback_config: Configuration used when the main configuration fails. 

225 with_spec_cache: Read and write the specs cache. 

226 hooks: List of ``(event, fn)`` pairs, see ``Object.configure``. 

227 

228 Returns: 

229 The configuration result. Its ``root`` is ``None`` if configuration failed. 

230 """ 

231 

232 ldr = Object( 

233 manifest_path, 

234 config_path, 

235 specs, 

236 raw_config, 

237 fallback_config, 

238 with_spec_cache, 

239 hooks, 

240 ) 

241 return ldr.configure() 

242 

243 

244def parse( 

245 manifest_path='', 

246 config_path='', 

247 specs: Optional[gws.SpecRuntime] = None, 

248) -> gws.ConfigResult: 

249 """Parse and validate the configuration without creating objects. 

250 

251 Args: 

252 manifest_path: Path to the application manifest. 

253 config_path: Path to the configuration file. 

254 specs: Specs to use. If not given, specs are created from the manifest. 

255 

256 Returns: 

257 The configuration result with the parsed config, errors and warnings. 

258 """ 

259 

260 ldr = Object( 

261 manifest_path, 

262 config_path, 

263 specs, 

264 ) 

265 return ldr.parse() 

266 

267 

268def initialize(specs: gws.SpecRuntime, config: gws.Config) -> gws.Root: 

269 """Create the root object and the application from a parsed configuration. 

270 

271 Args: 

272 specs: Specs runtime. 

273 config: Parsed application configuration. 

274 

275 Returns: 

276 The initialized, not yet activated root object. 

277 """ 

278 root = gws.create_root(specs) 

279 root.create_application(config) 

280 root.post_initialize() 

281 return root 

282 

283 

284def save(root: gws.Root, path=None) -> str: 

285 """Save the root object to a file, see ``gws.save_root``. 

286 

287 Args: 

288 root: Root object. 

289 path: File path. Defaults to ``config.pickle`` in the config directory. 

290 

291 Returns: 

292 The file path. 

293 

294 Raises: 

295 ``gws.ConfigurationError``: If the root cannot be saved. 

296 """ 

297 path = path or gws.c.ROOT_PICKLE_PATH 

298 gws.log.debug(f'writing config to {path!r}') 

299 try: 

300 return gws.save_root(root, path) 

301 except Exception as exc: 

302 raise gws.ConfigurationError('unable to save configuration') from exc 

303 

304 

305def load(path=None) -> gws.Root: 

306 """Load a saved root object, see ``gws.load_root``, and log the user, time and memory used. 

307 

308 Args: 

309 path: File path. Defaults to ``config.pickle`` in the config directory. 

310 

311 Returns: 

312 The root object. 

313 

314 Raises: 

315 ``gws.ConfigurationError``: If the root cannot be loaded or activated. 

316 """ 

317 ui = gws.lib.osx.user_info() 

318 path = path or gws.c.ROOT_PICKLE_PATH 

319 gws.log.info(f'loading config from {path!r}, user {ui["pw_name"]} ({ui["pw_uid"]}:{ui["pw_gid"]})') 

320 try: 

321 tm1 = _time_and_memory() 

322 root = gws.load_root(path) 

323 info = _info_string(root, tm1) 

324 gws.log.info(f'configuration loaded, {info}') 

325 return root 

326 except Exception as exc: 

327 raise gws.ConfigurationError('unable to load configuration') from exc 

328 

329 

330def real_config_path(config_path: str) -> str: 

331 """Find the configuration file. 

332 

333 Args: 

334 config_path: Comma-separated list of paths. Defaults to the ``GWS_CONFIG`` environment variable. 

335 If neither is given, the default paths are checked. 

336 

337 Returns: 

338 The first path that is an existing file, or an empty string. 

339 """ 

340 p = config_path or gws.env.GWS_CONFIG 

341 if p: 

342 for s in p.split(','): 

343 s = s.strip() 

344 if gws.u.is_file(s): 

345 return s 

346 return '' 

347 for p in _DEFAULT_CONFIG_PATHS: 

348 if gws.u.is_file(p): 

349 return p 

350 return '' 

351 

352 

353def real_manifest_path(manifest_path: str) -> str: 

354 """Find the application manifest. 

355 

356 Args: 

357 manifest_path: Manifest path. Defaults to the ``GWS_MANIFEST`` environment variable. 

358 If neither is given, the default path ``/data/MANIFEST.json`` is used if it exists. 

359 

360 Returns: 

361 The manifest path, or an empty string. 

362 """ 

363 p = manifest_path or gws.env.GWS_MANIFEST 

364 if p: 

365 return p 

366 for p in _DEFAULT_MANIFEST_PATHS: 

367 if gws.u.is_file(p): 

368 return p 

369 return '' 

370 

371 

372def log_report(cr: gws.ConfigResult): 

373 """Log a summary of the configuration result and all errors and warnings. 

374 

375 Args: 

376 cr: Configuration result. 

377 """ 

378 err_cnt = len(cr.errors) if cr.errors else 0 

379 warn_cnt = len(cr.warnings) if cr.warnings else 0 

380 ln = '*' * 80 

381 

382 if err_cnt == 0 and warn_cnt == 0: 

383 gws.log.info(ln) 

384 gws.log.info(f'configured: {cr.info}') 

385 gws.log.info(ln) 

386 return 

387 

388 if err_cnt == 0: 

389 gws.log.warning(ln) 

390 gws.log.warning(f'configured with warnings: {warn_cnt}, {cr.info}') 

391 gws.log.warning(ln) 

392 else: 

393 gws.log.error(ln) 

394 gws.log.error(f'configured with errors: {err_cnt}, warnings: {warn_cnt}, {cr.info}') 

395 gws.log.error(ln) 

396 

397 # cr.errors.sort(key=lambda ce: ce.message) 

398 

399 for n, cei in enumerate(cr.errors, 1): 

400 gws.log.error(f'{_ERROR_PREFIX}: {n} of {err_cnt}') 

401 _log_info(cei, gws.log.error, _ERROR_PREFIX) 

402 gws.log.error(f'{_ERROR_PREFIX}: ') 

403 

404 for n, cei in enumerate(cr.warnings, 1): 

405 gws.log.warning(f'{_WARNING_PREFIX}: {n} of {warn_cnt}') 

406 _log_info(cei, gws.log.warning, _WARNING_PREFIX) 

407 gws.log.warning(f'{_WARNING_PREFIX}: ') 

408 

409 if err_cnt == 0: 

410 gws.log.warning(ln) 

411 else: 

412 gws.log.error(ln) 

413 

414 

415def _log_info(cei: gws.ConfigErrorInfo, log_fn, prefix): 

416 """Log the details of an error or warning.""" 

417 ls = [] 

418 ls.append(cei.message) 

419 tab = ' ' * 4 

420 

421 if cei.path: 

422 ls.append(f'PATH: {cei.path}') 

423 if cei.line: 

424 ls.append(f'LINE: {cei.line}') 

425 if cei.value: 

426 ls.append(f'VALUE: {cei.value}') 

427 if cei.cause: 

428 ls.append(f'CAUSE: {cei.cause}') 

429 if cei.stack: 

430 for loc in cei.stack: 

431 p = [ 

432 loc.objectType, 

433 repr(loc.objectName) if loc.objectName else None, 

434 f'uid={loc.objectUid}' if loc.objectUid else None, 

435 ] 

436 p = '<' + ' '.join(gws.u.compact(p)) + '>' 

437 if loc.propName: 

438 p = f'{loc.propName!r} {p}' 

439 ls.append(f'{tab}in {p}') 

440 if cei.contextLines: 

441 ls.extend(cei.contextLines) 

442 

443 for s in ls: 

444 log_fn(f'{prefix}: {s}') 

445 

446 

447def _time_and_memory(): 

448 return gws.u.stime(), gws.lib.osx.process_rss_size() 

449 

450 

451def _info_string(root, tm1): 

452 tm2 = _time_and_memory() 

453 return 'objects: {:d}, time: {:d}s., memory: {:.2f} MB'.format( 

454 root.object_count() if root else 0, 

455 tm2[0] - tm1[0], 

456 tm2[1] - tm1[1], 

457 )