Coverage for gws-app/gws/config/loader.py: 23%

1"""Configuration loader: configure, save and load the root object.""" 

2 

3from typing import Optional 

4import sys 

5 

6import gws 

7import gws.spec.runtime 

8import gws.lib.osx 

9import gws.lib.dynimport 

10 

11from . import parser 

12 

13_ERROR_PREFIX = 'CONFIGURATION ERROR' 

14_WARNING_PREFIX = 'CONFIGURATION WARNING' 

15 

16_DEFAULT_CONFIG_PATHS = [ 

17 '/data/config.cx', 

18 '/data/config.json', 

19 '/data/config.yaml', 

20 '/data/config.py', 

21] 

22 

23_DEFAULT_MANIFEST_PATHS = [ 

24 '/data/MANIFEST.json', 

25] 

26 

27 

28class Object: 

29 """Configuration loader. 

30 

31 Holds the state of one configuration run: the specs, the parsed config, 

32 the root object and the collected errors and warnings. 

33 """ 

34 

35 ctx: gws.ConfigContext 

36 """Context with specs, errors, warnings and read paths.""" 

37 manifestPath: str 

38 """Path to the application manifest, or an empty string.""" 

39 configPath: str 

40 """Path to the configuration file, or an empty string.""" 

41 fallbackConfig: Optional[gws.Config] 

42 """Configuration used when the main configuration fails, if the manifest allows it.""" 

43 withSpecCache: bool 

44 """Read and write the specs cache.""" 

45 

46 def __init__( 

47 self, 

48 manifest_path='', 

49 config_path='', 

50 specs=None, 

51 raw_config=None, 

52 fallback_config=None, 

53 with_spec_cache=False, 

54 ): 

55 """Create a loader. 

56 

57 Args: 

58 manifest_path: Path to the application manifest. Defaults to ``GWS_MANIFEST`` or a default path. 

59 config_path: Path to the configuration file. Defaults to ``GWS_CONFIG`` or a default path. 

60 specs: Specs to use. If not given, specs are created from the manifest. 

61 raw_config: Configuration dict, used instead of the configuration file. 

62 fallback_config: Configuration used when the main configuration fails. 

63 with_spec_cache: Read and write the specs cache. 

64 """ 

65 self.tm1 = _time_and_memory() 

66 

67 self.ctx = gws.ConfigContext( 

68 errors=[], 

69 warnings=[], 

70 ) 

71 

72 self.manifestPath = real_manifest_path(manifest_path) 

73 if self.manifestPath: 

74 gws.log.info(f'using manifest {self.manifestPath!r}...') 

75 

76 self.configPath = real_config_path(config_path) 

77 self.rawConfig = raw_config 

78 self.fallbackConfig = fallback_config 

79 self.withSpecCache = with_spec_cache 

80 self.specs = specs 

81 

82 self.config = None 

83 self.root = None 

84 

85 def configure(self) -> gws.ConfigResult: 

86 """Parse the configuration and create the root object. 

87 

88 If no root can be created, the fallback config is used, provided the 

89 manifest sets ``withFallbackConfig``. If there are errors and the manifest 

90 sets ``withStrictConfig``, the root is discarded. 

91 

92 Returns: 

93 The configuration result. 

94 """ 

95 if not self._init_specs(): 

96 return self._result() 

97 

98 self.config = self._create_config() 

99 if self.config: 

100 self.root = self._create_root(self.config) 

101 

102 if not self.root and self.ctx.specs.manifest.withFallbackConfig and self.fallbackConfig: 

103 gws.log.warning(f'using fallback config') 

104 self.root = self._create_root(self.fallbackConfig) 

105 

106 if not self.root: 

107 return self._result() 

108 

109 if self.ctx.errors and self.ctx.specs.manifest.withStrictConfig: 

110 self.root = None 

111 return self._result() 

112 

113 self.root.configPaths = list(self.ctx.paths) 

114 return self._result() 

115 

116 def parse(self) -> gws.ConfigResult: 

117 """Parse and validate the configuration without creating objects. 

118 

119 Returns: 

120 The configuration result, without the root object. 

121 """ 

122 if not self._init_specs(): 

123 return self._result() 

124 

125 self.config = self._create_config() 

126 if not self.config: 

127 return self._result() 

128 

129 return self._result() 

130 

131 ## 

132 

133 def _init_specs(self): 

134 if self.specs: 

135 self.ctx.specs = self.specs 

136 return True 

137 

138 try: 

139 self.ctx.specs = gws.spec.runtime.create( 

140 manifest_path=self.manifestPath, 

141 read_cache=self.withSpecCache, 

142 write_cache=self.withSpecCache, 

143 ) 

144 return True 

145 except Exception as exc: 

146 gws.log.exception() 

147 self._error(exc) 

148 return False 

149 

150 def _create_config(self): 

151 if self.rawConfig: 

152 return parser.parse_app_dict(self.rawConfig, '', self.ctx) 

153 if not self.configPath: 

154 self._error(gws.ConfigurationError('no configuration file found')) 

155 return 

156 gws.log.info(f'using config {self.configPath!r}...') 

157 return parser.parse_app_from_path(self.configPath, self.ctx) 

158 

159 def _create_root(self, cfg): 

160 root = initialize(self.ctx.specs, cfg) 

161 if root: 

162 for ce in root.configErrors: 

163 self.ctx.errors.append(gws.ConfigErrorInfo(ce)) 

164 for cw in root.configWarnings: 

165 self.ctx.warnings.append(gws.ConfigErrorInfo(cw)) 

166 return root 

167 

168 def _error(self, exc): 

169 cei = gws.ConfigErrorInfo(message=str(exc)) 

170 if exc.__cause__: 

171 cei.cause = repr(exc.__cause__) 

172 self.ctx.errors.append(cei) 

173 

174 def _result(self): 

175 return gws.ConfigResult( 

176 errors=self.ctx.errors, 

177 warnings=self.ctx.warnings, 

178 root=self.root, 

179 config=self.config, 

180 info=_info_string(self.root, self.tm1), 

181 ) 

182 

183 

184def configure( 

185 manifest_path='', 

186 config_path='', 

187 specs: Optional[gws.SpecRuntime] = None, 

188 raw_config: dict | gws.Data = None, 

189 fallback_config: dict | gws.Data = None, 

190 with_spec_cache=False, 

191) -> gws.ConfigResult: 

192 """Parse the configuration and create the root object. 

193 

194 Args: 

195 manifest_path: Path to the application manifest. 

196 config_path: Path to the configuration file. 

197 specs: Specs to use. If not given, specs are created from the manifest. 

198 raw_config: Configuration dict, used instead of the configuration file. 

199 fallback_config: Configuration used when the main configuration fails. 

200 with_spec_cache: Read and write the specs cache. 

201 

202 Returns: 

203 The configuration result. Its ``root`` is ``None`` if configuration failed. 

204 """ 

205 

206 ldr = Object( 

207 manifest_path, 

208 config_path, 

209 specs, 

210 raw_config, 

211 fallback_config, 

212 with_spec_cache, 

213 ) 

214 return ldr.configure() 

215 

216 

217def parse( 

218 manifest_path='', 

219 config_path='', 

220 specs: Optional[gws.SpecRuntime] = None, 

221) -> gws.ConfigResult: 

222 """Parse and validate the configuration without creating objects. 

223 

224 Args: 

225 manifest_path: Path to the application manifest. 

226 config_path: Path to the configuration file. 

227 specs: Specs to use. If not given, specs are created from the manifest. 

228 

229 Returns: 

230 The configuration result with the parsed config, errors and warnings. 

231 """ 

232 

233 ldr = Object( 

234 manifest_path, 

235 config_path, 

236 specs, 

237 ) 

238 return ldr.parse() 

239 

240 

241def initialize(specs: gws.SpecRuntime, config: gws.Config) -> gws.Root: 

242 """Create the root object and the application from a parsed configuration. 

243 

244 Args: 

245 specs: Specs runtime. 

246 config: Parsed application configuration. 

247 

248 Returns: 

249 The initialized, not yet activated root object. 

250 """ 

251 root = gws.create_root(specs) 

252 root.create_application(config) 

253 root.post_initialize() 

254 return root 

255 

256 

257def save(root: gws.Root, path=None) -> str: 

258 """Save the root object to a file, see ``gws.save_root``. 

259 

260 Args: 

261 root: Root object. 

262 path: File path. Defaults to ``config.pickle`` in the config directory. 

263 

264 Returns: 

265 The file path. 

266 

267 Raises: 

268 ``gws.ConfigurationError``: If the root cannot be saved. 

269 """ 

270 path = path or gws.c.ROOT_PICKLE_PATH 

271 gws.log.debug(f'writing config to {path!r}') 

272 try: 

273 return gws.save_root(root, path) 

274 except Exception as exc: 

275 raise gws.ConfigurationError('unable to save configuration') from exc 

276 

277 

278def load(path=None) -> gws.Root: 

279 """Load a saved root object, see ``gws.load_root``, and log the user, time and memory used. 

280 

281 Args: 

282 path: File path. Defaults to ``config.pickle`` in the config directory. 

283 

284 Returns: 

285 The root object. 

286 

287 Raises: 

288 ``gws.ConfigurationError``: If the root cannot be loaded or activated. 

289 """ 

290 ui = gws.lib.osx.user_info() 

291 path = path or gws.c.ROOT_PICKLE_PATH 

292 gws.log.info(f'loading config from {path!r}, user {ui["pw_name"]} ({ui["pw_uid"]}:{ui["pw_gid"]})') 

293 try: 

294 tm1 = _time_and_memory() 

295 root = gws.load_root(path) 

296 info = _info_string(root, tm1) 

297 gws.log.info(f'configuration loaded, {info}') 

298 return root 

299 except Exception as exc: 

300 raise gws.ConfigurationError('unable to load configuration') from exc 

301 

302 

303def real_config_path(config_path: str) -> str: 

304 """Find the configuration file. 

305 

306 Args: 

307 config_path: Comma-separated list of paths. Defaults to the ``GWS_CONFIG`` environment variable. 

308 If neither is given, the default paths are checked. 

309 

310 Returns: 

311 The first path that is an existing file, or an empty string. 

312 """ 

313 p = config_path or gws.env.GWS_CONFIG 

314 if p: 

315 for s in p.split(','): 

316 s = s.strip() 

317 if gws.u.is_file(s): 

318 return s 

319 return '' 

320 for p in _DEFAULT_CONFIG_PATHS: 

321 if gws.u.is_file(p): 

322 return p 

323 return '' 

324 

325 

326def real_manifest_path(manifest_path: str) -> str: 

327 """Find the application manifest. 

328 

329 Args: 

330 manifest_path: Manifest path. Defaults to the ``GWS_MANIFEST`` environment variable. 

331 If neither is given, the default path ``/data/MANIFEST.json`` is used if it exists. 

332 

333 Returns: 

334 The manifest path, or an empty string. 

335 

336 Raises: 

337 ``gws.Error``: If an explicit or ``GWS_MANIFEST`` path does not exist. 

338 """ 

339 p = manifest_path or gws.env.GWS_MANIFEST 

340 if p: 

341 if not gws.u.is_file(p): 

342 raise gws.Error(f'manifest not found: {p!r}') 

343 return p 

344 for p in _DEFAULT_MANIFEST_PATHS: 

345 if gws.u.is_file(p): 

346 return p 

347 return '' 

348 

349 

350def log_report(cr: gws.ConfigResult): 

351 """Log a summary of the configuration result and all errors and warnings. 

352 

353 Args: 

354 cr: Configuration result. 

355 """ 

356 err_cnt = len(cr.errors) if cr.errors else 0 

357 warn_cnt = len(cr.warnings) if cr.warnings else 0 

358 ln = '*' * 80 

359 

360 if err_cnt == 0 and warn_cnt == 0: 

361 gws.log.info(ln) 

362 gws.log.info(f'configured: {cr.info}') 

363 gws.log.info(ln) 

364 return 

365 

366 if err_cnt == 0: 

367 gws.log.warning(ln) 

368 gws.log.warning(f'configured with warnings: {warn_cnt}, {cr.info}') 

369 gws.log.warning(ln) 

370 else: 

371 gws.log.error(ln) 

372 gws.log.error(f'configured with errors: {err_cnt}, warnings: {warn_cnt}, {cr.info}') 

373 gws.log.error(ln) 

374 

375 # cr.errors.sort(key=lambda ce: ce.message) 

376 

377 for n, cei in enumerate(cr.errors, 1): 

378 gws.log.error(f'{_ERROR_PREFIX}: {n} of {err_cnt}') 

379 _log_info(cei, gws.log.error, _ERROR_PREFIX) 

380 gws.log.error(f'{_ERROR_PREFIX}: ') 

381 

382 for n, cei in enumerate(cr.warnings, 1): 

383 gws.log.warning(f'{_WARNING_PREFIX}: {n} of {warn_cnt}') 

384 _log_info(cei, gws.log.warning, _WARNING_PREFIX) 

385 gws.log.warning(f'{_WARNING_PREFIX}: ') 

386 

387 if err_cnt == 0: 

388 gws.log.warning(ln) 

389 else: 

390 gws.log.error(ln) 

391 

392 

393def _log_info(cei: gws.ConfigErrorInfo, log_fn, prefix): 

394 """Log the details of an error or warning.""" 

395 ls = [] 

396 ls.append(cei.message) 

397 tab = ' ' * 4 

398 

399 if cei.path: 

400 ls.append(f'PATH: {cei.path}') 

401 if cei.line: 

402 ls.append(f'LINE: {cei.line}') 

403 if cei.value: 

404 ls.append(f'VALUE: {cei.value}') 

405 if cei.cause: 

406 ls.append(f'CAUSE: {cei.cause}') 

407 if cei.stack: 

408 for loc in cei.stack: 

409 p = [ 

410 loc.objectType, 

411 repr(loc.objectName) if loc.objectName else None, 

412 f'uid={loc.objectUid}' if loc.objectUid else None, 

413 ] 

414 p = '<' + ' '.join(gws.u.compact(p)) + '>' 

415 if loc.propName: 

416 p = f'{loc.propName!r} {p}' 

417 ls.append(f'{tab}in {p}') 

418 if cei.contextLines: 

419 ls.extend(cei.contextLines) 

420 

421 for s in ls: 

422 log_fn(f'{prefix}: {s}') 

423 

424 

425def _time_and_memory(): 

426 return gws.u.stime(), gws.lib.osx.process_rss_size() 

427 

428 

429def _info_string(root, tm1): 

430 tm2 = _time_and_memory() 

431 return 'objects: {:d}, time: {:d}s., memory: {:.2f} MB'.format( 

432 root.object_count() if root else 0, 

433 tm2[0] - tm1[0], 

434 tm2[1] - tm1[1], 

435 )