Coverage for gws-app/gws/config/parser.py: 17%

1"""Configuration parser: read configuration files and validate them against the specs."""

3from typing import Optional, cast

5import os

6import yaml

8import gws

9import gws.lib.jsonx

10import gws.lib.osx

11import gws.lib.datetimex

12import gws.lib.dynimport

13import gws.lib.vendor.jump

14import gws.lib.vendor.slon

15import gws.spec.runtime

17CONFIG_PATH_PATTERN = r'\.(py|json|yaml|yml|cx)$'

20def parse_from_path(path: str, as_type: str, ctx: gws.ConfigContext) -> Optional[gws.Config]:

21 """Read a configuration file and validate it as a given type.

23 Args:

24 path: Path to the configuration file.

25 as_type: Type of the configuration, e.g. ``gws.base.application.core.Config``.

26 ctx: Configuration context. Errors are added to it.

28 Returns:

29 The parsed configuration, or ``None`` if there were errors.

30 """

32 pp = _Parser(ctx)

33 val = pp.read_from_path(path)

34 d = pp.ensure_dict(val, path)

35 return pp.parse_dict(d, path, as_type) if d else None

38def parse_dict(dct: dict | gws.Data, path: str, as_type: str, ctx: gws.ConfigContext) -> Optional[gws.Config]:

39 """Validate a configuration dict as a given type.

41 Args:

42 dct: Configuration dict or ``gws.Data`` object.

43 path: Path to the configuration file, for error reporting.

44 as_type: Type of the configuration, e.g. ``gws.ext.config.layer``.

45 ctx: Configuration context. Errors are added to it.

47 Returns:

48 The parsed configuration, or ``None`` if there were errors.

49 """

51 pp = _Parser(ctx)

52 d = pp.ensure_dict(dct, path)

53 return pp.parse_dict(d, path, as_type) if d else None

56def parse_app_from_path(path: str, ctx: gws.ConfigContext) -> Optional[gws.Config]:

57 """Read and validate the application configuration file.

59 Sets the server time zone, then parses the application config and all

60 projects (inline ``projects``, ``projectPaths`` and files in ``projectDirs``).

62 Args:

63 path: Path to the application configuration file.

64 ctx: Configuration context. Errors are added to it.

66 Returns:

67 The parsed application configuration, or ``None`` if it could not be parsed.

68 """

70 pp = _Parser(ctx)

71 val = pp.read_from_path(path)

72 d = pp.ensure_dict(val, path)

73 return _parse_app_dict(d, path, pp) if d else None

76def parse_app_dict(dct: dict | gws.Data, path: str, ctx: gws.ConfigContext) -> Optional[gws.Config]:

77 """Validate an application configuration dict.

79 Works like ``parse_app_from_path``.

81 Args:

82 dct: Application configuration dict or ``gws.Data`` object.

83 path: Path to the configuration file, for error reporting.

84 ctx: Configuration context. Errors are added to it.

86 Returns:

87 The parsed application configuration, or ``None`` if it could not be parsed.

88 """

90 pp = _Parser(ctx)

91 d = pp.ensure_dict(dct, path)

92 return _parse_app_dict(d, path, pp) if d else None

95def read_from_path(path: str, ctx: gws.ConfigContext) -> Optional[dict]:

96 """Read a configuration file into a dict, without validating it.

98 The format is determined by the file extension.

100 Args:

101 path: Path to the configuration file.

102 ctx: Configuration context. Errors are added to it.

103

104 Returns:

105 The configuration dict, or ``None`` if the file could not be read or does not contain a dict.

106 """

107 pp = _Parser(ctx)

108 val = pp.read_from_path(path)

109 d = pp.ensure_dict(val, path)

110 return d

113##

116def _parse_app_dict(dct: dict, path, pp: '_Parser'):

117 """Set the time zone, parse the application config and collect all project configs."""

118 dct = gws.u.to_dict(dct)

119 if not isinstance(dct, dict):

120 _register_error(pp.ctx, f'app config must be a dict', path=path)

121 return

122

123 # the timezone must be set before everything else

124 tz = dct.get('server', {}).get('timeZone', '')

125 if tz:

126 if gws.lib.datetimex.is_valid_time_zone(tz):

127 gws.lib.datetimex.set_local_time_zone(tz)

128 else:

129 _register_error(pp.ctx, f'invalid time zone: {tz!r}', path=path)

130 gws.log.info(f'local time zone="{gws.lib.datetimex.time_zone()}"')

131

132 # remove 'projects' from the config, parse them later on

133 inline_projects = dct.pop('projects', [])

134

135 app_cfg = pp.parse_dict(dct, path, as_type='gws.base.application.core.Config')

136 if not app_cfg:

137 return

138

139 projects = []

140 for dcts in inline_projects:

141 projects.extend(_parse_projects(dcts, path, pp))

142

143 project_paths = list(app_cfg.get('projectPaths') or [])

144 project_dirs = list(app_cfg.get('projectDirs') or [])

145

146 all_project_paths = list(project_paths)

147 for dirname in project_dirs:

148 all_project_paths.extend(gws.lib.osx.find_files(dirname, CONFIG_PATH_PATTERN, deep=True))

149

150 for pth in sorted(set(all_project_paths)):

151 projects.extend(_parse_projects_from_path(pth, pp))

152

153 app_cfg.set('projectPaths', project_paths)

154 app_cfg.set('projectDirs', project_dirs)

155 app_cfg.set('projects', projects)

156

157 _save_debug(app_cfg, path, '.parsed.json')

158 return app_cfg

159

160

161def _parse_projects_from_path(path, pp: '_Parser'):

162 cfg_list = pp.read_from_path(path)

163 if not cfg_list:

164 return []

165 return _parse_projects(cfg_list, path, pp)

166

167

168def _parse_projects(cfg_list, path, pp: '_Parser'):

169 """Parse a project config or a (nested) list of them."""

170 ps = []

171

172 for c in _as_flat_list(cfg_list):

173 d = pp.ensure_dict(c, path)

174 if not d:

175 continue

176 prj_cfg = pp.parse_dict(d, path, 'gws.ext.config.project')

177 if prj_cfg:

178 ps.append(prj_cfg)

180 return ps

183##

186class _Parser:

187 """Reads configuration files and validates dicts, recording errors in the context."""

188

189 def __init__(self, ctx: gws.ConfigContext):

190 """Initialize the context and enable verbose errors.

191

192 Args:

193 ctx: Configuration context.

194 """

195 self.ctx = ctx

196 self.ctx.errors = ctx.errors or []

197 self.ctx.paths = ctx.paths or set()

198 self.ctx.readOptions = ctx.readOptions or set()

199 self.ctx.readOptions.add(gws.SpecReadOption.verboseErrors)

200

201 def ensure_dict(self, val, path):

202 """Convert a value to a plain dict.

203

204 Args:

205 val: Value to convert, a dict or a ``gws.Data`` object.

206 path: Path to the configuration file, for error reporting.

207

208 Returns:

209 A plain dict, or ``None`` if the value is ``None`` or not a dict.

210 """

211 if val is None:

212 return

213 d = _to_plain(val)

214 if not isinstance(d, dict):

215 _register_error(self.ctx, f'unsupported configuration type {type(val)!r}', path=path)

216 return

217 return d

218

219 def parse_dict(self, dct: dict, path: str, as_type: str) -> Optional[gws.Config]:

220 """Validate a dict against the specs.

221

222 Args:

223 dct: Configuration dict.

224 path: Path to the configuration file, recorded in the context.

225 as_type: Type of the configuration.

226

227 Returns:

228 The parsed configuration, or ``None`` if there were errors.

229 """

230 if not isinstance(dct, dict):

231 _register_error(self.ctx, 'unsupported configuration', path=path)

232 return

233 if path:

234 _register_path(self.ctx, path)

235 try:

236 cfg = self.ctx.specs.read(

237 dct,

238 as_type,

239 path=path,

240 options=self.ctx.readOptions,

241 )

242 return cast(gws.Config, cfg)

243 except gws.spec.runtime.ReadError as exc:

244 message, _, cei = exc.args

245 _register_error(self.ctx, f'parse error: {message}', cei=cei)

246

247 def read_from_path(self, path: str):

248 """Read a configuration file and convert the result to plain values.

249

250 Args:

251 path: Path to the configuration file.

252

253 Returns:

254 The configuration value, or ``None`` if the file could not be read.

255 """

256 if not os.path.isfile(path):

257 _register_error(self.ctx, f'file not found', path=path)

258 return

259

260 _register_path(self.ctx, path)

261 r = self.read2(path)

262

263 if r:

264 r = _to_plain(r)

265 _save_debug(r, path, '.src.json')

266 return r

267

268 def read2(self, path: str):

269 """Read a configuration file using the reader for its extension.

270

271 Args:

272 path: Path to the configuration file.

273

274 Returns:

275 The configuration value, or ``None`` on errors.

276 """

277 if path.endswith('.py'):

278 return self.read_py(path)

279 if path.endswith('.json'):

280 return self.read_json(path)

281 if path.endswith(('.yml', '.yaml')):

282 return self.read_yaml(path)

283 if path.endswith('.cx'):

284 return self.read_cx(path)

285

286 _register_error(self.ctx, 'unsupported configuration', path=path)

287

288 def read_py(self, path: str):

289 """Load a Python configuration file and call its ``main`` function with the context.

290

291 Args:

292 path: Path to the configuration file.

293

294 Returns:

295 The value returned by ``main``, or ``None`` on errors.

296 """

297 try:

298 fn = gws.lib.dynimport.load_file(path).get('main')

299 if not fn:

300 _register_error(self.ctx, f'no "main" function found', path=path)

301 return

302 return fn(self.ctx)

303 except Exception as exc:

304 gws.log.exception()

305 _register_error(self.ctx, f'python error: {exc}', path=path)

306

307 def read_json(self, path: str):

308 """Read a JSON configuration file.

309

310 Args:

311 path: Path to the configuration file.

312

313 Returns:

314 The decoded value, or ``None`` on errors.

315 """

316 try:

317 return gws.lib.jsonx.from_path(path)

318 except Exception as exc:

319 _register_error(self.ctx, f'json error: {exc}', path=path)

320

321 def read_yaml(self, path: str):

322 """Read a YAML configuration file.

323

324 Args:

325 path: Path to the configuration file.

326

327 Returns:

328 The decoded value, or ``None`` on errors.

329 """

330 try:

331 with open(path, encoding='utf8') as fp:

332 return yaml.safe_load(fp)

333 except Exception as exc:

334 _register_error(self.ctx, f'yaml error: {exc}', path=path)

335

336 def read_cx(self, path: str):

337 """Read a ``.cx`` configuration file.

338

339 The file is a ``jump`` template that renders to SLON. Included files are

340 recorded in the context. Template variables are ``true``, ``false``,

341 ``ctx`` and ``gws``.

342

343 Args:

344 path: Path to the configuration file.

345

346 Returns:

347 The decoded value, or ``None`` on errors.

348 """

349 err_cnt = [0]

350

351 def _error_handler(exc, path, line, env):

352 _register_syntax_error(self.ctx, path, gws.u.read_file(path), message=repr(exc), line=line)

353 err_cnt[0] += 1

354 return True

355

356 def _loader(cur_path, load_path):

357 if not os.path.isabs(load_path):

358 load_path = os.path.abspath(os.path.dirname(cur_path) + '/' + load_path)

359 _register_path(self.ctx, load_path)

360 return gws.u.read_file(load_path), load_path

361

362 try:

363 tpl = gws.lib.vendor.jump.compile_path(path, loader=_loader)

364 except gws.lib.vendor.jump.CompileError as exc:

365 _register_syntax_error(self.ctx, path, gws.u.read_file(exc.path), message=exc.message, line=exc.line)

366 return

367

368 args = {

369 'true': True,

370 'false': False,

371 'ctx': self.ctx,

372 'gws': gws,

373 }

374

375 slon = gws.lib.vendor.jump.call(tpl, args, error=_error_handler)

376 if err_cnt[0] > 0:

377 return

378

379 _save_debug(slon, path, '.src.slon')

380

381 try:

382 return gws.lib.vendor.slon.loads(slon, as_object=True)

383 except gws.lib.vendor.slon.SlonError as exc:

384 _register_syntax_error(self.ctx, path, slon, message=exc.args[0], line=exc.args[2])

387##

390def _register_path(ctx, path):

391 ctx.paths.add(path)

392

393

394def _register_error(ctx: gws.ConfigContext, message: str, **kwargs):

395 cei = kwargs.pop('cei', None) or gws.ConfigErrorInfo()

396 cei.message = message

397 cei.update(kwargs)

398 ctx.errors.append(cei)

399

400

401def _register_warning(ctx: gws.ConfigContext, message: str, **kwargs):

402 cei = kwargs.pop('cei', None) or gws.ConfigErrorInfo()

403 cei.message = message

404 cei.update(kwargs)

405 loc = f' in {cei.path!r}' if cei.path else ''

406 gws.log.warning(f'CONFIGURATION WARNING: {message}{loc}')

407 ctx.warnings.append(cei)

408

409

410def _register_syntax_error(ctx, path, src, message, line, context=10, cause=None):

411 """Add a syntax error with the surrounding source lines to the context."""

412 cei = gws.ConfigErrorInfo(

413 path=path,

414 line=line,

415 message=f'syntax error: {message}',

416 contextLines=[],

417 cause=cause,

418 )

419

420 for n, ln in enumerate(src.splitlines(), 1):

421 if n < line - context:

422 continue

423 if n > line + context:

424 break

425 ln = f'{n}: {ln}'

426 if n == line:

427 ln = f'>>> {ln}'

428 cei.contextLines.append(ln)

429

430 ctx.errors.append(cei)

431

432

433def _save_debug(src, src_path, ext):

434 """Write an intermediate parsing result to the config directory, for debugging."""

435 if ext.endswith('.json') and not isinstance(src, str):

436 src = gws.lib.jsonx.to_pretty_string(src)

437 path = gws.u.write_file(f'{gws.c.CONFIG_DIR}/{gws.u.to_uid(src_path)}{ext}', src)

438 return f'saved {path!r}'

439

440

441def _as_flat_list(ls):

442 if not isinstance(ls, (list, tuple)):

443 yield ls

444 else:

445 for x in ls:

446 yield from _as_flat_list(x)

447

448

449def _to_plain(val):

450 """Convert ``gws.Data`` objects to dicts recursively, values of keys starting with ``_`` are left as is."""

451 if isinstance(val, (list, tuple)):

452 return [_to_plain(x) for x in val]

453 if isinstance(val, gws.Data):

454 val = vars(val)

455 if isinstance(val, dict):

456 return {k: v if k.startswith('_') else _to_plain(v) for k, v in val.items()}

457 return val