Coverage for gws-app/gws/spec/reader.py: 48%

1"""Read and validate values according to spec types.""" 

2 

3import re 

4 

5import gws 

6import gws.lib.crs 

7import gws.lib.datetimex 

8import gws.lib.osx 

9import gws.lib.uom 

10 

11from . import core 

12 

13 

14class Reader: 

15 """Reads a raw value against a spec type and returns the parsed value. 

16 

17 Readers are selected first by type uid (built-in atoms and special types 

18 like ``gws.CrsName`` or ``gws.Duration``) and then by type kind. Objects 

19 are returned as ``gws.Data``. Some special types are converted, e.g. a 

20 ``gws.CrsName`` is read as an SRID and a ``gws.Duration`` as seconds. 

21 """ 

22 

23 atom = core.make_type({'c': core.c.ATOM}) 

24 """Placeholder type passed to readers of types that are not in the specs.""" 

25 

26 def __init__(self, runtime, path, options): 

27 """Create a reader. 

28 

29 Args: 

30 runtime: Spec runtime used to look up types. 

31 path: Path of the config file, used to resolve relative paths. 

32 options: A collection of ``gws.SpecReadOption`` values. 

33 """ 

34 

35 self.runtime = runtime 

36 self.path = path 

37 

38 options = set(options or []) 

39 

40 self.accept_extra_props = gws.SpecReadOption.acceptExtraProps in options 

41 self.case_insensitive = gws.SpecReadOption.caseInsensitive in options 

42 self.convert_values = gws.SpecReadOption.convertValues in options 

43 self.ignore_extra_props = gws.SpecReadOption.ignoreExtraProps in options 

44 self.allow_skip_required = gws.SpecReadOption.allowMissing in options 

45 self.verbose_errors = gws.SpecReadOption.verboseErrors in options 

46 

47 self.stack = None 

48 self.push = lambda _: ... 

49 self.pop = lambda: ... 

50 

51 def read(self, value, type_uid): 

52 """Read a value. 

53 

54 With the ``verboseErrors`` option, the reader tracks its position in 

55 the value and adds a ``gws.ConfigErrorInfo`` to the error. 

56 

57 Args: 

58 value: Raw value. 

59 type_uid: Type uid or extension name. 

60 

61 Returns: 

62 The parsed value. 

63 

64 Raises: 

65 ``core.ReadError``: If the value does not match the type. 

66 """ 

67 

68 if not self.verbose_errors: 

69 return self.read2(value, type_uid) 

70 

71 self.stack = [('', value, type_uid)] 

72 self.push = self.stack.append 

73 self.pop = self.stack.pop 

74 

75 try: 

76 return self.read2(value, type_uid) 

77 except core.ReadError as exc: 

78 raise self.add_config_error_info(exc) 

79 

80 def read2(self, value, type_uid): 

81 """Read a value without adding error information, used recursively by the readers. 

82 

83 Args: 

84 value: Raw value. 

85 type_uid: Type uid or extension name. 

86 

87 Returns: 

88 The parsed value. 

89 

90 Raises: 

91 ``core.ReadError``: If the value does not match the type or the type is unknown. 

92 """ 

93 

94 typ = self.runtime.get_type(type_uid) 

95 

96 if type_uid in _READERS: 

97 return _READERS[type_uid](self, value, typ or self.atom) 

98 

99 if not typ: 

100 raise core.ReadError(f'unknown type {type_uid!r}', value) 

101 

102 if typ.c not in _READERS: 

103 raise core.ReadError(f'unknown type category {typ.c!r}', value) 

104 

105 return _READERS[typ.c](self, value, typ) 

106 

107 def add_config_error_info(self, exc: Exception): 

108 """Add the value and the read stack to a read error. 

109 

110 Args: 

111 exc: Read error with the message and the value as arguments. 

112 

113 Returns: 

114 The same exception with a ``gws.ConfigErrorInfo`` as the third argument. 

115 """ 

116 

117 cei = gws.ConfigErrorInfo( 

118 value=_format_error_value(exc), 

119 path=self.path, 

120 stack=_prepare_error_stack(self.stack or []), 

121 ) 

122 exc.args = (exc.args[0], exc.args[1], cei) 

123 return exc 

124 

125 

126# atoms 

127 

128 

129def _read_any(r: Reader, val, typ: core.Type): 

130 return val 

131 

132 

133def _read_bool(r: Reader, val, typ: core.Type): 

134 if not r.convert_values: 

135 return _ensure(val, bool) 

136 try: 

137 return bool(val) 

138 except: 

139 raise core.ReadError('must be true or false', val) 

140 

141 

142def _read_bytes(r: Reader, val, typ: core.Type): 

143 try: 

144 if isinstance(val, str): 

145 return val.encode('utf8', errors='strict') 

146 return bytes(val) 

147 except: 

148 raise core.ReadError('must be a byte buffer', val) 

149 

150 

151def _read_float(r: Reader, val, typ: core.Type): 

152 if not r.convert_values: 

153 if isinstance(val, int): 

154 return float(val) 

155 return _ensure(val, float) 

156 try: 

157 return float(val) 

158 except: 

159 raise core.ReadError('must be a float', val) 

160 

161 

162def _read_int(r: Reader, val, typ: core.Type): 

163 if isinstance(val, bool): 

164 raise core.ReadError('must be an integer', val) 

165 if not r.convert_values: 

166 return _ensure(val, int) 

167 try: 

168 return int(val) 

169 except: 

170 raise core.ReadError('must be an integer', val) 

171 

172 

173def _read_str(r: Reader, val, typ: core.Type): 

174 if not r.convert_values: 

175 return _ensure(val, str) 

176 try: 

177 return _to_string(val) 

178 except: 

179 raise core.ReadError('must be a string', val) 

180 

181 

182# built-ins 

183 

184 

185def _read_raw_dict(r: Reader, val, typ: core.Type): 

186 return _ensure(val, dict) 

187 

188 

189def _read_dict(r: Reader, val, typ: core.Type): 

190 dct = {} 

191 for k, v in _ensure(val, dict).items(): 

192 dct[k] = r.read2(v, typ.tValue) 

193 return dct 

194 

195 

196def _read_raw_list(r: Reader, val, typ: core.Type): 

197 return _ensure(val, list) 

198 

199 

200def _read_list(r: Reader, val, typ: core.Type): 

201 lst = _read_any_list(r, val) 

202 res = [] 

203 for n, v in enumerate(lst): 

204 r.push((n, v, typ.tItem)) 

205 res.append(r.read2(v, typ.tItem)) 

206 r.pop() 

207 return res 

208 

209 

210def _read_set(r: Reader, val, typ: core.Type): 

211 lst = _read_list(r, val, typ) 

212 return set(lst) 

213 

214 

215def _read_tuple(r: Reader, val, typ: core.Type): 

216 lst = _read_any_list(r, val) 

217 

218 if len(lst) != len(typ.tItems): 

219 raise core.ReadError(f'expected: {_comma(typ.tItems)}', val) 

220 

221 res = [] 

222 for n, v in enumerate(lst): 

223 r.push((n, v, typ.tItems[n])) 

224 res.append(r.read2(v, typ.tItems[n])) 

225 r.pop() 

226 return res 

227 

228 

229def _read_any_list(r, val): 

230 if r.convert_values and isinstance(val, str): 

231 val = val.strip() 

232 val = [v.strip() for v in val.split(',')] if val else [] 

233 return _ensure(val, list) 

234 

235 

236def _read_literal(r: Reader, val, typ: core.Type): 

237 s = _read_any(r, val, typ) 

238 if s not in typ.literalValues: 

239 raise core.ReadError(f'invalid value: {s!r}, expected: {_comma(typ.literalValues)}', val) 

240 return s 

241 

242 

243def _read_optional(r: Reader, val, typ: core.Type): 

244 if val is None: 

245 return val 

246 return r.read2(val, typ.tTarget) 

247 

248 

249def _read_union(r: Reader, val, typ: core.Type): 

250 # @TODO no untyped unions yet 

251 raise core.ReadError('unions are not supported yet', val) 

252 

253 

254# our types 

255 

256 

257def _read_type(r: Reader, val, typ: core.Type): 

258 return r.read2(val, typ.tTarget) 

259 

260 

261def _read_enum(r: Reader, val, typ: core.Type): 

262 # NB: our Enums accept both names (for configs) and values (for api calls) 

263 # this prevents silly things like Enum{foo=bar bar=123} but we don't care 

264 # 

265 # the comparison is also case-insensitive 

266 # 

267 # this reader returns a value, it's up to the caller to convert it to the actual enum 

268 

269 def _lower(s): 

270 return s.lower() if isinstance(s, str) else s 

271 

272 lv = _lower(val) 

273 

274 for k, v in typ.enumValues.items(): 

275 if lv == _lower(k) or lv == _lower(v): 

276 return v 

277 raise core.ReadError(f'invalid value: {val!r}, expected: {_comma(typ.enumValues)}', val) 

278 

279 

280def _read_object(r: Reader, val, typ: core.Type): 

281 val = _ensure(val, dict) 

282 

283 if r.case_insensitive: 

284 val = {k.lower(): v for k, v in val.items()} 

285 else: 

286 val = dict(val) 

287 

288 res = {} 

289 

290 for prop_name, prop_type_uid in typ.tProperties.items(): 

291 prop_val = val.pop(prop_name.lower() if r.case_insensitive else prop_name, None) 

292 r.push((prop_name, prop_val, prop_type_uid)) 

293 res[prop_name] = r.read2(prop_val, prop_type_uid) 

294 r.pop() 

295 

296 unknown = [] 

297 

298 for k in val: 

299 if k not in typ.tProperties: 

300 # accept 'uid' for all objects 

301 if k == 'uid': 

302 res[k] = val[k] 

303 elif r.accept_extra_props: 

304 res[k] = val[k] 

305 elif r.ignore_extra_props: 

306 continue 

307 else: 

308 unknown.append(k) 

309 

310 if unknown: 

311 raise core.ReadError(f'unknown keys: {_comma(unknown)}, expected: {_comma(typ.tProperties)} for {typ.uid!r}', val) 

312 

313 return gws.Data(res) 

314 

315 

316def _read_property(r: Reader, val, typ: core.Type): 

317 if val is not None: 

318 return r.read2(val, typ.tValue) 

319 

320 if not typ.hasDefault: 

321 if r.allow_skip_required: 

322 return None 

323 raise core.ReadError(f'required property missing: {typ.ident!r} for {typ.tOwner!r}', None) 

324 

325 if typ.defaultValue is None: 

326 return None 

327 

328 # the default, if given, must match the type 

329 # NB, for Data objects, default={} will create an object with defaults 

330 return r.read2(typ.defaultValue, typ.tValue) 

331 

332 

333def _read_variant(r: Reader, val, typ: core.Type): 

334 val = _ensure(val, dict) 

335 if r.case_insensitive: 

336 val = {k.lower(): v for k, v in val.items()} 

337 

338 type_name = val.get(core.v.VARIANT_TAG, core.v.DEFAULT_VARIANT_TAG) 

339 target_type_uid = typ.tMembers.get(type_name) 

340 if not target_type_uid: 

341 raise core.ReadError(f'illegal type: {type_name!r}, expected: {_comma(typ.tMembers)}', val) 

342 return r.read2(val, target_type_uid) 

343 

344 

345# custom types 

346 

347 

348def _read_acl_str(r: Reader, val, typ: core.Type): 

349 try: 

350 return gws.u.parse_acl(val) 

351 except ValueError: 

352 raise core.ReadError(f'invalid ACL', val) 

353 

354 

355def _read_color(r: Reader, val, typ: core.Type): 

356 # @TODO: parse color values 

357 return _read_str(r, val, typ) 

358 

359 

360def _read_crs(r: Reader, val, typ: core.Type): 

361 crs = gws.lib.crs.get(val) 

362 if not crs: 

363 raise core.ReadError(f'invalid crs: {val!r}', val) 

364 return crs.srid 

365 

366 

367def _read_date(r: Reader, val, typ: core.Type): 

368 try: 

369 return gws.lib.datetimex.from_string(str(val)) 

370 except ValueError: 

371 raise core.ReadError(f'invalid date: {val!r}', val) 

372 

373 

374def _read_datetime(r: Reader, val, typ: core.Type): 

375 try: 

376 return gws.lib.datetimex.from_iso_string(str(val)) 

377 except ValueError: 

378 raise core.ReadError(f'invalid date: {val!r}', val) 

379 

380 

381def _read_dirpath(r: Reader, val, typ: core.Type): 

382 path = gws.lib.osx.abs_path(val, r.path) 

383 if not gws.u.is_dir(path): 

384 raise core.ReadError(f'directory not found: {path!r}', val) 

385 return path 

386 

387 

388def _read_duration(r: Reader, val, typ: core.Type): 

389 try: 

390 return gws.lib.datetimex.parse_duration(val) 

391 except ValueError: 

392 raise core.ReadError(f'invalid duration: {val!r}', val) 

393 

394 

395def _read_filepath(r: Reader, val, typ: core.Type): 

396 path = gws.lib.osx.abs_path(val, r.path) 

397 if not gws.lib.osx.is_abs_path(val): 

398 gws.log.warning(f'relative path, assuming {path!r} for {val!r}') 

399 if not gws.u.is_file(path): 

400 raise core.ReadError(f'file not found: {path!r}', val) 

401 return path 

402 

403 

404def _read_formatstr(r: Reader, val, typ: core.Type): 

405 # @TODO validate 

406 return _read_str(r, val, typ) 

407 

408 

409def _read_metadata(r: Reader, val, typ: core.Type): 

410 rr = r.allow_skip_required 

411 r.allow_skip_required = True 

412 res = gws.u.compact(_read_object(r, val, typ)) 

413 r.allow_skip_required = rr 

414 return res 

415 

416 

417def _read_uom_value(r: Reader, val, typ: core.Type): 

418 try: 

419 return gws.lib.uom.parse(val) 

420 except ValueError as e: 

421 raise core.ReadError(f'invalid value: {val!r}: {e!r}', val) 

422 

423 

424def _read_uom_point(r: Reader, val, typ: core.Type): 

425 try: 

426 return gws.lib.uom.parse_point(val) 

427 except ValueError as e: 

428 raise core.ReadError(f'invalid value: {val!r}: {e!r}', val) 

429 

430 

431def _read_uom_extent(r: Reader, val, typ: core.Type): 

432 try: 

433 return gws.lib.uom.parse_extent(val) 

434 except ValueError as e: 

435 raise core.ReadError(f'invalid value: {val!r}: {e!r}', val) 

436 

437 

438def _read_regex(r: Reader, val, typ: core.Type): 

439 try: 

440 re.compile(val) 

441 return val 

442 except re.error as e: 

443 raise core.ReadError(f'invalid regular expression: {val!r}: {e!r}', val) 

444 

445 

446def _read_url(r: Reader, val, typ: core.Type): 

447 u = _read_str(r, val, typ) 

448 if u.startswith(('http://', 'https://')): 

449 return u 

450 raise core.ReadError(f'invalid url: {val!r}', val) 

451 

452 

453# utils 

454 

455 

456def _ensure(val, cls): 

457 if isinstance(val, cls): 

458 return val 

459 if cls == list and isinstance(val, tuple): 

460 return list(val) 

461 if cls == dict and gws.u.is_data_object(val): 

462 return vars(val) 

463 raise core.ReadError(f'wrong type: {_classname(type(val))!r}, expected: {_classname(cls)!r}', val) 

464 

465 

466def _to_string(x): 

467 if isinstance(x, str): 

468 return x 

469 if isinstance(x, (bytes, bytearray)): 

470 return x.decode('utf8') 

471 raise ValueError() 

472 

473 

474def _classname(cls): 

475 try: 

476 return cls.__name__ 

477 except: 

478 return str(cls) 

479 

480 

481def _comma(ls): 

482 return repr(', '.join(sorted(str(x) for x in ls))) 

483 

484 

485## 

486 

487 

488def _format_error_value(exc): 

489 try: 

490 val = exc.args[1] 

491 except Exception: 

492 return '' 

493 

494 s = repr(val) 

495 if len(s) > 600: 

496 s = s[:600] + '...' 

497 return s 

498 

499 

500def _prepare_error_stack(stack): 

501 ls = [] 

502 

503 for name, value, type_uid in reversed(stack): 

504 obj_name = gws.u.get(value, 'name') or gws.u.get(value, 'title') 

505 ls.append( 

506 gws.ConfigLocation( 

507 objectUid=gws.u.get(value, 'uid'), 

508 objectType=type_uid or gws.u.get(value, 'type'), 

509 objectName=obj_name if isinstance(obj_name, str) else '', 

510 propName=str(name) if name is not None else '', 

511 ) 

512 ) 

513 

514 return ls 

515 

516 

517# 

518 

519_READERS = { 

520 'any': _read_any, 

521 'bool': _read_bool, 

522 'bytes': _read_bytes, 

523 'float': _read_float, 

524 'int': _read_int, 

525 'str': _read_str, 

526 'list': _read_raw_list, 

527 'dict': _read_raw_dict, 

528 core.c.CLASS: _read_object, 

529 core.c.DICT: _read_dict, 

530 core.c.ENUM: _read_enum, 

531 core.c.LIST: _read_list, 

532 core.c.LITERAL: _read_literal, 

533 core.c.OPTIONAL: _read_optional, 

534 core.c.PROPERTY: _read_property, 

535 core.c.SET: _read_set, 

536 core.c.TUPLE: _read_tuple, 

537 core.c.TYPE: _read_type, 

538 core.c.UNION: _read_union, 

539 core.c.VARIANT: _read_variant, 

540 'gws.AclStr': _read_acl_str, 

541 'gws.Color': _read_color, 

542 'gws.CrsName': _read_crs, 

543 'gws.DateStr': _read_date, 

544 'gws.DateTimeStr': _read_datetime, 

545 'gws.DirPath': _read_dirpath, 

546 'gws.Duration': _read_duration, 

547 'gws.FilePath': _read_filepath, 

548 'gws.FormatStr': _read_formatstr, 

549 'gws.UomValueStr': _read_uom_value, 

550 'gws.UomPointStr': _read_uom_point, 

551 'gws.UomSizeStr': _read_uom_point, 

552 'gws.UomExtentStr': _read_uom_extent, 

553 'gws.Metadata': _read_metadata, 

554 'gws.Regex': _read_regex, 

555 'gws.Url': _read_url, 

556}