Coverage for gws-app/gws/spec/reader.py: 41%

1"""Read and validate values according to spec types.""" 

2 

3import re 

4 

5import gws 

6import gws.lib.crs 

7import gws.lib.datetimex 

8import gws.lib.osx 

9import gws.lib.uom 

10 

11from . import core 

12 

13 

14class Reader: 

15 atom = core.make_type({'c': core.c.ATOM}) 

16 

17 def __init__(self, runtime, path, options): 

18 self.runtime = runtime 

19 self.path = path 

20 

21 options = set(options or []) 

22 

23 self.accept_extra_props = gws.SpecReadOption.acceptExtraProps in options 

24 self.case_insensitive = gws.SpecReadOption.caseInsensitive in options 

25 self.convert_values = gws.SpecReadOption.convertValues in options 

26 self.ignore_extra_props = gws.SpecReadOption.ignoreExtraProps in options 

27 self.allow_skip_required = gws.SpecReadOption.allowMissing in options 

28 self.verbose_errors = gws.SpecReadOption.verboseErrors in options 

29 

30 self.stack = None 

31 self.push = lambda _: ... 

32 self.pop = lambda: ... 

33 

34 def read(self, value, type_uid): 

35 if not self.verbose_errors: 

36 return self.read2(value, type_uid) 

37 

38 self.stack = [('', value, type_uid)] 

39 self.push = self.stack.append 

40 self.pop = self.stack.pop 

41 

42 try: 

43 return self.read2(value, type_uid) 

44 except core.ReadError as exc: 

45 raise self.add_error_details(exc) 

46 

47 def read2(self, value, type_uid): 

48 typ = self.runtime.get_type(type_uid) 

49 

50 if type_uid in _READERS: 

51 return _READERS[type_uid](self, value, typ or self.atom) 

52 

53 if not typ: 

54 raise core.ReadError(f'unknown type {type_uid!r}', value) 

55 

56 if typ.c not in _READERS: 

57 raise core.ReadError(f'unknown type category {typ.c!r}', value) 

58 

59 return _READERS[typ.c](self, value, typ) 

60 

61 def add_error_details(self, exc: Exception): 

62 details = { 

63 'formatted_value': _format_error_value(exc), 

64 'path': self.path, 

65 'formatted_stack': _format_error_stack(self.stack or []), 

66 } 

67 exc.args = (exc.args[0], exc.args[1], details) 

68 return exc 

69 

70 

71# atoms 

72 

73 

74def _read_any(r: Reader, val, typ: core.Type): 

75 return val 

76 

77 

78def _read_bool(r: Reader, val, typ: core.Type): 

79 if not r.convert_values: 

80 return _ensure(val, bool) 

81 try: 

82 return bool(val) 

83 except: 

84 raise core.ReadError('must be true or false', val) 

85 

86 

87def _read_bytes(r: Reader, val, typ: core.Type): 

88 try: 

89 if isinstance(val, str): 

90 return val.encode('utf8', errors='strict') 

91 return bytes(val) 

92 except: 

93 raise core.ReadError('must be a byte buffer', val) 

94 

95 

96def _read_float(r: Reader, val, typ: core.Type): 

97 if not r.convert_values: 

98 if isinstance(val, int): 

99 return float(val) 

100 return _ensure(val, float) 

101 try: 

102 return float(val) 

103 except: 

104 raise core.ReadError('must be a float', val) 

105 

106 

107def _read_int(r: Reader, val, typ: core.Type): 

108 if isinstance(val, bool): 

109 raise core.ReadError('must be an integer', val) 

110 if not r.convert_values: 

111 return _ensure(val, int) 

112 try: 

113 return int(val) 

114 except: 

115 raise core.ReadError('must be an integer', val) 

116 

117 

118def _read_str(r: Reader, val, typ: core.Type): 

119 if not r.convert_values: 

120 return _ensure(val, str) 

121 try: 

122 return _to_string(val) 

123 except: 

124 raise core.ReadError('must be a string', val) 

125 

126 

127# built-ins 

128 

129 

130def _read_raw_dict(r: Reader, val, typ: core.Type): 

131 return _ensure(val, dict) 

132 

133 

134def _read_dict(r: Reader, val, typ: core.Type): 

135 dct = {} 

136 for k, v in _ensure(val, dict).items(): 

137 dct[k] = r.read2(v, typ.tValue) 

138 return dct 

139 

140 

141def _read_raw_list(r: Reader, val, typ: core.Type): 

142 return _ensure(val, list) 

143 

144 

145def _read_list(r: Reader, val, typ: core.Type): 

146 lst = _read_any_list(r, val) 

147 res = [] 

148 for n, v in enumerate(lst): 

149 r.push((n, v, typ.tItem)) 

150 res.append(r.read2(v, typ.tItem)) 

151 r.pop() 

152 return res 

153 

154 

155def _read_set(r: Reader, val, typ: core.Type): 

156 lst = _read_list(r, val, typ) 

157 return set(lst) 

158 

159 

160def _read_tuple(r: Reader, val, typ: core.Type): 

161 lst = _read_any_list(r, val) 

162 

163 if len(lst) != len(typ.tItems): 

164 raise core.ReadError(f'expected: {_comma(typ.tItems)}', val) 

165 

166 res = [] 

167 for n, v in enumerate(lst): 

168 r.push((n, v, typ.tItems[n])) 

169 res.append(r.read2(v, typ.tItems[n])) 

170 r.pop() 

171 return res 

172 

173 

174def _read_any_list(r, val): 

175 if r.convert_values and isinstance(val, str): 

176 val = val.strip() 

177 val = [v.strip() for v in val.split(',')] if val else [] 

178 return _ensure(val, list) 

179 

180 

181def _read_literal(r: Reader, val, typ: core.Type): 

182 s = _read_any(r, val, typ) 

183 if s not in typ.literalValues: 

184 raise core.ReadError(f'invalid value: {s!r}, expected: {_comma(typ.literalValues)}', val) 

185 return s 

186 

187 

188def _read_optional(r: Reader, val, typ: core.Type): 

189 if val is None: 

190 return val 

191 return r.read2(val, typ.tTarget) 

192 

193 

194def _read_union(r: Reader, val, typ: core.Type): 

195 # @TODO no untyped unions yet 

196 raise core.ReadError('unions are not supported yet', val) 

197 

198 

199# our types 

200 

201 

202def _read_type(r: Reader, val, typ: core.Type): 

203 return r.read2(val, typ.tTarget) 

204 

205 

206def _read_enum(r: Reader, val, typ: core.Type): 

207 # NB: our Enums accept both names (for configs) and values (for api calls) 

208 # this prevents silly things like Enum{foo=bar bar=123} but we don't care 

209 # 

210 # the comparison is also case-insensitive 

211 # 

212 # this reader returns a value, it's up to the caller to convert it to the actual enum 

213 

214 def _lower(s): 

215 return s.lower() if isinstance(s, str) else s 

216 

217 lv = _lower(val) 

218 

219 for k, v in typ.enumValues.items(): 

220 if lv == _lower(k) or lv == _lower(v): 

221 return v 

222 raise core.ReadError(f'invalid value: {val!r}, expected: {_comma(typ.enumValues)}', val) 

223 

224 

225def _read_object(r: Reader, val, typ: core.Type): 

226 val = _ensure(val, dict) 

227 

228 if r.case_insensitive: 

229 val = {k.lower(): v for k, v in val.items()} 

230 else: 

231 val = dict(val) 

232 

233 res = {} 

234 

235 for prop_name, prop_type_uid in typ.tProperties.items(): 

236 prop_val = val.pop(prop_name.lower() if r.case_insensitive else prop_name, None) 

237 r.push((prop_name, prop_val, prop_type_uid)) 

238 res[prop_name] = r.read2(prop_val, prop_type_uid) 

239 r.pop() 

240 

241 unknown = [] 

242 

243 for k in val: 

244 if k not in typ.tProperties: 

245 if r.accept_extra_props: 

246 res[k] = val[k] 

247 elif r.ignore_extra_props: 

248 continue 

249 else: 

250 unknown.append(k) 

251 

252 if unknown: 

253 raise core.ReadError(f'unknown keys: {_comma(unknown)}, expected: {_comma(typ.tProperties)} for {typ.uid!r}', val) 

254 

255 return gws.Data(res) 

256 

257 

258def _read_property(r: Reader, val, typ: core.Type): 

259 if val is not None: 

260 return r.read2(val, typ.tValue) 

261 

262 if not typ.hasDefault: 

263 if r.allow_skip_required: 

264 return None 

265 raise core.ReadError(f'required property missing: {typ.ident!r} for {typ.tOwner!r}', None) 

266 

267 if typ.defaultValue is None: 

268 return None 

269 

270 # the default, if given, must match the type 

271 # NB, for Data objects, default={} will create an object with defaults 

272 return r.read2(typ.defaultValue, typ.tValue) 

273 

274 

275def _read_variant(r: Reader, val, typ: core.Type): 

276 val = _ensure(val, dict) 

277 if r.case_insensitive: 

278 val = {k.lower(): v for k, v in val.items()} 

279 

280 type_name = val.get(core.v.VARIANT_TAG, core.v.DEFAULT_VARIANT_TAG) 

281 target_type_uid = typ.tMembers.get(type_name) 

282 if not target_type_uid: 

283 raise core.ReadError(f'illegal type: {type_name!r}, expected: {_comma(typ.tMembers)}', val) 

284 return r.read2(val, target_type_uid) 

285 

286 

287# custom types 

288 

289 

290def _read_acl_str(r: Reader, val, typ: core.Type): 

291 try: 

292 return gws.u.parse_acl(val) 

293 except ValueError: 

294 raise core.ReadError(f'invalid ACL', val) 

295 

296 

297def _read_color(r: Reader, val, typ: core.Type): 

298 # @TODO: parse color values 

299 return _read_str(r, val, typ) 

300 

301 

302def _read_crs(r: Reader, val, typ: core.Type): 

303 crs = gws.lib.crs.get(val) 

304 if not crs: 

305 raise core.ReadError(f'invalid crs: {val!r}', val) 

306 return crs.srid 

307 

308 

309def _read_date(r: Reader, val, typ: core.Type): 

310 try: 

311 return gws.lib.datetimex.from_string(str(val)) 

312 except ValueError: 

313 raise core.ReadError(f'invalid date: {val!r}', val) 

314 

315 

316def _read_datetime(r: Reader, val, typ: core.Type): 

317 try: 

318 return gws.lib.datetimex.from_iso_string(str(val)) 

319 except ValueError: 

320 raise core.ReadError(f'invalid date: {val!r}', val) 

321 

322 

323def _read_dirpath(r: Reader, val, typ: core.Type): 

324 path = gws.lib.osx.abs_path(val, r.path) 

325 if not gws.u.is_dir(path): 

326 raise core.ReadError(f'directory not found: {path!r}, base {r.path!r}', val) 

327 return path 

328 

329 

330def _read_duration(r: Reader, val, typ: core.Type): 

331 try: 

332 return gws.lib.datetimex.parse_duration(val) 

333 except ValueError: 

334 raise core.ReadError(f'invalid duration: {val!r}', val) 

335 

336 

337def _read_filepath(r: Reader, val, typ: core.Type): 

338 path = gws.lib.osx.abs_path(val, r.path) 

339 if not gws.lib.osx.is_abs_path(val): 

340 gws.log.warning(f'relative path, assuming {path!r} for {val!r}') 

341 if not gws.u.is_file(path): 

342 raise core.ReadError(f'file not found: {path!r}, base {r.path!r}', val) 

343 return path 

344 

345 

346def _read_formatstr(r: Reader, val, typ: core.Type): 

347 # @TODO validate 

348 return _read_str(r, val, typ) 

349 

350 

351def _read_metadata(r: Reader, val, typ: core.Type): 

352 rr = r.allow_skip_required 

353 r.allow_skip_required = True 

354 res = gws.u.compact(_read_object(r, val, typ)) 

355 r.allow_skip_required = rr 

356 return res 

357 

358 

359def _read_uom_value(r: Reader, val, typ: core.Type): 

360 try: 

361 return gws.lib.uom.parse(val) 

362 except ValueError as e: 

363 raise core.ReadError(f'invalid value: {val!r}: {e!r}', val) 

364 

365 

366def _read_uom_point(r: Reader, val, typ: core.Type): 

367 try: 

368 return gws.lib.uom.parse_point(val) 

369 except ValueError as e: 

370 raise core.ReadError(f'invalid value: {val!r}: {e!r}', val) 

371 

372 

373def _read_uom_extent(r: Reader, val, typ: core.Type): 

374 try: 

375 return gws.lib.uom.parse_extent(val) 

376 except ValueError as e: 

377 raise core.ReadError(f'invalid value: {val!r}: {e!r}', val) 

378 

379 

380def _read_regex(r: Reader, val, typ: core.Type): 

381 try: 

382 re.compile(val) 

383 return val 

384 except re.error as e: 

385 raise core.ReadError(f'invalid regular expression: {val!r}: {e!r}', val) 

386 

387 

388def _read_url(r: Reader, val, typ: core.Type): 

389 u = _read_str(r, val, typ) 

390 if u.startswith(('http://', 'https://')): 

391 return u 

392 raise core.ReadError(f'invalid url: {val!r}', val) 

393 

394 

395# utils 

396 

397 

398def _ensure(val, cls): 

399 if isinstance(val, cls): 

400 return val 

401 if cls == list and isinstance(val, tuple): 

402 return list(val) 

403 if cls == dict and gws.u.is_data_object(val): 

404 return vars(val) 

405 raise core.ReadError(f'wrong type: {_classname(type(val))!r}, expected: {_classname(cls)!r}', val) 

406 

407 

408def _to_string(x): 

409 if isinstance(x, str): 

410 return x 

411 if isinstance(x, (bytes, bytearray)): 

412 return x.decode('utf8') 

413 raise ValueError() 

414 

415 

416def _classname(cls): 

417 try: 

418 return cls.__name__ 

419 except: 

420 return str(cls) 

421 

422 

423def _comma(ls): 

424 return repr(', '.join(sorted(str(x) for x in ls))) 

425 

426 

427## 

428 

429 

430def _format_error_value(exc): 

431 try: 

432 val = exc.args[1] 

433 except (AttributeError, IndexError): 

434 return '' 

435 

436 s = repr(val) 

437 if len(s) > 600: 

438 s = s[:600] + '...' 

439 return s 

440 

441 

442def _format_error_stack(stack): 

443 f = [] 

444 

445 for name, value, type_uid in reversed(stack): 

446 line = '' 

447 

448 if name: 

449 name = repr(name) 

450 line = f'item {name}' if name.isdigit() else name 

451 

452 obj = type_uid or 'object' 

453 for p in 'uid', 'title', 'type': 

454 try: 

455 s = value.get(p) 

456 if s is not None: 

457 obj += f' {p}={s!r}' 

458 break 

459 except AttributeError: 

460 pass 

461 

462 f.append(f'in {line} <{obj}>') 

463 

464 return f 

465 

466 

467# 

468 

469_READERS = { 

470 'any': _read_any, 

471 'bool': _read_bool, 

472 'bytes': _read_bytes, 

473 'float': _read_float, 

474 'int': _read_int, 

475 'str': _read_str, 

476 'list': _read_raw_list, 

477 'dict': _read_raw_dict, 

478 core.c.CLASS: _read_object, 

479 core.c.DICT: _read_dict, 

480 core.c.ENUM: _read_enum, 

481 core.c.LIST: _read_list, 

482 core.c.LITERAL: _read_literal, 

483 core.c.OPTIONAL: _read_optional, 

484 core.c.PROPERTY: _read_property, 

485 core.c.SET: _read_set, 

486 core.c.TUPLE: _read_tuple, 

487 core.c.TYPE: _read_type, 

488 core.c.UNION: _read_union, 

489 core.c.VARIANT: _read_variant, 

490 'gws.AclStr': _read_acl_str, 

491 'gws.Color': _read_color, 

492 'gws.CrsName': _read_crs, 

493 'gws.DateStr': _read_date, 

494 'gws.DateTimeStr': _read_datetime, 

495 'gws.DirPath': _read_dirpath, 

496 'gws.Duration': _read_duration, 

497 'gws.FilePath': _read_filepath, 

498 'gws.FormatStr': _read_formatstr, 

499 'gws.UomValueStr': _read_uom_value, 

500 'gws.UomPointStr': _read_uom_point, 

501 'gws.UomSizeStr': _read_uom_point, 

502 'gws.UomExtentStr': _read_uom_extent, 

503 'gws.Metadata': _read_metadata, 

504 'gws.Regex': _read_regex, 

505 'gws.Url': _read_url, 

506}