Coverage for gws-app/gws/spec/reader.py: 43%

1"""Read and validate values according to spec types.""" 

2 

3import re 

4 

5import gws 

6import gws.lib.crs 

7import gws.lib.datetimex 

8import gws.lib.osx 

9import gws.lib.uom 

10 

11from . import core 

12 

13 

14class Reader: 

15 atom = core.make_type({'c': core.c.ATOM}) 

16 

17 def __init__(self, runtime, path, options): 

18 self.runtime = runtime 

19 self.path = path 

20 

21 options = set(options or []) 

22 

23 self.accept_extra_props = gws.SpecReadOption.acceptExtraProps in options 

24 self.case_insensitive = gws.SpecReadOption.caseInsensitive in options 

25 self.convert_values = gws.SpecReadOption.convertValues in options 

26 self.ignore_extra_props = gws.SpecReadOption.ignoreExtraProps in options 

27 self.allow_skip_required = gws.SpecReadOption.allowMissing in options 

28 self.verbose_errors = gws.SpecReadOption.verboseErrors in options 

29 

30 self.stack = None 

31 self.push = lambda _: ... 

32 self.pop = lambda: ... 

33 

34 def read(self, value, type_uid): 

35 if not self.verbose_errors: 

36 return self.read2(value, type_uid) 

37 

38 self.stack = [('', value, type_uid)] 

39 self.push = self.stack.append 

40 self.pop = self.stack.pop 

41 

42 try: 

43 return self.read2(value, type_uid) 

44 except core.ReadError as exc: 

45 raise self.add_config_error_info(exc) 

46 

47 def read2(self, value, type_uid): 

48 typ = self.runtime.get_type(type_uid) 

49 

50 if type_uid in _READERS: 

51 return _READERS[type_uid](self, value, typ or self.atom) 

52 

53 if not typ: 

54 raise core.ReadError(f'unknown type {type_uid!r}', value) 

55 

56 if typ.c not in _READERS: 

57 raise core.ReadError(f'unknown type category {typ.c!r}', value) 

58 

59 return _READERS[typ.c](self, value, typ) 

60 

61 def add_config_error_info(self, exc: Exception): 

62 cei = gws.ConfigErrorInfo( 

63 value=_format_error_value(exc), 

64 path=self.path, 

65 stack=_prepare_error_stack(self.stack or []), 

66 ) 

67 exc.args = (exc.args[0], exc.args[1], cei) 

68 return exc 

69 

70 

71# atoms 

72 

73 

74def _read_any(r: Reader, val, typ: core.Type): 

75 return val 

76 

77 

78def _read_bool(r: Reader, val, typ: core.Type): 

79 if not r.convert_values: 

80 return _ensure(val, bool) 

81 try: 

82 return bool(val) 

83 except: 

84 raise core.ReadError('must be true or false', val) 

85 

86 

87def _read_bytes(r: Reader, val, typ: core.Type): 

88 try: 

89 if isinstance(val, str): 

90 return val.encode('utf8', errors='strict') 

91 return bytes(val) 

92 except: 

93 raise core.ReadError('must be a byte buffer', val) 

94 

95 

96def _read_float(r: Reader, val, typ: core.Type): 

97 if not r.convert_values: 

98 if isinstance(val, int): 

99 return float(val) 

100 return _ensure(val, float) 

101 try: 

102 return float(val) 

103 except: 

104 raise core.ReadError('must be a float', val) 

105 

106 

107def _read_int(r: Reader, val, typ: core.Type): 

108 if isinstance(val, bool): 

109 raise core.ReadError('must be an integer', val) 

110 if not r.convert_values: 

111 return _ensure(val, int) 

112 try: 

113 return int(val) 

114 except: 

115 raise core.ReadError('must be an integer', val) 

116 

117 

118def _read_str(r: Reader, val, typ: core.Type): 

119 if not r.convert_values: 

120 return _ensure(val, str) 

121 try: 

122 return _to_string(val) 

123 except: 

124 raise core.ReadError('must be a string', val) 

125 

126 

127# built-ins 

128 

129 

130def _read_raw_dict(r: Reader, val, typ: core.Type): 

131 return _ensure(val, dict) 

132 

133 

134def _read_dict(r: Reader, val, typ: core.Type): 

135 dct = {} 

136 for k, v in _ensure(val, dict).items(): 

137 dct[k] = r.read2(v, typ.tValue) 

138 return dct 

139 

140 

141def _read_raw_list(r: Reader, val, typ: core.Type): 

142 return _ensure(val, list) 

143 

144 

145def _read_list(r: Reader, val, typ: core.Type): 

146 lst = _read_any_list(r, val) 

147 res = [] 

148 for n, v in enumerate(lst): 

149 r.push((n, v, typ.tItem)) 

150 res.append(r.read2(v, typ.tItem)) 

151 r.pop() 

152 return res 

153 

154 

155def _read_set(r: Reader, val, typ: core.Type): 

156 lst = _read_list(r, val, typ) 

157 return set(lst) 

158 

159 

160def _read_tuple(r: Reader, val, typ: core.Type): 

161 lst = _read_any_list(r, val) 

162 

163 if len(lst) != len(typ.tItems): 

164 raise core.ReadError(f'expected: {_comma(typ.tItems)}', val) 

165 

166 res = [] 

167 for n, v in enumerate(lst): 

168 r.push((n, v, typ.tItems[n])) 

169 res.append(r.read2(v, typ.tItems[n])) 

170 r.pop() 

171 return res 

172 

173 

174def _read_any_list(r, val): 

175 if r.convert_values and isinstance(val, str): 

176 val = val.strip() 

177 val = [v.strip() for v in val.split(',')] if val else [] 

178 return _ensure(val, list) 

179 

180 

181def _read_literal(r: Reader, val, typ: core.Type): 

182 s = _read_any(r, val, typ) 

183 if s not in typ.literalValues: 

184 raise core.ReadError(f'invalid value: {s!r}, expected: {_comma(typ.literalValues)}', val) 

185 return s 

186 

187 

188def _read_optional(r: Reader, val, typ: core.Type): 

189 if val is None: 

190 return val 

191 return r.read2(val, typ.tTarget) 

192 

193 

194def _read_union(r: Reader, val, typ: core.Type): 

195 # @TODO no untyped unions yet 

196 raise core.ReadError('unions are not supported yet', val) 

197 

198 

199# our types 

200 

201 

202def _read_type(r: Reader, val, typ: core.Type): 

203 return r.read2(val, typ.tTarget) 

204 

205 

206def _read_enum(r: Reader, val, typ: core.Type): 

207 # NB: our Enums accept both names (for configs) and values (for api calls) 

208 # this prevents silly things like Enum{foo=bar bar=123} but we don't care 

209 # 

210 # the comparison is also case-insensitive 

211 # 

212 # this reader returns a value, it's up to the caller to convert it to the actual enum 

213 

214 def _lower(s): 

215 return s.lower() if isinstance(s, str) else s 

216 

217 lv = _lower(val) 

218 

219 for k, v in typ.enumValues.items(): 

220 if lv == _lower(k) or lv == _lower(v): 

221 return v 

222 raise core.ReadError(f'invalid value: {val!r}, expected: {_comma(typ.enumValues)}', val) 

223 

224 

225def _read_object(r: Reader, val, typ: core.Type): 

226 val = _ensure(val, dict) 

227 

228 if r.case_insensitive: 

229 val = {k.lower(): v for k, v in val.items()} 

230 else: 

231 val = dict(val) 

232 

233 res = {} 

234 

235 for prop_name, prop_type_uid in typ.tProperties.items(): 

236 prop_val = val.pop(prop_name.lower() if r.case_insensitive else prop_name, None) 

237 r.push((prop_name, prop_val, prop_type_uid)) 

238 res[prop_name] = r.read2(prop_val, prop_type_uid) 

239 r.pop() 

240 

241 unknown = [] 

242 

243 for k in val: 

244 if k not in typ.tProperties: 

245 # accept 'uid' for all objects 

246 if k == 'uid': 

247 res[k] = val[k] 

248 elif r.accept_extra_props: 

249 res[k] = val[k] 

250 elif r.ignore_extra_props: 

251 continue 

252 else: 

253 unknown.append(k) 

254 

255 if unknown: 

256 raise core.ReadError(f'unknown keys: {_comma(unknown)}, expected: {_comma(typ.tProperties)} for {typ.uid!r}', val) 

257 

258 return gws.Data(res) 

259 

260 

261def _read_property(r: Reader, val, typ: core.Type): 

262 if val is not None: 

263 return r.read2(val, typ.tValue) 

264 

265 if not typ.hasDefault: 

266 if r.allow_skip_required: 

267 return None 

268 raise core.ReadError(f'required property missing: {typ.ident!r} for {typ.tOwner!r}', None) 

269 

270 if typ.defaultValue is None: 

271 return None 

272 

273 # the default, if given, must match the type 

274 # NB, for Data objects, default={} will create an object with defaults 

275 return r.read2(typ.defaultValue, typ.tValue) 

276 

277 

278def _read_variant(r: Reader, val, typ: core.Type): 

279 val = _ensure(val, dict) 

280 if r.case_insensitive: 

281 val = {k.lower(): v for k, v in val.items()} 

282 

283 type_name = val.get(core.v.VARIANT_TAG, core.v.DEFAULT_VARIANT_TAG) 

284 target_type_uid = typ.tMembers.get(type_name) 

285 if not target_type_uid: 

286 raise core.ReadError(f'illegal type: {type_name!r}, expected: {_comma(typ.tMembers)}', val) 

287 return r.read2(val, target_type_uid) 

288 

289 

290# custom types 

291 

292 

293def _read_acl_str(r: Reader, val, typ: core.Type): 

294 try: 

295 return gws.u.parse_acl(val) 

296 except ValueError: 

297 raise core.ReadError(f'invalid ACL', val) 

298 

299 

300def _read_color(r: Reader, val, typ: core.Type): 

301 # @TODO: parse color values 

302 return _read_str(r, val, typ) 

303 

304 

305def _read_crs(r: Reader, val, typ: core.Type): 

306 crs = gws.lib.crs.get(val) 

307 if not crs: 

308 raise core.ReadError(f'invalid crs: {val!r}', val) 

309 return crs.srid 

310 

311 

312def _read_date(r: Reader, val, typ: core.Type): 

313 try: 

314 return gws.lib.datetimex.from_string(str(val)) 

315 except ValueError: 

316 raise core.ReadError(f'invalid date: {val!r}', val) 

317 

318 

319def _read_datetime(r: Reader, val, typ: core.Type): 

320 try: 

321 return gws.lib.datetimex.from_iso_string(str(val)) 

322 except ValueError: 

323 raise core.ReadError(f'invalid date: {val!r}', val) 

324 

325 

326def _read_dirpath(r: Reader, val, typ: core.Type): 

327 path = gws.lib.osx.abs_path(val, r.path) 

328 if not gws.u.is_dir(path): 

329 raise core.ReadError(f'directory not found: {path!r}', val) 

330 return path 

331 

332 

333def _read_duration(r: Reader, val, typ: core.Type): 

334 try: 

335 return gws.lib.datetimex.parse_duration(val) 

336 except ValueError: 

337 raise core.ReadError(f'invalid duration: {val!r}', val) 

338 

339 

340def _read_filepath(r: Reader, val, typ: core.Type): 

341 path = gws.lib.osx.abs_path(val, r.path) 

342 if not gws.lib.osx.is_abs_path(val): 

343 gws.log.warning(f'relative path, assuming {path!r} for {val!r}') 

344 if not gws.u.is_file(path): 

345 raise core.ReadError(f'file not found: {path!r}', val) 

346 return path 

347 

348 

349def _read_formatstr(r: Reader, val, typ: core.Type): 

350 # @TODO validate 

351 return _read_str(r, val, typ) 

352 

353 

354def _read_metadata(r: Reader, val, typ: core.Type): 

355 rr = r.allow_skip_required 

356 r.allow_skip_required = True 

357 res = gws.u.compact(_read_object(r, val, typ)) 

358 r.allow_skip_required = rr 

359 return res 

360 

361 

362def _read_uom_value(r: Reader, val, typ: core.Type): 

363 try: 

364 return gws.lib.uom.parse(val) 

365 except ValueError as e: 

366 raise core.ReadError(f'invalid value: {val!r}: {e!r}', val) 

367 

368 

369def _read_uom_point(r: Reader, val, typ: core.Type): 

370 try: 

371 return gws.lib.uom.parse_point(val) 

372 except ValueError as e: 

373 raise core.ReadError(f'invalid value: {val!r}: {e!r}', val) 

374 

375 

376def _read_uom_extent(r: Reader, val, typ: core.Type): 

377 try: 

378 return gws.lib.uom.parse_extent(val) 

379 except ValueError as e: 

380 raise core.ReadError(f'invalid value: {val!r}: {e!r}', val) 

381 

382 

383def _read_regex(r: Reader, val, typ: core.Type): 

384 try: 

385 re.compile(val) 

386 return val 

387 except re.error as e: 

388 raise core.ReadError(f'invalid regular expression: {val!r}: {e!r}', val) 

389 

390 

391def _read_url(r: Reader, val, typ: core.Type): 

392 u = _read_str(r, val, typ) 

393 if u.startswith(('http://', 'https://')): 

394 return u 

395 raise core.ReadError(f'invalid url: {val!r}', val) 

396 

397 

398# utils 

399 

400 

401def _ensure(val, cls): 

402 if isinstance(val, cls): 

403 return val 

404 if cls == list and isinstance(val, tuple): 

405 return list(val) 

406 if cls == dict and gws.u.is_data_object(val): 

407 return vars(val) 

408 raise core.ReadError(f'wrong type: {_classname(type(val))!r}, expected: {_classname(cls)!r}', val) 

409 

410 

411def _to_string(x): 

412 if isinstance(x, str): 

413 return x 

414 if isinstance(x, (bytes, bytearray)): 

415 return x.decode('utf8') 

416 raise ValueError() 

417 

418 

419def _classname(cls): 

420 try: 

421 return cls.__name__ 

422 except: 

423 return str(cls) 

424 

425 

426def _comma(ls): 

427 return repr(', '.join(sorted(str(x) for x in ls))) 

428 

429 

430## 

431 

432 

433def _format_error_value(exc): 

434 try: 

435 val = exc.args[1] 

436 except Exception: 

437 return '' 

438 

439 s = repr(val) 

440 if len(s) > 600: 

441 s = s[:600] + '...' 

442 return s 

443 

444 

445def _prepare_error_stack(stack): 

446 ls = [] 

447 

448 for name, value, type_uid in reversed(stack): 

449 obj_name = gws.u.get(value, 'name') or gws.u.get(value, 'title') 

450 ls.append( 

451 gws.ConfigLocation( 

452 objectUid=gws.u.get(value, 'uid'), 

453 objectType=type_uid or gws.u.get(value, 'type'), 

454 objectName=obj_name if isinstance(obj_name, str) else '', 

455 propName=str(name) if name is not None else '', 

456 ) 

457 ) 

458 

459 return ls 

460 

461 

462# 

463 

464_READERS = { 

465 'any': _read_any, 

466 'bool': _read_bool, 

467 'bytes': _read_bytes, 

468 'float': _read_float, 

469 'int': _read_int, 

470 'str': _read_str, 

471 'list': _read_raw_list, 

472 'dict': _read_raw_dict, 

473 core.c.CLASS: _read_object, 

474 core.c.DICT: _read_dict, 

475 core.c.ENUM: _read_enum, 

476 core.c.LIST: _read_list, 

477 core.c.LITERAL: _read_literal, 

478 core.c.OPTIONAL: _read_optional, 

479 core.c.PROPERTY: _read_property, 

480 core.c.SET: _read_set, 

481 core.c.TUPLE: _read_tuple, 

482 core.c.TYPE: _read_type, 

483 core.c.UNION: _read_union, 

484 core.c.VARIANT: _read_variant, 

485 'gws.AclStr': _read_acl_str, 

486 'gws.Color': _read_color, 

487 'gws.CrsName': _read_crs, 

488 'gws.DateStr': _read_date, 

489 'gws.DateTimeStr': _read_datetime, 

490 'gws.DirPath': _read_dirpath, 

491 'gws.Duration': _read_duration, 

492 'gws.FilePath': _read_filepath, 

493 'gws.FormatStr': _read_formatstr, 

494 'gws.UomValueStr': _read_uom_value, 

495 'gws.UomPointStr': _read_uom_point, 

496 'gws.UomSizeStr': _read_uom_point, 

497 'gws.UomExtentStr': _read_uom_extent, 

498 'gws.Metadata': _read_metadata, 

499 'gws.Regex': _read_regex, 

500 'gws.Url': _read_url, 

501}