Coverage for gws-app/gws/plugin/alkis/data/norbit.py: 97%

1"""Read data from Norbit plugin tables."""

3import gws

4import gws.base.database

5import gws.lib.datetimex

6import gws.lib.sa as sa

7import gws.plugin.postgres.provider

9from . import gid

10from . import types as dt

13class Object(dt.Reader):

14 """Reader for ALKIS source tables written by the norBIT ALKIS import.

16 There is one table per object type, named after the class in lower case,

17 with one column per attribute. Struct attributes are stored in columns

18 prefixed with the attribute name, lists are stored as arrays. The reader

19 converts each row into an instance of the ``gid`` class, using a reader

20 method per attribute, chosen by the attribute type.

21 """

23 STD_READERS = {

24 'Area': 'as_float',

25 'Boolean': 'as_bool',

26 'CharacterString': 'as_str',

27 'Integer': 'as_int',

28 'DateTime': 'as_date',

29 'Date': 'as_date',

31 'AX_Lagebezeichnung': 'as_ax_lagebezeichnung',

32 'AX_Buchung_HistorischesFlurstueck': 'as_ax_buchung_historischesflurstueck',

33 'AX_RelativeHoehe': 'as_ax_relativehoehe',

34 }

35 """Reader method names for basic and special attribute types."""

37 def __init__(self, provider: gws.plugin.postgres.provider.Object, schema='public'):

38 """Create a reader and set up attribute readers for all GeoInfoDok classes.

40 Args:

41 provider: Database provider.

42 schema: Schema with the source tables.

43 """

45 self.db = provider

46 self.schema = schema

48 self.readers = {}

50 for meta in gid.METADATA.values():

51 if meta['kind'] not in {'struct', 'object'}:

52 continue

53 d = {}

54 for sup in meta['supers']:

55 d.update(self.readers.get(sup, {}))

56 for attr in meta['attributes']:

57 d[attr['name']] = [

58 attr['name'],

59 attr['name'].lower(),

60 getattr(gid, attr['type'], None),

61 self.get_reader(attr)

62 ]

64 self.readers[meta['name']] = d

66 def get_reader(self, attr):

67 """Return the reader method for an attribute.

69 Args:

70 attr: Attribute metadata from the ``gid`` schema.

72 Returns:

73 A bound reader method. Unknown types are read as strings.

74 """

76 typ = attr['type']

77 is_list = attr['list']

79 std = self.STD_READERS.get(typ)

80 if std:

81 return getattr(self, std + '_list' if is_list else std)

83 meta = gid.METADATA.get(typ)

84 if meta:

85 if meta['kind'] == 'object':

86 return self.as_ref_list if is_list else self.as_ref

87 if meta['kind'] == 'struct':

88 return self.as_struct_list if is_list else self.as_struct

89 if meta['kind'] == 'enum':

90 return self.as_enum_list if is_list else self.as_enum

92 return self.as_str_list if is_list else self.as_str

94 ##

96 def count(self, cls, table_name=None):

97 # NB not using db.count to avoid schema introspection

98 sql = f"SELECT COUNT(*) FROM {self.schema}.{table_name or cls.__name__.lower()}"

99 try:

100 with self.db.begin(nested=True) as conn:

101 rs = list(conn.execute(sa.text(sql)))

102 return rs[0][0]

103 except sa.Error:

104 return 0

105

106 def read_all(self, cls, table_name=None, uids=None):

107 sql = f"SELECT * FROM {self.schema}.{table_name or cls.__name__.lower()}"

108 if uids:

109 sql += ' WHERE gml_id = ANY(:uids)'

110 sql = sa.text(sql).bindparams(uids=uids)

111 else:

112 sql = sa.text(sql)

113

114 with self.db.begin() as conn:

115 for row in conn.execute(sql, execution_options={'stream_results': True}):

116 r = gws.u.to_dict(row)

117 o = self.as_struct(cls, '', r)

118 o.identifikator = r.get('gml_id')

119 o.geom = r.get('wkb_geometry', '')

120 yield o

121

122 ##

123

124 def as_struct(self, cls, prop, r):

125 """Read a struct attribute.

126

127 A column ``<prop>_<attribute>`` is used if present, otherwise the plain

128 attribute column.

129

130 Args:

131 cls: GeoInfoDok class of the attribute.

132 prop: Column name prefix of the struct attributes.

133 r: Table row as a dict.

134

135 Returns:

136 An instance of ``cls``.

137 """

138

139 d = {}

140

141 for attr_name, attr_low_name, attr_cls, fn in self.readers[cls.__name__].values():

142 comp_key = prop.lower() + '_' + attr_low_name

143 if comp_key in r:

144 d[attr_name] = fn(attr_cls, comp_key, r)

145 else:

146 d[attr_name] = fn(attr_cls, attr_low_name, r)

147

148 o = cls()

149 vars(o).update(d)

150 return o

151

152 def as_struct_list(self, cls, prop, r):

153 """Read a list of structs from parallel array columns.

154

155 Args:

156 cls: GeoInfoDok class of the attribute.

157 prop: Column name prefix of the struct attributes.

158 r: Table row as a dict.

159

160 Returns:

161 A list of ``cls`` instances.

162 """

163

164 d = {}

165

166 for attr_name, attr_low_name, attr_cls, fn in self.readers[cls.__name__].values():

167 comp_key = prop.lower() + '_' + attr_low_name

168 if comp_key in r:

169 a = _array(r[comp_key])

170 else:

171 a = _array(r.get(attr_low_name))

172 if a:

173 d[attr_low_name] = a

174

175 objs = []

176

177 for vals in zip(*d.values()):

178 r2 = dict(zip(d, vals))

179 o = self.as_struct(cls, '', r2)

180 objs.append(o)

181

182 return objs

183

184 def as_ref(self, cls, prop, r):

185 """Read a reference to another object.

186

187 Args:

188 cls: GeoInfoDok class of the attribute.

189 prop: Column name.

190 r: Table row as a dict.

191

192 Returns:

193 The identifier of the referenced object.

194 """

195

196 return r.get(prop)

197

198 def as_ref_list(self, cls, prop, r):

199 """Read a list of references to other objects.

200

201 Args:

202 cls: GeoInfoDok class of the attribute.

203 prop: Column name.

204 r: Table row as a dict.

205

206 Returns:

207 A list of object identifiers.

208 """

209

210 return _array(r.get(prop))

211

212 def as_enum(self, cls, prop, r):

213 """Read a code list value.

214

215 Args:

216 cls: GeoInfoDok class of the attribute.

217 prop: Column name.

218 r: Table row as a dict.

219

220 Returns:

221 An ``EnumPair``, or ``None`` if the value is not in the code list.

222 """

223

224 v = str(r.get(prop))

225 if v in cls.VALUES:

226 return dt.EnumPair(v, cls.VALUES[v])

227

228 def as_enum_list(self, cls, prop, r):

229 """Read a list of code list values.

230

231 Args:

232 cls: GeoInfoDok class of the attribute.

233 prop: Column name.

234 r: Table row as a dict.

235

236 Returns:

237 A list of ``EnumPair`` objects. Values not in the code list are skipped.

238 """

239

240 ls = []

241

242 for v in _array(r.get(prop)):

243 v = str(v)

244 if v in cls.VALUES:

245 ls.append(dt.EnumPair(v, cls.VALUES[v]))

246

247 return ls

248

249 ##

250

251 def as_ax_lagebezeichnung(self, cls, prop, r):

252 """Read an ``AX_Lagebezeichnung`` attribute.

253

254 Args:

255 cls: GeoInfoDok class of the attribute.

256 prop: Column name.

257 r: Table row as a dict.

258

259 Returns:

260 The unencoded location string if present, otherwise an ``AX_VerschluesselteLagebezeichnung``.

261 """

262

263 if r.get('unverschluesselt'):

264 return r.get('unverschluesselt')

265 return self.as_struct(gid.AX_VerschluesselteLagebezeichnung, prop, r)

266

267 def as_ax_buchung_historischesflurstueck_list(self, cls, prop, r):

268 """Read a list of ``AX_Buchung_HistorischesFlurstueck`` structs.

269

270 Args:

271 cls: GeoInfoDok class of the attribute.

272 prop: Column name.

273 r: Table row as a dict.

274

275 Returns:

276 A list of ``AX_Buchung_HistorischesFlurstueck`` instances with their ``buchungsblattbezirk`` set.

277 """

278

279 # this one is stored as

280 # "blattart": [xxxx],

281 # "buchungsart": ["xxxx"],

282 # ....etc

283 # "buchungsblattbezirk_bezirk": ["xxxx"],

284 # "buchungsblattbezirk_land": ["xx"],

285 # "buchungsblattkennzeichen": ["xxxx"],

286 # "buchungsblattnummermitbuchstabenerweiterung": ["xxxx"],

287

288 bbs = self.as_struct_list(gid.AX_Buchung_HistorischesFlurstueck, '', r)

289 bzs = self.as_struct_list(gid.AX_Buchungsblattbezirk_Schluessel, 'buchungsblattbezirk', r)

290 objs = []

291 for bb, bz in zip(bbs, bzs):

292 bb.buchungsblattbezirk = bz

293 objs.append(bb)

294 return objs

295

296 def as_ax_relativehoehe_list(self, cls, prop, r):

297 """Read a list of ``AX_RelativeHoehe`` structs.

298

299 Args:

300 cls: GeoInfoDok class of the attribute.

301 prop: Column name prefix of the struct attributes.

302 r: Table row as a dict.

303

304 Returns:

305 The ``hoehe`` of the first struct that has one, or ``None``.

306 """

307

308 # GeoInfoDok 7 'objekthoehe' is a list of heights with reference points,

309 # GeoInfoDok 6 has a single height, so we take the first one

310 for o in self.as_struct_list(cls, prop, r):

311 if o.hoehe is not None:

312 return o.hoehe

313

314 ##

315

316 def as_str(self, cls, prop, r):

317 """Read a string.

318

319 Args:

320 cls: GeoInfoDok class of the attribute.

321 prop: Column name.

322 r: Table row as a dict.

323

324 Returns:

325 A stripped string, or ``None`` if the value is empty.

326 """

327

328 return _str(r.get(prop))

329

330 def as_str_list(self, cls, prop, r):

331 """Read a list of strings.

332

333 Args:

334 cls: GeoInfoDok class of the attribute.

335 prop: Column name.

336 r: Table row as a dict.

337

338 Returns:

339 A list of stripped strings, empty values become ``None``.

340 """

341

342 return [_str(v) for v in _array(r.get(prop))]

343

344 def as_bool(self, cls, prop, r):

345 """Read a boolean.

346

347 Args:

348 cls: GeoInfoDok class of the attribute.

349 prop: Column name.

350 r: Table row as a dict.

351

352 Returns:

353 A boolean, or ``None`` if the value is missing.

354 """

355

356 return _bool(r.get(prop))

357

358 def as_bool_list(self, cls, prop, r):

359 """Read a list of booleans.

360

361 Args:

362 cls: GeoInfoDok class of the attribute.

363 prop: Column name.

364 r: Table row as a dict.

365

366 Returns:

367 A list of booleans.

368 """

369

370 return [_bool(v) for v in _array(r.get(prop))]

371

372 def as_int(self, cls, prop, r):

373 """Read an integer.

374

375 Args:

376 cls: GeoInfoDok class of the attribute.

377 prop: Column name.

378 r: Table row as a dict.

379

380 Returns:

381 An integer, or ``None`` if the value is missing.

382 """

383

384 return _int(r.get(prop))

385

386 def as_int_list(self, cls, prop, r):

387 """Read a list of integers.

388

389 Args:

390 cls: GeoInfoDok class of the attribute.

391 prop: Column name.

392 r: Table row as a dict.

393

394 Returns:

395 A list of integers.

396 """

397

398 return [_int(v) for v in _array(r.get(prop))]

399

400 def as_float(self, cls, prop, r):

401 """Read a float.

402

403 Args:

404 cls: GeoInfoDok class of the attribute.

405 prop: Column name.

406 r: Table row as a dict.

407

408 Returns:

409 A float, or ``None`` if the value is missing.

410 """

411

412 return _float(r.get(prop))

413

414 def as_float_list(self, cls, prop, r):

415 """Read a list of floats.

416

417 Args:

418 cls: GeoInfoDok class of the attribute.

419 prop: Column name.

420 r: Table row as a dict.

421

422 Returns:

423 A list of floats.

424 """

425

426 return [_float(v) for v in _array(r.get(prop))]

427

428 def as_date(self, cls, prop, r):

429 """Read a date or a date and time.

430

431 ISO strings are parsed, other values are returned as is.

432

433 Args:

434 cls: GeoInfoDok class of the attribute.

435 prop: Column name.

436 r: Table row as a dict.

437

438 Returns:

439 A date or datetime, or ``None`` if the value is empty.

440 """

441

442 return _datetime(r.get(prop))

443

444 def as_date_list(self, cls, prop, r):

445 """Read a list of dates.

446

447 Args:

448 cls: GeoInfoDok class of the attribute.

449 prop: Column name.

450 r: Table row as a dict.

451

452 Returns:

453 A list of dates or datetimes.

454 """

455

456 return [_datetime(v) for v in _array(r.get(prop))]

457

458

459def _str(v):

460 """Convert to a stripped string, ``None`` if empty."""

461

462 if v is not None:

463 v = str(v).strip()

464 if v:

465 return v

466

467

468def _bool(v):

469 return bool(v) if v is not None else None

470

471

472def _int(v):

473 return int(v) if v is not None else None

474

475

476def _float(v):

477 return float(v) if v is not None else None

478

479

480def _datetime(v):

481 """Convert an ISO string to a datetime, pass other values through."""

482

483 if not v:

484 return None

485 if isinstance(v, str):

486 return gws.lib.datetimex.from_iso_string(v)

487 return v

488

489

490def _array(val):

491 """Wrap a scalar into a list, ``None`` becomes an empty list."""

492

493 if isinstance(val, list):

494 return val

495 if val is None:

496 return []

497 return [val]