Coverage for gws-app/gws/plugin/alkis/data/norbit6.py: 0%

1"""Read data from Norbit plugin tables (GeoInfoDok 6)."""

3import gws

4import gws.base.database

5import gws.lib.datetimex

6import gws.lib.sa as sa

7import gws.plugin.postgres.provider

9from .geo_info_dok import gid6 as gid

10from . import types as dt

13class Object(dt.Reader):

14 """Reader for ALKIS source tables written by the norBIT ALKIS import (GeoInfoDok 6).

16 There is one table per object type, named after the class in lower case,

17 with one column per attribute. Struct attributes are stored in columns

18 prefixed with the attribute name, lists are stored as arrays. The reader

19 converts each row into an instance of the ``gid6`` class, using a reader

20 method per attribute, chosen by the attribute type.

21 """

23 STD_READERS = {

24 'Area': 'as_float',

25 'Boolean': 'as_bool',

26 'CharacterString': 'as_str',

27 'Integer': 'as_int',

28 'DateTime': 'as_date',

29 'Date': 'as_date',

31 'AX_Lagebezeichnung': 'as_ax_lagebezeichnung',

32 'AX_Buchung_HistorischesFlurstueck': 'as_ax_buchung_historischesflurstueck',

33 }

34 """Reader method names for basic and special attribute types."""

36 def __init__(self, provider: gws.plugin.postgres.provider.Object, schema='public'):

37 """Create a reader and set up attribute readers for all GeoInfoDok classes.

39 Args:

40 provider: Database provider.

41 schema: Schema with the source tables.

42 """

44 self.db = provider

45 self.schema = schema

47 self.readers = {}

49 for meta in gid.METADATA.values():

50 if meta['kind'] not in {'struct', 'object'}:

51 continue

52 d = {}

53 for sup in meta['supers']:

54 d.update(self.readers.get(sup, {}))

55 for attr in meta['attributes']:

56 d[attr['name']] = [

57 attr['name'],

58 attr['name'].lower(),

59 getattr(gid, attr['type'], None),

60 self.get_reader(attr)

61 ]

63 self.readers[meta['name']] = d

65 def get_reader(self, attr):

66 """Return the reader method for an attribute.

68 Args:

69 attr: Attribute metadata from the ``gid6`` schema.

71 Returns:

72 A bound reader method. Unknown types are read as strings.

73 """

75 typ = attr['type']

76 is_list = attr['list']

78 std = self.STD_READERS.get(typ)

79 if std:

80 return getattr(self, std + '_list' if is_list else std)

82 meta = gid.METADATA.get(typ)

83 if meta:

84 if meta['kind'] == 'object':

85 return self.as_ref_list if is_list else self.as_ref

86 if meta['kind'] == 'struct':

87 return self.as_struct_list if is_list else self.as_struct

88 if meta['kind'] == 'enum':

89 return self.as_enum_list if is_list else self.as_enum

91 return self.as_str_list if is_list else self.as_str

93 ##

95 def count(self, cls, table_name=None):

96 # NB not using db.count to avoid schema introspection

97 sql = f"SELECT COUNT(*) FROM {self.schema}.{table_name or cls.__name__.lower()}"

98 with self.db.connect() as conn:

99 try:

100 rs = list(conn.execute(sa.text(sql)))

101 return rs[0][0]

102 except sa.Error:

103 conn.rollback()

104 return 0

105

106 def read_all(self, cls, table_name=None, uids=None):

107 sql = f"SELECT * FROM {self.schema}.{table_name or cls.__name__.lower()}"

108 if uids:

109 sql += ' WHERE gml_id IN (:uids)'

110 sql = sa.text(sql).bindparams(uids=uids)

111 else:

112 sql = sa.text(sql)

113

114 with self.db.connect() as conn:

115 for row in conn.execute(sql, execution_options={'stream_results': True}):

116 r = gws.u.to_dict(row)

117 o = self.as_struct(cls, '', r)

118 o.identifikator = r.get('gml_id')

119 o.geom = r.get('wkb_geometry', '')

120 yield o

121

122 ##

123

124 def as_struct(self, cls, prop, r):

125 """Read a struct attribute.

126

127 A column ``<prop>_<attribute>`` is used if present, otherwise the plain

128 attribute column.

129

130 Args:

131 cls: GeoInfoDok class of the attribute.

132 prop: Column name prefix of the struct attributes.

133 r: Table row as a dict.

134

135 Returns:

136 An instance of ``cls``.

137 """

138

139 d = {}

140

141 for attr_name, attr_low_name, attr_cls, fn in self.readers[cls.__name__].values():

142 comp_key = prop.lower() + '_' + attr_low_name

143 if comp_key in r:

144 d[attr_name] = fn(attr_cls, comp_key, r)

145 else:

146 d[attr_name] = fn(attr_cls, attr_low_name, r)

147

148 o = cls()

149 vars(o).update(d)

150 return o

151

152 def as_struct_list(self, cls, prop, r):

153 """Read a list of structs from parallel array columns.

154

155 Args:

156 cls: GeoInfoDok class of the attribute.

157 prop: Column name prefix of the struct attributes.

158 r: Table row as a dict.

159

160 Returns:

161 A list of ``cls`` instances.

162 """

163

164 d = {}

165

166 for attr_name, attr_low_name, attr_cls, fn in self.readers[cls.__name__].values():

167 comp_key = prop.lower() + '_' + attr_low_name

168 if comp_key in r:

169 a = _array(r[comp_key])

170 else:

171 a = _array(r.get(attr_low_name))

172 if a:

173 d[attr_low_name] = a

174

175 objs = []

176

177 for vals in zip(*d.values()):

178 r2 = dict(zip(d, vals))

179 o = self.as_struct(cls, '', r2)

180 objs.append(o)

181

182 return objs

183

184 def as_ref(self, cls, prop, r):

185 """Read a reference to another object.

186

187 Args:

188 cls: GeoInfoDok class of the attribute.

189 prop: Column name.

190 r: Table row as a dict.

191

192 Returns:

193 The identifier of the referenced object.

194 """

195

196 return r.get(prop)

197

198 def as_ref_list(self, cls, prop, r):

199 """Read a list of references to other objects.

200

201 Args:

202 cls: GeoInfoDok class of the attribute.

203 prop: Column name.

204 r: Table row as a dict.

205

206 Returns:

207 A list of object identifiers.

208 """

209

210 return _array(r.get(prop))

211

212 def as_enum(self, cls, prop, r):

213 """Read a code list value.

214

215 Args:

216 cls: GeoInfoDok class of the attribute.

217 prop: Column name.

218 r: Table row as a dict.

219

220 Returns:

221 An ``EnumPair``, or ``None`` if the value is not in the code list.

222 """

223

224 v = str(r.get(prop))

225 if v in cls.VALUES:

226 return dt.EnumPair(v, cls.VALUES[v])

227

228 def as_enum_list(self, cls, prop, r):

229 """Read a list of code list values.

230

231 Args:

232 cls: GeoInfoDok class of the attribute.

233 prop: Column name.

234 r: Table row as a dict.

235

236 Returns:

237 A list of ``EnumPair`` objects. Values not in the code list are skipped.

238 """

239

240 ls = []

241

242 for v in _array(r.get(prop)):

243 v = str(v)

244 if v in cls.VALUES:

245 ls.append(dt.EnumPair(v, cls.VALUES[v]))

246

247 return ls

248

249 ##

250

251 def as_ax_lagebezeichnung(self, cls, prop, r):

252 """Read an ``AX_Lagebezeichnung`` attribute.

253

254 Args:

255 cls: GeoInfoDok class of the attribute.

256 prop: Column name.

257 r: Table row as a dict.

258

259 Returns:

260 The unencoded location string if present, otherwise an ``AX_VerschluesselteLagebezeichnung``.

261 """

262

263 if r.get('unverschluesselt'):

264 return r.get('unverschluesselt')

265 return self.as_struct(gid.AX_VerschluesselteLagebezeichnung, prop, r)

266

267 def as_ax_buchung_historischesflurstueck_list(self, cls, prop, r):

268 """Read a list of ``AX_Buchung_HistorischesFlurstueck`` structs.

269

270 Args:

271 cls: GeoInfoDok class of the attribute.

272 prop: Column name.

273 r: Table row as a dict.

274

275 Returns:

276 A list of ``AX_Buchung_HistorischesFlurstueck`` instances with their ``buchungsblattbezirk`` set.

277 """

278

279 # this one is stored as

280 # "blattart": [xxxx],

281 # "buchungsart": ["xxxx"],

282 # ....etc

283 # "buchungsblattbezirk_bezirk": ["xxxx"],

284 # "buchungsblattbezirk_land": ["xx"],

285 # "buchungsblattkennzeichen": ["xxxx"],

286 # "buchungsblattnummermitbuchstabenerweiterung": ["xxxx"],

287

288 bbs = self.as_struct_list(gid.AX_Buchung_HistorischesFlurstueck, '', r)

289 bzs = self.as_struct_list(gid.AX_Buchungsblattbezirk_Schluessel, 'buchungsblattbezirk', r)

290 objs = []

291 for bb, bz in zip(bbs, bzs):

292 bb.buchungsblattbezirk = bz

293 objs.append(bb)

294 return objs

295

296 ##

297

298 def as_str(self, cls, prop, r):

299 """Read a string.

300

301 Args:

302 cls: GeoInfoDok class of the attribute.

303 prop: Column name.

304 r: Table row as a dict.

305

306 Returns:

307 A stripped string, or ``None`` if the value is empty.

308 """

309

310 return _str(r.get(prop))

311

312 def as_str_list(self, cls, prop, r):

313 """Read a list of strings.

314

315 Args:

316 cls: GeoInfoDok class of the attribute.

317 prop: Column name.

318 r: Table row as a dict.

319

320 Returns:

321 A list of stripped strings, empty values become ``None``.

322 """

323

324 return [_str(v) for v in _array(r.get(prop))]

325

326 def as_bool(self, cls, prop, r):

327 """Read a boolean.

328

329 Args:

330 cls: GeoInfoDok class of the attribute.

331 prop: Column name.

332 r: Table row as a dict.

333

334 Returns:

335 A boolean, or ``None`` if the value is missing.

336 """

337

338 return _bool(r.get(prop))

339

340 def as_bool_list(self, cls, prop, r):

341 """Read a list of booleans.

342

343 Args:

344 cls: GeoInfoDok class of the attribute.

345 prop: Column name.

346 r: Table row as a dict.

347

348 Returns:

349 A list of booleans.

350 """

351

352 return [_bool(v) for v in _array(r.get(prop))]

353

354 def as_int(self, cls, prop, r):

355 """Read an integer.

356

357 Args:

358 cls: GeoInfoDok class of the attribute.

359 prop: Column name.

360 r: Table row as a dict.

361

362 Returns:

363 An integer, or ``None`` if the value is missing.

364 """

365

366 return _int(r.get(prop))

367

368 def as_int_list(self, cls, prop, r):

369 """Read a list of integers.

370

371 Args:

372 cls: GeoInfoDok class of the attribute.

373 prop: Column name.

374 r: Table row as a dict.

375

376 Returns:

377 A list of integers.

378 """

379

380 return [_int(v) for v in _array(r.get(prop))]

381

382 def as_float(self, cls, prop, r):

383 """Read a float.

384

385 Args:

386 cls: GeoInfoDok class of the attribute.

387 prop: Column name.

388 r: Table row as a dict.

389

390 Returns:

391 A float, or ``None`` if the value is missing.

392 """

393

394 return _float(r.get(prop))

395

396 def as_float_list(self, cls, prop, r):

397 """Read a list of floats.

398

399 Args:

400 cls: GeoInfoDok class of the attribute.

401 prop: Column name.

402 r: Table row as a dict.

403

404 Returns:

405 A list of floats.

406 """

407

408 return [_float(v) for v in _array(r.get(prop))]

409

410 def as_date(self, cls, prop, r):

411 """Read a date or a date and time.

412

413 ISO strings are parsed, other values are returned as is.

414

415 Args:

416 cls: GeoInfoDok class of the attribute.

417 prop: Column name.

418 r: Table row as a dict.

419

420 Returns:

421 A date or datetime, or ``None`` if the value is empty.

422 """

423

424 return _datetime(r.get(prop))

425

426 def as_date_list(self, cls, prop, r):

427 """Read a list of dates.

428

429 Args:

430 cls: GeoInfoDok class of the attribute.

431 prop: Column name.

432 r: Table row as a dict.

433

434 Returns:

435 A list of dates or datetimes.

436 """

437

438 return [_datetime(v) for v in _array(r.get(prop))]

439

440

441def _str(v):

442 """Convert to a stripped string, ``None`` if empty."""

443

444 if v is not None:

445 v = str(v).strip()

446 if v:

447 return v

448

449

450def _bool(v):

451 return bool(v) if v is not None else None

452

453

454def _int(v):

455 return int(v) if v is not None else None

456

457

458def _float(v):

459 return float(v) if v is not None else None

460

461

462def _datetime(v):

463 """Convert an ISO string to a datetime, pass other values through."""

464

465 if not v:

466 return None

467 if isinstance(v, str):

468 return gws.lib.datetimex.from_iso_string(v)

469 return v

470

471

472def _array(val):

473 """Wrap a scalar into a list, ``None`` becomes an empty list."""

474

475 if isinstance(val, list):

476 return val

477 if val is None:

478 return []

479 return [val]