Coverage for gws-app/gws/plugin/gekos/index.py: 0%

1"""GekoS index, loaded from gek-online.""" 

2 

3 

4import math 

5 

6import gws 

7import gws.lib.shape 

8import gws.config.util 

9import gws.lib.crs 

10import gws.lib.xmlx 

11import gws.lib.net 

12import gws.lib.sa as sa 

13from . import core 

14 

15 

16""" 

17Gekos-Online can be called with different "instance" parameters or no "instance" at all 

18 

19The xml structure is like this: 

20  

21 <?xml version="1.0" encoding="ISO-8859-1" standalone="yes"?> 

22 <OnlineTreffer> 

23 <Vorgang> 

24 <AntragsartID>..</AntragsartID> 

25 <SystemNr>...</SystemNr> 

26 <X>407000.000</X> 

27 <Y>5716000.000</Y> 

28 <ObjectID>1</ObjectID> 

29 <Verfahren>...</Verfahren> 

30 <AntragsartBez>...</AntragsartBez> 

31 <Darstellung>...</Darstellung> 

32 <Massnahme>....</Massnahme> 

33 <Tooltip>...</Tooltip> 

34 <UrlFV>...</UrlFV> 

35 <UrlOL>...</UrlOL> 

36 </Vorgang> 

37 <Vorgang> 

38 .... 

39  

40ObjectID appears to be unique within an instance, so we generate a PK = instance_ObjectID  

41  

42""" 

43 

44 

45class Object(gws.Node): 

46 """GekoS index. 

47 

48 Loads GekoS records from gek-online and stores them in a PostGIS table. 

49 """ 

50 

51 db: gws.DatabaseProvider 

52 """Database provider for the index table.""" 

53 tableName: str 

54 """Name of the index table.""" 

55 position: core.PositionConfig 

56 """Position correction for points, or ``None``.""" 

57 crs: gws.Crs 

58 """CRS of the GekoS coordinates.""" 

59 

60 def configure(self): 

61 gws.config.util.configure_database_provider_for(self, ext_type='postgres') 

62 self.tableName = self.cfg('tableName') 

63 self.crs = gws.lib.crs.get(self.cfg('crs')) 

64 self.position = self.cfg('position') 

65 

66 def create(self): 

67 """Load the records of all sources and recreate the index table. 

68 

69 The table is dropped, created again and filled with the records. 

70 """ 

71 recs = self._collect() 

72 self._write(recs) 

73 

74 def _collect(self): 

75 """Load and transform the records of all sources.""" 

76 recs = [] 

77 

78 for source in self.cfg('sources'): 

79 rs = self._load(source) 

80 gws.log.info(f'loaded {len(rs)} records from {source.instance!r}') 

81 rs = self._transform(rs, source.instance) 

82 recs.extend(rs) 

83 

84 return recs 

85 

86 def _load(self, source: core.SourceConfig): 

87 """Load the records of a gek-online source as dicts.""" 

88 

89 res = gws.lib.net.http_request(source.url, params=dict(source.params or {})) 

90 res.raise_if_failed() 

91 xml = gws.lib.xmlx.from_string((res.text or '').strip(), gws.XmlOptions(removeNamespaces=True)) 

92 

93 rs = [] 

94 

95 for node in xml.findall('Vorgang'): 

96 rec = {} 

97 for tag in node: 

98 rec[tag.name] = tag.text 

99 rs.append(rec) 

100 

101 return rs 

102 

103 def _transform(self, recs, instance_name): 

104 """Add uids and point geometries to records, skipping records without coordinates or ``ObjectID`` and duplicate uids.""" 

105 

106 recs2 = [] 

107 points = set() 

108 uids = set() 

109 

110 for rec in recs: 

111 if 'X' not in rec or 'Y' not in rec: 

112 continue 

113 

114 if 'ObjectID' not in rec: 

115 gws.log.warning(f'record without ObjectID ignored in {instance_name=}') 

116 continue 

117 

118 xy = self._free_point( 

119 float(rec.pop('X')), 

120 float(rec.pop('Y')), 

121 points, 

122 ) 

123 points.add(xy) 

124 

125 rec['instance'] = instance_name 

126 

127 uid = instance_name + '_' + str(rec['ObjectID']) 

128 if uid in uids: 

129 gws.log.warning(f'non-unique {uid=} ignored') 

130 continue 

131 uids.add(uid) 

132 rec['uid'] = uid 

133 

134 shape = gws.lib.shape.from_geojson( 

135 {'type': 'Point', 'coordinates': xy}, 

136 self.crs, 

137 ) 

138 rec['wkb_geometry'] = shape.to_ewkb_hex() 

139 

140 recs2.append(rec) 

141 

142 return recs2 

143 

144 def _free_point(self, x, y, points): 

145 """Apply the position correction to a point.""" 

146 

147 if not self.position: 

148 return x, y 

149 

150 # move a point by specified offsets 

151 

152 x = round(x, 3) + self.position.offsetX 

153 y = round(y, 3) + self.position.offsetY 

154 

155 if (x, y) not in points: 

156 return x, y 

157 

158 # if two or more points share the same XY, 

159 # arrange them in a circle around XY 

160 

161 distance = self.position.distance 

162 angle = self.position.angle 

163 

164 if not distance or not angle: 

165 return x, y 

166 

167 for a in range(0, 360, angle): 

168 a = math.radians(a) 

169 xa = round(x + distance * math.cos(a), 3) 

170 ya = round(y + distance * math.sin(a), 3) 

171 

172 if (xa, ya) not in points: 

173 return xa, ya 

174 

175 return x, y 

176 

177 def _write(self, recs): 

178 """Recreate the index table and insert the records.""" 

179 columns = [ 

180 sa.Column('uid', sa.Text, primary_key=True), 

181 sa.Column('ObjectID', sa.Text), 

182 sa.Column('AntragsartBez', sa.Text), 

183 sa.Column('AntragsartID', sa.Integer, index=True), 

184 sa.Column('Darstellung', sa.Text), 

185 sa.Column('Massnahme', sa.Text), 

186 sa.Column('SystemNr', sa.Text), 

187 sa.Column('status', sa.Text), 

188 sa.Column('Tooltip', sa.Text), 

189 sa.Column('UrlFV', sa.Text), 

190 sa.Column('UrlOL', sa.Text), 

191 sa.Column('Verfahren', sa.Text), 

192 sa.Column('instance', sa.Text), 

193 sa.Column('wkb_geometry', sa.geo.Geometry(geometry_type='POINT', srid=self.crs.srid), index=True), 

194 ] 

195 

196 schema, name = self.db.split_table_name(self.tableName) 

197 sa_meta = sa.MetaData(schema=schema) 

198 table = sa.Table(name, sa_meta, *columns, schema=schema) 

199 

200 with self.db.begin() as conn: 

201 table.drop(conn.saConn, checkfirst=True) 

202 table.create(conn.saConn) 

203 if recs: 

204 conn.execute(sa.insert(table), recs) 

205 

206 gws.log.info(f'saved {len(recs)} records in {schema}.{name}')