Coverage for gws-app/gws/plugin/gekos/index.py: 0%

1"""GekoS index, loaded from gek-online.""" 

2 

3 

4import math 

5 

6import gws 

7import gws.lib.shape 

8import gws.config.util 

9import gws.lib.crs 

10import gws.lib.xmlx 

11import gws.lib.net 

12import gws.lib.sa as sa 

13from . import core 

14 

15 

16""" 

17Gekos-Online can be called with different "instance" parameters or no "instance" at all 

18 

19The xml structure is like this: 

20  

21 <?xml version="1.0" encoding="ISO-8859-1" standalone="yes"?> 

22 <OnlineTreffer> 

23 <Vorgang> 

24 <AntragsartID>..</AntragsartID> 

25 <SystemNr>...</SystemNr> 

26 <X>407000.000</X> 

27 <Y>5716000.000</Y> 

28 <ObjectID>1</ObjectID> 

29 <Verfahren>...</Verfahren> 

30 <AntragsartBez>...</AntragsartBez> 

31 <Darstellung>...</Darstellung> 

32 <Massnahme>....</Massnahme> 

33 <Tooltip>...</Tooltip> 

34 <UrlFV>...</UrlFV> 

35 <UrlOL>...</UrlOL> 

36 </Vorgang> 

37 <Vorgang> 

38 .... 

39  

40ObjectID appears to be unique within an instance, so we generate a PK = instance_ObjectID  

41  

42""" 

43 

44 

45class Object(gws.Node): 

46 """GekoS index. 

47 

48 Loads GekoS records from gek-online and stores them in a PostGIS table. 

49 """ 

50 

51 db: gws.DatabaseProvider 

52 """Database provider for the index table.""" 

53 tableName: str 

54 """Name of the index table.""" 

55 position: core.PositionConfig 

56 """Position correction for points, or ``None``.""" 

57 crs: gws.Crs 

58 """CRS of the GekoS coordinates.""" 

59 

60 def configure(self): 

61 gws.config.util.configure_database_provider_for(self, ext_type='postgres') 

62 self.tableName = self.cfg('tableName') 

63 self.crs = gws.lib.crs.get(self.cfg('crs')) 

64 self.position = self.cfg('position') 

65 

66 def create(self): 

67 """Load the records of all sources and recreate the index table. 

68 

69 The table is dropped, created again and filled with the records. 

70 """ 

71 recs = self._collect() 

72 self._write(recs) 

73 

74 def _collect(self): 

75 """Load and transform the records of all sources.""" 

76 recs = [] 

77 

78 for source in self.cfg('sources'): 

79 rs = self._load(source) 

80 gws.log.info(f'loaded {len(rs)} records from {source.instance!r}') 

81 rs = self._transform(rs, source.instance) 

82 recs.extend(rs) 

83 

84 return recs 

85 

86 def _load(self, source: core.SourceConfig): 

87 """Load the records of a gek-online source as dicts.""" 

88 

89 res = gws.lib.net.http_request(source.url, params=dict(source.params or {}), verify=False) 

90 res.raise_if_failed() 

91 xml = gws.lib.xmlx.from_string((res.text or '').strip(), gws.XmlOptions(removeNamespaces=True)) 

92 

93 rs = [] 

94 

95 for node in xml.findall('Vorgang'): 

96 rec = {} 

97 for tag in node: 

98 rec[tag.name] = tag.text 

99 rs.append(rec) 

100 

101 return rs 

102 

103 def _transform(self, recs, instance_name): 

104 """Add uids and point geometries to records, skipping duplicate uids.""" 

105 

106 recs2 = [] 

107 points = set() 

108 uids = set() 

109 

110 for rec in recs: 

111 if 'X' not in rec or 'Y' not in rec: 

112 continue 

113 

114 xy = self._free_point( 

115 float(rec.pop('X')), 

116 float(rec.pop('Y')), 

117 points, 

118 ) 

119 points.add(xy) 

120 

121 rec['instance'] = instance_name 

122 

123 uid = instance_name + '_' + str(rec['ObjectID']) 

124 if uid in uids: 

125 gws.log.warning(f'non-unique {uid=} ignored') 

126 continue 

127 uids.add(uid) 

128 rec['uid'] = uid 

129 

130 shape = gws.lib.shape.from_geojson( 

131 {'type': 'Point', 'coordinates': xy}, 

132 self.crs, 

133 ) 

134 rec['wkb_geometry'] = shape.to_ewkb_hex() 

135 

136 recs2.append(rec) 

137 

138 return recs2 

139 

140 def _free_point(self, x, y, points): 

141 """Apply the position correction to a point.""" 

142 

143 if not self.position: 

144 return x, y 

145 

146 # move a point by specified offsets 

147 

148 x = round(x, 3) + self.position.offsetX 

149 y = round(y, 3) + self.position.offsetY 

150 

151 if (x, y) not in points: 

152 return x, y 

153 

154 # if two or more points share the same XY, 

155 # arrange them in a circle around XY 

156 

157 distance = self.position.distance 

158 angle = self.position.angle 

159 

160 if not distance: 

161 return x, y 

162 

163 for a in range(0, 360, angle): 

164 a = math.radians(a) 

165 xa = round(x + distance * math.cos(a)) 

166 ya = round(y + distance * math.sin(a)) 

167 

168 if (xa, ya) not in points: 

169 return xa, ya 

170 

171 return x, y 

172 

173 def _write(self, recs): 

174 """Recreate the index table and insert the records.""" 

175 columns = [ 

176 sa.Column('uid', sa.Text, primary_key=True), 

177 sa.Column('ObjectID', sa.Text), 

178 sa.Column('AntragsartBez', sa.Text), 

179 sa.Column('AntragsartID', sa.Integer, index=True), 

180 sa.Column('Darstellung', sa.Text), 

181 sa.Column('Massnahme', sa.Text), 

182 sa.Column('SystemNr', sa.Text), 

183 sa.Column('status', sa.Text), 

184 sa.Column('Tooltip', sa.Text), 

185 sa.Column('UrlFV', sa.Text), 

186 sa.Column('UrlOL', sa.Text), 

187 sa.Column('Verfahren', sa.Text), 

188 sa.Column('instance', sa.Text), 

189 sa.Column('wkb_geometry', sa.geo.Geometry(geometry_type='POINT', srid=self.crs.srid), index=True), 

190 ] 

191 

192 schema, name = self.db.split_table_name(self.tableName) 

193 sa_meta = sa.MetaData(schema=schema) 

194 table = sa.Table(name, sa_meta, *columns, schema=schema) 

195 

196 with self.db.connect() as conn: 

197 table.drop(conn.saConn, checkfirst=True) 

198 table.create(conn.saConn) 

199 conn.commit() 

200 conn.execute(sa.insert(table).values(recs)) 

201 conn.commit() 

202 

203 gws.log.info(f'saved {len(recs)} records in {schema}.{name}')