Coverage for gws-app/gws/plugin/gekos/index.py: 0%

1"""Query the Gekos-Online server and create the index.""" 

2 

3 

4import math 

5 

6import gws 

7import gws.base.shape 

8import gws.config.util 

9import gws.lib.crs 

10import gws.lib.xmlx 

11import gws.lib.net 

12import gws.lib.sa as sa 

13from . import core 

14 

15 

16""" 

17Gekos-Online can be called with different "instance" parameters or no "instance" at all 

18 

19The xml structure is like this: 

20  

21 <?xml version="1.0" encoding="ISO-8859-1" standalone="yes"?> 

22 <OnlineTreffer> 

23 <Vorgang> 

24 <AntragsartID>..</AntragsartID> 

25 <SystemNr>...</SystemNr> 

26 <X>407000.000</X> 

27 <Y>5716000.000</Y> 

28 <ObjectID>1</ObjectID> 

29 <Verfahren>...</Verfahren> 

30 <AntragsartBez>...</AntragsartBez> 

31 <Darstellung>...</Darstellung> 

32 <Massnahme>....</Massnahme> 

33 <Tooltip>...</Tooltip> 

34 <UrlFV>...</UrlFV> 

35 <UrlOL>...</UrlOL> 

36 </Vorgang> 

37 <Vorgang> 

38 .... 

39  

40ObjectID appears to be unique within an instance, so we generate a PK = instance_ObjectID  

41  

42""" 

43 

44 

45class Object(gws.Node): 

46 db: gws.DatabaseProvider 

47 tableName: str 

48 position: core.PositionConfig 

49 crs: gws.Crs 

50 

51 def configure(self): 

52 gws.config.util.configure_database_provider_for(self, ext_type='postgres') 

53 self.tableName = self.cfg('tableName') 

54 self.crs = gws.lib.crs.get(self.cfg('crs')) 

55 self.position = self.cfg('position') 

56 

57 def create(self): 

58 recs = self._collect() 

59 self._write(recs) 

60 

61 def _collect(self): 

62 recs = [] 

63 

64 for source in self.cfg('sources'): 

65 rs = self._load(source) 

66 gws.log.info(f'loaded {len(rs)} records from {source.instance!r}') 

67 rs = self._transform(rs, source.instance) 

68 recs.extend(rs) 

69 

70 return recs 

71 

72 def _load(self, source: core.SourceConfig): 

73 """Load XML from GekOnline and create record dicts.""" 

74 

75 res = gws.lib.net.http_request(source.url, params=dict(source.params or {}), verify=False) 

76 res.raise_if_failed() 

77 xml = gws.lib.xmlx.from_string((res.text or '').strip()) 

78 

79 rs = [] 

80 

81 for node in xml.findall('Vorgang'): 

82 rec = {} 

83 for tag in node: 

84 rec[tag.name] = tag.text 

85 rs.append(rec) 

86 

87 return rs 

88 

89 def _transform(self, recs, instance_name): 

90 """Compute geometries and uids for record dicts.""" 

91 

92 recs2 = [] 

93 points = set() 

94 uids = set() 

95 

96 for rec in recs: 

97 if 'X' not in rec or 'Y' not in rec: 

98 continue 

99 

100 xy = self._free_point( 

101 float(rec.pop('X')), 

102 float(rec.pop('Y')), 

103 points, 

104 ) 

105 points.add(xy) 

106 

107 rec['instance'] = instance_name 

108 

109 uid = instance_name + '_' + str(rec['ObjectID']) 

110 if uid in uids: 

111 gws.log.warning(f'non-unique {uid=} ignored') 

112 continue 

113 uids.add(uid) 

114 rec['uid'] = uid 

115 

116 shape = gws.base.shape.from_geojson( 

117 {'type': 'Point', 'coordinates': xy}, 

118 self.crs, 

119 ) 

120 rec['wkb_geometry'] = shape.to_ewkb_hex() 

121 

122 recs2.append(rec) 

123 

124 return recs2 

125 

126 def _free_point(self, x, y, points): 

127 """Move points around, according to the 'position' config.""" 

128 

129 if not self.position: 

130 return x, y 

131 

132 # move a point by specified offsets 

133 

134 x = round(x, 3) + self.position.offsetX 

135 y = round(y, 3) + self.position.offsetY 

136 

137 if (x, y) not in points: 

138 return x, y 

139 

140 # if two or more points share the same XY, 

141 # arrange them in a circle around XY 

142 

143 distance = self.position.distance 

144 angle = self.position.angle 

145 

146 if not distance: 

147 return x, y 

148 

149 for a in range(0, 360, angle): 

150 a = math.radians(a) 

151 xa = round(x + distance * math.cos(a)) 

152 ya = round(y + distance * math.sin(a)) 

153 

154 if (xa, ya) not in points: 

155 return xa, ya 

156 

157 return x, y 

158 

159 def _write(self, recs): 

160 columns = [ 

161 sa.Column('uid', sa.Text, primary_key=True), 

162 sa.Column('ObjectID', sa.Text), 

163 sa.Column('AntragsartBez', sa.Text), 

164 sa.Column('AntragsartID', sa.Integer, index=True), 

165 sa.Column('Darstellung', sa.Text), 

166 sa.Column('Massnahme', sa.Text), 

167 sa.Column('SystemNr', sa.Text), 

168 sa.Column('status', sa.Text), 

169 sa.Column('Tooltip', sa.Text), 

170 sa.Column('UrlFV', sa.Text), 

171 sa.Column('UrlOL', sa.Text), 

172 sa.Column('Verfahren', sa.Text), 

173 sa.Column('instance', sa.Text), 

174 sa.Column('wkb_geometry', sa.geo.Geometry(geometry_type='POINT', srid=self.crs.srid), index=True), 

175 ] 

176 

177 schema, name = self.db.split_table_name(self.tableName) 

178 sa_meta = sa.MetaData(schema=schema) 

179 table = sa.Table(name, sa_meta, *columns, schema=schema) 

180 

181 with self.db.connect() as conn: 

182 table.drop(conn.saConn, checkfirst=True) 

183 table.create(conn.saConn) 

184 conn.commit() 

185 conn.execute(sa.insert(table).values(recs)) 

186 conn.commit() 

187 

188 gws.log.info(f'saved {len(recs)} records in {schema}.{name}')