Coverage for gws-app/gws/base/web/wsgi.py: 80%

1"""Basic WSGI request/response handling."""

3import gzip

4import io

5import os

6from typing import cast

8import urllib.parse

9import werkzeug.formparser

10import werkzeug.utils

11import werkzeug.wrappers

12import werkzeug.wsgi

14import gws

15import gws.lib.jsonx

16import gws.lib.mime

17import gws.lib.vendor.umsgpack as umsgpack

19from . import error

22class Responder(gws.WebResponder):

23 def __init__(self, **kwargs):

24 if 'wz' in kwargs:

25 self._wz = kwargs['wz']

26 else:

27 self._wz = werkzeug.wrappers.Response(**kwargs)

28 self.status = self._wz.status_code

30 def __repr__(self):

31 return f'<Responder {self._wz}>'

33 def send_response(self, environ, start_response):

34 return self._wz(environ, start_response)

36 def set_cookie(self, key, value, **kwargs):

37 self._wz.set_cookie(key, value, **kwargs)

39 def delete_cookie(self, key, **kwargs):

40 self._wz.delete_cookie(key, **kwargs)

42 def add_header(self, key, value):

43 self._wz.headers.add(key, value)

45 def set_status(self, status):

46 self._wz.status_code = int(status)

47 self.status = self._wz.status_code

49 def set_body(self, body):

50 if isinstance(body, str):

51 body = body.encode('utf-8')

52 self._wz.set_data(body)

55class Requester(gws.WebRequester):

56 _STRUCT_JSON = 'json'

57 _STRUCT_MSGPACK = 'msgpack'

59 _struct_mime = {

60 _STRUCT_JSON: 'application/json',

61 _STRUCT_MSGPACK: 'application/msgpack',

62 }

64 def __init__(self, root: gws.Root, environ: dict, site: gws.WebSite, **kwargs):

65 if 'wz' in kwargs:

66 self._wz = kwargs['wz']

67 else:

68 self._wz = werkzeug.wrappers.Request(environ)

70 # this is also set in nginx (see server/ini), but we need this for unzipping (see data() below)

71 self.maxContentLength = int(root.app.cfg('server.web.maxRequestLength') or 1) * 1024 * 1024

72 self._wz.max_content_length = self.maxContentLength

74 self.root = root

75 self.site = site

77 self.environ = self._wz.environ

78 self.method = cast(gws.RequestMethod, self._wz.method.upper())

80 self.scheme = 'http'

81 self.host = ''

82 self.port = 0

83 self.ip = self.environ.get('REMOTE_ADDR', '')

84 self.isSecure = False

86 self.session = root.app.authMgr.guestSession

87 self.user = root.app.authMgr.guestUser

89 self.isGet = self.method == gws.RequestMethod.GET

90 self.isPost = self.method == gws.RequestMethod.POST

91 self.isForm = False

92 self.isApi = False

94 self.structInput = None

95 self.structOutput = None

97 self.contentTypeHeader = self.header('content-type', '').lower().split(';')[0].strip()

98 self.contentType = gws.lib.mime.get(self.contentTypeHeader) or gws.lib.mime.BIN

100 if self.isPost:

101 if self.contentTypeHeader == 'application/x-www-form-urlencoded' or self.contentTypeHeader == 'multipart/form-data':

102 self.isForm = True

103 else:

104 self.structInput = self._struct_type(self.contentTypeHeader)

105 if self.structInput:

106 self.isApi = True

107 self.structOutput = self._struct_type(self.header('accept')) or self.structInput

108

109 self._parsed_params = {}

110 self._parsed_params_lc = {}

111 self._parsed_query_params = {}

112 self._parsed_struct = {}

113 self._parsed_command = ''

114 self._parsed_path = ''

115 self._parsed = False

116 self._raw_post_data = None

117 self._uid = gws.u.mstime()

118

119 if self.root.app.developer_option('request.log_all'):

120 u = {

121 'method': self.method,

122 'path': self._wz.path,

123 'query': self._wz.query_string,

124 'headers': self._wz.headers,

125 'environ': self._wz.environ,

126 }

127 gws.u.write_debug_file(f'request_{self._uid}', ''.join(f'{k}={v!r}\n' for k, v in u.items()))

128

129 def __repr__(self):

130 return f'<Requester {self._wz}>'

131

132 def parse(self):

133 self._parse()

134

135 def params(self):

136 self._parse()

137 return self._parsed_params

138

139 def query_params(self):

140 self._parse()

141 return self._parsed_query_params

142

143 def path(self):

144 self._parse()

145 return self._parsed_path

146

147 def struct(self):

148 self._parse()

149 return self._parsed_struct

150

151 def command(self):

152 self._parse()

153 return self._parsed_command

154

155 def data(self):

156 if not self.isPost:

157 return b''

158

159 if self._raw_post_data is not None:

160 return self._raw_post_data

161

162 cl = self.header('content-length')

163 if not cl:

164 self._raw_post_data = b''

165 return self._raw_post_data

166 try:

167 cl = int(cl)

168 except ValueError as exc:

169 raise error.BadRequest('invalid content-length header') from exc

170 if cl == 0:

171 self._raw_post_data = b''

172 return self._raw_post_data

173 if cl > self.maxContentLength:

174 raise error.RequestEntityTooLarge(f'content-length header too large: {cl}')

175

176 data = self._wz.get_data(as_text=False, cache=False, parse_form_data=False)

177

178 if self.root.app.developer_option('request.log_all'):

179 gws.u.write_debug_file(f'request_{self._uid}.data', data)

180

181 if self.header('content-encoding') == 'gzip':

182 try:

183 with gzip.GzipFile(fileobj=io.BytesIO(data)) as fp:

184 data = fp.read(self.maxContentLength)

185 except OSError as exc:

186 raise error.BadRequest('gzip data error') from exc

187

188 self._raw_post_data = data

189 return data

190

191 def text(self):

192 data = self.data()

193 if not data:

194 return ''

195

196 charset = self._wz.mimetype_params.get('charset', 'utf-8')

197 try:

198 return data.decode(encoding=charset, errors='strict')

199 except UnicodeDecodeError as exc:

200 raise error.BadRequest('text data decoding error') from exc

201

202 def form(self):

203 if not self.isForm:

204 return []

205 data = self.data()

206 if not data:

207 return []

208

209 try:

210 stream = io.BytesIO(data)

211 opts = self._wz.mimetype_params

212

213 # Fix for Qt multipart/form-data boundaries

214 # Qt boundaries start with "boundary_.oOo._" and are base64-encoded

215 # https://github.com/qt/qtbase/blob/04b7fc2de3c97174a725bbd4fdc0f6e496c85861/src/network/access/qhttpmultipart.cpp#L400

216 # Werkzeug header parser does not understand base64 chars "/+="

217 if data.startswith(b'--boundary_.oOo._'):

218 boundary = data[2 : data.find(b'\r\n')].decode('ascii')

219 opts['boundary'] = boundary

220

221 parser = werkzeug.formparser.FormDataParser(silent=False)

222 _, form, files = parser.parse(

223 stream,

224 mimetype=self.contentTypeHeader,

225 content_length=len(data),

226 options=opts,

227 )

228 return list(form.items()) + list(files.items())

229 except Exception as exc:

230 raise error.BadRequest(f'form decode error: {exc}') from exc

231

232 def env(self, key, default=''):

233 return self._wz.environ.get(key, default)

234

235 def has_param(self, key):

236 self._parse()

237 return key.lower() in self._parsed_params_lc

238

239 def param(self, key, default=''):

240 self._parse()

241 return self._parsed_params_lc.get(key.lower(), default)

242

243 def header(self, key, default=''):

244 return self._wz.headers.get(key, default)

245

246 def cookie(self, key, default=''):

247 return self._wz.cookies.get(key, default)

248

249 def content_responder(self, response):

250 args: dict = {

251 'mimetype': response.mime,

252 'status': response.status or 200,

253 'headers': {},

254 'direct_passthrough': False,

255 }

256

257 if response.contentFilename:

258 if response.contentFilename.isascii():

259 args['headers']['Content-Disposition'] = f'attachment; filename="{response.contentFilename}"'

260 else:

261 args['headers']['Content-Disposition'] = 'attachment; filename="{}"; filename*=UTF-8\'\'{}'.format(

262 response.contentFilename.encode('ascii', errors='replace').decode('ascii'),

263 urllib.parse.quote(response.contentFilename, safe=''),

264 )

265 args['mimetype'] = args['mimetype'] or gws.lib.mime.for_path(response.contentFilename)

266

267 if response.contentPath:

268 args['response'] = werkzeug.wsgi.wrap_file(self.environ, open(response.contentPath, 'rb'))

269 args['headers']['Content-Length'] = str(os.path.getsize(response.contentPath))

270 args['mimetype'] = args['mimetype'] or gws.lib.mime.for_path(response.contentPath)

271 args['direct_passthrough'] = True

272 else:

273 args['response'] = response.content

274

275 if response.headers:

276 args['headers'].update(response.headers)

277

278 return Responder(**args)

279

280 def redirect_responder(self, response):

281 wz = werkzeug.utils.redirect(response.location, response.status or 302)

282 if response.headers:

283 wz.headers.update(response.headers)

284 return Responder(wz=wz)

285

286 def api_responder(self, response):

287 typ = self.structOutput or self._STRUCT_JSON

288 return Responder(

289 response=self._encode_struct(response, typ),

290 mimetype=self._struct_mime[typ],

291 status=response.status or 200,

292 )

293

294 def error_responder(self, exc):

295 err = exc if isinstance(exc, error.HTTPException) else error.InternalServerError()

296 return Responder(wz=err.get_response(self._wz.environ))

297

298 ##

299

300 def absolute_url_for(self, path, **params):

301 return self.site.url_for(self, path, 'absolute', **params)

302

303 def relative_url_for(self, path, **params):

304 return self.site.url_for(self, path, 'relative', **params)

305

306 def canonical_url_for(self, path, **params):

307 return self.site.url_for(self, path, 'canonical', **params)

308

309 ##

310

311 def set_session(self, sess):

312 self.session = sess

313 self.user = sess.user

314

315 ##

316

317 _CMD_PARAM_NAME = 'cmd'

318

319 def _parse(self):

320 if not self._parsed:

321 self._parsed = True

322 self._parse2()

323

324 def _parse2(self):

325 self._parse_origin()

326

327 # the server only understands requests to /_ or /_/commandName

328 # GET params can be given as query string or encoded in the path

329 # like _/commandName/param1/value1/param2/value2 etc

330

331 path = self._wz.path

332 path_parts = None

333

334 if path == gws.c.SERVER_ENDPOINT:

335 # example.com/_

336 # the cmd param is expected to be in the query string or json

337 cmd = ''

338 elif path.startswith(gws.c.SERVER_ENDPOINT + '/'):

339 # example.com/_/someCommand

340 # the cmd param is in the url

341 path_parts = path.split('/')

342 cmd = path_parts[2]

343 path_parts = path_parts[3:]

344 self._parsed_path = '/'.join(path_parts)

345 else:

346 raise error.NotFound(f'invalid request path: {path!r}')

347

348 if self.structInput:

349 self._parsed_struct = self._decode_struct(self.structInput)

350 self._parsed_command = cmd or self._parsed_struct.pop(self._CMD_PARAM_NAME, '')

351 else:

352 d = dict(self._wz.args)

353 if path_parts:

354 for n in range(1, len(path_parts), 2):

355 d[path_parts[n - 1]] = path_parts[n]

356 self._parsed_command = cmd or d.pop(self._CMD_PARAM_NAME, '')

357 self._parsed_params = d

358 self._parsed_params_lc = {k.lower(): v for k, v in d.items()}

359 self._parsed_query_params = dict(self._wz.args)

360

361 def _parse_origin(self):

362 scheme = 'https' if self.site.ssl or self._wz.is_secure else 'http'

363 host = self.environ.get('HTTP_HOST', '')

364 ip = self.environ.get('REMOTE_ADDR', '')

365 fwd_port = ''

366

367 if self.site.proxyCount > 0:

368 fwd = self._parse_proxy_headers()

369 if fwd['host']:

370 host = fwd['host']

371 if fwd['port']:

372 fwd_port = fwd['port']

373 if fwd['scheme'] and not self.site.ssl:

374 scheme = fwd['scheme']

375 if fwd['ip']:

376 ip = fwd['ip']

377

378 host, _, port = host.strip().lower().partition(':')

379 port = fwd_port or port

380

381 try:

382 port = int(port)

383 except ValueError:

384 port = 0

385

386 if self.site.hostnames and host not in self.site.hostnames:

387 raise error.BadRequest(f'invalid host {host!r}')

388

389 self.scheme = scheme

390 self.isSecure = scheme == 'https'

391 self.host = host

392 self.port = port

393 self.ip = ip

394

395 def _parse_proxy_headers(self):

396 # read the client's scheme, host, port and address from the X-Forwarded headers

397 # each value is returned as given, or as None if it cannot be trusted or is absent,

398 # the caller is expected to provide the fallbacks

399 #

400 # X-Forwarded-For is a list to which every proxy appends the address it sees,

401 # so with 'proxyCount' proxies in front of us, the client is the 'proxyCount'-th

402 # element from the right. everything to the left of it originates from the client

403 # and is not trustworthy, therefore a list which is too short yields no address at all

404 #

405 # X-Forwarded-Host, -Port and -Proto are normally set, not appended, by the outermost

406 # proxy, and their lists are therefore shorter than the number of proxies.

407 # for these, the leftmost element is used

408 #

409 # @TODO 'proxyCount' counts the proxies, it does not identify them. a request which

410 # arrives over a shorter path, e.g. a monitoring client which bypasses the proxies,

411 # is resolved against the wrong element. verifying the peer requires a list of

412 # trusted proxy addresses

413

414 d: dict = {'scheme': None, 'host': None, 'port': None, 'ip': None}

415

416 p = self.environ.get('HTTP_X_FORWARDED_HOST', '').split(',')[0].strip()

417 if p:

418 d['host'] = p

419

420 p = self.environ.get('HTTP_X_FORWARDED_PORT', '').split(',')[0].strip()

421 if p:

422 d['port'] = p

423

424 p = self.environ.get('HTTP_X_FORWARDED_PROTO', '').split(',')[0].strip().lower()

425 if p in ('http', 'https'):

426 d['scheme'] = p

427

428 lst = [s.strip() for s in self.environ.get('HTTP_X_FORWARDED_FOR', '').split(',')]

429 if len(lst) >= self.site.proxyCount:

430 p = lst[-self.site.proxyCount]

431 if p:

432 d['ip'] = p

433

434 return d

435

436 def _struct_type(self, header):

437 if header:

438 header = header.lower()

439 if header.startswith(self._struct_mime[self._STRUCT_JSON]):

440 return self._STRUCT_JSON

441 if header.startswith(self._struct_mime[self._STRUCT_MSGPACK]):

442 return self._STRUCT_MSGPACK

443

444 def _encode_struct(self, data, typ):

445 if typ == self._STRUCT_JSON:

446 return gws.lib.jsonx.to_string(data)

447 if typ == self._STRUCT_MSGPACK:

448 return umsgpack.dumps(data, default=gws.u.to_dict)

449 raise ValueError(f'invalid struct type {typ!r}')

450

451 def _decode_struct(self, typ):

452 if typ == self._STRUCT_JSON:

453 try:

454 data = gws.u.require(self.data())

455 s = data.decode(encoding='utf-8', errors='strict')

456 return gws.lib.jsonx.from_string(s)

457 except Exception as exc:

458 raise error.BadRequest('malformed json request') from exc

459

460 if typ == self._STRUCT_MSGPACK:

461 try:

462 data = gws.u.require(self.data())

463 return umsgpack.loads(data)

464 except Exception as exc:

465 raise error.BadRequest('malformed msgpack request') from exc

466

467 raise ValueError(f'invalid struct type {typ!r}')