Coverage for gws-app/gws/base/web/wsgi.py: 81%

1"""WSGI requester and responder based on Werkzeug."""

3import gzip

4import io

5import os

6from typing import cast

8import urllib.parse

9import werkzeug.formparser

10import werkzeug.utils

11import werkzeug.wrappers

12import werkzeug.wsgi

14import gws

15import gws.lib.jsonx

16import gws.lib.mime

17import gws.lib.vendor.umsgpack as umsgpack

19from . import error

22class Responder(gws.WebResponder):

23 """Web responder that wraps a Werkzeug response."""

25 def __init__(self, **kwargs):

26 """Create a responder.

28 Args:

29 **kwargs: Either ``wz``, an existing Werkzeug response, or arguments for a new ``werkzeug.wrappers.Response``.

30 """

31 if 'wz' in kwargs:

32 self._wz = kwargs['wz']

33 else:

34 self._wz = werkzeug.wrappers.Response(**kwargs)

35 self.status = self._wz.status_code

37 def __repr__(self):

38 return f'<Responder {self._wz}>'

40 def send_response(self, environ, start_response):

41 return self._wz(environ, start_response)

43 def set_cookie(self, key, value, **kwargs):

44 self._wz.set_cookie(key, value, **kwargs)

46 def delete_cookie(self, key, **kwargs):

47 self._wz.delete_cookie(key, **kwargs)

49 def add_header(self, key, value):

50 self._wz.headers.add(key, value)

52 def set_status(self, status):

53 self._wz.status_code = int(status)

54 self.status = self._wz.status_code

56 def set_body(self, body):

57 if isinstance(body, str):

58 body = body.encode('utf-8')

59 self._wz.set_data(body)

62class Requester(gws.WebRequester):

63 """Web requester that wraps a Werkzeug request."""

65 _STRUCT_JSON = 'json'

66 _STRUCT_MSGPACK = 'msgpack'

68 _struct_mime = {

69 _STRUCT_JSON: 'application/json',

70 _STRUCT_MSGPACK: 'application/msgpack',

71 }

73 def __init__(self, root: gws.Root, environ: dict, site: gws.WebSite, **kwargs):

74 """Create a requester.

76 The user is the guest user until a session is set. POST requests with a

77 form content type are form requests, POST requests with a JSON or

78 MessagePack body are API requests.

80 Args:

81 root: Object tree root.

82 environ: WSGI environment.

83 site: Web site the request is processed for.

84 **kwargs: ``wz``, an existing Werkzeug request to use instead of creating one.

85 """

86 if 'wz' in kwargs:

87 self._wz = kwargs['wz']

88 else:

89 self._wz = werkzeug.wrappers.Request(environ)

91 # this is also set in nginx (see server/ini), but we need this for unzipping (see data() below)

92 self.maxContentLength = int(root.app.cfg('server.web.maxRequestLength') or 1) * 1024 * 1024

93 self._wz.max_content_length = self.maxContentLength

95 self.root = root

96 self.site = site

98 self.environ = self._wz.environ

99 self.method = cast(gws.RequestMethod, self._wz.method.upper())

100

101 self.scheme = 'http'

102 self.host = ''

103 self.port = 0

104 self.ip = self.environ.get('REMOTE_ADDR', '')

105 self.isSecure = False

106

107 self.session = root.app.authMgr.guestSession

108 self.user = root.app.authMgr.guestUser

109

110 self.isGet = self.method == gws.RequestMethod.GET

111 self.isPost = self.method == gws.RequestMethod.POST

112 self.isForm = False

113 self.isApi = False

114

115 self.structInput = None

116 self.structOutput = None

117

118 self.contentTypeHeader = self.header('content-type', '').lower().split(';')[0].strip()

119 self.contentType = gws.lib.mime.get(self.contentTypeHeader) or gws.lib.mime.BIN

120

121 if self.isPost:

122 if self.contentTypeHeader == 'application/x-www-form-urlencoded' or self.contentTypeHeader == 'multipart/form-data':

123 self.isForm = True

124 else:

125 self.structInput = self._struct_type(self.contentTypeHeader)

126 if self.structInput:

127 self.isApi = True

128 self.structOutput = self._struct_type(self.header('accept')) or self.structInput

129

130 self._parsed_params = {}

131 self._parsed_params_lc = {}

132 self._parsed_query_params = {}

133 self._parsed_struct = {}

134 self._parsed_command = ''

135 self._parsed_path = ''

136 self._parsed = False

137 self._raw_post_data = None

138 self._uid = gws.u.mstime()

139

140 if self.root.app.developer_option('request.log_all'):

141 u = {

142 'method': self.method,

143 'path': self._wz.path,

144 'query': self._wz.query_string,

145 'headers': self._wz.headers,

146 'environ': self._wz.environ,

147 }

148 gws.u.write_debug_file(f'request_{self._uid}', ''.join(f'{k}={v!r}\n' for k, v in u.items()))

149

150 def __repr__(self):

151 return f'<Requester {self._wz}>'

152

153 def parse(self):

154 self._parse()

155

156 def params(self):

157 self._parse()

158 return self._parsed_params

159

160 def query_params(self):

161 self._parse()

162 return self._parsed_query_params

163

164 def path(self):

165 self._parse()

166 return self._parsed_path

167

168 def struct(self):

169 self._parse()

170 return self._parsed_struct

171

172 def command(self):

173 self._parse()

174 return self._parsed_command

175

176 def data(self):

177 if not self.isPost:

178 return b''

179

180 if self._raw_post_data is not None:

181 return self._raw_post_data

182

183 cl = self.header('content-length')

184 if not cl:

185 self._raw_post_data = b''

186 return self._raw_post_data

187 try:

188 cl = int(cl)

189 except ValueError as exc:

190 raise error.BadRequest('invalid content-length header') from exc

191 if cl == 0:

192 self._raw_post_data = b''

193 return self._raw_post_data

194 if cl > self.maxContentLength:

195 raise error.RequestEntityTooLarge(f'content-length header too large: {cl}')

196

197 data = self._wz.get_data(as_text=False, cache=False, parse_form_data=False)

198

199 if self.root.app.developer_option('request.log_all'):

200 gws.u.write_debug_file(f'request_{self._uid}.data', data)

201

202 if self.header('content-encoding') == 'gzip':

203 try:

204 with gzip.GzipFile(fileobj=io.BytesIO(data)) as fp:

205 data = fp.read(self.maxContentLength)

206 except OSError as exc:

207 raise error.BadRequest('gzip data error') from exc

208

209 self._raw_post_data = data

210 return data

211

212 def text(self):

213 data = self.data()

214 if not data:

215 return ''

216

217 charset = self._wz.mimetype_params.get('charset', 'utf-8')

218 try:

219 return data.decode(encoding=charset, errors='strict')

220 except UnicodeDecodeError as exc:

221 raise error.BadRequest('text data decoding error') from exc

222

223 def form(self):

224 if not self.isForm:

225 return []

226 data = self.data()

227 if not data:

228 return []

229

230 try:

231 stream = io.BytesIO(data)

232 opts = self._wz.mimetype_params

233

234 # Fix for Qt multipart/form-data boundaries

235 # Qt boundaries start with "boundary_.oOo._" and are base64-encoded

236 # https://github.com/qt/qtbase/blob/04b7fc2de3c97174a725bbd4fdc0f6e496c85861/src/network/access/qhttpmultipart.cpp#L400

237 # Werkzeug header parser does not understand base64 chars "/+="

238 if data.startswith(b'--boundary_.oOo._'):

239 boundary = data[2 : data.find(b'\r\n')].decode('ascii')

240 opts['boundary'] = boundary

241

242 parser = werkzeug.formparser.FormDataParser(silent=False)

243 _, form, files = parser.parse(

244 stream,

245 mimetype=self.contentTypeHeader,

246 content_length=len(data),

247 options=opts,

248 )

249 return list(form.items()) + list(files.items())

250 except Exception as exc:

251 raise error.BadRequest(f'form decode error: {exc}') from exc

252

253 def env(self, key, default=''):

254 return self._wz.environ.get(key, default)

255

256 def has_param(self, key):

257 self._parse()

258 return key.lower() in self._parsed_params_lc

259

260 def param(self, key, default=''):

261 self._parse()

262 return self._parsed_params_lc.get(key.lower(), default)

263

264 def header(self, key, default=''):

265 return self._wz.headers.get(key, default)

266

267 def cookie(self, key, default=''):

268 return self._wz.cookies.get(key, default)

269

270 def content_responder(self, response):

271 args: dict = {

272 'mimetype': response.mimeType,

273 'status': response.status or 200,

274 'headers': {},

275 'direct_passthrough': False,

276 }

277

278 if response.contentFilename:

279 if response.contentFilename.isascii():

280 args['headers']['Content-Disposition'] = f'attachment; filename="{response.contentFilename}"'

281 else:

282 args['headers']['Content-Disposition'] = 'attachment; filename="{}"; filename*=UTF-8\'\'{}'.format(

283 response.contentFilename.encode('ascii', errors='replace').decode('ascii'),

284 urllib.parse.quote(response.contentFilename, safe=''),

285 )

286 args['mimetype'] = args['mimetype'] or gws.lib.mime.for_path(response.contentFilename)

287

288 if response.contentPath:

289 args['response'] = werkzeug.wsgi.wrap_file(self.environ, open(response.contentPath, 'rb'))

290 args['headers']['Content-Length'] = str(os.path.getsize(response.contentPath))

291 args['mimetype'] = args['mimetype'] or gws.lib.mime.for_path(response.contentPath)

292 args['direct_passthrough'] = True

293 else:

294 args['response'] = response.content

295

296 if response.headers:

297 args['headers'].update(response.headers)

298

299 return Responder(**args)

300

301 def redirect_responder(self, response):

302 wz = werkzeug.utils.redirect(response.location, response.status or 302)

303 if response.headers:

304 wz.headers.update(response.headers)

305 return Responder(wz=wz)

306

307 def api_responder(self, response):

308 typ = self.structOutput or self._STRUCT_JSON

309 return Responder(

310 response=self._encode_struct(response, typ),

311 mimetype=self._struct_mime[typ],

312 status=response.status or 200,

313 )

314

315 def error_responder(self, exc):

316 err = exc if isinstance(exc, error.HTTPException) else error.InternalServerError()

317 return Responder(wz=err.get_response(self._wz.environ))

318

319 ##

320

321 def absolute_url_for(self, path, **params):

322 return self.site.url_for(self, path, 'absolute', **params)

323

324 def relative_url_for(self, path, **params):

325 return self.site.url_for(self, path, 'relative', **params)

326

327 def canonical_url_for(self, path, **params):

328 return self.site.url_for(self, path, 'canonical', **params)

329

330 ##

331

332 def set_session(self, sess):

333 self.session = sess

334 self.user = sess.user

335

336 ##

337

338 _CMD_PARAM_NAME = 'cmd'

339

340 def _parse(self):

341 """Parse the request once, on the first access to the parsed data."""

342 if not self._parsed:

343 self._parsed = True

344 self._parse2()

345

346 def _parse2(self):

347 """Parse the origin, the command, the parameters and the structured payload."""

348 self._parse_origin()

349

350 # the server only understands requests to /_ or /_/commandName

351 # GET params can be given as query string or encoded in the path

352 # like _/commandName/param1/value1/param2/value2 etc

353

354 path = self._wz.path

355 path_parts = None

356

357 if path == gws.c.SERVER_ENDPOINT:

358 # example.com/_

359 # the cmd param is expected to be in the query string or json

360 cmd = ''

361 elif path.startswith(gws.c.SERVER_ENDPOINT + '/'):

362 # example.com/_/someCommand

363 # the cmd param is in the url

364 path_parts = path.split('/')

365 cmd = path_parts[2]

366 path_parts = path_parts[3:]

367 self._parsed_path = '/'.join(path_parts)

368 else:

369 raise error.NotFound(f'invalid request path: {path!r}')

370

371 if self.structInput:

372 self._parsed_struct = self._decode_struct(self.structInput)

373 self._parsed_command = cmd or self._parsed_struct.pop(self._CMD_PARAM_NAME, '')

374 else:

375 d = dict(self._wz.args)

376 if path_parts:

377 for n in range(1, len(path_parts), 2):

378 d[path_parts[n - 1]] = path_parts[n]

379 self._parsed_command = cmd or d.pop(self._CMD_PARAM_NAME, '')

380 self._parsed_params = d

381 self._parsed_params_lc = {k.lower(): v for k, v in d.items()}

382 self._parsed_query_params = dict(self._wz.args)

383

384 def _parse_origin(self):

385 """Determine the scheme, host, port and client address, and check the host.

386

387 If the site has proxies, the values are taken from the ``X-Forwarded-*``

388 headers. Requests for a host not in the site host names are rejected.

389 """

390 scheme = 'https' if self.site.ssl or self._wz.is_secure else 'http'

391 host = self.environ.get('HTTP_HOST', '')

392 ip = self.environ.get('REMOTE_ADDR', '')

393 fwd_port = ''

394

395 if self.site.proxyCount > 0:

396 fwd = self._parse_proxy_headers()

397 if fwd['host']:

398 host = fwd['host']

399 if fwd['port']:

400 fwd_port = fwd['port']

401 if fwd['scheme'] and not self.site.ssl:

402 scheme = fwd['scheme']

403 if fwd['ip']:

404 ip = fwd['ip']

405

406 host, _, port = host.strip().lower().partition(':')

407 port = fwd_port or port

408

409 try:

410 port = int(port)

411 except ValueError:

412 port = 0

413

414 if self.site.hostnames and host not in self.site.hostnames:

415 raise error.BadRequest(f'invalid host {host!r}')

416

417 self.scheme = scheme

418 self.isSecure = scheme == 'https'

419 self.host = host

420 self.port = port

421 self.ip = ip

422

423 def _parse_proxy_headers(self):

424 """Read the client's scheme, host, port and address from the ``X-Forwarded-*`` headers."""

425 # read the client's scheme, host, port and address from the X-Forwarded headers

426 # each value is returned as given, or as None if it cannot be trusted or is absent,

427 # the caller is expected to provide the fallbacks

428 #

429 # X-Forwarded-For is a list to which every proxy appends the address it sees,

430 # so with 'proxyCount' proxies in front of us, the client is the 'proxyCount'-th

431 # element from the right. everything to the left of it originates from the client

432 # and is not trustworthy, therefore a list which is too short yields no address at all

433 #

434 # X-Forwarded-Host, -Port and -Proto are normally set, not appended, by the outermost

435 # proxy, and their lists are therefore shorter than the number of proxies.

436 # for these, the leftmost element is used

437 #

438 # @TODO 'proxyCount' counts the proxies, it does not identify them. a request which

439 # arrives over a shorter path, e.g. a monitoring client which bypasses the proxies,

440 # is resolved against the wrong element. verifying the peer requires a list of

441 # trusted proxy addresses

442

443 d: dict = {'scheme': None, 'host': None, 'port': None, 'ip': None}

444

445 p = self.environ.get('HTTP_X_FORWARDED_HOST', '').split(',')[0].strip()

446 if p:

447 d['host'] = p

448

449 p = self.environ.get('HTTP_X_FORWARDED_PORT', '').split(',')[0].strip()

450 if p:

451 d['port'] = p

452

453 p = self.environ.get('HTTP_X_FORWARDED_PROTO', '').split(',')[0].strip().lower()

454 if p in ('http', 'https'):

455 d['scheme'] = p

456

457 lst = [s.strip() for s in self.environ.get('HTTP_X_FORWARDED_FOR', '').split(',')]

458 if len(lst) >= self.site.proxyCount:

459 p = lst[-self.site.proxyCount]

460 if p:

461 d['ip'] = p

462

463 return d

464

465 def _struct_type(self, header):

466 """Return the structured data type (json or msgpack) for a content type header."""

467 if header:

468 header = header.lower()

469 if header.startswith(self._struct_mime[self._STRUCT_JSON]):

470 return self._STRUCT_JSON

471 if header.startswith(self._struct_mime[self._STRUCT_MSGPACK]):

472 return self._STRUCT_MSGPACK

473

474 def _encode_struct(self, data, typ):

475 """Encode data as JSON or MessagePack."""

476 if typ == self._STRUCT_JSON:

477 return gws.lib.jsonx.to_string(data)

478 if typ == self._STRUCT_MSGPACK:

479 return umsgpack.dumps(data, default=gws.u.to_dict)

480 raise ValueError(f'invalid struct type {typ!r}')

481

482 def _decode_struct(self, typ):

483 """Decode the POST data as JSON or MessagePack."""

484 if typ == self._STRUCT_JSON:

485 try:

486 data = gws.u.require(self.data())

487 s = data.decode(encoding='utf-8', errors='strict')

488 return gws.lib.jsonx.from_string(s)

489 except Exception as exc:

490 raise error.BadRequest('malformed json request') from exc

491

492 if typ == self._STRUCT_MSGPACK:

493 try:

494 data = gws.u.require(self.data())

495 return umsgpack.loads(data)

496 except Exception as exc:

497 raise error.BadRequest('malformed msgpack request') from exc

498

499 raise ValueError(f'invalid struct type {typ!r}')