Coverage for gws-app/gws/lib/net/__init_

1"""URL utilities and an HTTP client.

3URLs:

5``parse_url`` splits a URL into a ``Url`` object (scheme, host, port, path, query parameters and so on),

6``make_url`` builds a URL string from such an object or a dict. ``make_qs``, ``add_params``,

7``extract_params`` and the ``quote_*`` functions handle query strings and URL quoting.

9HTTP:

11``http_request`` sends a request with ``requests``, using one session per process, so that

12connections to the same host are reused. It does not raise on failures; instead, it returns an

13``HTTPResponse`` with ``ok`` set to ``False``. Connection errors, timeouts and other request

14errors get the pseudo status codes 900, 901 and 999. ``HTTPResponse.raise_if_failed`` turns a

15failed response into an ``HTTPError`` subclass. The response text is decoded with the charset

16from the content type header, or as UTF-8 or ISO-8859-1 if there is none.

18Example::

20 u = gws.lib.net.parse_url('https://example.com/wms?SERVICE=WMS')

21 url = gws.lib.net.add_params(u.url, REQUEST='GetCapabilities')

23 res = gws.lib.net.http_request(url, timeout=10)

24 res.raise_if_failed()

25 xml = res.text

26"""

28from typing import Optional

30import re

31import requests

32import requests.adapters

33import urllib.parse

35import gws

36import gws.lib.osx

39class Error(gws.Error):

40 """Network error."""

42 pass

45class HTTPError(Error):

46 """HTTP request failed."""

48 pass

51class Timeout(HTTPError):

52 """HTTP request timed out."""

54 pass

57class ConnectionError(HTTPError):

58 """HTTP connection failed."""

60 pass

63class GenericError(HTTPError):

64 """HTTP request failed for another reason."""

66 pass

69_STATUS_CONNECTION_ERROR = 900

70_STATUS_TIMEOUT = 901

71_STATUS_GENERIC_ERROR = 999

74class Url(gws.Data):

75 """Components of a URL, as returned by ``parse_url``."""

77 fragment: str

78 """Fragment, without ``#``."""

79 hostname: str

80 """Host name."""

81 netloc: str

82 """Network location, including user name, password and port."""

83 params: dict

84 """Query parameters. For repeated parameters, the first value is used."""

85 password: str

86 """Password, unquoted."""

87 path: str

88 """Path."""

89 pathparts: gws.lib.osx.ParsePathResult

90 """Components of the path."""

91 port: int

92 """Port, ``0`` if not given."""

93 qsl: list

94 """Query parameters as a list of ``(name, value)`` pairs."""

95 query: str

96 """Query string, without ``?``."""

97 scheme: str

98 """Scheme, e.g. ``https``."""

99 url: str

100 """The parsed URL; with a ``//`` prefix if the original URL had no scheme."""

101 username: str

102 """User name, unquoted."""

103

104

105def parse_url(url: str, **kwargs) -> Url:

106 """Parse a URL.

107

108 A URL without ``//`` is treated as starting with a host name.

109

110 Args:

111 url: URL string.

112 **kwargs: Values to set on the ``Url`` object after parsing.

113

114 Returns:

115 A Url object.

116 """

117

118 if not is_abs_url(url):

119 url = '//' + url

120

121 us = urllib.parse.urlsplit(url)

122

123 u = Url(

124 fragment=us.fragment or '',

125 hostname=us.hostname or '',

126 netloc=us.netloc or '',

127 params={},

128 password=us.password or '',

129 path=us.path or '',

130 pathparts=gws.lib.osx.ParsePathResult(),

131 port=0,

132 qsl=[],

133 query=us.query or '',

134 scheme=us.scheme or '',

135 url=url,

136 username=us.username or '',

137 )

138

139 if us.port:

140 try:

141 u.port = int(us.port)

142 except ValueError:

143 pass

144

145 if u.path:

146 u.pathparts = gws.lib.osx.parse_path(u.path)

147

148 if u.query:

149 u.qsl = urllib.parse.parse_qsl(u.query)

150 for k, v in u.qsl:

151 u.params.setdefault(k, v)

152

153 if u.username:

154 u.username = unquote(u.username)

155 u.password = unquote(u.get('password') or '')

156

157 u.update(**kwargs)

158 return u

159

160

161_DEFAULT_PORTS = {'http': '80', 'https': '443'}

162

163

164def make_url(u: Optional[Url | dict] = None, **kwargs) -> str:

165 """Build a URL string.

166

167 Uses ``scheme``, ``hostname``, ``port``, ``username``, ``password``, ``path``, ``params``

168 and ``fragment``; other keys, like ``query`` or ``url``, are ignored. The port is omitted

169 if it is the default port of the scheme.

170

171 Args:

172 u: URL components, a Url object or a dict.

173 **kwargs: Components that override the values from ``u``.

174

175 Returns:

176 The URL.

177 """

178

179 p = gws.u.merge({}, u, kwargs)

180

181 s = ''

182

183 scheme = p.get('scheme', '').lower()

184 if scheme:

185 s += scheme + ':'

186

187 host = p.get('hostname', '')

188 port = p.get('port', '')

189 path = p.get('path', '')

190

191 if scheme or host:

192 s += '//'

193

194 if host:

195 username = p.get('username', '')

196 if username:

197 s += quote_param(username) + ':' + quote_param(p.get('password', '')) + '@'

198

199 s += host

200 if port and str(port) != _DEFAULT_PORTS.get(scheme):

201 s += ':' + str(port)

202

203 if path:

204 s += '/' + quote_path(path.lstrip('/'))

205

206 params = p.get('params')

207 if params:

208 s += '?' + make_qs(params)

209

210 fragment = p.get('fragment', '')

211 if fragment:

212 s += '#' + fragment.lstrip('#')

213

214 return s

215

216

217def parse_qs(x) -> dict:

218 """Parse a query string.

219

220 Args:

221 x: Query string.

222

223 Returns:

224 A dict that maps each parameter name to a list of its values.

225 """

226

227 return urllib.parse.parse_qs(x)

228

229

230def make_qs(x) -> str:

231 """Convert a dict or a list of pairs to a query string.

232

233 Values are encoded as UTF-8 and quoted. Booleans become ``true`` and ``false``,

234 lists and other iterables are joined with commas, other values are converted with ``str``.

235

236 Args:

237 x: A dict, an object that can be converted to a dict, or a list of ``(name, value)`` pairs.

238

239 Returns:

240 The query string, without ``?``.

241 """

242

243 p = []

244 items = x if isinstance(x, (list, tuple)) else gws.u.to_dict(x).items()

245

246 def _value(v):

247 if isinstance(v, (bytes, bytearray)):

248 return v

249 if isinstance(v, str):

250 return v.encode('utf8')

251 if v is True:

252 return b'true'

253 if v is False:

254 return b'false'

255 try:

256 return b','.join(_value(y) for y in v)

257 except TypeError:

258 return str(v).encode('utf8')

259

260 for k, v in items:

261 k = urllib.parse.quote_from_bytes(_value(k))

262 v = urllib.parse.quote_from_bytes(_value(v))

263 p.append(k + '=' + v)

264

265 return '&'.join(p)

266

267

268def quote_param(s: str) -> str:

269 """Quote a string for use in a URL, including slashes.

270

271 Args:

272 s: A string.

273

274 Returns:

275 The quoted string.

276 """

277

278 return urllib.parse.quote(s, safe='')

279

280

281def quote_path(s: str) -> str:

282 """Quote a URL path, leaving slashes as they are.

283

284 Args:

285 s: A path.

286

287 Returns:

288 The quoted path.

289 """

290

291 return urllib.parse.quote(s, safe='/')

292

293

294def unquote(s: str) -> str:

295 """Unquote a URL-quoted string.

296

297 Args:

298 s: A quoted string.

299

300 Returns:

301 The unquoted string.

302 """

303

304 return urllib.parse.unquote(s)

305

306

307def add_params(url: str, params: dict = None, **kwargs) -> str:

308 """Add query parameters to a URL.

309

310 Existing parameters with the same names are replaced.

311

312 Args:

313 url: A URL.

314 params: Parameters to add.

315 **kwargs: More parameters to add.

316

317 Returns:

318 The new URL.

319 """

320

321 u = parse_url(url)

322 if params:

323 u.params.update(params)

324 u.params.update(kwargs)

325 return make_url(u)

326

327

328def make_relative_url(path: str, params: dict = None, **kwargs) -> str:

329 """Build a URL from a path and query parameters, without a scheme and host.

330

331 Args:

332 path: URL path; a leading slash is added if needed.

333 params: Query parameters.

334 **kwargs: More query parameters.

335

336 Returns:

337 The URL.

338 """

339

340 s = '/' + quote_path(path.lstrip('/'))

341 p = {}

342 if params:

343 p.update(params)

344 p.update(kwargs)

345 if p:

346 s += '?' + make_qs(p)

347 return s

348

349

350def extract_params(url: str) -> tuple[str, dict]:

351 """Split the query parameters off a URL.

352

353 Args:

354 url: A URL.

355

356 Returns:

357 The URL without the query string, and the query parameters.

358 """

359

360 u = parse_url(url)

361 params = u.params

362 u.params = None

363 return make_url(u), params

364

365

366def is_abs_url(url):

367 """Check if a URL has a host part, i.e. starts with ``//`` or a scheme and ``//``.

368

369 Args:

370 url: A URL.

371

372 Returns:

373 A truthy match object if the URL is absolute, ``None`` otherwise.

374 """

375

376 return re.match(r'^([a-zA-Z][a-zA-Z0-9+.-]*:|)//', url)

379##

382class HTTPResponse:

383 """Response of ``http_request``.

384

385 Attributes:

386 ok: ``True`` if the request succeeded with a 2xx status.

387 url: Request URL.

388 content: Response body.

389 content_type: Content type, without parameters.

390 content_encoding: Charset from the content type header, or ``None``.

391 status_code: HTTP status code, or a pseudo status code for requests that failed without a response.

392 """

393

394 def __init__(

395 self,

396 ok: bool,

397 url: str,

398 res: requests.Response = None,

399 text: str = None,

400 status_code=0,

401 ):

402 """Create a response.

403

404 Args:

405 ok: ``True`` if the request succeeded.

406 url: Request URL.

407 res: The ``requests`` response, if there is one.

408 text: Content of a response without ``res``, e.g. an error message. It is stored as UTF-8 plain text.

409 status_code: Status code of a response without ``res``.

410 """

411 self.ok = ok

412 self.url = url

413 if res is not None:

414 self.content_type, self.content_encoding = _parse_content_type(res.headers)

415 self.content = res.content

416 self.status_code = res.status_code

417 else:

418 self.content_type, self.content_encoding = 'text/plain', 'utf8'

419 self.content = text.encode('utf8') if text is not None else b''

420 self.status_code = status_code

421

422 @property

423 def text(self) -> str:

424 """Response body as text, decoded on first access.

425

426 Returns:

427 The decoded body.

428 """

429

430 if not hasattr(self, '_text'):

431 setattr(self, '_text', _get_text(self.content, self.content_encoding))

432 return getattr(self, '_text')

433

434 def raise_if_failed(self):

435 """Raise an error if the request failed.

436

437 Raises:

438 ``ConnectionError``: If the connection failed.

439 ``Timeout``: If the request timed out.

440 ``GenericError``: If the request failed for another reason, without a response.

441 ``HTTPError``: If the response status is not 2xx.

442 """

443

444 if self.ok:

445 return

446 if self.status_code == _STATUS_CONNECTION_ERROR:

447 raise ConnectionError(self.text)

448 if self.status_code == _STATUS_TIMEOUT:

449 raise Timeout(self.text)

450 if self.status_code == _STATUS_GENERIC_ERROR:

451 raise GenericError(self.text)

452 raise HTTPError(f'HTTP error: {self.status_code}')

453

454

455def _get_text(content, encoding) -> str:

456 """Decode content with the given encoding, falling back to UTF-8 and ISO-8859-1."""

457

458 if encoding:

459 try:

460 return str(content, encoding=encoding, errors='strict')

461 except UnicodeDecodeError:

462 pass

463

464 # some folks serve utf8 content without a header, in which case requests thinks it's ISO-8859-1

465 # (see http://docs.python-requests.org/en/master/user/advanced/#encodings)

466 #

467 # 'apparent_encoding' is not always reliable

468 #

469 # therefore when there's no header, we try utf8 first, and then ISO-8859-1

470

471 try:

472 return str(content, encoding='utf8', errors='strict')

473 except UnicodeDecodeError:

474 pass

475

476 try:

477 return str(content, encoding='ISO-8859-1', errors='strict')

478 except UnicodeDecodeError:

479 pass

480

481 # both failed, do utf8 with replace

482

483 gws.log.warning(f'decode failed')

484 return str(content, encoding='utf8', errors='replace')

485

486

487def _parse_content_type_header(header):

488 """Split a content type header into the lowercase type and a dict of parameters."""

489

490 parts = header.split(';')

491 ctype = parts[0].strip().lower()

492 params: dict[str, str] = {}

493

494 for part in parts[1:]:

495 part = part.strip()

496 if '=' in part:

497 k, v = part.split('=', 1)

498 k = k.strip().lower()

499 v = v.strip()

500 # strip matched quotes (single or double)

501 if len(v) >= 2 and v[0] == v[-1] and v[0] in ('"', "'"):

502 v = v[1:-1]

503 params[k] = v

504

505 return ctype, params

506

507

508def _parse_content_type(headers):

509 """Return the content type and the valid charset, or ``None``, from response headers."""

510

511 # copied from requests.utils.get_encoding_from_headers, but with no ISO-8859-1 default

512

513 header = headers.get('content-type')

514 if not header:

515 # https://www.w3.org/Protocols/rfc2616/rfc2616-sec7.html#sec7.2.1

516 return 'application/octet-stream', None

517

518 ctype, params = _parse_content_type_header(header)

519 if 'charset' not in params:

520 return ctype, None

521

522 # make sure this is a valid python encoding

523 enc = params['charset']

524 try:

525 str(b'.', encoding=enc, errors='strict')

526 except LookupError:

527 gws.log.warning(f'invalid content-type encoding {enc!r}')

528 return ctype, None

529

530 return ctype, enc

531

532

533##

534

535# @TODO locking for caches

536

537

538def http_request(url, **kwargs) -> HTTPResponse:

539 """Send an HTTP request.

540

541 Failures are logged and returned as a response with ``ok`` set to ``False``, they do not raise.

542 By default, the connect and read timeouts are 60 seconds, a GBD WebSuite ``User-Agent`` header is sent,

543 and certificates are verified with the ``requests`` default (``REQUESTS_CA_BUNDLE``, ``CURL_CA_BUNDLE`` or ``certifi``).

544

545 Args:

546 url: Request URL.

547 **kwargs: Options for ``requests.Session.request``. ``method`` is the HTTP method (``GET`` by default),

548 ``params`` are added to the URL, a numeric ``timeout`` applies to both connect and read.

549

550 Returns:

551 The response.

552 """

553

554 kwargs = dict(kwargs)

555

556 if 'params' in kwargs:

557 url = add_params(url, kwargs.pop('params'))

558

559 method = kwargs.pop('method', 'GET').upper()

560

561 gws.debug.time_start(f'HTTP_{method}={url!r}')

562 res = _http_request(method, url, kwargs)

563 gws.debug.time_end()

564

565 return res

566

567

568_DEFAULT_CONNECT_TIMEOUT = 60

569_DEFAULT_READ_TIMEOUT = 60

570

571_USER_AGENT = f'GBD WebSuite (https://gbd-websuite.de)'

572

573_POOL_SIZE = 16

574"""Max. keep-alive connections per host and process."""

575

576_session: requests.Session | None = None

577

578

579def _get_session() -> requests.Session:

580 """Return the HTTP session of this process, creating it if needed."""

581 # one session per process: reuses TCP/TLS connections to the same host across requests

582 global _session

583 if _session is None:

584 s = requests.Session()

585 adapter = requests.adapters.HTTPAdapter(pool_connections=_POOL_SIZE, pool_maxsize=_POOL_SIZE)

586 s.mount('http://', adapter)

587 s.mount('https://', adapter)

588 _session = s

589 return _session

590

591

592def _http_request(method, url, kwargs) -> HTTPResponse:

593 """Send a request with default options and wrap the result in an ``HTTPResponse``."""

594

595 kwargs['stream'] = False

596

597 timeout = kwargs.get('timeout', (_DEFAULT_CONNECT_TIMEOUT, _DEFAULT_READ_TIMEOUT))

598 if isinstance(timeout, (int, float)):

599 timeout = int(timeout), int(timeout)

600 kwargs['timeout'] = timeout

601

602 if 'headers' not in kwargs:

603 kwargs['headers'] = {}

604 kwargs['headers'].setdefault('User-Agent', _USER_AGENT)

605

606 try:

607 res = _get_session().request(method, url, **kwargs)

608 if 200 <= res.status_code < 300:

609 gws.log.debug(f'HTTP_OK_{method}: url={url!r} status={res.status_code!r}')

610 return HTTPResponse(ok=True, url=url, res=res)

611 gws.log.error(f'HTTP_FAILED_{method}: ({res.status_code!r}) url={url!r}')

612 return HTTPResponse(ok=False, url=url, res=res)

613 except requests.ConnectionError as exc:

614 gws.log.error(f'HTTP_FAILED_{method}: (ConnectionError) url={url!r}')

615 return HTTPResponse(ok=False, url=url, text=repr(exc), status_code=_STATUS_CONNECTION_ERROR)

616 except requests.Timeout as exc:

617 gws.log.error(f'HTTP_FAILED_{method}: (Timeout) url={url!r}')

618 return HTTPResponse(ok=False, url=url, text=repr(exc), status_code=_STATUS_TIMEOUT)

619 except requests.RequestException as exc:

620 gws.log.error(f'HTTP_FAILED_{method}: (Generic: {exc!r}) url={url!r}')

621 return HTTPResponse(ok=False, url=url, text=repr(exc), status_code=_STATUS_GENERIC_ERROR)

Coverage for gws-app/gws/lib/net/init.py: 87%

252 statements