Coverage for gws-app/gws/core/util.py: 71%

1"""General utilities, available as ``gws.u``."""

3import fcntl

4import hashlib

5import json

6import os

7import pickle

8import random

9import re

10import subprocess

11import sys

12import threading

13import time

14import urllib.parse

15from typing import Optional, TypeVar, Union, cast

17from . import const, log

20def is_data_object(x) -> bool:

21 """Check if the argument is a ``Data`` object.

23 This is a placeholder, replaced by ``gws.is_data_object`` when ``gws`` is imported.

25 Args:

26 x: A value.

28 Returns:

29 ``True`` if the value is a ``Data`` object.

30 """

31 return False

34def to_data_object(x):

35 """Convert a value to a ``Data`` object.

37 This is a placeholder, replaced by ``gws.to_data_object`` when ``gws`` is imported.

39 Args:

40 x: A value.

42 Returns:

43 A ``Data`` object.

44 """

45 pass

48def exit(code: int = 255):

49 """Exit the application.

51 Args:

52 code: Exit code.

53 """

55 sys.exit(code)

58T = TypeVar('T')

61def require(value: Optional[T], message: str = '') -> T:

62 """Return the value if it is not ``None``, otherwise raise an error.

64 Args:

65 value: A value.

66 message: Error message.

68 Returns:

69 The value.

71 Raises:

72 ValueError: If the value is ``None``.

73 """

74 if value is None:

75 raise ValueError(message or 'unexpected None value')

76 return value

79##

81# @TODO use ABC

84def is_list(x):

85 """Check if the value is a list or a tuple.

87 Args:

88 x: A value.

90 Returns:

91 ``True`` if the value is a list or a tuple.

92 """

93 return isinstance(x, (list, tuple))

96def is_dict(x):

97 """Check if the value is a dict.

99 Args:

100 x: A value.

101

102 Returns:

103 ``True`` if the value is a dict.

104 """

105 return isinstance(x, dict)

106

107

108def is_bytes(x):

109 """Check if the value is ``bytes`` or ``bytearray``.

110

111 Args:

112 x: A value.

113

114 Returns:

115 ``True`` if the value is ``bytes`` or ``bytearray``.

116 """

117 return isinstance(x, (bytes, bytearray))

118 # @TODO how to handle bytes-alikes?

119 # return hasattr(x, 'decode')

120

121

122def is_atom(x):

123 """Check if the value is ``None`` or a scalar (number, bool, string or bytes).

124

125 Args:

126 x: A value.

127

128 Returns:

129 ``True`` if the value is an atom.

130 """

131 return x is None or isinstance(x, (int, float, bool, str, bytes))

132

133

134def is_empty(x) -> bool:

135 """Check if the value is empty.

136

137 A value is empty if it is ``None``, has zero length, or is an object without attributes.

138

139 Args:

140 x: A value.

141

142 Returns:

143 ``True`` if the value is empty.

144 """

145

146 if x is None:

147 return True

148 try:

149 return len(x) == 0

150 except TypeError:

151 pass

152 try:

153 return not vars(x)

154 except TypeError:

155 pass

156 return False

159##

162def get(x, key, default=None):

163 """Get a nested value or attribute from a structure.

164

165 Args:

166 x: A dict, list, ``Data`` or any object.

167 key: A list or a dot-separated string of nested keys. List elements are addressed by numeric keys.

168 default: The default value.

169

170 Returns:

171 The value if it exists and the default otherwise.

172 """

173

174 if not x:

175 return default

176 if isinstance(key, str):

177 key = key.split('.')

178 try:

179 return _get(x, key)

180 except (KeyError, IndexError, AttributeError, ValueError):

181 return default

182

183

184def has(x, key) -> bool:

185 """Check if a nested value or attribute exists in a structure.

186

187 Args:

188 x: A dict, list, ``Data`` or any object.

189 key: A list or a dot-separated string of nested keys.

190

191 Returns:

192 ``True`` if the key exists.

193 """

194

195 if not x:

196 return False

197 if isinstance(key, str):

198 key = key.split('.')

199 try:

200 _get(x, key)

201 return True

202 except (KeyError, IndexError, AttributeError, ValueError):

203 return False

204

205

206def _get(x, keys):

207 """Follow the keys into a structure, raise an error if a key is missing."""

208 for k in keys:

209 if is_dict(x):

210 x = x[k]

211 elif is_list(x):

212 x = x[int(k)]

213 elif is_data_object(x):

214 # special case: raise a KeyError if the attribute is truly missing in a Data

215 # (and not just equals to None)

216 x = vars(x)[k]

217 else:

218 x = getattr(x, k)

219 return x

220

221

222def pop(x, key, default=None):

223 """Remove a key from a dict or a ``Data`` object and return its value.

224

225 Args:

226 x: A dict or ``Data``.

227 key: The key.

228 default: Value to return if the key is missing or ``x`` is of another type.

229

230 Returns:

231 The value or the default.

232 """

233 if is_dict(x):

234 return x.pop(key, default)

235 if is_data_object(x):

236 return vars(x).pop(key, default)

237 return default

238

239

240def pick(x, *keys):

241 """Return a copy of a dict or a ``Data`` object with the given keys only.

242

243 Args:

244 x: A dict or ``Data``.

245 *keys: Keys to keep.

246

247 Returns:

248 A new object of the same type, or an empty dict if ``x`` is of another type.

249 """

250

251 def _pick(d):

252 r = {}

253 for k in keys:

254 if k in d:

255 r[k] = d[k]

256 return r

257

258 if is_dict(x):

259 return _pick(x)

260 if is_data_object(x):

261 return type(x)(_pick(vars(x)))

262 return {}

263

264

265def omit(x, *keys):

266 """Return a copy of a dict or a ``Data`` object without the given keys.

267

268 Args:

269 x: A dict or ``Data``.

270 *keys: Keys to remove.

271

272 Returns:

273 A new object of the same type, or an empty dict if ``x`` is of another type.

274 """

275

276 def _omit(d):

277 r = {}

278 for k, v in d.items():

279 if k not in keys:

280 r[k] = d[k]

281 return r

282

283 if is_dict(x):

284 return _omit(x)

285 if is_data_object(x):

286 return type(x)(_omit(vars(x)))

287 return {}

288

289

290def collect(pairs):

291 """Group values by keys.

292

293 Args:

294 pairs: An iterable of ``(key, value)`` pairs. Pairs with a ``None`` key are skipped.

295

296 Returns:

297 A dict mapping each key to the list of its values.

298 """

299 m = {}

300

301 for key, val in pairs:

302 if key is not None:

303 m.setdefault(key, []).append(val)

304

305 return m

306

307

308def first(it):

309 """Return the first element of an iterable.

310

311 Args:

312 it: An iterable.

313

314 Returns:

315 The first element, or ``None`` if the iterable is empty.

316 """

317 for x in it:

318 return x

319

320

321def first_not_none(*args):

322 """Return the first argument that is not ``None``.

323

324 Args:

325 *args: Values.

326

327 Returns:

328 The first value that is not ``None``, or ``None``.

329 """

330 for a in args:

331 if a is not None:

332 return a

333

334

335def merge(*args, **kwargs) -> Union[dict, 'Data']:

336 """Create a new dict or ``Data`` object by merging values from dicts, ``Data`` objects and keyword args.

337

338 Later values override earlier ones, unless they are ``None``.

339

340 Args:

341 *args: Dicts or ``Data`` objects. Empty values are skipped.

342 **kwargs: Keyword args, merged last.

343

344 Returns:

345 A new object of the same type as the first argument, or a dict if it is a dict or ``None``.

346 """

347

348 def _merge(arg):

349 for k, v in to_dict(arg).items():

350 if v is not None:

351 m[k] = v

352

353 m = {}

354

355 for a in args:

356 if a:

357 _merge(a)

358 if kwargs:

359 _merge(kwargs)

360

361 if not args or isinstance(args[0], dict) or args[0] is None:

362 return m

363 return type(args[0])(m)

364

365

366def compact(x):

367 """Remove all ``None`` values from a collection.

368

369 Args:

370 x: A dict, ``Data`` or an iterable.

371

372 Returns:

373 A new dict or ``Data`` object, or a list for other iterables.

374 """

375

376 if is_dict(x):

377 return {k: v for k, v in x.items() if v is not None}

378 if is_data_object(x):

379 d = {k: v for k, v in vars(x).items() if v is not None}

380 return type(x)(d)

381 return [v for v in x if v is not None]

382

383

384def strip(x):

385 """Strip all strings and remove empty values from a collection.

386

387 Args:

388 x: A dict, ``Data`` or an iterable.

389

390 Returns:

391 A new dict or ``Data`` object, or a list for other iterables.

392 """

393

394 def _strip(v):

395 if isinstance(v, (str, bytes, bytearray)):

396 return v.strip()

397 return v

398

399 def _dict(x1):

400 d = {}

401 for k, v in x1.items():

402 v = _strip(v)

403 if not is_empty(v):

404 d[k] = v

405 return d

406

407 if is_dict(x):

408 return _dict(x)

409 if is_data_object(x):

410 return type(x)(_dict(vars(x)))

411

412 r = [_strip(v) for v in x]

413 return [v for v in r if not is_empty(v)]

414

415

416def uniq(x):

417 """Remove duplicate elements from a collection, keeping the order.

418

419 Args:

420 x: An iterable.

421

422 Returns:

423 A list of unique elements.

424 """

425

426 s = set()

427 r = []

428

429 for y in x:

430 try:

431 if y not in s:

432 s.add(y)

433 r.append(y)

434 except TypeError:

435 if y not in r:

436 r.append(y)

438 return r

441##

444def to_int(x) -> int:

445 """Convert a value to an int.

446

447 Args:

448 x: A value.

449

450 Returns:

451 The int value, or 0 if the conversion fails.

452 """

453

454 try:

455 return int(x)

456 except:

457 return 0

458

459

460def to_rounded_int(x) -> int:

461 """Round a float and convert a value to an int.

462

463 Args:

464 x: A value.

465

466 Returns:

467 The int value, or 0 if the conversion fails.

468 """

469

470 try:

471 if isinstance(x, float):

472 return int(round(x))

473 return int(x)

474 except:

475 return 0

476

477

478def to_float(x) -> float:

479 """Convert a value to a float.

480

481 Args:

482 x: A value.

483

484 Returns:

485 The float value, or 0.0 if the conversion fails.

486 """

487

488 try:

489 return float(x)

490 except:

491 return 0.0

492

493

494def clamp(x, lo, hi):

495 """Limit a value to a range.

496

497 Args:

498 x: A value.

499 lo: Lower bound.

500 hi: Upper bound.

501

502 Returns:

503 ``lo`` if the value is less than ``lo``, ``hi`` if it is greater than ``hi``, otherwise the value.

504 """

505

506 return max(lo, min(x, hi))

507

508

509def to_str(x, encodings: list[str] = None) -> str:

510 """Convert a value to a string.

511

512 Bytes are decoded, ``None`` becomes an empty string, other values are converted with ``str``.

513

514 Args:

515 x: A value.

516 encodings: A list of acceptable encodings. If the value is bytes, try each encoding

517 and return the first result that decodes without errors. If none succeeds,

518 decode as UTF-8 and ignore errors.

519

520 Returns:

521 A string.

522 """

523

524 if isinstance(x, str):

525 return x

526 if x is None:

527 return ''

528 if not is_bytes(x):

529 return str(x)

530 if encodings:

531 for enc in encodings:

532 try:

533 return x.decode(encoding=enc, errors='strict')

534 except UnicodeDecodeError:

535 pass

536 return x.decode(encoding='utf-8', errors='ignore')

537

538

539def to_bytes(x, encoding='utf8') -> bytes:

540 """Convert a value to bytes by converting it to a string and encoding it.

541

542 Args:

543 x: A value. Bytes are returned as is, ``None`` becomes empty bytes.

544 encoding: The encoding.

545

546 Returns:

547 Bytes.

548 """

549

550 if is_bytes(x):

551 return bytes(x)

552 if x is None:

553 return b''

554 if not isinstance(x, str):

555 x = str(x)

556 return x.encode(encoding or 'utf8')

557

558

559def to_list(x, delimiter: str = ',') -> list:

560 """Convert a value to a list.

561

562 Args:

563 x: A value. A string (or bytes) is split by the delimiter, the parts are stripped and empty parts removed.

564 A number or a bool becomes a one-element list, other iterables are converted to a list.

565 delimiter: The delimiter. If empty, a string becomes a one-element list.

566

567 Returns:

568 A list. Empty values and values that cannot be converted give an empty list.

569 """

570

571 if isinstance(x, list):

572 return x

573 if is_empty(x):

574 return []

575 if is_bytes(x):

576 x = to_str(x)

577 if isinstance(x, str):

578 if delimiter:

579 ls = [s.strip() for s in x.split(delimiter)]

580 return [s for s in ls if s]

581 return [x]

582 if isinstance(x, (int, float, bool)):

583 return [x]

584 try:

585 return [s for s in x]

586 except TypeError:

587 return []

588

589

590def to_dict(x) -> dict:

591 """Convert a value to a dict.

592

593 Args:

594 x: A dict, ``None``, a named tuple or an object.

595

596 Returns:

597 The dict itself, an empty dict for ``None``, the fields of a named tuple, or the object's ``vars``.

598

599 Raises:

600 ValueError: If the value cannot be converted.

601 """

602

603 if is_dict(x):

604 return x

605 if x is None:

606 return {}

607 try:

608 f = getattr(x, '_asdict', None)

609 if f:

610 return f()

611 return vars(x)

612 except TypeError:

613 raise ValueError(f'cannot convert {x!r} to dict')

614

615

616def to_json_value(x) -> Union[dict, list, str, int, float, bool, None]:

617 """Recursively convert a value to a JSON serializable type.

618

619 Args:

620 x: A value. Dicts, lists and ``Data`` objects are converted recursively, other non-scalar values are converted with ``str``.

621

622 Returns:

623 A JSON serializable value.

624 """

625

626 if is_atom(x):

627 return x

628 if is_dict(x):

629 return {k: to_json_value(v) for k, v in x.items()}

630 if is_list(x):

631 return [to_json_value(v) for v in x]

632 if is_data_object(x):

633 return {k: to_json_value(v) for k, v in vars(x).items()}

634 return str(x)

635

636

637def to_upper_dict(x) -> dict:

638 """Convert a value to a dict with upper-case keys.

639

640 Args:

641 x: A value accepted by ``to_dict``.

642

643 Returns:

644 A new dict.

645 """

646 x = to_dict(x)

647 return {k.upper(): v for k, v in x.items()}

648

649

650def to_lower_dict(x) -> dict:

651 """Convert a value to a dict with lower-case keys.

652

653 Args:

654 x: A value accepted by ``to_dict``.

655

656 Returns:

657 A new dict.

658 """

659 x = to_dict(x)

660 return {k.lower(): v for k, v in x.items()}

661

662

663##

664

665_UID_DE_TRANS = {

666 ord('ä'): 'ae',

667 ord('ö'): 'oe',

668 ord('ü'): 'ue',

669 ord('ß'): 'ss',

670}

671

672

673def to_uid(x) -> str:

674 """Convert a value to a uid.

675

676 The value is converted to a lower-case string, German umlauts are transliterated,

677 and runs of other characters are replaced with underscores.

678

679 Args:

680 x: A value.

681

682 Returns:

683 A string of ``a-z``, ``0-9`` and ``_``, or an empty string for an empty value.

684 """

685

686 if not x:

687 return ''

688 x = to_str(x).lower().strip().translate(_UID_DE_TRANS)

689 x = re.sub(r'[^a-z0-9]+', '_', x)

690 return x.strip('_')

693##

696def parse_acl(acl):

697 """Parse an ACL config into an ACL.

698

699 Args:

700 acl: An ACL config. Can be given as a string ``allow X, allow Y, deny Z``,

701 or as a list of dicts ``{ role X type allow }, { role Y type deny }``,

702 or it can already be an ACL ``[1 X], [0 Y]``,

703 or it can be ``None``.

704

705 Returns:

706 Access list, empty for an empty value.

707

708 Raises:

709 ValueError: If the ACL config is invalid.

710 """

711

712 if not acl:

713 return []

714

715 a = 'allow'

716 d = 'deny'

717 bits = {const.ALLOW, const.DENY}

718 err = 'invalid ACL'

719

720 access = []

721

722 if isinstance(acl, str):

723 for p in acl.strip().split(','):

724 s = p.strip().split()

725 if len(s) != 2:

726 raise ValueError(err)

727 if s[0] == a:

728 access.append((const.ALLOW, s[1]))

729 elif s[0] == d:

730 access.append((const.DENY, s[1]))

731 else:

732 raise ValueError(err)

733 return access

734

735 if not isinstance(acl, list):

736 raise ValueError(err)

737

738 if isinstance(acl[0], (list, tuple)):

739 try:

740 if all(len(s) == 2 and s[0] in bits for s in acl):

741 return acl

742 except (TypeError, IndexError):

743 pass

744 raise ValueError(err)

745

746 if isinstance(acl[0], dict):

747 for s in acl:

748 tk = s.get('type', '')

749 rk = s.get('role', '')

750 if not isinstance(rk, str):

751 raise ValueError(err)

752 if tk == a:

753 access.append((const.ALLOW, rk))

754 elif tk == d:

755 access.append((const.DENY, rk))

756 else:

757 raise ValueError(err)

758 return access

759

760 raise ValueError(err)

761

762

763##

764

765UID_DELIMITER = '::'

766

767

768def join_uid(parent_uid, object_uid):

769 """Join a parent uid and an object uid with ``UID_DELIMITER``.

770

771 If either uid is already joined, only its last part is used.

772

773 Args:

774 parent_uid: Parent uid.

775 object_uid: Object uid.

776

777 Returns:

778 The joined uid.

779 """

780 p = parent_uid.split(UID_DELIMITER)

781 u = object_uid.split(UID_DELIMITER)

782 return p[-1] + UID_DELIMITER + u[-1]

783

784

785def split_uid(joined_uid: str) -> tuple[str, str]:

786 """Split a joined uid at the first ``UID_DELIMITER``.

787

788 Args:

789 joined_uid: Joined uid.

790

791 Returns:

792 A tuple of the parent uid and the object uid. If there is no delimiter, the parent uid is empty.

793 """

794 p, sep, u = joined_uid.partition(UID_DELIMITER)

795 if not sep:

796 return '', joined_uid

797 return p, u

800##

803def is_file(path):

804 """Check if the path is an existing file.

805

806 Args:

807 path: File path.

808

809 Returns:

810 ``True`` if the file exists.

811 """

812 return os.path.isfile(path)

813

814

815def is_dir(path):

816 """Check if the path is an existing directory.

817

818 Args:

819 path: Directory path.

820

821 Returns:

822 ``True`` if the directory exists.

823 """

824 return os.path.isdir(path)

825

826

827def read_file(path: str) -> str:

828 """Read a UTF-8 text file.

829

830 Args:

831 path: File path.

832

833 Returns:

834 The file content.

835

836 Raises:

837 Exception: Errors from opening or reading the file are logged and raised again.

838 """

839 try:

840 with open(path, 'rt', encoding='utf8') as fp:

841 return fp.read()

842 except Exception as exc:

843 log.debug(f'error reading {path=} {exc=}')

844 raise

845

846

847def read_file_b(path: str) -> bytes:

848 """Read a binary file.

849

850 Args:

851 path: File path.

852

853 Returns:

854 The file content.

855

856 Raises:

857 Exception: Errors from opening or reading the file are logged and raised again.

858 """

859 try:

860 with open(path, 'rb') as fp:

861 return fp.read()

862 except Exception as exc:

863 log.debug(f'error reading {path=} {exc=}')

864 raise

865

866

867def write_file(path: str, s: str, user: int = None, group: int = None):

868 """Write a text file atomically, via a temporary file in the same directory.

869

870 Args:

871 path: File path.

872 s: Text content, encoded as UTF-8.

873 user: File owner, defaults to ``const.UID``.

874 group: File group, defaults to ``const.GID``.

875

876 Returns:

877 The file path.

878

879 Raises:

880 Exception: Write errors are logged and raised again.

881 """

882

883 return write_file_b(path, s.encode('utf8'), user, group)

884

885

886def write_file_b(path: str, s: str | bytes, user: int = None, group: int = None):

887 """Write a binary file atomically, via a temporary file in the same directory.

888

889 Args:

890 path: File path.

891 s: Content. A string is encoded as UTF-8.

892 user: File owner, defaults to ``const.UID``.

893 group: File group, defaults to ``const.GID``.

894

895 Returns:

896 The file path.

897

898 Raises:

899 Exception: Write errors are logged and raised again.

900 """

901

902 if isinstance(s, str):

903 s = s.encode('utf8')

904 tmp = f'{path}.{random_string(32)}.tmp'

905 try:

906 with open(tmp, 'wb') as fp:

907 fp.write(s)

908 chown_default(tmp, user, group)

909 os.replace(tmp, path)

910 return path

911 except Exception as exc:

912 log.debug(f'error writing {path=} {exc=}')

913 try:

914 os.unlink(tmp)

915 except OSError:

916 pass

917 raise

918

919

920def write_debug_file(path: str, s: str | bytes):

921 """Write a file to the debug directory ``<VAR_DIR>/debug``.

922

923 Errors are logged and ignored.

924

925 Args:

926 path: File path, relative to the debug directory.

927 s: Content. A string is encoded as UTF-8.

928 """

929

930 if isinstance(s, str):

931 s = s.encode('utf8')

932 try:

933 d = ensure_dir(f'{const.VAR_DIR}/debug')

934 with open(f'{d}/{path}', 'wb') as fp:

935 fp.write(s)

936 except Exception as exc:

937 log.debug(f'error writing debug {path=} {exc=}')

938

939

940def basename(path):

941 """Return the base name of a path.

942

943 Args:

944 path: A path.

945

946 Returns:

947 The base name.

948 """

949 return os.path.basename(path)

950

951

952def dirname(path):

953 """Return the directory part of a path.

954

955 Args:

956 path: A path.

957

958 Returns:

959 The directory name.

960 """

961 return os.path.dirname(path)

962

963

964def ensure_dir(dir_path: str, base_dir: str = None, mode: int = 0o755, user: int = None, group: int = None) -> str:

965 """Check if a (possibly nested) directory exists and create it if it does not.

966

967 Args:

968 dir_path: Path to a directory. Must be absolute without ``base_dir`` and relative with it.

969 base_dir: Base directory.

970 mode: Directory creation mode.

971 user: Directory owner, defaults to ``const.UID``.

972 group: Directory group, defaults to ``const.GID``.

973

974 Returns:

975 The path to the directory.

976

977 Raises:

978 ValueError: If the path is absolute with ``base_dir``, or relative without it.

979 """

980

981 if base_dir:

982 if os.path.isabs(dir_path):

983 raise ValueError(f'cannot use an absolute path {dir_path!r} with a base dir')

984 bpath = cast(bytes, os.path.join(base_dir.encode('utf8'), dir_path.encode('utf8')))

985 else:

986 if not os.path.isabs(dir_path):

987 raise ValueError(f'cannot use a relative path {dir_path!r} without a base dir')

988 bpath = dir_path.encode('utf8')

989

990 if os.path.isdir(bpath):

991 return bpath.decode('utf8')

992

993 parts = []

994

995 for p in bpath.split(b'/'):

996 parts.append(p)

997 path = b'/'.join(parts)

998 if path and not os.path.isdir(path):

999 try:

1000 os.mkdir(path, mode)

1001 chown_default(path, user, group)

1002 except FileExistsError:

1003 pass

1004

1005 return bpath.decode('utf8')

1006

1007

1008def ensure_system_dirs():

1009 """Create all system directories listed in ``const.ALL_DIRS``."""

1010 for d in const.ALL_DIRS:

1011 ensure_dir(d)

1012

1013

1014def chown_default(path, user=None, group=None):

1015 """Change the owner of a path, ignoring errors.

1016

1017 Args:

1018 path: A path.

1019 user: Owner, defaults to ``const.UID``.

1020 group: Group, defaults to ``const.GID``.

1021 """

1022 try:

1023 os.chown(path, user or const.UID, group or const.GID)

1024 except OSError:

1025 pass

1026

1027

1028_ephemeral_state = dict(

1029 next_check=0,

1030 check_interval=2 * 3600,

1031 max_age=2 * 3600,

1032)

1033

1034

1035def ephemeral_path(name: str) -> str:

1036 """Return a new unique path in the ephemeral directory.

1037

1038 The path is not created. Ephemeral paths are removed by ``ephemeral_cleanup`` after two hours.

1039

1040 Args:

1041 name: Base name, appended to a unique prefix.

1042

1043 Returns:

1044 The path.

1045 """

1046

1047 ephemeral_cleanup()

1048 name = str(os.getpid()) + '_' + random_string(64) + '_' + name

1049 return const.EPHEMERAL_DIR + '/' + name

1050

1051

1052def ephemeral_dir(name: str) -> str:

1053 """Create a directory in the ephemeral directory, if it does not exist yet.

1054

1055 Args:

1056 name: Directory name.

1057

1058 Returns:

1059 The directory path.

1060 """

1061

1062 ephemeral_cleanup()

1063 return ensure_dir(const.EPHEMERAL_DIR + '/' + name)

1064

1065

1066def ephemeral_cleanup(force=False):

1067 """Remove ephemeral files and empty directories older than two hours.

1068

1069 Throttled to once per two hours in each process, unless ``force`` is set.

1070

1071 Args:

1072 force: Run even if the last run was recent.

1073

1074 Returns:

1075 The detached ``find`` process, or ``None`` if throttled.

1076 """

1077

1078 ts = stime()

1079

1080 if ts < _ephemeral_state['next_check'] and not force:

1081 return

1082

1083 _ephemeral_state['next_check'] = ts + _ephemeral_state['check_interval']

1084

1085 cutoff = ts - _ephemeral_state['max_age']

1086 cmd = [

1087 'find',

1088 const.EPHEMERAL_DIR,

1089 '-mindepth',

1090 '1',

1091 '!',

1092 '-newermt',

1093 f'@{cutoff}',

1094 '(',

1095 '-type',

1096 'f',

1097 '-o',

1098 '-type',

1099 'd',

1100 '-empty',

1101 ')',

1102 '-delete',

1103 ]

1104 return subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

1105

1106

1107def random_string(size: int) -> str:

1108 """Generate a random alphanumeric string.

1109

1110 Args:

1111 size: String length.

1112

1113 Returns:

1114 The string.

1115 """

1116

1117 a = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'

1118 r = random.SystemRandom()

1119 return ''.join(r.choice(a) for _ in range(size))

1120

1121

1122class _FormatMapDefault:

1123 """Mapping for ``str.format_map`` that returns a default for missing or ``None`` values."""

1124

1125 def __init__(self, d, default):

1126 self.d = d

1127 self.default = default

1128

1129 def __getitem__(self, item):

1130 val = self.d.get(item)

1131 return val if val is not None else self.default

1132

1133

1134def format_map(fmt: str, x: Union[dict, 'Data'], default: str = '') -> str:

1135 """Format a string with values from a dict or a ``Data`` object.

1136

1137 Args:

1138 fmt: Format string with ``{name}`` placeholders.

1139 x: A dict or ``Data`` object with the values.

1140 default: Replacement for missing or ``None`` values.

1141

1142 Returns:

1143 The formatted string.

1144 """

1145 return fmt.format_map(_FormatMapDefault(x, default))

1146

1147

1148def sha256(x) -> str:

1149 """Compute the SHA-256 hash of a value.

1150

1151 Bytes, strings and numbers are hashed directly, other values are converted to JSON first,

1152 with sorted keys and ``Data`` objects as dicts.

1153

1154 Args:

1155 x: A value.

1156

1157 Returns:

1158 The hex digest.

1159 """

1160

1161 def _bytes(x):

1162 if is_bytes(x):

1163 return bytes(x)

1164 if isinstance(x, (int, float, bool)):

1165 return str(x).encode('utf8')

1166 if isinstance(x, str):

1167 return x.encode('utf8')

1168

1169 def _default(x):

1170 if is_data_object(x):

1171 return vars(x)

1172 return str(x)

1173

1174 c = _bytes(x)

1175 if c is None:

1176 j = json.dumps(x, default=_default, sort_keys=True, ensure_ascii=True)

1177 c = j.encode('utf8')

1178

1179 return hashlib.sha256(c).hexdigest()

1180

1181

1182class cached_property:

1183 """Decorator for a cached property.

1184

1185 The value is computed on first access and stored as an instance attribute with the same name.

1186 """

1187

1188 def __init__(self, fn):

1189 """Create the descriptor.

1190

1191 Args:

1192 fn: The getter function.

1193 """

1194 self._fn = fn

1195 self.__doc__ = getattr(fn, '__doc__')

1196

1197 def __get__(self, obj, objtype=None):

1198 """Compute the value and store it on the instance."""

1199 value = self._fn(obj)

1200 setattr(obj, self._fn.__name__, value)

1201 return value

1202

1203

1204# application lock/globals are global to one application

1205# server locks lock the whole server

1206# server globals are pickled in /tmp

1207

1208

1209_app_locks: dict[str, threading.RLock] = {}

1210_app_master_lock = threading.Lock()

1211

1212

1213def app_lock(name=''):

1214 """Return a reentrant lock, local to this process.

1215

1216 Locks with the same name are shared.

1217

1218 Args:

1219 name: Lock name.

1220

1221 Returns:

1222 A ``threading.RLock``.

1223 """

1224 with _app_master_lock:

1225 lock = _app_locks.get(name)

1226 if lock is None:

1227 lock = _app_locks[name] = threading.RLock()

1228 return lock

1229

1230

1231_app_globals: dict = {}

1232

1233

1234def get_app_global(name, init_fn):

1235 """Get a process-wide global value, creating it if needed.

1236

1237 Args:

1238 name: Value name.

1239 init_fn: Function without arguments that creates the value. It is called once, under an app lock.

1240

1241 Returns:

1242 The value.

1243 """

1244 if name in _app_globals:

1245 return _app_globals[name]

1246

1247 with app_lock(name):

1248 if name not in _app_globals:

1249 _app_globals[name] = init_fn()

1250

1251 return _app_globals[name]

1252

1253

1254def set_app_global(name, value):

1255 """Set a process-wide global value.

1256

1257 Args:

1258 name: Value name.

1259 value: The value.

1260

1261 Returns:

1262 The value.

1263 """

1264 with app_lock(name):

1265 _app_globals[name] = value

1266 return _app_globals[name]

1267

1268

1269def delete_app_global(name):

1270 """Delete a process-wide global value.

1271

1272 Args:

1273 name: Value name.

1274 """

1275 with app_lock(name):

1276 _app_globals.pop(name, None)

1279##

1282def serialize_to_path(obj, path):

1283 """Pickle an object to a file atomically.

1284

1285 Args:

1286 obj: An object.

1287 path: File path.

1288

1289 Returns:

1290 The file path.

1291 """

1292 tmp = path + random_string(64)

1293 with open(tmp, 'wb') as fp:

1294 pickle.dump(obj, fp)

1295 os.replace(tmp, path)

1296 chown_default(path)

1297 return path

1298

1299

1300def unserialize_from_path(path):

1301 """Load a pickled object from a file.

1302

1303 Args:

1304 path: File path.

1305

1306 Returns:

1307 The object.

1308 """

1309 with open(path, 'rb') as fp:

1310 return pickle.load(fp)

1311

1312

1313_server_globals = {}

1314

1315

1316def get_ephemeral_content(name: str, init_fn) -> bytes:

1317 """Get bytes from the ephemeral content cache, creating them if needed.

1318

1319 The content is stored in the ephemeral directory and created under a server lock,

1320 so it is shared between processes until the ephemeral cleanup removes it.

1321

1322 Args:

1323 name: Content name.

1324 init_fn: Function without arguments that returns the content.

1325

1326 Returns:

1327 The content.

1328 """

1329 uid = to_uid(name)

1330 path = ephemeral_dir('content') + '/' + uid

1331

1332 def _get():

1333 if not os.path.isfile(path):

1334 return

1335 try:

1336 return read_file_b(path)

1337 except OSError:

1338 return

1339

1340 b = _get()

1341 if b is not None:

1342 return b

1343

1344 with server_lock(f'ephemeral_content_{uid}'):

1345 b = _get()

1346 if b is not None:

1347 return b

1348

1349 b = init_fn()

1350 write_file_b(path, b)

1351 return b

1352

1353

1354def get_cached_object(name: str, life_time: int, init_fn):

1355 """Get an object from the object cache, creating it if needed.

1356

1357 The object is pickled in ``const.OBJECT_CACHE_DIR`` and created under a server lock.

1358 A cached object older than ``life_time``, or ``None``, is created again.

1359 Load and store errors are logged and ignored.

1360

1361 Args:

1362 name: Object name.

1363 life_time: Life time in seconds.

1364 init_fn: Function without arguments that creates the object.

1365

1366 Returns:

1367 The object.

1368 """

1369 uid = to_uid(name)

1370 path = const.OBJECT_CACHE_DIR + '/' + uid

1371

1372 def _get():

1373 if not os.path.isfile(path):

1374 return

1375 try:

1376 age = int(time.time() - os.stat(path).st_mtime)

1377 except OSError:

1378 return

1379 if age < life_time:

1380 try:

1381 obj = unserialize_from_path(path)

1382 log.debug(f'get_cached_object {uid!r} {life_time=} {age=} - loaded')

1383 return obj

1384 except:

1385 log.exception(f'get_cached_object {uid!r} LOAD ERROR')

1386

1387 obj = _get()

1388 if obj is not None:

1389 return obj

1390

1391 with server_lock(uid):

1392 obj = _get()

1393 if obj is not None:

1394 return obj

1395

1396 obj = init_fn()

1397 try:

1398 serialize_to_path(obj, path)

1399 log.debug(f'get_cached_object {uid!r} - stored')

1400 except:

1401 log.exception(f'get_cached_object {uid!r} STORE ERROR')

1402

1403 return obj

1404

1405

1406def get_server_global(name: str, init_fn):

1407 """Get a server-wide global value, creating it if needed.

1408

1409 The value is kept in memory and pickled in ``const.GLOBALS_DIR``, so other processes load it

1410 instead of creating it again. It is created under a server lock.

1411 Load and store errors are logged and ignored.

1412

1413 Args:

1414 name: Value name.

1415 init_fn: Function without arguments that creates the value.

1416

1417 Returns:

1418 The value.

1419 """

1420 uid = to_uid(name)

1421 path = const.GLOBALS_DIR + '/' + uid

1422

1423 def _get():

1424 if uid in _server_globals:

1425 log.debug(f'get_server_global {uid!r} - found')

1426 return True

1427

1428 if os.path.isfile(path):

1429 try:

1430 _server_globals[uid] = unserialize_from_path(path)

1431 log.debug(f'get_server_global {uid!r} - loaded')

1432 return True

1433 except:

1434 log.exception(f'get_server_global {uid!r} LOAD ERROR')

1435

1436 if _get():

1437 return _server_globals[uid]

1438

1439 with server_lock(uid):

1440 if _get():

1441 return _server_globals[uid]

1442

1443 _server_globals[uid] = init_fn()

1444

1445 try:

1446 serialize_to_path(_server_globals[uid], path)

1447 log.debug(f'get_server_global {uid!r} - stored')

1448 except:

1449 log.exception(f'get_server_global {uid!r} STORE ERROR')

1450

1451 return _server_globals[uid]

1452

1453

1454class LockBusyError(Exception):

1455 """Raised when a server lock cannot be acquired within the timeout."""

1456

1457 pass

1458

1459

1460class _FileLock:

1461 """Inter-process lock based on ``flock`` on a file in ``const.LOCKS_DIR``."""

1462

1463 _PAUSE = 0.05

1464

1465 def __init__(self, uid, timeout):

1466 """Create the lock.

1467

1468 Args:

1469 uid: Lock identifier.

1470 timeout: Seconds to wait for the lock.

1471 """

1472 self.uid = to_uid(uid)

1473 self.path = const.LOCKS_DIR + '/' + self.uid

1474 self.timeout = timeout

1475 self.fp = None

1476

1477 def __enter__(self):

1478 self.acquire()

1479 return self

1480

1481 def __exit__(self, exc_type, exc_val, exc_tb):

1482 self.release()

1483

1484 def acquire(self):

1485 """Acquire the lock and write the process id into the lock file.

1486

1487 Raises:

1488 LockBusyError: If the lock is not acquired within the timeout.

1489 """

1490 ts = time.time()

1491 self.fp = os.open(self.path, os.O_CREAT | os.O_RDWR)

1492

1493 while True:

1494 try:

1495 fcntl.flock(self.fp, fcntl.LOCK_EX | fcntl.LOCK_NB)

1496 except OSError:

1497 pass

1498 else:

1499 os.ftruncate(self.fp, 0)

1500 os.write(self.fp, str(os.getpid()).encode('ascii'))

1501 log.debug(f'server lock {self.uid!r}: acquired')

1502 return

1503

1504 t = time.time() - ts

1505

1506 if t >= self.timeout:

1507 pid = self._holder_pid()

1508 os.close(self.fp)

1509 self.fp = None

1510 log.debug(f'server lock {self.uid!r}: BUSY {pid=}')

1511 raise LockBusyError(f'server lock {self.uid!r}: busy {pid=}')

1512

1513 time.sleep(self._PAUSE)

1514

1515 def release(self):

1516 """Release the lock. Errors are logged and ignored."""

1517 if self.fp is None:

1518 return

1519 try:

1520 fcntl.flock(self.fp, fcntl.LOCK_UN)

1521 os.close(self.fp)

1522 log.debug(f'server lock {self.uid!r}: released')

1523 except OSError as exc:

1524 log.exception(f'server lock {self.uid!r}: RELEASE ERROR {exc!r}')

1525 self.fp = None

1526

1527 def _holder_pid(self):

1528 """Return the process id stored in the lock file, or ``?``."""

1529 if self.fp is None:

1530 return '?'

1531 try:

1532 os.lseek(self.fp, 0, os.SEEK_SET)

1533 return os.read(self.fp, 64).decode('ascii') or '?'

1534 except OSError:

1535 return '?'

1536

1537

1538def server_lock(uid, timeout: float = 60):

1539 """Create an inter-process lock, to be used as a context manager.

1540

1541 The lock is acquired when the ``with`` block is entered.

1542

1543 Example::

1544

1545 with gws.u.server_lock('my_task', timeout=5):

1546 ...

1547

1548 Args:

1549 uid: Lock identifier.

1550 timeout: Seconds to wait for the lock. ``0`` means a single attempt.

1551

1552 Returns:

1553 The lock object.

1554

1555 Raises:

1556 LockBusyError: If the lock is not acquired within ``timeout``.

1557 """

1558 return _FileLock(uid, timeout)

1561##

1564def action_url_path(name: str, **kwargs) -> str:

1565 """Build a server URL path for an action.

1566

1567 Example::

1568

1569 action_url_path('owsService', serviceUid='wms', projectUid='') # '/_/owsService/serviceUid/wms'

1570

1571 Args:

1572 name: Action command name.

1573 **kwargs: Parameters, appended as ``/key/value`` path segments. Empty values are skipped.

1574

1575 Returns:

1576 The URL path.

1577 """

1578 ls = []

1579

1580 for k, v in kwargs.items():

1581 if not is_empty(v):

1582 ls.append(urllib.parse.quote(k))

1583 ls.append(urllib.parse.quote(to_str(v)))

1584

1585 path = const.SERVER_ENDPOINT + '/' + name

1586 if ls:

1587 path += '/' + '/'.join(ls)

1588 return path

1591##

1594def utime() -> float:

1595 """Return the Unix time as a float number.

1596

1597 Returns:

1598 Seconds since the epoch.

1599 """

1600 return time.time()

1601

1602

1603def stime() -> int:

1604 """Return the Unix time as an integer number of seconds.

1605

1606 Returns:

1607 Seconds since the epoch.

1608 """

1609 return int(time.time())

1610

1611

1612def sleep(n: float):

1613 """Sleep for a number of seconds.

1614

1615 Args:

1616 n: Seconds.

1617 """

1618 time.sleep(n)

1619

1620

1621def mstime() -> int:

1622 """Return the Unix time as an integer number of milliseconds.

1623

1624 Returns:

1625 Milliseconds since the epoch.

1626 """

1627 return int(time.time() * 1000)

1628

1629

1630def microtime() -> int:

1631 """Return the Unix time as an integer number of microseconds.

1632

1633 Returns:

1634 Microseconds since the epoch.

1635 """

1636 return int(time.time() * 1000000)