Coverage for gws-app/gws/core/util.py: 63%

1"""General utilities, available as ``gws.u``."""

3import fcntl

4import hashlib

5import json

6import os

7import pickle

8import random

9import re

10import subprocess

11import sys

12import threading

13import time

14import urllib.parse

15from typing import Optional, TypeVar, Union, cast

17from . import const, log

20def is_data_object(x) -> bool:

21 """Check if the argument is a ``Data`` object.

23 This is a placeholder, replaced by ``gws.is_data_object`` when ``gws`` is imported.

25 Args:

26 x: A value.

28 Returns:

29 ``True`` if the value is a ``Data`` object.

30 """

31 return False

34def to_data_object(x):

35 """Convert a value to a ``Data`` object.

37 This is a placeholder, replaced by ``gws.to_data_object`` when ``gws`` is imported.

39 Args:

40 x: A value.

42 Returns:

43 A ``Data`` object.

44 """

45 pass

48def exit(code: int = 255):

49 """Exit the application.

51 Args:

52 code: Exit code.

53 """

55 sys.exit(code)

58T = TypeVar('T')

61def require(value: Optional[T], message: str = '') -> T:

62 """Return the value if it is not ``None``, otherwise raise an error.

64 Args:

65 value: A value.

66 message: Error message.

68 Returns:

69 The value.

71 Raises:

72 ValueError: If the value is ``None``.

73 """

74 if value is None:

75 raise ValueError(message or 'unexpected None value')

76 return value

79##

81# @TODO use ABC

84def is_list(x):

85 """Check if the value is a list or a tuple.

87 Args:

88 x: A value.

90 Returns:

91 ``True`` if the value is a list or a tuple.

92 """

93 return isinstance(x, (list, tuple))

96def is_dict(x):

97 """Check if the value is a dict.

99 Args:

100 x: A value.

101

102 Returns:

103 ``True`` if the value is a dict.

104 """

105 return isinstance(x, dict)

106

107

108def is_bytes(x):

109 """Check if the value is ``bytes`` or ``bytearray``.

110

111 Args:

112 x: A value.

113

114 Returns:

115 ``True`` if the value is ``bytes`` or ``bytearray``.

116 """

117 return isinstance(x, (bytes, bytearray))

118 # @TODO how to handle bytes-alikes?

119 # return hasattr(x, 'decode')

120

121

122def is_atom(x):

123 """Check if the value is ``None`` or a scalar (number, bool, string or bytes).

124

125 Args:

126 x: A value.

127

128 Returns:

129 ``True`` if the value is an atom.

130 """

131 return x is None or isinstance(x, (int, float, bool, str, bytes))

132

133

134def is_empty(x) -> bool:

135 """Check if the value is empty.

136

137 A value is empty if it is ``None``, has zero length, or is an object without attributes.

138

139 Args:

140 x: A value.

141

142 Returns:

143 ``True`` if the value is empty.

144 """

145

146 if x is None:

147 return True

148 try:

149 return len(x) == 0

150 except TypeError:

151 pass

152 try:

153 return not vars(x)

154 except TypeError:

155 pass

156 return False

159##

162def get(x, key, default=None):

163 """Get a nested value or attribute from a structure.

164

165 Args:

166 x: A dict, list, ``Data`` or any object.

167 key: A list or a dot-separated string of nested keys. List elements are addressed by numeric keys.

168 default: The default value.

169

170 Returns:

171 The value if it exists and the default otherwise.

172 """

173

174 if not x:

175 return default

176 if isinstance(key, str):

177 key = key.split('.')

178 try:

179 return _get(x, key)

180 except (KeyError, IndexError, AttributeError, ValueError):

181 return default

182

183

184def has(x, key) -> bool:

185 """Check if a nested value or attribute exists in a structure.

186

187 Args:

188 x: A dict, list, ``Data`` or any object.

189 key: A list or a dot-separated string of nested keys.

190

191 Returns:

192 ``True`` if the key exists.

193 """

194

195 if not x:

196 return False

197 if isinstance(key, str):

198 key = key.split('.')

199 try:

200 _get(x, key)

201 return True

202 except (KeyError, IndexError, AttributeError, ValueError):

203 return False

204

205

206def _get(x, keys):

207 """Follow the keys into a structure, raise an error if a key is missing."""

208 for k in keys:

209 if is_dict(x):

210 x = x[k]

211 elif is_list(x):

212 x = x[int(k)]

213 elif is_data_object(x):

214 # special case: raise a KeyError if the attribute is truly missing in a Data

215 # (and not just equals to None)

216 x = vars(x)[k]

217 else:

218 x = getattr(x, k)

219 return x

220

221

222def pop(x, key, default=None):

223 """Remove a key from a dict or a ``Data`` object and return its value.

224

225 Args:

226 x: A dict or ``Data``.

227 key: The key.

228 default: Value to return if the key is missing or ``x`` is of another type.

229

230 Returns:

231 The value or the default.

232 """

233 if is_dict(x):

234 return x.pop(key, default)

235 if is_data_object(x):

236 return vars(x).pop(key, default)

237 return default

238

239

240def pick(x, *keys):

241 """Return a copy of a dict or a ``Data`` object with the given keys only.

242

243 Args:

244 x: A dict or ``Data``.

245 *keys: Keys to keep.

246

247 Returns:

248 A new object of the same type, or an empty dict if ``x`` is of another type.

249 """

250 def _pick(d):

251 r = {}

252 for k in keys:

253 if k in d:

254 r[k] = d[k]

255 return r

256

257 if is_dict(x):

258 return _pick(x)

259 if is_data_object(x):

260 return type(x)(_pick(vars(x)))

261 return {}

262

263

264def omit(x, *keys):

265 """Return a copy of a dict or a ``Data`` object without the given keys.

266

267 Args:

268 x: A dict or ``Data``.

269 *keys: Keys to remove.

270

271 Returns:

272 A new object of the same type, or an empty dict if ``x`` is of another type.

273 """

274 def _omit(d):

275 r = {}

276 for k, v in d.items():

277 if k not in keys:

278 r[k] = d[k]

279 return r

280

281 if is_dict(x):

282 return _omit(x)

283 if is_data_object(x):

284 return type(x)(_omit(vars(x)))

285 return {}

286

287

288def collect(pairs):

289 """Group values by keys.

290

291 Args:

292 pairs: An iterable of ``(key, value)`` pairs. Pairs with a ``None`` key are skipped.

293

294 Returns:

295 A dict mapping each key to the list of its values.

296 """

297 m = {}

298

299 for key, val in pairs:

300 if key is not None:

301 m.setdefault(key, []).append(val)

302

303 return m

304

305

306def first(it):

307 """Return the first element of an iterable.

308

309 Args:

310 it: An iterable.

311

312 Returns:

313 The first element, or ``None`` if the iterable is empty.

314 """

315 for x in it:

316 return x

317

318

319def first_not_none(*args):

320 """Return the first argument that is not ``None``.

321

322 Args:

323 *args: Values.

324

325 Returns:

326 The first value that is not ``None``, or ``None``.

327 """

328 for a in args:

329 if a is not None:

330 return a

331

332

333def merge(*args, **kwargs) -> Union[dict, 'Data']:

334 """Create a new dict or ``Data`` object by merging values from dicts, ``Data`` objects and keyword args.

335

336 Later values override earlier ones, unless they are ``None``.

337

338 Args:

339 *args: Dicts or ``Data`` objects. Empty values are skipped.

340 **kwargs: Keyword args, merged last.

341

342 Returns:

343 A new object of the same type as the first argument, or a dict if it is a dict or ``None``.

344 """

345

346 def _merge(arg):

347 for k, v in to_dict(arg).items():

348 if v is not None:

349 m[k] = v

350

351 m = {}

352

353 for a in args:

354 if a:

355 _merge(a)

356 if kwargs:

357 _merge(kwargs)

358

359 if not args or isinstance(args[0], dict) or args[0] is None:

360 return m

361 return type(args[0])(m)

362

363

364def compact(x):

365 """Remove all ``None`` values from a collection.

366

367 Args:

368 x: A dict, ``Data`` or an iterable.

369

370 Returns:

371 A new dict or ``Data`` object, or a list for other iterables.

372 """

373

374 if is_dict(x):

375 return {k: v for k, v in x.items() if v is not None}

376 if is_data_object(x):

377 d = {k: v for k, v in vars(x).items() if v is not None}

378 return type(x)(d)

379 return [v for v in x if v is not None]

380

381

382def strip(x):

383 """Strip all strings and remove empty values from a collection.

384

385 Args:

386 x: A dict, ``Data`` or an iterable.

387

388 Returns:

389 A new dict or ``Data`` object, or a list for other iterables.

390 """

391

392 def _strip(v):

393 if isinstance(v, (str, bytes, bytearray)):

394 return v.strip()

395 return v

396

397 def _dict(x1):

398 d = {}

399 for k, v in x1.items():

400 v = _strip(v)

401 if not is_empty(v):

402 d[k] = v

403 return d

404

405 if is_dict(x):

406 return _dict(x)

407 if is_data_object(x):

408 return type(x)(_dict(vars(x)))

409

410 r = [_strip(v) for v in x]

411 return [v for v in r if not is_empty(v)]

412

413

414def uniq(x):

415 """Remove duplicate elements from a collection, keeping the order.

416

417 Args:

418 x: An iterable.

419

420 Returns:

421 A list of unique elements.

422 """

423

424 s = set()

425 r = []

426

427 for y in x:

428 try:

429 if y not in s:

430 s.add(y)

431 r.append(y)

432 except TypeError:

433 if y not in r:

434 r.append(y)

436 return r

439##

442def to_int(x) -> int:

443 """Convert a value to an int.

444

445 Args:

446 x: A value.

447

448 Returns:

449 The int value, or 0 if the conversion fails.

450 """

451

452 try:

453 return int(x)

454 except:

455 return 0

456

457

458def to_rounded_int(x) -> int:

459 """Round a float and convert a value to an int.

460

461 Args:

462 x: A value.

463

464 Returns:

465 The int value, or 0 if the conversion fails.

466 """

467

468 try:

469 if isinstance(x, float):

470 return int(round(x))

471 return int(x)

472 except:

473 return 0

474

475

476def to_float(x) -> float:

477 """Convert a value to a float.

478

479 Args:

480 x: A value.

481

482 Returns:

483 The float value, or 0.0 if the conversion fails.

484 """

485

486 try:

487 return float(x)

488 except:

489 return 0.0

490

491

492def to_str(x, encodings: list[str] = None) -> str:

493 """Convert a value to a string.

494

495 Bytes are decoded, ``None`` becomes an empty string, other values are converted with ``str``.

496

497 Args:

498 x: A value.

499 encodings: A list of acceptable encodings. If the value is bytes, try each encoding

500 and return the first result that decodes without errors. If none succeeds,

501 decode as UTF-8 and ignore errors.

502

503 Returns:

504 A string.

505 """

506

507 if isinstance(x, str):

508 return x

509 if x is None:

510 return ''

511 if not is_bytes(x):

512 return str(x)

513 if encodings:

514 for enc in encodings:

515 try:

516 return x.decode(encoding=enc, errors='strict')

517 except UnicodeDecodeError:

518 pass

519 return x.decode(encoding='utf-8', errors='ignore')

520

521

522def to_bytes(x, encoding='utf8') -> bytes:

523 """Convert a value to bytes by converting it to a string and encoding it.

524

525 Args:

526 x: A value. Bytes are returned as is, ``None`` becomes empty bytes.

527 encoding: The encoding.

528

529 Returns:

530 Bytes.

531 """

532

533 if is_bytes(x):

534 return bytes(x)

535 if x is None:

536 return b''

537 if not isinstance(x, str):

538 x = str(x)

539 return x.encode(encoding or 'utf8')

540

541

542def to_list(x, delimiter: str = ',') -> list:

543 """Convert a value to a list.

544

545 Args:

546 x: A value. A string (or bytes) is split by the delimiter, the parts are stripped and empty parts removed.

547 A number or a bool becomes a one-element list, other iterables are converted to a list.

548 delimiter: The delimiter. If empty, a string becomes a one-element list.

549

550 Returns:

551 A list. Empty values and values that cannot be converted give an empty list.

552 """

553

554 if isinstance(x, list):

555 return x

556 if is_empty(x):

557 return []

558 if is_bytes(x):

559 x = to_str(x)

560 if isinstance(x, str):

561 if delimiter:

562 ls = [s.strip() for s in x.split(delimiter)]

563 return [s for s in ls if s]

564 return [x]

565 if isinstance(x, (int, float, bool)):

566 return [x]

567 try:

568 return [s for s in x]

569 except TypeError:

570 return []

571

572

573def to_dict(x) -> dict:

574 """Convert a value to a dict.

575

576 Args:

577 x: A dict, ``None``, a named tuple or an object.

578

579 Returns:

580 The dict itself, an empty dict for ``None``, the fields of a named tuple, or the object's ``vars``.

581

582 Raises:

583 ValueError: If the value cannot be converted.

584 """

585

586 if is_dict(x):

587 return x

588 if x is None:

589 return {}

590 try:

591 f = getattr(x, '_asdict', None)

592 if f:

593 return f()

594 return vars(x)

595 except TypeError:

596 raise ValueError(f'cannot convert {x!r} to dict')

597

598

599def to_json_value(x) -> Union[dict, list, str, int, float, bool, None]:

600 """Recursively convert a value to a JSON serializable type.

601

602 Args:

603 x: A value. Dicts, lists and ``Data`` objects are converted recursively, other non-scalar values are converted with ``str``.

604

605 Returns:

606 A JSON serializable value.

607 """

608

609 if is_atom(x):

610 return x

611 if is_dict(x):

612 return {k: to_json_value(v) for k, v in x.items()}

613 if is_list(x):

614 return [to_json_value(v) for v in x]

615 if is_data_object(x):

616 return {k: to_json_value(v) for k, v in vars(x).items()}

617 return str(x)

618

619

620def to_upper_dict(x) -> dict:

621 """Convert a value to a dict with upper-case keys.

622

623 Args:

624 x: A value accepted by ``to_dict``.

625

626 Returns:

627 A new dict.

628 """

629 x = to_dict(x)

630 return {k.upper(): v for k, v in x.items()}

631

632

633def to_lower_dict(x) -> dict:

634 """Convert a value to a dict with lower-case keys.

635

636 Args:

637 x: A value accepted by ``to_dict``.

638

639 Returns:

640 A new dict.

641 """

642 x = to_dict(x)

643 return {k.lower(): v for k, v in x.items()}

644

645

646##

647

648_UID_DE_TRANS = {

649 ord('ä'): 'ae',

650 ord('ö'): 'oe',

651 ord('ü'): 'ue',

652 ord('ß'): 'ss',

653}

654

655

656def to_uid(x) -> str:

657 """Convert a value to a uid.

658

659 The value is converted to a lower-case string, German umlauts are transliterated,

660 and runs of other characters are replaced with underscores.

661

662 Args:

663 x: A value.

664

665 Returns:

666 A string of ``a-z``, ``0-9`` and ``_``, or an empty string for an empty value.

667 """

668

669 if not x:

670 return ''

671 x = to_str(x).lower().strip().translate(_UID_DE_TRANS)

672 x = re.sub(r'[^a-z0-9]+', '_', x)

673 return x.strip('_')

674

675

676def to_lines(txt: str, comment: str = None) -> list[str]:

677 """Convert a multiline string into a list of strings.

678

679 Args:

680 txt: A string.

681 comment: Comment marker. If given, everything from the marker to the end of the line is removed.

682

683 Returns:

684 A list of stripped, non-empty lines.

685 """

686

687 ls = []

688

689 for s in txt.splitlines():

690 if comment and comment in s:

691 s = s.split(comment)[0]

692 s = s.strip()

693 if s:

694 ls.append(s)

696 return ls

699##

702def parse_acl(acl):

703 """Parse an ACL config into an ACL.

704

705 Args:

706 acl: An ACL config. Can be given as a string ``allow X, allow Y, deny Z``,

707 or as a list of dicts ``{ role X type allow }, { role Y type deny }``,

708 or it can already be an ACL ``[1 X], [0 Y]``,

709 or it can be ``None``.

710

711 Returns:

712 Access list, empty for an empty value.

713

714 Raises:

715 ValueError: If the ACL config is invalid.

716 """

717

718 if not acl:

719 return []

720

721 a = 'allow'

722 d = 'deny'

723 bits = {const.ALLOW, const.DENY}

724 err = 'invalid ACL'

725

726 access = []

727

728 if isinstance(acl, str):

729 for p in acl.strip().split(','):

730 s = p.strip().split()

731 if len(s) != 2:

732 raise ValueError(err)

733 if s[0] == a:

734 access.append((const.ALLOW, s[1]))

735 elif s[0] == d:

736 access.append((const.DENY, s[1]))

737 else:

738 raise ValueError(err)

739 return access

740

741 if not isinstance(acl, list):

742 raise ValueError(err)

743

744 if isinstance(acl[0], (list, tuple)):

745 try:

746 if all(len(s) == 2 and s[0] in bits for s in acl):

747 return acl

748 except (TypeError, IndexError):

749 pass

750 raise ValueError(err)

751

752 if isinstance(acl[0], dict):

753 for s in acl:

754 tk = s.get('type', '')

755 rk = s.get('role', '')

756 if not isinstance(rk, str):

757 raise ValueError(err)

758 if tk == a:

759 access.append((const.ALLOW, rk))

760 elif tk == d:

761 access.append((const.DENY, rk))

762 else:

763 raise ValueError(err)

764 return access

765

766 raise ValueError(err)

767

768

769##

770

771UID_DELIMITER = '::'

772

773

774def join_uid(parent_uid, object_uid):

775 """Join a parent uid and an object uid with ``UID_DELIMITER``.

776

777 If either uid is already joined, only its last part is used.

778

779 Args:

780 parent_uid: Parent uid.

781 object_uid: Object uid.

782

783 Returns:

784 The joined uid.

785 """

786 p = parent_uid.split(UID_DELIMITER)

787 u = object_uid.split(UID_DELIMITER)

788 return p[-1] + UID_DELIMITER + u[-1]

789

790

791def split_uid(joined_uid: str) -> tuple[str, str]:

792 """Split a joined uid at the first ``UID_DELIMITER``.

793

794 Args:

795 joined_uid: Joined uid.

796

797 Returns:

798 A tuple of the parent uid and the object uid. If there is no delimiter, the object uid is empty.

799 """

800 p, _, u = joined_uid.partition(UID_DELIMITER)

801 return p, u

804##

807def is_file(path):

808 """Check if the path is an existing file.

809

810 Args:

811 path: File path.

812

813 Returns:

814 ``True`` if the file exists.

815 """

816 return os.path.isfile(path)

817

818

819def is_dir(path):

820 """Check if the path is an existing directory.

821

822 Args:

823 path: Directory path.

824

825 Returns:

826 ``True`` if the directory exists.

827 """

828 return os.path.isdir(path)

829

830

831def read_file(path: str) -> str:

832 """Read a UTF-8 text file.

833

834 Args:

835 path: File path.

836

837 Returns:

838 The file content.

839

840 Raises:

841 Exception: Errors from opening or reading the file are logged and raised again.

842 """

843 try:

844 with open(path, 'rt', encoding='utf8') as fp:

845 return fp.read()

846 except Exception as exc:

847 log.debug(f'error reading {path=} {exc=}')

848 raise

849

850

851def read_file_b(path: str) -> bytes:

852 """Read a binary file.

853

854 Args:

855 path: File path.

856

857 Returns:

858 The file content.

859

860 Raises:

861 Exception: Errors from opening or reading the file are logged and raised again.

862 """

863 try:

864 with open(path, 'rb') as fp:

865 return fp.read()

866 except Exception as exc:

867 log.debug(f'error reading {path=} {exc=}')

868 raise

869

870

871def write_file(path: str, s: str, user: int = None, group: int = None):

872 """Write a text file atomically, via a temporary file in the same directory.

873

874 Args:

875 path: File path.

876 s: Text content, encoded as UTF-8.

877 user: File owner, defaults to ``const.UID``.

878 group: File group, defaults to ``const.GID``.

879

880 Returns:

881 The file path.

882

883 Raises:

884 Exception: Write errors are logged and raised again.

885 """

886

887 return write_file_b(path, s.encode('utf8'), user, group)

888

889

890def write_file_b(path: str, s: str | bytes, user: int = None, group: int = None):

891 """Write a binary file atomically, via a temporary file in the same directory.

892

893 Args:

894 path: File path.

895 s: Content. A string is encoded as UTF-8.

896 user: File owner, defaults to ``const.UID``.

897 group: File group, defaults to ``const.GID``.

898

899 Returns:

900 The file path.

901

902 Raises:

903 Exception: Write errors are logged and raised again.

904 """

905

906 if isinstance(s, str):

907 s = s.encode('utf8')

908 tmp = f'{path}.{random_string(32)}.tmp'

909 try:

910 with open(tmp, 'wb') as fp:

911 fp.write(s)

912 chown_default(tmp, user, group)

913 os.replace(tmp, path)

914 return path

915 except Exception as exc:

916 log.debug(f'error writing {path=} {exc=}')

917 try:

918 os.unlink(tmp)

919 except OSError:

920 pass

921 raise

922

923

924def write_debug_file(path: str, s: str | bytes):

925 """Write a file to the debug directory ``<VAR_DIR>/debug``.

926

927 Errors are logged and ignored.

928

929 Args:

930 path: File path, relative to the debug directory.

931 s: Content. A string is encoded as UTF-8.

932 """

933

934 if isinstance(s, str):

935 s = s.encode('utf8')

936 try:

937 d = ensure_dir(f'{const.VAR_DIR}/debug')

938 with open(f'{d}/{path}', 'wb') as fp:

939 fp.write(s)

940 except Exception as exc:

941 log.debug(f'error writing debug {path=} {exc=}')

942

943

944def dirname(path):

945 """Return the directory part of a path.

946

947 Args:

948 path: A path.

949

950 Returns:

951 The directory name.

952 """

953 return os.path.dirname(path)

954

955

956def ensure_dir(dir_path: str, base_dir: str = None, mode: int = 0o755, user: int = None, group: int = None) -> str:

957 """Check if a (possibly nested) directory exists and create it if it does not.

958

959 Args:

960 dir_path: Path to a directory. Must be absolute without ``base_dir`` and relative with it.

961 base_dir: Base directory.

962 mode: Directory creation mode.

963 user: Directory owner, defaults to ``const.UID``.

964 group: Directory group, defaults to ``const.GID``.

965

966 Returns:

967 The path to the directory.

968

969 Raises:

970 ValueError: If the path is absolute with ``base_dir``, or relative without it.

971 """

972

973 if base_dir:

974 if os.path.isabs(dir_path):

975 raise ValueError(f'cannot use an absolute path {dir_path!r} with a base dir')

976 bpath = cast(bytes, os.path.join(base_dir.encode('utf8'), dir_path.encode('utf8')))

977 else:

978 if not os.path.isabs(dir_path):

979 raise ValueError(f'cannot use a relative path {dir_path!r} without a base dir')

980 bpath = dir_path.encode('utf8')

981

982 if os.path.isdir(bpath):

983 return bpath.decode('utf8')

984

985 parts = []

986

987 for p in bpath.split(b'/'):

988 parts.append(p)

989 path = b'/'.join(parts)

990 if path and not os.path.isdir(path):

991 try:

992 os.mkdir(path, mode)

993 except FileExistsError:

994 pass

995

996 chown_default(bpath, user, group)

997 return bpath.decode('utf8')

998

999

1000def ensure_system_dirs():

1001 """Create all system directories listed in ``const.ALL_DIRS``."""

1002 for d in const.ALL_DIRS:

1003 ensure_dir(d)

1004

1005

1006def chown_default(path, user=None, group=None):

1007 """Change the owner of a path, ignoring errors.

1008

1009 Args:

1010 path: A path.

1011 user: Owner, defaults to ``const.UID``.

1012 group: Group, defaults to ``const.GID``.

1013 """

1014 try:

1015 os.chown(path, user or const.UID, group or const.GID)

1016 except OSError:

1017 pass

1018

1019

1020_ephemeral_state = dict(

1021 next_check=0,

1022 check_interval=2 * 3600,

1023 max_age=2 * 3600,

1024)

1025

1026

1027def ephemeral_path(name: str) -> str:

1028 """Return a new unique path in the ephemeral directory.

1029

1030 The path is not created. Ephemeral paths are removed by ``ephemeral_cleanup`` after two hours.

1031

1032 Args:

1033 name: Base name, appended to a unique prefix.

1034

1035 Returns:

1036 The path.

1037 """

1038

1039 ephemeral_cleanup()

1040 name = str(os.getpid()) + '_' + random_string(64) + '_' + name

1041 return const.EPHEMERAL_DIR + '/' + name

1042

1043

1044def ephemeral_dir(name: str) -> str:

1045 """Create a directory in the ephemeral directory, if it does not exist yet.

1046

1047 Args:

1048 name: Directory name.

1049

1050 Returns:

1051 The directory path.

1052 """

1053

1054 ephemeral_cleanup()

1055 return ensure_dir(const.EPHEMERAL_DIR + '/' + name)

1056

1057

1058def ephemeral_cleanup(force=False):

1059 """Remove ephemeral files and empty directories older than two hours.

1060

1061 Throttled to once per two hours in each process, unless ``force`` is set.

1062

1063 Args:

1064 force: Run even if the last run was recent.

1065

1066 Returns:

1067 The detached ``find`` process, or ``None`` if throttled.

1068 """

1069

1070 ts = stime()

1071

1072 if ts < _ephemeral_state['next_check'] and not force:

1073 return

1074

1075 _ephemeral_state['next_check'] = ts + _ephemeral_state['check_interval']

1076

1077 cutoff = ts - _ephemeral_state['max_age']

1078 cmd = [

1079 'find', const.EPHEMERAL_DIR, '-mindepth', '1',

1080 '!', '-newermt', f'@{cutoff}', '(', '-type', 'f', '-o', '-type', 'd', '-empty', ')',

1081 '-delete',

1082 ]

1083 return subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)

1084

1085

1086def random_string(size: int) -> str:

1087 """Generate a random alphanumeric string.

1088

1089 Args:

1090 size: String length.

1091

1092 Returns:

1093 The string.

1094 """

1095

1096 a = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'

1097 r = random.SystemRandom()

1098 return ''.join(r.choice(a) for _ in range(size))

1099

1100

1101class _FormatMapDefault:

1102 """Mapping for ``str.format_map`` that returns a default for missing or ``None`` values."""

1103 def __init__(self, d, default):

1104 self.d = d

1105 self.default = default

1106

1107 def __getitem__(self, item):

1108 val = self.d.get(item)

1109 return val if val is not None else self.default

1110

1111

1112def format_map(fmt: str, x: Union[dict, 'Data'], default: str = '') -> str:

1113 """Format a string with values from a dict or a ``Data`` object.

1114

1115 Args:

1116 fmt: Format string with ``{name}`` placeholders.

1117 x: A dict or ``Data`` object with the values.

1118 default: Replacement for missing or ``None`` values.

1119

1120 Returns:

1121 The formatted string.

1122 """

1123 return fmt.format_map(_FormatMapDefault(x, default))

1124

1125

1126def sha256(x) -> str:

1127 """Compute the SHA-256 hash of a value.

1128

1129 Bytes, strings and numbers are hashed directly, other values are converted to JSON first,

1130 with sorted keys and ``Data`` objects as dicts.

1131

1132 Args:

1133 x: A value.

1134

1135 Returns:

1136 The hex digest.

1137 """

1138 def _bytes(x):

1139 if is_bytes(x):

1140 return bytes(x)

1141 if isinstance(x, (int, float, bool)):

1142 return str(x).encode('utf8')

1143 if isinstance(x, str):

1144 return x.encode('utf8')

1145

1146 def _default(x):

1147 if is_data_object(x):

1148 return vars(x)

1149 return str(x)

1150

1151 c = _bytes(x)

1152 if c is None:

1153 j = json.dumps(x, default=_default, sort_keys=True, ensure_ascii=True)

1154 c = j.encode('utf8')

1155

1156 return hashlib.sha256(c).hexdigest()

1157

1158

1159class cached_property:

1160 """Decorator for a cached property.

1161

1162 The value is computed on first access and stored as an instance attribute with the same name.

1163 """

1164

1165 def __init__(self, fn):

1166 """Create the descriptor.

1167

1168 Args:

1169 fn: The getter function.

1170 """

1171 self._fn = fn

1172 self.__doc__ = getattr(fn, '__doc__')

1173

1174 def __get__(self, obj, objtype=None):

1175 """Compute the value and store it on the instance."""

1176 value = self._fn(obj)

1177 setattr(obj, self._fn.__name__, value)

1178 return value

1179

1180

1181# application lock/globals are global to one application

1182# server locks lock the whole server

1183# server globals are pickled in /tmp

1184

1185

1186_app_locks: dict[str, threading.RLock] = {}

1187_app_master_lock = threading.Lock()

1188

1189

1190def app_lock(name=''):

1191 """Return a reentrant lock, local to this process.

1192

1193 Locks with the same name are shared.

1194

1195 Args:

1196 name: Lock name.

1197

1198 Returns:

1199 A ``threading.RLock``.

1200 """

1201 with _app_master_lock:

1202 lock = _app_locks.get(name)

1203 if lock is None:

1204 lock = _app_locks[name] = threading.RLock()

1205 return lock

1206

1207

1208_app_globals: dict = {}

1209

1210

1211def get_app_global(name, init_fn):

1212 """Get a process-wide global value, creating it if needed.

1213

1214 Args:

1215 name: Value name.

1216 init_fn: Function without arguments that creates the value. It is called once, under an app lock.

1217

1218 Returns:

1219 The value.

1220 """

1221 if name in _app_globals:

1222 return _app_globals[name]

1223

1224 with app_lock(name):

1225 if name not in _app_globals:

1226 _app_globals[name] = init_fn()

1227

1228 return _app_globals[name]

1229

1230

1231def set_app_global(name, value):

1232 """Set a process-wide global value.

1233

1234 Args:

1235 name: Value name.

1236 value: The value.

1237

1238 Returns:

1239 The value.

1240 """

1241 with app_lock(name):

1242 _app_globals[name] = value

1243 return _app_globals[name]

1244

1245

1246def delete_app_global(name):

1247 """Delete a process-wide global value.

1248

1249 Args:

1250 name: Value name.

1251 """

1252 with app_lock(name):

1253 _app_globals.pop(name, None)

1256##

1259def serialize_to_path(obj, path):

1260 """Pickle an object to a file atomically.

1261

1262 Args:

1263 obj: An object.

1264 path: File path.

1265

1266 Returns:

1267 The file path.

1268 """

1269 tmp = path + random_string(64)

1270 with open(tmp, 'wb') as fp:

1271 pickle.dump(obj, fp)

1272 os.replace(tmp, path)

1273 chown_default(path)

1274 return path

1275

1276

1277def unserialize_from_path(path):

1278 """Load a pickled object from a file.

1279

1280 Args:

1281 path: File path.

1282

1283 Returns:

1284 The object.

1285 """

1286 with open(path, 'rb') as fp:

1287 return pickle.load(fp)

1288

1289

1290_server_globals = {}

1291

1292

1293def get_ephemeral_content(name: str, init_fn) -> bytes:

1294 """Get bytes from the ephemeral content cache, creating them if needed.

1295

1296 The content is stored in the ephemeral directory and created under a server lock,

1297 so it is shared between processes until the ephemeral cleanup removes it.

1298

1299 Args:

1300 name: Content name.

1301 init_fn: Function without arguments that returns the content.

1302

1303 Returns:

1304 The content.

1305 """

1306 uid = to_uid(name)

1307 path = ephemeral_dir('content') + '/' + uid

1308

1309 def _get():

1310 if not os.path.isfile(path):

1311 return

1312 try:

1313 return read_file_b(path)

1314 except OSError:

1315 return

1316

1317 b = _get()

1318 if b is not None:

1319 return b

1320

1321 with server_lock(f'ephemeral_content_{uid}'):

1322 b = _get()

1323 if b is not None:

1324 return b

1325

1326 b = init_fn()

1327 write_file_b(path, b)

1328 return b

1329

1330def get_cached_object(name: str, life_time: int, init_fn):

1331 """Get an object from the object cache, creating it if needed.

1332

1333 The object is pickled in ``const.OBJECT_CACHE_DIR`` and created under a server lock.

1334 A cached object older than ``life_time``, or a falsy one, is created again.

1335 Load and store errors are logged and ignored.

1336

1337 Args:

1338 name: Object name.

1339 life_time: Life time in seconds.

1340 init_fn: Function without arguments that creates the object.

1341

1342 Returns:

1343 The object.

1344 """

1345 uid = to_uid(name)

1346 path = const.OBJECT_CACHE_DIR + '/' + uid

1347

1348 def _get():

1349 if not os.path.isfile(path):

1350 return

1351 try:

1352 age = int(time.time() - os.stat(path).st_mtime)

1353 except OSError:

1354 return

1355 if age < life_time:

1356 try:

1357 obj = unserialize_from_path(path)

1358 log.debug(f'get_cached_object {uid!r} {life_time=} {age=} - loaded')

1359 return obj

1360 except:

1361 log.exception(f'get_cached_object {uid!r} LOAD ERROR')

1362

1363 obj = _get()

1364 if obj:

1365 return obj

1366

1367 with server_lock(uid):

1368 obj = _get()

1369 if obj:

1370 return obj

1371

1372 obj = init_fn()

1373 try:

1374 serialize_to_path(obj, path)

1375 log.debug(f'get_cached_object {uid!r} - stored')

1376 except:

1377 log.exception(f'get_cached_object {uid!r} STORE ERROR')

1378

1379 return obj

1380

1381

1382def get_server_global(name: str, init_fn):

1383 """Get a server-wide global value, creating it if needed.

1384

1385 The value is kept in memory and pickled in ``const.GLOBALS_DIR``, so other processes load it

1386 instead of creating it again. It is created under a server lock.

1387 Load and store errors are logged and ignored.

1388

1389 Args:

1390 name: Value name.

1391 init_fn: Function without arguments that creates the value.

1392

1393 Returns:

1394 The value.

1395 """

1396 uid = to_uid(name)

1397 path = const.GLOBALS_DIR + '/' + uid

1398

1399 def _get():

1400 if uid in _server_globals:

1401 log.debug(f'get_server_global {uid!r} - found')

1402 return True

1403

1404 if os.path.isfile(path):

1405 try:

1406 _server_globals[uid] = unserialize_from_path(path)

1407 log.debug(f'get_server_global {uid!r} - loaded')

1408 return True

1409 except:

1410 log.exception(f'get_server_global {uid!r} LOAD ERROR')

1411

1412 if _get():

1413 return _server_globals[uid]

1414

1415 with server_lock(uid):

1416 if _get():

1417 return _server_globals[uid]

1418

1419 _server_globals[uid] = init_fn()

1420

1421 try:

1422 serialize_to_path(_server_globals[uid], path)

1423 log.debug(f'get_server_global {uid!r} - stored')

1424 except:

1425 log.exception(f'get_server_global {uid!r} STORE ERROR')

1426

1427 return _server_globals[uid]

1428

1429

1430class LockBusyError(Exception):

1431 """Raised when a server lock cannot be acquired within the timeout."""

1432

1433 pass

1434

1435

1436class _FileLock:

1437 """Inter-process lock based on ``flock`` on a file in ``const.LOCKS_DIR``."""

1438 _PAUSE = 0.05

1439

1440 def __init__(self, uid, timeout):

1441 """Create the lock.

1442

1443 Args:

1444 uid: Lock identifier.

1445 timeout: Seconds to wait for the lock.

1446 """

1447 self.uid = to_uid(uid)

1448 self.path = const.LOCKS_DIR + '/' + self.uid

1449 self.timeout = timeout

1450 self.fp = None

1451

1452 def __enter__(self):

1453 self.acquire()

1454

1455 def __exit__(self, exc_type, exc_val, exc_tb):

1456 self.release()

1457

1458 def acquire(self):

1459 """Acquire the lock and write the process id into the lock file.

1460

1461 Raises:

1462 LockBusyError: If the lock is not acquired within the timeout.

1463 """

1464 ts = time.time()

1465 self.fp = os.open(self.path, os.O_CREAT | os.O_RDWR)

1466

1467 while True:

1468 try:

1469 fcntl.flock(self.fp, fcntl.LOCK_EX | fcntl.LOCK_NB)

1470 except OSError:

1471 pass

1472 else:

1473 os.ftruncate(self.fp, 0)

1474 os.write(self.fp, str(os.getpid()).encode('ascii'))

1475 log.debug(f'server lock {self.uid!r}: acquired')

1476 return

1477

1478 t = time.time() - ts

1479

1480 if t >= self.timeout:

1481 pid = self._holder_pid()

1482 os.close(self.fp)

1483 self.fp = None

1484 log.debug(f'server lock {self.uid!r}: BUSY {pid=}')

1485 raise LockBusyError(f'server lock {self.uid!r}: busy {pid=}')

1486

1487 time.sleep(self._PAUSE)

1488

1489 def release(self):

1490 """Release the lock. Errors are logged and ignored."""

1491 if self.fp is None:

1492 return

1493 try:

1494 fcntl.flock(self.fp, fcntl.LOCK_UN)

1495 os.close(self.fp)

1496 log.debug(f'server lock {self.uid!r}: released')

1497 except OSError as exc:

1498 log.exception(f'server lock {self.uid!r}: RELEASE ERROR {exc!r}')

1499 self.fp = None

1500

1501 def _holder_pid(self):

1502 """Return the process id stored in the lock file, or ``?``."""

1503 if self.fp is None:

1504 return '?'

1505 try:

1506 os.lseek(self.fp, 0, os.SEEK_SET)

1507 return os.read(self.fp, 64).decode('ascii') or '?'

1508 except OSError:

1509 return '?'

1510

1511

1512def server_lock(uid, timeout: float = 60):

1513 """Create an inter-process lock, to be used as a context manager.

1514

1515 The lock is acquired when the ``with`` block is entered.

1516

1517 Example::

1518

1519 with gws.u.server_lock('my_task', timeout=5):

1520 ...

1521

1522 Args:

1523 uid: Lock identifier.

1524 timeout: Seconds to wait for the lock. ``0`` means a single attempt.

1525

1526 Returns:

1527 The lock object.

1528

1529 Raises:

1530 LockBusyError: If the lock is not acquired within ``timeout``.

1531 """

1532 return _FileLock(uid, timeout)

1535##

1538def action_url_path(name: str, **kwargs) -> str:

1539 """Build a server URL path for an action.

1540

1541 Example::

1542

1543 action_url_path('owsService', serviceUid='wms', projectUid='') # '/_/owsService/serviceUid/wms'

1544

1545 Args:

1546 name: Action command name.

1547 **kwargs: Parameters, appended as ``/key/value`` path segments. Empty values are skipped.

1548

1549 Returns:

1550 The URL path.

1551 """

1552 ls = []

1553

1554 for k, v in kwargs.items():

1555 if not is_empty(v):

1556 ls.append(urllib.parse.quote(k))

1557 ls.append(urllib.parse.quote(to_str(v)))

1558

1559 path = const.SERVER_ENDPOINT + '/' + name

1560 if ls:

1561 path += '/' + '/'.join(ls)

1562 return path

1565##

1568def utime() -> float:

1569 """Return the Unix time as a float number.

1570

1571 Returns:

1572 Seconds since the epoch.

1573 """

1574 return time.time()

1575

1576

1577def stime() -> int:

1578 """Return the Unix time as an integer number of seconds.

1579

1580 Returns:

1581 Seconds since the epoch.

1582 """

1583 return int(time.time())

1584

1585

1586def sleep(n: float):

1587 """Sleep for a number of seconds.

1588

1589 Args:

1590 n: Seconds.

1591 """

1592 time.sleep(n)

1593

1594

1595def mstime() -> int:

1596 """Return the Unix time as an integer number of milliseconds.

1597

1598 Returns:

1599 Milliseconds since the epoch.

1600 """

1601 return int(time.time() * 1000)

1602

1603

1604def microtime() -> int:

1605 """Return the Unix time as an integer number of microseconds.

1606

1607 Returns:

1608 Microseconds since the epoch.

1609 """

1610 return int(time.time() * 1000000)