Coverage for gws-app/gws/plugin/model_field/file/__init__.py: 87%

1"""File field. 

2 

3A field for files attached to features of a database model. The file content 

4is stored in a database column (``contentColumn``), optionally with the file 

5name in another column (``nameColumn``). A ``pathColumn`` can be configured 

6and is read, but storing and serving files from the filesystem is not 

7implemented yet. The model must have a primary key. 

8 

9The field value is a ``FileValue``. When features are selected, only the 

10content length is read, not the content itself. In the client props, a file 

11is described by a ``ServerFileProps`` object with a label, extension, size and 

12URLs of the ``webFile`` command for downloading the file and, for raster 

13images, for a thumbnail preview. Previews are generated on request and cached 

14as ephemeral content. Without a configured widget, the field uses a ``file`` 

15widget. 

16 

17Example:: 

18 

19 fields+ { 

20 name "photo" 

21 type "file" 

22 contentColumn "photo_content" 

23 nameColumn "photo_name" 

24 } 

25""" 

26 

27from typing import Optional, cast 

28 

29import gws 

30import gws.base.database.model 

31import gws.base.model.field 

32import gws.lib.image 

33import gws.lib.mime 

34import gws.lib.sa as sa 

35 

36 

37@gws.ext.config.modelField('file') 

38class Config(gws.base.model.field.Config): 

39 """Field for files stored in the database or the filesystem.""" 

40 

41 contentColumn: str = '' 

42 """Column name for the file content, if stored in the database.""" 

43 pathColumn: str = '' 

44 """Column name for the file path, if stored in the filesystem.""" 

45 nameColumn: str = '' 

46 """Column name for the file name, if stored in the database or filesystem.""" 

47 

48 

49@gws.ext.props.modelField('file') 

50class Props(gws.base.model.field.Props): 

51 pass 

52 

53 

54class FileInputProps(gws.Data): 

55 """File uploaded from the client.""" 

56 

57 content: bytes 

58 """File content.""" 

59 name: str 

60 """File name.""" 

61 

62 

63class ServerFileProps(gws.Data): 

64 """File description sent to the client.""" 

65 

66 downloadUrl: str 

67 """URL to download the file, empty if there is no project.""" 

68 extension: str 

69 """File extension derived from the MIME type.""" 

70 label: str 

71 """Label to display, the file name.""" 

72 previewUrl: str 

73 """URL of a thumbnail preview, empty if the file cannot be previewed.""" 

74 size: int 

75 """File size in bytes.""" 

76 

77 

78class ClientFileProps(gws.Data): 

79 """File sent from the client.""" 

80 

81 name: str 

82 """File name.""" 

83 content: bytes 

84 """File content.""" 

85 

86 

87class FileValue(gws.Data): 

88 """Value of a file field.""" 

89 

90 content: bytes 

91 """File content, if loaded.""" 

92 name: str 

93 """File name.""" 

94 path: str 

95 """File path in the filesystem.""" 

96 size: int 

97 """File size in bytes.""" 

98 

99 

100_PREVIEW_SIZE = 120, 120 

101_PREVIEW_MIME = gws.lib.mime.PNG 

102_PREVIEW_MAX_PIXELS = 40_000_000 

103_PREVIEW_BIG_FILE_SIZE = 1024 * 1024 

104 

105 

106@gws.ext.object.modelField('file') 

107class Object(gws.base.model.field.Object): 

108 """File field object. 

109 

110 Stores files attached to features in columns of a database model and 

111 describes them to the client for download and preview. 

112 """ 

113 

114 model: gws.DatabaseModel 

115 """The model of this field.""" 

116 

117 attributeType = gws.AttributeType.file 

118 

119 contentColumn: Optional[sa.Column] = None 

120 """Column for the file content.""" 

121 pathColumn: Optional[sa.Column] = None 

122 """Column for the file path.""" 

123 nameColumn: Optional[sa.Column] = None 

124 """Column for the file name.""" 

125 

126 def __getstate__(self): 

127 return gws.u.omit(vars(self), 'cols') 

128 

129 def post_configure(self): 

130 self.configure_columns() 

131 

132 def activate(self): 

133 self.configure_columns() 

134 

135 def configure_columns(self): 

136 """Resolve the configured content, path and name columns of the model. 

137 

138 The columns are SQLAlchemy objects. They are resolved after configuration 

139 and again on activation. 

140 

141 Raises: 

142 ``gws.ConfigurationError``: If neither ``contentColumn`` nor ``pathColumn`` is set, 

143 or the model has no primary key. 

144 """ 

145 model = cast(gws.base.database.model.Object, self.model) 

146 

147 p = self.cfg('contentColumn') 

148 self.contentColumn = model.column(p) if p else None 

149 

150 p = self.cfg('pathColumn') 

151 self.pathColumn = model.column(p) if p else None 

152 

153 p = self.cfg('nameColumn') 

154 self.nameColumn = model.column(p) if p else None 

155 

156 if self.contentColumn is None and self.pathColumn is None: 

157 raise gws.ConfigurationError('contentColumn or pathColumn must be set') 

158 

159 if not self.model.uidName: 

160 raise gws.ConfigurationError('file fields require a primary key') 

161 

162 def configure_widget(self): 

163 if not super().configure_widget(): 

164 self.widget = self.root.create_shared(gws.ext.object.modelWidget, type='file') 

165 return True 

166 

167 ## 

168 

169 def before_select(self, mc): 

170 mc.dbSelect.columns.extend(self.select_columns(mc)) 

171 

172 def after_select(self, features, mc): 

173 for feature in features: 

174 self.from_record(feature, mc) 

175 

176 def before_create(self, feature, mc): 

177 self.to_record(feature, mc) 

178 

179 def before_update(self, feature, mc): 

180 self.to_record(feature, mc) 

181 

182 def from_record(self, feature, mc): 

183 feature.set(self.name, self.load_value(feature.record.attributes, mc)) 

184 

185 def to_record(self, feature, mc): 

186 if not mc.user.can_write(self): 

187 return 

188 

189 # @TODO store in the filesystem 

190 

191 fv = cast(FileValue, feature.get(self.name)) 

192 if not fv: 

193 return 

194 if self.contentColumn is not None: 

195 feature.record.attributes[self.contentColumn.name] = fv.content 

196 if self.nameColumn is not None: 

197 feature.record.attributes[self.nameColumn.name] = fv.name 

198 

199 # @TODO merge with scalar_field? 

200 

201 def from_props(self, feature, mc): 

202 value = feature.props.attributes.get(self.name) 

203 if value is not None: 

204 value = self.prop_to_python(feature, value, mc) 

205 if value is not None: 

206 feature.set(self.name, value) 

207 

208 def to_props(self, feature, mc): 

209 if not mc.user.can_read(self): 

210 return 

211 value = feature.get(self.name) 

212 if value is not None: 

213 value = self.python_to_prop(feature, value, mc) 

214 if value is not None: 

215 feature.props.attributes[self.name] = value 

216 

217 ## 

218 

219 def can_preview(self, mime_type) -> bool: 

220 """Check if a thumbnail preview can be made for a MIME type. 

221 

222 Args: 

223 mime_type: The MIME type. 

224 

225 Returns: 

226 True for image types except SVG. 

227 """ 

228 return mime_type.startswith('image/') and mime_type != gws.lib.mime.SVG 

229 

230 def prop_to_python(self, feature, value, mc) -> FileValue: 

231 try: 

232 return FileValue( 

233 content=gws.u.get(value, 'content'), 

234 name=gws.u.get(value, 'name'), 

235 ) 

236 except ValueError: 

237 return gws.ErrorValue 

238 

239 def python_to_prop(self, feature, value, mc) -> ServerFileProps: 

240 fv = cast(FileValue, value) 

241 

242 mime_type = self.get_mime_type(fv) 

243 ext = gws.lib.mime.extension_for(mime_type) 

244 

245 p = ServerFileProps( 

246 # @TODO use a template 

247 label=fv.name or '', 

248 extension=ext, 

249 size=fv.size or 0, 

250 previewUrl='', 

251 downloadUrl='', 

252 ) 

253 

254 if not mc.project: 

255 return p 

256 

257 name = fv.name or f'gws.{ext}' 

258 

259 url_args = dict( 

260 projectUid=mc.project.uid, 

261 modelUid=self.model.uid, 

262 fieldName=self.name, 

263 featureUid=feature.uid(), 

264 ) 

265 

266 if self.can_preview(mime_type): 

267 p.previewUrl = gws.u.action_url_path('webFile', preview=1, **url_args) + '/' + name 

268 

269 p.downloadUrl = gws.u.action_url_path('webFile', **url_args) + '/' + name 

270 

271 return p 

272 

273 ## 

274 

275 def get_mime_type(self, fv: FileValue) -> str: 

276 """Determine the MIME type of a file from its path or name. 

277 

278 Args: 

279 fv: The file value. 

280 

281 Returns: 

282 The MIME type, or the generic binary type if it cannot be determined. 

283 """ 

284 if fv.path: 

285 return gws.lib.mime.for_path(fv.path) 

286 if fv.name: 

287 return gws.lib.mime.for_path(fv.name) 

288 # @TODO guess mime from content? 

289 return gws.lib.mime.BIN 

290 

291 def handle_web_file_request(self, feature_uid: str, preview: bool, mc: gws.ModelContext) -> Optional[gws.ContentResponse]: 

292 """Serve the file of a feature, or its thumbnail preview. 

293 

294 Called by the ``webFile`` command. Only files stored in the database are served. 

295 Thumbnails are PNG images, generated from the content and cached as ephemeral 

296 content keyed by the content checksum. 

297 

298 Args: 

299 feature_uid: Uid of the feature. 

300 preview: Return a thumbnail preview instead of the file. 

301 mc: The model context. 

302 

303 Returns: 

304 The file content or the preview, or None if the user may not read the field, 

305 the feature or the file is not found, or the file cannot be previewed. 

306 

307 Raises: 

308 ``gws.NotFoundError``: If the content for a preview cannot be loaded or the 

309 thumbnail cannot be created. 

310 """ 

311 if not mc.user.can_read(self): 

312 return 

313 

314 if self.contentColumn is None: 

315 # @TODO serve files stored in the filesystem 

316 return 

317 

318 search = gws.SearchQuery(uids=[feature_uid]) 

319 if preview: 

320 # for small files, fetch content md5 and content, for big files only md5 

321 search.extraColumns = [ 

322 sa.func.md5(self.contentColumn).label(f'{self.name}_preview_md5'), 

323 sa.case( 

324 (sa.func.length(self.contentColumn) < _PREVIEW_BIG_FILE_SIZE, self.contentColumn), 

325 else_=sa.null(), 

326 ).label(f'{self.name}_preview_content'), 

327 ] 

328 else: 

329 search.extraColumns = [self.contentColumn] 

330 

331 features = self.model.find_features(search, mc) 

332 if not features: 

333 return 

334 

335 feature = features[0] 

336 

337 fv = cast(FileValue, feature.get(self.name)) 

338 if not fv: 

339 return 

340 

341 if not preview: 

342 # download complete file content 

343 mime_type = self.get_mime_type(fv) 

344 return gws.ContentResponse( 

345 content=fv.content, 

346 contentFilename=fv.name or f'gws.{gws.lib.mime.extension_for(mime_type)}', 

347 mimeType=mime_type, 

348 ) 

349 

350 # preview 

351 

352 if not self.can_preview(self.get_mime_type(fv)): 

353 return 

354 

355 md5 = feature.record.attributes.get(f'{self.name}_preview_md5') 

356 if not md5: 

357 return 

358 

359 def make_preview(): 

360 content = feature.record.attributes.get(f'{self.name}_preview_content') 

361 

362 if content is None: 

363 # big file 

364 search = gws.SearchQuery(uids=[feature.uid()]) 

365 search.extraColumns = [self.contentColumn] 

366 features = self.model.find_features(search, mc) 

367 if not features: 

368 raise gws.NotFoundError(f'file preview: no feature {feature.uid()!r}') 

369 fv = cast(FileValue, features[0].get(self.name)) 

370 if not fv or fv.content is None: 

371 raise gws.NotFoundError(f'file preview: no content for {feature.uid()!r}') 

372 content = fv.content 

373 

374 try: 

375 return gws.lib.image.thumbnail( 

376 content, 

377 _PREVIEW_SIZE, 

378 max_pixels=_PREVIEW_MAX_PIXELS, 

379 mime_type=_PREVIEW_MIME, 

380 ) 

381 except gws.lib.image.Error as exc: 

382 raise gws.NotFoundError(f'file preview: {exc}') from exc 

383 

384 cache_key = gws.u.sha256( 

385 [ 

386 self.model.uid, 

387 self.name, 

388 feature.uid(), 

389 md5, 

390 _PREVIEW_SIZE, 

391 _PREVIEW_MIME, 

392 ] 

393 ) 

394 

395 return gws.ContentResponse( 

396 content=gws.u.get_ephemeral_content(f'preview_{cache_key}', make_preview), 

397 mimeType=_PREVIEW_MIME, 

398 ) 

399 

400 ## 

401 

402 def select_columns(self, mc): 

403 """Return the columns to add to a select statement. 

404 

405 The content column is not selected; only its length is, labeled ``<name>_length``. 

406 

407 Args: 

408 mc: The model context. 

409 

410 Returns: 

411 A list of column expressions. 

412 """ 

413 cs = [] 

414 

415 if self.contentColumn is not None: 

416 cs.append(sa.func.length(self.contentColumn).label(f'{self.name}_length')) 

417 if self.pathColumn is not None: 

418 cs.append(self.pathColumn) 

419 if self.nameColumn is not None: 

420 cs.append(self.nameColumn) 

421 

422 return cs 

423 

424 def load_value(self, attributes: dict, mc) -> Optional[FileValue]: 

425 """Create a file value from the attributes of a database record. 

426 

427 Args: 

428 attributes: Record attributes. 

429 mc: The model context. 

430 

431 Returns: 

432 The file value, or None if no file columns are configured. 

433 """ 

434 d = {} 

435 

436 if self.contentColumn is not None: 

437 d['size'] = attributes.get(f'{self.name}_length') 

438 d['content'] = attributes.get(self.contentColumn.name) 

439 if self.pathColumn is not None: 

440 d['path'] = attributes.get(self.pathColumn.name) 

441 if self.nameColumn is not None: 

442 d['name'] = attributes.get(self.nameColumn.name) 

443 

444 if d: 

445 return FileValue(**d)