"""CioWarehouse module: a Chrysalio add-on to manage digital warehouses."""
from sys import exit as sys_exit
from logging import getLogger
from os import makedirs, scandir
from os.path import exists, join, dirname
from shutil import rmtree
from time import time
from mimetypes import init as mimetypes_init
from json import loads
from collections import OrderedDict
from pyramid.config import Configurator
from chrysalio.initialize import Initialize
from chrysalio.lib.utils import deltatime_label
from chrysalio.lib.config import settings_get_list, settings_get_namespace
from chrysalio.includes.cache import cache_user_access, cache_global_item
from chrysalio.includes.cache import cache_namespace
from chrysalio.includes.modules.models import DBModule
from chrysalio.modules import Module
from .relaxng import RELAXNG_CIOWAREHOUSE
from .security import PRINCIPALS_CIOWAREHOUSE
from .menu import MENU_CIOWAREHOUSE
from .lib.i18n import _, translate
from .lib.utils import CIOWAREHOUSE_NS, CACHE_REGION_USER, CACHE_REGION_GLOBAL
from .lib.utils import MIMETYPES_DIR, FILE_RIGHTS_INDEX, META_RIGHTS_INDEX
from .lib.utils import build_callback
from .lib.warehouse import Warehouse
from .lib.fileinfo import FileInfo
from .handlers.handler_directory import HandlerDirectory
from .handlers.handler_html import HandlerHtml
from .handlers.handler_image import HandlerImage
from .handlers.handler_audio import HandlerAudio
from .handlers.handler_video import HandlerVideo
from .handlers.handler_pdf import HandlerPdf
from .handlers.handler_plain import HandlerPlain
from .models.populate import xml2db as _xml2db, db2xml as _db2xml
from .models.dbmetafield import DBMetafield
from .models.dbindexfield import INDEXFIELD_BUILTIN, DBIndexfield
from .models.dbwarehouse import DBWarehouse, DBWarehouseUser, DBWarehouseGroup
LOG = getLogger(__name__)
RIGHT_STRENGTH = {None: 0, 'reader': 1, 'writer': 2, 'writer-admin': 3}
BROWSE_AREA = (
'browse_all', 'browse_favorite', 'browse_warehouse',
'browse_directory_root', 'browse_directory', 'glance_directory',
'browse_filter_all', 'browse_filter_favorite',
'browse_filter_warehouse', 'browse_filter_directory',
'browse_filter_directory_root')
HANDLER_PANEL_AREA = BROWSE_AREA + (
'warehouse_index', 'file_view', 'file_edit', 'file_move', 'build_view',
'cioset_remove', 'cioset_add', 'cioset_rearrange')
# =============================================================================
[docs]
def includeme(configurator):
"""Function to include CioWarehouse module.
:type configurator: pyramid.config.Configurator
:param configurator:
Object used to do configuration declaration within the application.
"""
# Registration
Module.register(configurator, ModuleCioWarehouse)
# Mime types
mimetypes_init((join(MIMETYPES_DIR, 'mime.types'),))
if not isinstance(configurator, Configurator):
return
# Cache
if 'cache_user' not in configurator.registry or \
'cache_global' not in configurator.registry:
sys_exit(translate(_('*** You must register a cache manager.')))
# Permissions
configurator.include('ciowarehouse.security')
# Routes
configurator.include('ciowarehouse.routes')
# Translations
configurator.add_translation_dirs(join(dirname(__file__), 'Locale'))
# Views
static_dir = join(dirname(__file__), 'Static')
Initialize(configurator).add_static_views(__package__, (
('css', join(static_dir, 'Css')),
('js', join(static_dir, 'Js')),
('images', join(static_dir, 'Images')),
('mimetypes', MIMETYPES_DIR),
('pdfjs', join(static_dir, 'Pdfjs'))))
configurator.scan('ciowarehouse.views')
# =============================================================================
[docs]
class ModuleCioWarehouse(Module):
"""Class for CioWarehouse module.
:param str config_ini:
Absolute path to the configuration file (e.g. development.ini).
This module has the following attributes:
* ``locations``: a dictionary of absolute paths of root directories
* ``restful``: a dictionary defining the RESTful parameters
"""
name = _('Warehouse')
implements = (
'warehouse', 'handler:directory', 'handler:image', 'handler:plain',
'handler:pdf', 'handler:html')
dependencies = ('cioservice',)
relaxng = RELAXNG_CIOWAREHOUSE
xml2db = (_xml2db,)
db2xml = (_db2xml,)
areas = {
'ciowarehouse.browse': _('Warehouse browsing'),
'ciowarehouse.warehouse': _('Warehouse index')}
_DBModule = DBModule
# -------------------------------------------------------------------------
def __init__(self, config_ini):
"""Constructor method."""
super(ModuleCioWarehouse, self).__init__(config_ini)
# Read locations
settings = self._settings(config_ini)
self.locations = settings_get_list(settings, 'locations')
if not self.locations:
sys_exit(translate(_(
'*** ${n}: "locations" is missing.', {'n': self.uid})))
self.locations = {
k.split(':')[0]: k.split(':')[1]
for k in self.locations if ':' in k}
if not self.locations:
sys_exit(translate(_(
'*** ${n}: "locations" must be a list of WAREHOUSE_ID:PATH.',
{'n': self.uid})))
# Create location directories
for location in self.locations:
if not exists(self.locations[location]):
try:
makedirs(self.locations[location])
except OSError:
sys_exit(translate(_(
'*** ${n}: Unable to create location "${l}".',
{'n': self.uid, 'l': self.locations[location]})))
# Retrieve RESTful parameters
self.restful = settings_get_namespace(settings, 'restful')
# Retrieve handler parameters
self._handler_config = settings_get_namespace(settings, 'handler')
# Warehouses
self._warehouses = {}
# -------------------------------------------------------------------------
[docs]
def populate(self, args, registry, dbsession):
"""Method called by populate script to complete the operation.
See: :meth:`chrysalio.modules.Module.populate`
"""
# pylint: disable = too-many-branches
# Clean up handler installation
handler_root = self._handler_config.get('root')
if handler_root and exists(handler_root):
rmtree(handler_root)
# Populate warehouses
LOG.info(translate(_('====== Updating warehouses')))
is_ok = True
for dbwarehouse in dbsession.query(DBWarehouse):
LOG.info('......{0:.<32}'.format(dbwarehouse.warehouse_id))
if dbwarehouse.location not in self.locations:
LOG.warning(translate(_(
'Location ${l} for warehouse ${w} does not exist.',
{'l': dbwarehouse.location,
'w': dbwarehouse.warehouse_id})))
continue
start = time()
# Warehouse object creation
warehouse = Warehouse(registry, dbwarehouse, self.locations)
if not exists(warehouse.root) or not scandir(warehouse.root):
error = warehouse.vcs.clone()
if error:
is_ok = False
LOG.error(error)
continue
else:
error = warehouse.vcs.pull()
if error:
is_ok = False
LOG.error(error)
continue
# Add forgotten files
if warehouse.vcs.is_dirty():
warehouse.vcs.add()
warehouse.vcs.commit(
translate(_('Automatic integration.')),
registry.settings['site.uid'])
# Remove locks
if '--remove-locks' in args.extra or (
hasattr(args, 'remove_locks') and args.remove_locks):
warehouse.unlock_all()
# Skip refresh
if '--skip-refresh' in args.extra or (
hasattr(args, 'skip_refresh') and args.skip_refresh):
continue
# Indexing
if '--reindex' in args.extra or (
hasattr(args, 'reindex') and args.reindex):
LOG.info(translate(_('Erasing indexes')))
warehouse.index_erase()
LOG.info(translate(_('Indexing')))
warehouse.index_update_all(dbsession)
# Creating thumbnails
if '--recreate-thumbnails' in args.extra or (
hasattr(args, 'recreate_thumbnails') and
args.recreate_thumbnails):
LOG.info(translate(_('Erasing thumbnails')))
warehouse.thumbnails_erase()
LOG.info(translate(_('Creating thumbnails')))
warehouse.thumbnails_update_all(registry=registry)
warehouse.refreshed()
duration = int(time() - start)
if duration:
LOG.info(translate(_('Done in ${d}', {
'd': deltatime_label(duration)})))
return translate(_('Warehouses have errors')) if not is_ok else None
# -------------------------------------------------------------------------
[docs]
def activate(self, registry, dbsession):
"""Method to activate the module.
See: :meth:`chrysalio.modules.Module.activate`
"""
# Security
if PRINCIPALS_CIOWAREHOUSE[0] not in registry['principals']:
registry['principals'].append(PRINCIPALS_CIOWAREHOUSE[0])
# Menu
if 'menu' in registry and MENU_CIOWAREHOUSE not in registry['menu']:
registry['menu'].insert(1, MENU_CIOWAREHOUSE)
# Metadata fields
registry['metafields'] = self.metafields(registry, dbsession)
# Index fields
registry['indexfields'] = self.indexfields(registry, dbsession)
# Fields for list
fields = registry['indexfields']
registry['listfields'] = [
{'id': k, 'label': fields[k]['label'],
'type': fields[k]['field_type'], 'class': fields[k]['in_class']}
for k in sorted(fields, key=lambda k: fields[k]['in_list'])
if fields[k]['in_list']]
registry['listfields'] = tuple(registry['listfields'])
# Fields for cards
registry['cardfields'] = [
{'id': k, 'label': fields[k]['label'],
'type': fields[k]['field_type'], 'class': fields[k]['in_class']}
for k in sorted(fields, key=lambda k: fields[k]['in_cards'])
if fields[k]['in_cards']]
registry['cardfields'] = tuple(registry['cardfields'])
# Handlers
if 'handlers' not in registry:
registry['handlers'] = []
handler_ids = [k.uid for k in registry['handlers']]
for handler in (HandlerDirectory, HandlerImage, HandlerAudio,
HandlerVideo, HandlerHtml, HandlerPdf, HandlerPlain):
if handler.uid not in handler_ids:
registry['handlers'].append(handler())
registry['handlers'] = tuple(registry['handlers'])
# Handler initialization and panel registration
for handler in registry['handlers']:
handler.initialize(dbsession, self._handler_config)
if handler.panel is not None:
panel = handler.panel.register(
registry, handler.panel, HANDLER_PANEL_AREA)
panel.handler = handler
FileInfo.register(registry, FileInfo, BROWSE_AREA)
# Callback
registry['modules']['cioservice'].build_manager.add_callback(
'ciowarehouse', build_callback)
# -------------------------------------------------------------------------
[docs]
def deactivate(self, registry, dbsession):
"""Method to deactivate the module.
See: :meth:`chrysalio.modules.Module.deactivate`
"""
# Security
if PRINCIPALS_CIOWAREHOUSE[0] in registry['principals']:
registry['principals'].remove(PRINCIPALS_CIOWAREHOUSE[0])
# Menu
if 'menu' in registry and MENU_CIOWAREHOUSE in registry['menu']:
registry['menu'].remove(MENU_CIOWAREHOUSE)
# Panels
for handler in registry.get('handlers', ''):
panel_id = 'handler_{0}'.format(handler.uid)
if 'panels' in registry and panel_id in registry['panels']:
del registry['panels'][panel_id]
if 'panels' in registry and 'fileinfo' in registry['panels']:
del registry['panels']['fileinfo']
# Registry entries
for entry in (
'metafields', 'indexfields', 'listfields', 'cardfields',
'handlers'):
if entry in registry:
del registry[entry]
# Warehouses
registry['cache_global'].clear(namespace=CIOWAREHOUSE_NS)
for warehouse_id in self._warehouses:
registry['cache_global'].clear(
namespace=cache_namespace(CIOWAREHOUSE_NS, warehouse_id))
self._warehouses = {}
# -------------------------------------------------------------------------
[docs]
def configuration_route(self, request):
"""Return the route to configure this module.
:type request: pyramid.request.Request
:param request:
Current request.
"""
return request.route_path('ciowarehouse_view')
# -------------------------------------------------------------------------
@cache_user_access(CIOWAREHOUSE_NS, CACHE_REGION_USER)
@classmethod
def warehouse_access(cls, request, warehouse):
"""Return a tuple defining the authorization of the user on the
warehouse.
:type request: pyramid.request.Request
:param request:
Current request.
:type warehouse: .lib.warehouse.Warehouse
:param warehouse:
Current warehouse object.
:rtype: tuple
:return:
A tuple with index ``FILE_RIGHTS_INDEX`` and ``META_RIGHTS_INDEX``.
"""
if warehouse is None:
return (None, 'reader')
# Compute rights
user_id = request.session['user']['user_id'] \
if 'user' in request.session else ''
groups = request.session['user']['groups'] \
if 'user' in request.session else ()
dbwrh_user = request.dbsession.query(DBWarehouseUser)\
.filter_by(warehouse_id=warehouse.uid, user_id=user_id).first()
right_list = [(dbwrh_user.file_rights, dbwrh_user.meta_rights)] \
if dbwrh_user else []
right_list += request.dbsession.query(
DBWarehouseGroup.file_rights, DBWarehouseGroup.meta_rights)\
.filter_by(warehouse_id=warehouse.uid)\
.filter(DBWarehouseGroup.group_id.in_(groups)).all()
rights = [None, 'reader']
for item in right_list:
if RIGHT_STRENGTH[item[0]] > RIGHT_STRENGTH[rights[0]]:
rights[0] = item[0]
if RIGHT_STRENGTH[item[1]] > RIGHT_STRENGTH[rights[1]]:
rights[1] = item[1]
i_creator = request.has_permission('warehouse-create')
# Compute access
if warehouse.access == 'closed':
access = ('reader-admin' if i_creator else None, 'reader')
elif warehouse.access == 'readonly':
access = ('reader-admin' if i_creator else 'reader', 'reader')
elif warehouse.access == 'restricted-readonly':
access = (
'reader-admin' if i_creator else (
'reader' if rights[0] is not None else None), 'reader')
elif warehouse.access == 'free' or i_creator:
access = ('writer-admin' if i_creator else 'writer', 'writer')
else:
access = tuple(rights)
return access
# -------------------------------------------------------------------------
@cache_global_item(CIOWAREHOUSE_NS, CACHE_REGION_GLOBAL, warehouse_access)
def warehouse(self, request, warehouse_id):
"""Return the warehouse with ID ``warehouse_id`` or ``None``.
:type request: pyramid.request.Request
:param request:
Current request.
:param str warehouse_id:
ID of the warehouse to return.
:rtype: :class:`.lib.warehouse.Warehouse` or ``None``
Thanks to the decorator, the global cache ``ciowrh-<warehouse_id>``
stores file lists.
The user cache ``ciowrh-<warehouse_id>`` is a tuple with
following index:
* ``FILE_RIGTS_INDEX``: ``'reader'``, ``'reader-admin'``,
``'writer'``, ``'writer-admin'`` or ``None`` (not authorized)
* ``META_RIGHTS_INDEX``: ``'reader'`` or ``'writer'``
"""
if not warehouse_id:
return None
dbwarehouse = request.dbsession.query(DBWarehouse).filter_by(
warehouse_id=warehouse_id).first()
if dbwarehouse is None or dbwarehouse.location not in self.locations:
return None
warehouse = Warehouse(request.registry, dbwarehouse, self.locations)
if not exists(warehouse.root):
return None
return warehouse
# -------------------------------------------------------------------------
[docs]
def warehouse_admin(self, request, warehouse, access=None):
"""Return ``True`` if the user administrates the warehouse.
:type request: pyramid.request.Request
:param request:
Current request.
:type warehouse: lib.warehouse.Warehouse
:param warehouse:
Current warehouse object..
:param tuple access: (optional)
Already retrieved access tuple.
:rtype: bool
"""
if access is None:
access = self.warehouse_access(request, warehouse)
return (access[FILE_RIGHTS_INDEX] or '')[-5:] == 'admin'
# -------------------------------------------------------------------------
[docs]
def warehouse_file_writer(self, request, warehouse, access=None):
"""Return ``True`` if the user can write files in this warehouse.
:type request: pyramid.request.Request
:param request:
Current request.
:type warehouse: lib.warehouse.Warehouse
:param warehouse:
Current warehouse object.
:param tuple access: (optional)
Already retrieved access tuple.
:rtype: bool
"""
if access is None:
access = self.warehouse_access(request, warehouse)
return (access[FILE_RIGHTS_INDEX] or '')[:6] == 'writer'
# -------------------------------------------------------------------------
# -------------------------------------------------------------------------
[docs]
def warehouse_root(self, request, warehouse_id):
"""Return the root directory of the warehouse with ID ``warehouse_id``
or ``None``.
:type request: pyramid.request.Request
:param request:
Current request.
:param str warehouse_id:
ID of the warehouse to return.
:rtype: str
"""
if warehouse_id not in self._warehouses:
dbwarehouse = request.dbsession.query(DBWarehouse).filter_by(
warehouse_id=warehouse_id).first()
if dbwarehouse is None:
return None
self._warehouses[warehouse_id] = Warehouse(
request.registry, dbwarehouse, self.locations)
return self._warehouses[warehouse_id].root
# -------------------------------------------------------------------------
[docs]
def warehouse_forget(self, request, warehouse_id=None):
"""Remove warehouse from list.
:type request: pyramid.request.Request
:param request:
Current request.
:param str warehouse_id: (optional)
ID of the warehouse to forget.
"""
if warehouse_id is None:
request.registry['cache_global'].clear(namespace=CIOWAREHOUSE_NS)
for uid in self._warehouses:
namespace = cache_namespace(CIOWAREHOUSE_NS, uid)
request.registry['cache_global'].clear(namespace=namespace)
request.registry['cache_user'].clear(
request, namespace=namespace)
self._warehouses = {}
else:
self.cache_clear(request, warehouse_id)
if warehouse_id in self._warehouses:
del self._warehouses[warehouse_id]
# -------------------------------------------------------------------------
[docs]
@classmethod
def cache_clear(cls, request, warehouse_id):
"""Clear file and metadata cache for a warehouse.
:type request: pyramid.request.Request
:param request:
Current request.
:param str warehouse_id:
ID of the warehouse.
"""
namespace = cache_namespace(CIOWAREHOUSE_NS, warehouse_id)
request.registry['cache_global'].clear(namespace=CIOWAREHOUSE_NS)
request.registry['cache_global'].clear(namespace=namespace)
request.registry['cache_user'].clear(request, namespace=namespace)
# -------------------------------------------------------------------------
# -------------------------------------------------------------------------
[docs]
@classmethod
def indexfields(cls, registry, dbsession):
"""Return a dictionary where each key is an index field ID and each
entry a dictionary with keys ``'label'``, ``'field_type'``,
``'whoosh_type'``, ``'in_condition'`` and ``'in_result'``.
:type registry: pyramid.registry.Registry
:param registry:
Application registry.
:type dbsession: sqlalchemy.orm.session.Session
:param dbsession:
SQLAlchemy session.
:rtype: dict
"""
fields = {}
for dbitem in dbsession.query(DBIndexfield):
stored = dbitem.stored or \
dbitem.indexfield_id in INDEXFIELD_BUILTIN
fields[dbitem.indexfield_id] = {
'label': loads(dbitem.i18n_label),
'field_type': dbitem.field_type,
'whoosh_type': {
'file_name': 'ID',
'file_type': 'ID',
'file_size': 'NUMERIC',
'file_date': 'NUMERIC',
'vcs_author': 'ID',
'vcs_date': 'NUMERIC',
'vcs_message': 'STEMS',
'id': 'ID',
'stems': 'STEMS',
'keyword': 'KEYWORD',
'integer': 'NUMERIC',
'decimal': 'NUMERIC',
'boolean': 'BOOLEAN',
'time': 'NUMERIC',
'date': 'DATETIME',
'list': 'TEXT',
'palette': 'ID'}.get(dbitem.field_type, 'TEXT'),
'in_filter': dbitem.in_filter,
'stored': stored,
'in_list': dbitem.in_list or 0,
'in_cards': dbitem.in_cards or 0,
'in_class': dbitem.in_class}
if dbitem.field_type in ('list', 'palette'):
choices = [
(k.value, loads(k.i18n_label)) for k in dbitem.choices]
fields[dbitem.indexfield_id]['choices'] = dict(choices)
# Defaults
if not fields:
languages = settings_get_list(
registry.settings, 'languages', ('en',))
fields = {
'file_name': {
'label': {
k: translate(_('File name'), k) for k in languages},
'field_type': 'file_name',
'whoosh_type': 'ID',
'in_filter': 1,
'stored': True,
'in_list': 1, 'in_cards': 1,
'in_class': None},
'file_size': {
'label': {k: translate(_('Size'), k) for k in languages},
'field_type': 'file_size',
'whoosh_type': 'NUMERIC',
'in_filter': 0,
'stored': True,
'in_list': 2, 'in_cards': 0,
'in_class': 'cioOptional'},
'file_date': {
'label': {
k: translate(_('Modified'), k) for k in languages},
'field_type': 'file_date',
'whoosh_type': 'NUMERIC',
'in_filter': 0,
'stored': True,
'in_list': 3, 'in_cards': 0,
'in_class': 'cioOptional'},
'shared': {
'label': {
k: translate(_('Shared'), k) for k in languages},
'field_type': 'boolean',
'whoosh_type': 'BOOLEAN',
'in_filter': 2,
'stored': True,
'in_list': 0, 'in_cards': 0,
'in_class': None}}
return fields