Source code for ciowarehouse.views.browse

"""View callables to browse or search into one warehouse or several
warehouses."""

from os.path import join
from datetime import datetime

from sqlalchemy import or_
from colander import SchemaNode, Mapping, String, Length, Boolean
from colander import Integer, Float, Date, DateTime

from pyramid.view import view_config
from pyramid.httpexceptions import HTTPFound, HTTPForbidden

from chrysalio.lib.form import SameAs, get_action, Form
from chrysalio.lib.paging import PAGE_SIZES, Paging
from chrysalio.lib.filter import Filter
from chrysalio.lib.i18n import translate_field
from chrysalio.lib.utils import age, make_digest, size_label
from chrysalio.helpers.literal import Literal
from chrysalio.includes.cache import cache_namespace
from chrysalio.views import BaseView
from chrysalio.models.dbgroup import DBGroup
from cioservice.models.dbjob import DBJob
from ..lib.i18n import _
from ..lib.utils import CIOWAREHOUSE_NS, CACHE_REGION_GLOBAL, THUMBNAIL_SMALL
from ..lib.utils import HERE, mimetype_url, sort_key, files2response
from ..lib.utils import file_ids2fullnames
from ..lib.wfile import WFile
from ..lib.wjob import WJob
from ..models.dbwarehouse import DBWarehouse, DBWarehouseUser, DBWarehouseGroup
from ..models.dbsharing import DBSharing, DBSharingFile


ALL_SCOPES = (
    ('directory', _('in the directory')), ('warehouse', _('in the warehouse')),
    ('favorite', _('in the favorites')), ('all', _('everywhere')))
TOP_SCOPES = (
    ('favorite', _('in the favorites')), ('all', _('everywhere')))
AUTOCOMPLETE_LIMIT = 10


# =============================================================================
[docs]class BrowseView(BaseView): """Class to manage warehouse browsing. :type request: pyramid.request.Request :param request: Current request. """ # ------------------------------------------------------------------------- def __init__(self, request): """Constructor method.""" super(BrowseView, self).__init__(request) self._fileinfo = request.registry['panels']['fileinfo'] self._wfile = WFile(request) self._wjob = WJob(request) # -------------------------------------------------------------------------
[docs] @view_config( route_name='browse_directory', renderer='ciowarehouse:Templates/browse.pt') @view_config( route_name='browse_directory_root', renderer='ciowarehouse:Templates/browse.pt') @view_config( route_name='browse_warehouse', renderer='ciowarehouse:Templates/browse.pt') @view_config( route_name='browse_favorite', renderer='ciowarehouse:Templates/browse.pt') @view_config( route_name='browse_all', renderer='ciowarehouse:Templates/browse.pt') @view_config( route_name='glance_directory', renderer='ciowarehouse:Templates/browse.pt') @view_config(route_name='browse_directory', renderer='json', xhr=True) def browse(self): """Browse one or several warehouses.""" # pylint: disable = too-many-locals # Fix route warehouse_id = self._request.matchdict.get('warehouse_id') scope, route = self._fixed_route(warehouse_id) if route is not None: return HTTPFound(route) # Warehouse ciowarehouse = self._request.registry['modules']['ciowarehouse'] warehouse = ciowarehouse.warehouse(self._request, warehouse_id) if warehouse_id and warehouse is None: raise HTTPForbidden(comment=_('This warehouse is not accessible!')) # Trail & current directory trail, directory = self._trail(scope, warehouse) # Action i_writer = ciowarehouse.warehouse_file_writer(self._request, warehouse) action, file_ids = get_action(self._request) if self._action_global(warehouse, directory, action, i_writer): return {} # Filter wfilter = self._filter( 'warehouse:{0}/{1}'.format(warehouse_id, directory), action) # Paging paging, defaults = self._paging(scope, warehouse, directory, wfilter) # Form groups = () if action[:3] != 'grp' else [ (k.group_id, k.label(self._request)) for k in self._request.dbsession.query(DBGroup)] form = self._form(warehouse, defaults, action, groups) # Other actions if action[:4] == 'dnl!': action = files2response( self._request, file_ids2fullnames(self._request, paging, file_ids), join(warehouse.root, directory) if warehouse and directory else None) if action: return action action = self._fileinfo.action( self._request, warehouse, paging, form, action) action = self._action_button(warehouse, paging, form, action, file_ids) # Refresh warehouse if warehouse is not None: warehouse.full_refresh(self._request, in_thread=True, force=False) return { 'action': action, 'files': file_ids, 'form': form, 'scope': scope, 'warehouse_id': warehouse_id, 'directory': directory, 'wfilter': wfilter, 'paging': paging, 'trail': trail, 'back_route': self._breadcrumbs(scope, warehouse), 'indexfields': self._indexfields(warehouse), 'i_writer': i_writer, 'fileinfo': self._fileinfo, 'groups': groups, 'jobs': self._wjob.available(warehouse), 'seeds': warehouse.seeds(self._request) if warehouse else (), 'metafields': self._request.registry['metafields'], 'join': join, 'translate_field': translate_field, 'age': age, 'datetime': datetime, 'mimetype_url': mimetype_url, 'size_label': size_label, 'PAGE_SIZES': PAGE_SIZES, 'SCOPES': TOP_SCOPES if warehouse is None else ALL_SCOPES, 'HERE': HERE, 'THUMBNAIL_SMALL': THUMBNAIL_SMALL, 'TOP_SCOPES': [k[0] for k in TOP_SCOPES], 'download_max_size': warehouse.download_max_size if warehouse else 0}
# -------------------------------------------------------------------------
[docs] @view_config( route_name='browse_filter_all', renderer='json', xhr=True) @view_config( route_name='browse_filter_favorite', renderer='json', xhr=True) @view_config( route_name='browse_filter_warehouse', renderer='json', xhr=True) @view_config( route_name='browse_filter_directory', renderer='json', xhr=True) @view_config( route_name='browse_filter_directory_root', renderer='json', xhr=True) def browse_filter(self): """Return a dictionary to autocomplete a filter field.""" field_id = self._request.params.get('field') if field_id not in self._request.registry['indexfields'] or \ not self._request.registry['indexfields'][field_id]['stored']: return [] scope = self._request.matched_route.name.split('_')[2] value = self._request.params.get('value') # Whoosh query path = self._request.matchdict.get('path') if scope == 'directory': fieldnames = ('directory', field_id) wquery = 'directory:"{0}" AND {1}:({2}*)'.format( join(*path) if path else '.', field_id, value) else: fieldnames = (field_id,) wquery = '{0}:({1}*)'.format(field_id, value) wquery = wquery.replace('**', '*') # Directory and file list dirs, files = [], [] for wid in self._warehouses_in_scope( scope, self._request.matchdict.get('warehouse_id')): warehouse = self._request.registry['modules']['ciowarehouse']\ .warehouse(self._request, wid) if warehouse is not None: hits = warehouse.index_search( self._request, fieldnames, wquery, limit=AUTOCOMPLETE_LIMIT) dirs += hits[0] files += hits[1] # Sort and extract string hits = tuple({k[field_id] for k in sorted( dirs + files, key=lambda k: k['score'], reverse=True)})[ :AUTOCOMPLETE_LIMIT] # Simplify keywords if self._request.registry['indexfields'][field_id][ 'field_type'] == 'keyword': simple_hits = set() for hit in hits: for word in hit.split(','): word = word.strip() if word.startswith(value): simple_hits.add(word) hits = tuple(simple_hits)[:AUTOCOMPLETE_LIMIT] return hits
# ------------------------------------------------------------------------- def _fixed_route(self, warehouse_id): """Possibly fix route according to scope. :param str warehouse_id: Current warehouse ID. :rtype: tuple :return: A tuple such as ``(scope, route)``. """ if 'modules_off' not in self._request.registry or \ 'ciowarehouse' in self._request.registry['modules_off']: raise HTTPForbidden(comment=_( 'The module "ciowarehouse" is not activated.')) route_name = self._request.matched_route.name scope = self._request.POST.get('scope') or route_name.split('_')[1] if scope == 'all' and route_name != 'browse_all': return scope, self._request.route_path('browse_all') if scope == 'favorite' and route_name != 'browse_favorite': return scope, self._request.route_path('browse_favorite') if scope == 'warehouse' and route_name != 'browse_warehouse': return scope, self._request.route_path( 'browse_warehouse', warehouse_id=warehouse_id) if scope == 'directory' and route_name not in ( 'browse_directory', 'glance_directory'): return scope, self._request.route_path( 'browse_directory', warehouse_id=warehouse_id, path='') return scope, None # ------------------------------------------------------------------------- def _trail(self, scope, warehouse): """Compute the trail or label of the page. :param str scope: Scope of the search. :type warehouse: .lib.warehouse.Warehouse :param warehouse: Current warehouse object, possibly none. :rtype: tuple :return: A tuple such as ``(html_trail, directory)``. """ if scope == 'directory': return warehouse.file_trail( self._request, warehouse.directory_path( self._request, self._request.matchdict.get('path', []))) translate = self._request.localizer.translate if scope == 'warehouse': return Literal('<span>{0}</span>'.format(translate(_( 'Search in the warehouse "${l}"', {'l': warehouse.label(self._request)})))), None if scope == 'favorite': return Literal('<span>{0}</span>'.format(translate( _('Search in the favorite warehouses')))), None return Literal('<span>{0}</span>'.format(translate( _('Search in the all warehouses')))), None # ------------------------------------------------------------------------- def _filter(self, filter_id, action): """Create the filter object for the search. :param str filter_id: Filter ID. :param str action: The possibly current action. :rtype: chrysalio.lib.filter.Filter """ inputs = self._request.registry['cache_global'].get( 'filter', CIOWAREHOUSE_NS, CACHE_REGION_GLOBAL) if inputs is None: inputs = [] fields = self._request.registry['indexfields'] for field_id in sorted( fields, key=lambda k: fields[k]['in_filter']): if not fields[field_id]['in_filter']: continue field_type = fields[field_id]['field_type'] values = None if field_type in ('integer', 'decimal'): values = 0 elif field_type in ('boolean', 'image'): values = True elif field_type in ('list', 'palette'): values = [('', ' ')] + [ (k[0], translate_field(self._request, k[1])) for k in fields[field_id]['choices'].items()] elif fields[field_id]['stored']: values = '' inputs.append(( field_id, translate_field( self._request, fields[field_id]['label']), fields[field_id]['in_filter'] < 0, values)) self._request.registry['cache_global'].set( 'filter', inputs, CIOWAREHOUSE_NS, CACHE_REGION_GLOBAL) wfilter = Filter( self._request, filter_id, inputs, remove=action[:4] == 'crm!' and action[4:] or None) if action == 'crm!all': wfilter.clear() return wfilter # ------------------------------------------------------------------------- def _paging(self, scope, warehouse, directory, wfilter): """Return the current paging. :param str scope: Scope of the search. :type warehouse: .:lib.warehouse.Warehouse :param warehouse: Warehouse object of the possibly current warehouse. :param str directory: Path of the possibly directory. :type wfilter: chrysalio.lib.filter.Filter :param wfilter: Current filter. :rtype: tuple :return: A tuple such as ``(paging, defaults)``. """ # Clean up sharing DBSharing.purge_expired(self._request, self._request.dbsession) # Set paging parameters paging_id = 'warehouses' if warehouse is None \ else 'warehouse:{0}'.format(warehouse.uid) defaults = Paging.params( self._request, paging_id, '-score' if scope != 'directory' else '+file_name') defaults['scope'] = scope sort = Paging.get_sort(self._request, paging_id) # Retrieve file list from cache namespace = cache_namespace(CIOWAREHOUSE_NS, warehouse.uid) \ if warehouse else CIOWAREHOUSE_NS key = make_digest('{0}|{1}|{2}|{3}'.format( scope, directory or '', str(wfilter), sort)) if 'ciowarehouse' not in self._request.session: self._request.session['ciowarehouse'] = {} self._request.session['ciowarehouse']['current_cache'] = ( key, namespace) files = self._request.registry['cache_global'].get( key, namespace, CACHE_REGION_GLOBAL) # Possibly, populate the cache if files is None: files = self._file_list(scope, warehouse, directory, wfilter, sort) self._request.registry['cache_global'].set( key, files, namespace, CACHE_REGION_GLOBAL) # Possibly, filter by groups if not self._request.registry['modules'][ 'ciowarehouse'].warehouse_admin(self._request, warehouse): user_groups = set(self._request.session['user']['groups']) files = [ k for k in files if not k['only_groups'] or (user_groups & k['only_groups'])] # Create paging paging = Paging(self._request, paging_id, files, defaults) # Thumbnail information self._wfile.detect_thumbnails(paging) return paging, defaults # ------------------------------------------------------------------------- def _form(self, warehouse, defaults, action, groups): """Return a schema object to manage file information. :type warehouse: .lib.warehouse.Warehouse :param warehouse: Current warehouse object if exists. :param dict defaults: Default values for the form. :param str action: Current action. :param list groups: List of available groups as tuple such as ``(group_id, group_label)``. :rtype: chrysalio.lib.form.Form """ # pylint: disable = too-many-branches, too-many-statements schema = None force_defaults = False # Renaming a file if action[:4] == 'ren?': defaults['new_name'] = self._fileinfo.value( self._request, 'file_name') force_defaults = True elif action[:4] == 'ren!': schema = SchemaNode(Mapping()) schema.add(SchemaNode( String(), name='new_name', validator=Length(max=128))) # Updating groups elif action[:4] == 'grp?': for group_id in self._fileinfo.value(self._request, 'only_groups'): defaults[group_id] = True force_defaults = True elif action[:4] == 'grp!': schema = SchemaNode(Mapping()) for group in groups: schema.add(SchemaNode(Boolean(), name=group[0], missing=False)) # Updating metadata elif action[:4] == 'mta?': defaults.update(self._fileinfo.value(self._request, 'metadata')) force_defaults = True elif action[:4] == 'mta!': schema = SchemaNode(Mapping()) for field_id, field in self._fileinfo.value( self._request, 'metafields').items(): if field['type'] == 'boolean': schema.add(SchemaNode( Boolean(), name=field_id, missing=None)) elif field['type'] == 'integer': schema.add(SchemaNode( Integer(), name=field_id, missing=None)) elif field['type'] == 'decimal': schema.add(SchemaNode( Float(), name=field_id, missing=None)) elif field['type'] == 'datetime': schema.add(SchemaNode( DateTime(), name=field_id, missing=None)) elif field['type'] == 'date': schema.add(SchemaNode( Date(), name=field_id, missing=None)) else: schema.add(SchemaNode( String(), name=field_id, missing=None)) # Sharing elif action[:4] == 'shr?': defaults['message'] = self._request.localizer.translate(_( '${n} shares with you:', {'n': self._request.session['user']['name']})) force_defaults = True elif action[:4] == 'shr!': schema = SchemaNode(Mapping()) schema.add(SchemaNode( String(), name='message', validator=Length(max=255))) schema.add(SchemaNode( String(), name='password1', validator=Length(min=2), missing=None)) schema.add(SchemaNode( String(), name='password2', missing=None, validator=SameAs(self._request, 'password1', _( 'The two passwords are not identical.')))) schema.add(SchemaNode(Date(), name='expiration', missing=None)) # Action button elif action[:3] == 'job' and warehouse is not None: service = self._request.registry['services'][ warehouse.job(self._request, action[4:])['service_id']] dbjob = self._request.dbsession.query(DBJob).filter_by( job_id=action[4:]).first() schema = SchemaNode(Mapping()) service.values_schema(schema, defaults, dbjob, False) force_defaults = True # Form form = Form( self._request, schema=schema, defaults=defaults, force_defaults=force_defaults) form.forget('filter_value') if action and action[3] == '!' and form.validate(): form.forget('#') return form # ------------------------------------------------------------------------- def _action_global(self, warehouse, directory, action, i_writer): """Execute global action. :type warehouse: .lib.warehouse.Warehouse :param warehouse: Current warehouse object if exists. :param str directory: Current directory if exists. :param str action: Current action. :param bool i_writer: ``True`` if the user is authorized to write into the warehouse. :rtype: bool :return: ``True`` if it is an AJAX call. """ # Import files via Ajax if self._request.is_xhr: if i_writer: self._wfile.upload_all(warehouse, directory) return True # Clear cache (see also browse.pt) # if action == 'clr!' and warehouse: # self._request.registry['modules']['ciowarehouse'].cache_clear( # self._request, warehouse.uid) # log_info( # self._request, 'ciowarehouse_clear_cache', warehouse.uid) # Import files if action == 'imp!' and warehouse and i_writer: self._wfile.upload_all(warehouse, directory) # Paste elif action == 'pst!' and warehouse and i_writer: self._wfile.clipboard_paste(warehouse, directory) # Create a directory elif action == 'dir!-' and warehouse and i_writer: self._wfile.make_directory(warehouse, directory) # Create a new file elif action[:4] == 'sed!' and warehouse and i_writer: self._wfile.new_file(warehouse, directory, action[4:]) return False # ------------------------------------------------------------------------- def _action_button(self, warehouse, paging, form, action, file_ids): """Execute the action of the `Action` button. :type warehouse: .lib.warehouse.Warehouse :param warehouse: Current warehouse object if exists. :type paging: chrysalio.lib.paging.Paging :param paging: Paging containing all the files. :type form: chrysalio.lib.form.Form :param form: Current form. :param str action: Current action. :param list file_ids: List of file IDs. :rtype: str :return: Return a possibly new action. """ # Copy/cut if action[:4] in ('cpy!', 'cut!'): self._wfile.clipboard_copy(paging, file_ids, action[:4] == 'cut!') # Removal elif action[:4] == 'rmv!': self._wfile.remove(paging, file_ids) # Sharing elif action[:4] == 'shr!': if form.validate(): self._sharing_create(warehouse, paging, file_ids, form.values) else: action = 'shr?#' # Job elif action[:3] == 'job': action, build_id = self._wjob.prepare( warehouse, paging, form, action) if build_id is not None and action and action[3] == '!': self._wjob.run(build_id) return action # ------------------------------------------------------------------------- def _breadcrumbs(self, scope, warehouse): """Set the breadcrumbs. :param str scope: Scope of the search. :type warehouse: .lib.warehouse.Warehouse :param warehouse: Current warehouse object, possibly none. :return: Back route. """ from_route = self._request.breadcrumbs.current_route_name() if scope == 'directory': label = warehouse.label(self._request) elif scope == 'warehouse': label = self._request.localizer.translate(_( 'Search in "${l}"', {'l': warehouse.label(self._request)})) else: label = self._request.localizer.translate(_('Advanced Search')) if (scope == 'directory' and from_route in ( 'browse_directory', 'glance_directory', 'browse_directory_root', 'warehouse_view')) or ( scope in ('all', 'favorite', 'warehouse') and from_route in ( 'browse_all', 'browse_favorite', 'browse_warehouse')): self._request.breadcrumbs.pop() from_route = self._request.breadcrumbs.current_route_name() from_path = self._request.breadcrumbs.current_path() self._request.breadcrumbs(label, root_chunks=3) return from_path if from_route in ( 'browse_all', 'browse_favorite', 'browse_warehouse', 'browse_directory') else None # ------------------------------------------------------------------------- def _indexfields(self, warehouse): """Retrieve the appropriate list of index fields. :type warehouse: .lib.warehouse.Warehouse :param warehouse: Current warehouse object, possibly none. :rtype: dict :return: A dictionary with keys ``'list'`` and ``'cards'``. """ if warehouse is not None: return { 'list': warehouse.listfields, 'cards': warehouse.cardfields} return { 'list': self._request.registry['listfields'], 'cards': self._request.registry['cardfields']} # ------------------------------------------------------------------------- def _warehouses_in_scope(self, scope, warehouse_id): """Find ID of warehouses in the scope. :param str scope: Scope of the search. :param str warehouse_id: ID of warehouse if exists. :rtype: list """ # One warehouse if warehouse_id is not None: return (warehouse_id,) # All warehouses user_id = self._request.session['user']['user_id'] if scope == 'all': dbquery = self._request.dbsession.query( DBWarehouse.warehouse_id) if not self._request.has_permission('warehouse-create'): groups = set(self._request.session['user']['groups']) dbquery = dbquery.outerjoin( DBWarehouseUser, DBWarehouseGroup).filter(or_( DBWarehouseUser.user_id == user_id, DBWarehouseGroup.group_id.in_(groups), DBWarehouse.access.in_(('free', 'readonly')))) return [k[0] for k in dbquery] # Favorite warehouses dbquery = self._request.dbsession.query( DBWarehouse.warehouse_id).outerjoin(DBWarehouseUser).filter( DBWarehouseUser.user_id == user_id, DBWarehouseUser.favorite) return [k[0] for k in dbquery] # ------------------------------------------------------------------------- def _file_list(self, scope, warehouse, directory, wfilter, sort): """Execute the search and return a list of files. :param str scope: Scope of the search. :type warehouse: .lib.warehouse.Warehouse :param warehouse: Possible object of current warehouse. :param str directory: Possible relative path in the warehouse of the current directory. :type wfilter: chrysalio.lib.filter.Filter :param wfilter: Possible filter to apply. :param str sort: Sort criteria. :rtype: list """ # Whoosh query fieldnames, wquery = wfilter.whoosh() if scope == 'directory': fieldnames += ('directory',) wquery = 'directory:"{0}"{1}{2}'.format( directory, ' AND ' if wquery else '', wquery) # Directory and file list dirs, files = [], [] warehouse_ids = self._warehouses_in_scope( scope, warehouse.uid if warehouse is not None else None) for wid in warehouse_ids: if warehouse is None: warehouse = self._request.registry['modules']['ciowarehouse']\ .warehouse(self._request, wid) if warehouse is not None: hits = warehouse.index_search( self._request, fieldnames, wquery) dirs += hits[0] files += hits[1] warehouse = None # Sort if sort[1:] not in self._request.registry['indexfields'] and \ sort[1:] not in ('score', 'warehouse_id'): return dirs + files if sort[1:] != 'file_name': dirs, files = [], dirs + files if sort[1:] in ('score', 'warehouse_id') or self._request.registry[ 'indexfields'][sort[1:]]['whoosh_type'] != 'ID': dirs = sorted( dirs, key=lambda k: k.get(sort[1:], ''), reverse=sort[0] == '-') files = sorted( files, key=lambda k: k.get(sort[1:], ''), reverse=sort[0] == '-') else: dirs = sorted( dirs, key=lambda k: sort_key(k.get(sort[1:], '')), reverse=sort[0] == '-') files = sorted( files, key=lambda k: sort_key(k.get(sort[1:], '')), reverse=sort[0] == '-') files = dirs + files return files # ------------------------------------------------------------------------- def _sharing_create(self, warehouse, paging, file_ids, values): """Save a sharing. :type warehouse: .lib.warehouse.Warehouse :param warehouse: Current warehouse object if exists. :type paging: chrysalio.lib.paging.Paging :param paging: Paging containing all the files. :param list file_ids: List of file IDs. :param dict values: Values such as ``message``, ``password1`` and ``expiration``. """ # Create the sharing dbsharing = DBSharing( sharing_id=make_digest(datetime.now().isoformat()), message=values.get('message'), expiration=values.get('expiration')) dbsharing.set_password(values.get('password1')) self._request.dbsession.add(dbsharing) # Add files file_dict = {} for pfile in paging: if pfile['file_id'] not in file_ids: continue dbsharing.files.append(DBSharingFile( file_id=pfile['file_id'], warehouse_id=pfile['warehouse_id'], directory=pfile['directory'], file_name=pfile['file_name'])) if pfile['warehouse_id'] not in file_dict: file_dict[pfile['warehouse_id']] = [ (pfile['directory'], pfile['file_name'])] else: file_dict[pfile['warehouse_id']].append( (pfile['directory'], pfile['file_name'])) pfile['shared'] = True # Update index for warehouse_id, files in file_dict.items(): warehouse = self._request.registry['modules']['ciowarehouse']\ .warehouse(self._request, warehouse_id) if warehouse is not None: warehouse.index_update( self._request.dbsession, files, self._request, force=True) self._request.registry['modules']['ciowarehouse'].cache_clear( self._request, warehouse.uid) # Update file info if self._fileinfo.is_open(self._request): self._fileinfo.prepare_rendering( self._request, warehouse, paging.get_item( 'file_id', self._fileinfo.file_id(self._request)), True) self._request.session.flash( _('The sharing link is: ${l}', {'l': self._request.route_url( 'sharing_download', sharing_id=dbsharing.sharing_id)}), 'persistent')