[O7hh0T6D] create CKAN plugin containing shared functionality for Dat…

…a and Publications

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include ckanext/qgov/common/resources *

build.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+
+install () {
+    echo "Deploying $1 to local Apache instance..."
+    . /usr/lib/ckan/bin/activate
+    easy_install "$1"
+    sudo apachectl graceful
+}
+
+if [ "$1" = "check" ]; then
+    . /usr/lib/ckan/bin/activate
+    pip install pyflakes pylint
+    python -m pyflakes ckanext-qgov
+    python -m pylint ckanext
+    exit
+fi
+
+VERSION=$1
+if [ "$1" = "install" ]; then
+    VERSION=$2
+fi
+if [ "$VERSION" = "" ]; then
+    VERSION=0.0.1
+fi
+ARTIFACT=ckanext_qgov-$VERSION-py2.7.egg
+
+echo "Building CKAN extension..."
+python setup.py build -f
+
+echo "Packaging CKAN extension..."
+cp setup.py setup.py.bak
+sed -i -e "s/@BUILD-LABEL@/$VERSION/" setup.py
+python setup.py bdist_egg --skip-build --dist-dir=target
+
+echo "Cleaning up..."
+mv setup.py.bak setup.py
+rm -rf build
+
+if [ "$1" = "install" -o "$2" = "all" ]; then
+    install target/$ARTIFACT
+fi
+

ckanext/__init__.py

@@ -0,0 +1,7 @@
+"""Empty namespace, just exists to keep things tidy"""
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/qgov/__init__.py

@@ -0,0 +1,7 @@
+"""Empty namespace, just exists to keep things tidy"""
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/qgov/common/__init__.py

@@ -0,0 +1,7 @@
+"""Empty namespace, just exists to keep things tidy"""
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/qgov/common/anti_csrf.py

@@ -0,0 +1,98 @@
+import ckan.lib.base as base
+import re
+from re import DOTALL, IGNORECASE, MULTILINE
+from genshi.template import MarkupTemplate
+from logging import getLogger
+from pylons import request, response
+
+LOG = getLogger(__name__)
+
+RAW_RENDER = base.render
+RAW_RENDER_JINJA = base.render_jinja2
+RAW_BEFORE = base.BaseController.__before__
+
+POST_FORM = re.compile(r'(<form [^>]*method="post"[^>]*>)([^<]*\s<)', IGNORECASE | MULTILINE)
+FORM_TOKEN = re.compile(r'<input type="hidden" name="token" value="([0-9a-f]+)"/>')
+API_URL = re.compile(r'^/api\b.*')
+
+def is_logged_in():
+    return request.cookies.get("auth_tkt")
+
+# Insert token into applicable responses
+
+def anti_csrf_render(template_name, extra_vars=None, cache_key=None, cache_type=None, cache_expire=None, method='xhtml', loader_class=MarkupTemplate, cache_force=None, renderer=None):
+    html = apply_token(RAW_RENDER(template_name, extra_vars, cache_key, cache_type, cache_expire, method, loader_class, cache_force, renderer))
+    return html
+
+def anti_csrf_render_jinja2(template_name, extra_vars=None):
+    html = apply_token(RAW_RENDER_JINJA(template_name, extra_vars))
+    return html
+
+def apply_token(html):
+    if not is_logged_in() or not POST_FORM.search(html):
+        return html
+
+    token_match = FORM_TOKEN.search(html)
+    if token_match:
+        token = token_match.group(1)
+    else:
+        token = get_server_token()
+
+    def insert_form_token(form_match):
+        return form_match.group(1) + '<input type="hidden" name="token" value="'+token+'"/>' + form_match.group(2)
+
+    return POST_FORM.sub(insert_form_token, html)
+
+def get_server_token():
+    if request.environ['webob.adhoc_attrs'].has_key('server_token'):
+        token = request.server_token
+    elif request.cookies.has_key("token"):
+        token = request.cookies.pop("token")
+    else:
+        import binascii, os
+        token = binascii.hexlify(os.urandom(32))
+        response.set_cookie("token", token, max_age=600, httponly=True)
+
+    if token is None or token.strip() == "":
+        csrf_fail("Server token is blank")
+
+    request.server_token = token
+    return token
+
+# Check token on applicable requests
+
+def is_request_exempt():
+    return not is_logged_in() or API_URL.match(request.path) or request.method in {'GET', 'HEAD', 'OPTIONS'}
+
+def anti_csrf_before(obj, action, **params):
+    if not is_request_exempt() and get_server_token() != get_post_token():
+        csrf_fail("Could not match session token with form token")
+
+    RAW_BEFORE(obj, action)
+
+def csrf_fail(message):
+    from pylons.controllers.util import abort
+    LOG.error(message)
+    abort(403, "Your form submission could not be validated")
+
+def get_post_token():
+    if request.environ['webob.adhoc_attrs'].has_key('token'):
+        return request.token
+
+    postTokens = request.POST.getall('token')
+    if not postTokens:
+        csrf_fail("Missing CSRF token in form submission")
+    elif len(postTokens) > 1:
+        csrf_fail("More than one CSRF token in form submission")
+    else:
+        request.token = postTokens[0]
+
+    # drop token from request so it doesn't populate resource extras
+    del request.POST['token']
+
+    return request.token
+
+def intercept_csrf():
+    base.render = anti_csrf_render
+    base.render_jinja2 = anti_csrf_render_jinja2
+    base.BaseController.__before__ = anti_csrf_before

ckanext/qgov/common/authenticator.py

@@ -0,0 +1,49 @@
+from ckan.lib.authenticator import UsernamePasswordAuthenticator
+from ckan.model import User, Session
+
+from sqlalchemy import Column, types
+from sqlalchemy.ext.declarative import declarative_base
+
+from zope.interface import implements
+from repoze.who.interfaces import IAuthenticator
+
+Base = declarative_base()
+
+import logging
+log = logging.getLogger(__name__)
+
+def intercept_authenticator():
+    UsernamePasswordAuthenticator.authenticate = QGOVAuthenticator().authenticate
+
+class QGOVAuthenticator(UsernamePasswordAuthenticator):
+    implements(IAuthenticator)
+
+    def authenticate(self, environ, identity):
+        if not 'login' in identity or not 'password' in identity:
+            return None
+        user = User.by_name(identity.get('login'))
+        if user is None:
+            log.debug('Login failed - username %r not found', identity.get('login'))
+            return None
+
+        qgovUser = Session.query(QGOVUser).filter_by(name = identity.get('login')).first()
+        if qgovUser.login_attempts >= 10:
+            log.debug('Login as %r failed - account is locked', identity.get('login'))
+        elif user.validate_password(identity.get('password')):
+            # reset attempt count to 0
+            qgovUser.login_attempts = 0
+            Session.commit()
+            return user.name
+        else:
+            log.debug('Login as %r failed - password not valid', identity.get('login'))
+
+        qgovUser.login_attempts += 1
+        Session.commit()
+        return None
+
+class QGOVUser(Base):
+    __tablename__ = 'user'
+    __mapper_args__ = {'include_properties' : ['id', 'name', 'login_attempts']}
+    id = Column(types.UnicodeText, primary_key=True)
+    name = Column(types.UnicodeText, nullable=False, unique=True)
+    login_attempts = Column(types.SmallInteger)

ckanext/qgov/common/controller.py

@@ -0,0 +1,30 @@
+import re
+from ckan.lib.base import BaseController, c, render, request
+from ckan.controllers.storage import StorageController
+from ckan.controllers.user import UserController
+from pylons.controllers.util import abort
+
+from ckan.model import Session
+from ckanext.qgov.common.authenticator import QGOVUser
+
+ALLOWED_EXTENSIONS = re.compile(r'.*((\.csv)|(\.xls)|(\.txt)|(\.kmz)|(\.xlsx)|(\.pdf)|(\.shp)|(\.tab)|(\.jp2)|(\.esri)|(\.gdb)|(\.jpg)|(\.tif)|(\.tiff)|(\.jpeg)|(\.xml)|(\.kml)|(\.doc)|(\.docx)|(\.rtf))$', re.I)
+class QGOVController(BaseController):
+
+    def upload_handle(self):
+        params = dict(request.params.items())
+        originalFilename = params.get('file').filename
+        if ALLOWED_EXTENSIONS.search(originalFilename):
+            return StorageController.upload_handle(StorageController())
+        abort(403, 'This file type is not supported. If possible, upload the file in another format.\nIf you continue to have problems, email Smart Service—\n[email protected]')
+
+    def logged_in(self):
+        controller = UserController()
+        if not c.user:
+            # a number of failed login attempts greater than 10
+            # indicates that the locked user is associated with the current request
+            qgovUser = Session.query(QGOVUser).filter(QGOVUser.login_attempts > 10).first()
+            if qgovUser:
+                qgovUser.login_attempts = 10
+                Session.commit()
+                return controller.login('account-locked')
+        return controller.logged_in()

ckanext/qgov/common/plugin.py

@@ -0,0 +1,109 @@
+import os
+from logging import getLogger
+
+from ckan.plugins import implements, SingletonPlugin, IConfigurer, IRoutes, ITemplateHelpers
+from ckanext.qgov.common.stats import Stats
+
+LOG = getLogger(__name__)
+
+from ckan.lib.base import h
+
+import ckan.lib.formatters as formatters
+
+def random_tags():
+    import random
+    tags = h.unselected_facet_items('tags', limit=15)
+    random.shuffle(tags)
+    return tags
+
+def format_resource_filesize(size):
+    return formatters.localised_filesize(int(size))
+
+def group_id_for(group_name):
+    import ckan.model as model
+    group = model.Group.get(group_name)
+
+    if group and group.is_active():
+        return group.id
+    else:
+        LOG.error("%s group was not found or not active", group_name)
+        return None
+
+def user_password_validator(key, data, errors, context):
+    from ckan.lib.navl.dictization_functions import Missing
+    from pylons import config
+    from pylons.i18n import _
+    import re
+
+    password_min_length = int(config.get('password_min_length', '10'))
+    password_patterns = config.get('password_patterns', r'.*[0-9].*,.*[a-z].*,.*[A-Z].*,.*[-`~!@#$%^&*()_+=|\\/ ].*').split(',')
+
+    value = data[key]
+
+    if value is None or value == '' or isinstance(value, Missing):
+        raise ValueError(_('You must provide a password'))
+    if not len(value) >= password_min_length:
+        errors[('password',)].append(_('Your password must be %s characters or longer' % password_min_length))
+    for policy in password_patterns:
+        if not re.search(policy, value):
+            errors[('password',)].append(_('Must contain at least one number, lowercase letter, capital letter, and symbol'))
+
+class QGOVPlugin(SingletonPlugin):
+    """Apply custom functions for Queensland Government portals.
+
+    ``IConfigurer`` allows us to add/modify configuration options.
+    ``IRoutes`` allows us to add new URLs, or override existing URLs.
+    ``ITemplateHelpers`` allows us to provide helper functions to templates.
+    """
+    implements(IConfigurer, inherit=True)
+    implements(IRoutes, inherit=True)
+    #~ implements(ITemplateHelpers, inherit=True)
+
+    def __init__(self, **kwargs):
+        import ckan.logic.validators as validators
+        validators.user_password_validator = user_password_validator
+        import anti_csrf, authenticator
+        anti_csrf.intercept_csrf()
+        authenticator.intercept_authenticator()
+
+    def get_helpers(self):
+        """ A dictionary of extra helpers that will be available
+        to provide QGOV-specific helpers to the templates.
+        """
+
+        helper_dict = {}
+        helper_dict['random_tags'] = random_tags
+        helper_dict['group_id_for'] = group_id_for
+        helper_dict['format_resource_filesize'] = format_resource_filesize
+        helper_dict['top_organisations'] = Stats.top_organisations
+        helper_dict['top_categories'] = Stats.top_categories
+        helper_dict['resource_count'] = Stats.resource_count
+        helper_dict['resource_report'] = Stats.resource_report
+        helper_dict['resource_org_count'] = Stats.resource_org_count
+
+        return helper_dict
+
+    def update_config(self, config):
+        """Use our custom list of licences, and disable some unwanted features
+        """
+
+        here = os.path.dirname(__file__)
+        config['licenses_group_url'] = 'file://'+os.path.join(here, 'resources', 'qgov-licences.json')
+        config['ckan.template_title_deliminater'] = '|'
+
+        # block unwanted content
+        config['openid_enabled'] = False
+        return config
+
+    def before_map(self, routeMap):
+        """ Use our custom controller, and disable some unwanted URLs
+        """
+        routeMap.connect('/storage/upload_handle', controller='ckanext.qgov.common.controller:QGOVController', action='upload_handle')
+        routeMap.connect('/user/logged_in', controller='ckanext.qgov.common.controller:QGOVController', action='logged_in')
+
+        # block unwanted content
+        routeMap.connect('/user', controller='error', action='404')
+        routeMap.connect('/user/register', controller='error', action='404')
+        routeMap.connect('/user/followers/{username:.*}', controller='error', action='404')
+        routeMap.connect('/api/action/follow{action:.*}', controller='error', action='404')
+        return routeMap