build assets on application startup, instead of during the build

This will make it possible (in a subsequent commit) to augment the asset
sources by dropping new files in and/or setting a config option. That
way Beaker installations can add site-specific customisations without
having to rebuild Beaker's entire asset tree.

The asset building tools (uglify-js, ycssmin, lessc) are now runtime
requirements rather than build-time requirements.

We can also no longer be lazy and just install the generated/
subdirectory plus the .webassets-manifest file. Instead, we extract
a list of all required source files from the assets environment itself,
and use that during the package build to copy over the necessary files.

Ultimately the asset sources should not be bundled at all (Fedora now
requires this) but that is a longer term goal.

Bug: 1012224
Change-Id: Id8e2883269287a120236d52f0863e9ab0e7f4e4c

.gitignore

@@ -6,14 +6,10 @@ rpm-build
 *.egg-info
 *.bz2
 Harness/build
-Server/assets/.webassets-cache/
-Server/assets/.webassets-manifest
-Server/assets/generated/
+Server/assets-cache/
 Server/build
 Server/build.out
 Server/devdata.sqlite
-Server/bkr/server/static/.webassets-cache/
-Server/bkr/server/static/webassets-external/
 Common/build
 Client/build
 Client/build.out

IntegrationTests/server-test.cfg

@@ -36,6 +36,7 @@ beaker.ks_meta = ''
 beaker.kernel_options = 'noverifyssl'
 beaker.kernel_options_post = ''
 basepath.assets = '../Server/assets'
+basepath.assets_cache = '../Server/assets-cache'
 assets.auto_build = True
 beaker.deprecated_job_group_permissions.on = True
 #Set this to True to enable qpid testing

Server/apache/beaker-server.conf

@@ -11,6 +11,7 @@ RewriteRule   ^/bkr$  /bkr/   [R]
 
 Alias /static /usr/share/bkr/server/static
 Alias /bkr/static /usr/share/bkr/server/static
+Alias /bkr/assets/generated /var/cache/beaker/assets
 Alias /bkr/assets /usr/share/bkr/server/assets
 Redirect permanent /bkr/apidoc http://beaker-project.org/docs/server-api
 Alias /logs /var/www/beaker/logs
@@ -51,9 +52,18 @@ WSGIScriptAlias /bkr/ /usr/share/bkr/beaker-server.wsgi/bkr/
     </IfModule>
 </Directory>
 
-# Generated assets have a content hash in their filename so they can
-# safely be cached forever.
-<Directory /usr/share/bkr/server/assets/generated>
+<Directory /var/cache/beaker/assets>
+    <IfModule mod_authz_core.c>
+        # Apache 2.4
+        Require all granted
+    </IfModule>
+    <IfModule !mod_authz_core.c>
+        # Apache 2.2
+        Order deny,allow
+        Allow from all
+    </IfModule>
+    # Generated assets have a content hash in their filename so they can
+    # safely be cached forever.
     ExpiresActive on
     ExpiresDefault "access plus 1 year"
 </Directory>

Server/assets/style.less

@@ -23,7 +23,7 @@
 //@baseLineHeight:        3ex;
 
 // overrides for FontAwesome variables
-@FontAwesomePath: "generated/font-awesome-3.2.1";
+@FontAwesomePath: "font-awesome/font";
 
 // heading sizes (Bootstrap's are too generous)
 h1, h2, h3 { line-height: @baseLineHeight * 2; }

Server/bkr/server/app.py

@@ -4,7 +4,8 @@
 # the Free Software Foundation; either version 2 of the License, or
 # (at your option) any later version.
 
-from flask import Flask
+import os
+from flask import Flask, send_from_directory
 from turbogears import config
 
 # This is NOT the right way to do this, we should just use SCRIPT_NAME properly instead.
@@ -14,4 +15,20 @@ def add_url_rule(self, rule, *args, **kwargs):
         prefixed_rule = config.get('server.webpath', '').rstrip('/') + rule
         return super(PrefixedFlask, self).add_url_rule(prefixed_rule, *args, **kwargs)
 
-app = PrefixedFlask('bkr.server', static_folder='../../assets')
+app = PrefixedFlask('bkr.server')
+
+# URL rules for serving static assets. The same URL paths are mapped in the 
+# Apache config, so in production the Python application will never see these 
+# requests at all. This is just for the development server.
+
+@app.route('/assets/generated/<path:filename>')
+def assets_generated(filename):
+    return send_from_directory(
+            os.path.abspath(config.get('basepath.assets_cache')),
+            filename)
+
+@app.route('/assets/<path:filename>')
+def assets(filename):
+    return send_from_directory(
+            os.path.abspath(config.get('basepath.assets')),
+            filename)

Server/bkr/server/assets.py

@@ -6,6 +6,7 @@
 
 import os
 import webassets
+from webassets.bundle import get_all_bundle_files
 from webassets.filter.jst import JST
 import threading
 from turbogears import config
@@ -23,14 +24,19 @@ def input(self, in_, out, source_path, **kwargs):
 _env = None
 _env_lock = threading.Lock()
 
-def _create_env(**kwargs):
-    env = webassets.Environment(url='/assets', manifest='file', **kwargs)
+def _create_env(source_dir, output_dir, **kwargs):
+    # some pieces of webassets assume the directories are absolute
+    source_dir = os.path.abspath(source_dir)
+    output_dir = os.path.abspath(output_dir)
+    env = webassets.Environment(directory=output_dir, url='/assets/generated',
+            manifest='cache', **kwargs)
+    env.append_path(source_dir, url='/assets')
     env.config['UGLIFYJS_EXTRA_ARGS'] = ['--mangle', '--compress']
     env.register('css',
             'style.less',
             filters=['less', 'cssrewrite', YCSSMin()],
-            output='generated/beaker-%(version)s.css',
-            depends=['*.less', 'bootstrap/less/*.less'])
+            output='beaker-%(version)s.css',
+            depends=['*.less', 'bootstrap/less/*.less', 'font-awesome/less/*.less'])
     env.register('js',
             # third-party
             'bootstrap/js/bootstrap-transition.js',
@@ -49,23 +55,23 @@ def _create_env(**kwargs):
                 'jst/*/*.html',
                 'jst/*.html',
                 filters=[JST(template_function='_.template')],
-                output='generated/beaker-jst-%(version)s.js'),
+                output='beaker-jst-%(version)s.js'),
             'local-datetime.js',
             'link-tabs-to-anchor.js',
             'beaker-typeaheads.js',
             'recipe-tasks.js',
             'access-policy.js',
             filters=['uglifyjs'],
-            output='generated/beaker-%(version)s.js')
+            output='beaker-%(version)s.js')
     return env
 
 def _create_runtime_env():
-    directory = config.get('basepath.assets',
-            # default location is at the base of our source tree
-            os.path.join(os.path.dirname(__file__), '..', '..', 'assets'))
+    source_dir = config.get('basepath.assets')
+    output_dir = config.get('basepath.assets_cache')
     debug = config.get('assets.debug')
     auto_build = config.get('assets.auto_build')
-    return _create_env(directory=directory, debug=debug, auto_build=auto_build)
+    return _create_env(source_dir=source_dir, output_dir=output_dir,
+            debug=debug, auto_build=auto_build)
 
 def get_assets_env():
     global _env
@@ -75,7 +81,39 @@ def get_assets_env():
                 _env = _create_runtime_env()
     return _env
 
-def build_assets(directory):
-    env = _create_env(directory=directory, debug=False, auto_build=False)
+def build_assets():
+    env = get_assets_env()
     for bundle in env:
-        bundle.build()
+        # force=True here is a workaround for the timestamp-based caching 
+        # behaviour in webassets. If the site.less symlink is changed to point 
+        # at a different source file, with a different mod time, webassets will 
+        # never notice if the mod time is still older than the mod time of the 
+        # output file (which it normally would be).
+        # The timestamp-based updater is still useful when auto_build=True is 
+        # used during development though.
+        bundle.build(force=True)
+
+def list_asset_sources(source_dir):
+    """
+    Returns a list of paths (relative to the given source_dir) of all asset 
+    sources files defined in the assets environment.
+    """
+    # This is called during package build, so we create a new env specially to 
+    # refer to the given source dir.
+    # We aren't going to produce any generated files so output_dir is unused.
+    source_dir = os.path.abspath(source_dir)
+    env = _create_env(source_dir=source_dir, output_dir='/unused',
+            debug=False, auto_build=False)
+    paths = []
+    for bundle in env:
+        for path in get_all_bundle_files(bundle, env):
+            paths.append(os.path.relpath(path, source_dir))
+    # font-awesome is currently not managed by webassets because webassets 
+    # breaks on non-UTF8 input files
+    paths.extend([
+        'font-awesome/font/fontawesome-webfont.eot',
+        'font-awesome/font/fontawesome-webfont.svg',
+        'font-awesome/font/fontawesome-webfont.ttf',
+        'font-awesome/font/fontawesome-webfont.woff',
+    ])
+    return paths

Server/bkr/server/config/app.cfg

@@ -70,6 +70,7 @@ autoreload.on = False
 
 # Assets configuration
 basepath.assets = '/usr/share/bkr/server/assets'
+basepath.assets_cache = '/var/cache/beaker/assets'
 assets.debug = False
 assets.auto_build = False
 

Server/bkr/server/wsgi.py

@@ -32,7 +32,7 @@
 import cherrypy._cpwsgi
 from cherrypy.filters.basefilter import BaseFilter
 from flask import Flask
-from bkr.server import identity
+from bkr.server import identity, assets
 from bkr.server.app import app
 
 log = logging.getLogger(__name__)
@@ -82,6 +82,10 @@ def restart_transaction_patched(args):
         resource.setrlimit(resource.RLIMIT_AS, (config.get('rlimit_as'),
                                                 config.get('rlimit_as')))
 
+    # Build assets. If assets.auto_build is True in the config, this will also
+    # happen on page request. Otherwise, it only happens once at startup here.
+    assets.build_assets()
+
     # workaround for TGMochiKit initialisation
     # https://sourceforge.net/p/turbogears1/tickets/34/
     import tgmochikit

Server/dev.cfg

@@ -56,5 +56,6 @@ tg.strict_parameters = True
 beaker.reliable_distro_tag = 'RELEASED'
 
 basepath.assets = './assets'
+basepath.assets_cache = './assets-cache'
 assets.debug = False
 assets.auto_build = True

Server/setup.py

@@ -15,7 +15,7 @@
 import sys
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'Common'))
 sys.path.insert(1, os.path.dirname(__file__))
-from bkr.server.assets import build_assets
+from bkr.server import assets
 
 description = (
   "Beaker is a system for full stack software integration testing "
@@ -54,9 +54,6 @@ class Build(_build, object):
     # These are set in finalize_options()
     substitutions = {'@DATADIR@': None, '@LOCALEDIR@': None}
     subRE = re.compile('(' + '|'.join(substitutions.keys()) + ')+')
-    sub_commands = _build.sub_commands + [
-        ('build_assets', None),
-    ]
 
     def initialize_options(self):
         self.install_data = None
@@ -137,23 +134,6 @@ def byte_compile(self, files):
             else:
                 log.debug("skipping byte-compilation of %s", kid_file)
 
-class BuildAssets(Command):
-    description = 'build web assets'
-    user_options = []
-
-    def initialize_options(self):
-        self.build_base = None
-
-    def finalize_options(self):
-        self.set_undefined_options('build', ('build_base', 'build_base'))
-        self.source_dir = 'assets'
-        self.build_dir = os.path.join(self.build_base, 'assets')
-
-    def run(self):
-        self.copy_tree(self.source_dir, self.build_dir)
-        log.info('building assets in %s', self.build_dir)
-        build_assets(self.build_dir)
-
 class Install(_install):
     sub_commands = _install.sub_commands + [
         ('install_assets', None),
@@ -164,25 +144,19 @@ class InstallAssets(Command):
     user_options = []
 
     def initialize_options(self):
-        self.build_base = None
         self.install_data = None
 
     def finalize_options(self):
-        self.set_undefined_options('install',
-                ('build_base', 'build_base'),
-                ('install_data', 'install_data'))
-        self.build_dir = os.path.join(self.build_base, 'assets')
+        self.set_undefined_options('install', ('install_data', 'install_data'))
         self.install_dir = os.path.join(self.install_data, 'bkr/server/assets')
+        self.source_dir = 'assets'
 
     def run(self):
-        manifest_name = '.webassets-manifest'
-        self.mkpath(self.install_dir, mode=0755)
-        self.copy_file(
-                os.path.join(self.build_dir, manifest_name),
-                os.path.join(self.install_dir, manifest_name))
-        self.copy_tree(
-                os.path.join(self.build_dir, 'generated'),
-                os.path.join(self.install_dir, 'generated'))
+        for filename in assets.list_asset_sources(self.source_dir):
+            source_path = os.path.join(self.source_dir, filename)
+            dest_path = os.path.join(self.install_dir, filename)
+            self.mkpath(os.path.dirname(dest_path), mode=0755)
+            self.copy_file(source_path, dest_path)
 
 
 def find_data_recursive(dest_dir, src_dir, exclude=frozenset()):
@@ -203,6 +177,7 @@ def find_data_recursive(dest_dir, src_dir, exclude=frozenset()):
     ("/etc/logrotate.d", ["logrotate.d/beaker"]),
     ("/usr/share/bkr", filter(os.path.isfile, glob.glob("apache/*.wsgi"))),
     ("/var/log/beaker", []),
+    ("/var/cache/beaker/assets", []),
     ("/var/www/beaker/logs", []),
     ("/var/www/beaker/rpms", []),
     ("/var/www/beaker/repos", []),
@@ -232,7 +207,6 @@ def find_data_recursive(dest_dir, src_dir, exclude=frozenset()):
     cmdclass = {
         'build': Build,
         'build_py': build_py_and_kid,
-        'build_assets': BuildAssets,
         'install': Install,
         'install_assets': InstallAssets,
     },

beaker.spec

@@ -75,10 +75,6 @@ BuildRequires:  pkgconfig(systemd)
 
 %if %{with server}
 BuildRequires:  python-kid
-BuildRequires:  python-webassets
-BuildRequires:  /usr/bin/lessc
-BuildRequires:  /usr/bin/cssmin
-BuildRequires:  /usr/bin/uglifyjs
 # These runtime dependencies are needed at build time as well, because
 # the unit tests and Sphinx autodoc import the server code as part of the
 # build process.
@@ -102,6 +98,7 @@ BuildRequires:  python-netaddr
 BuildRequires:  ovirt-engine-sdk
 BuildRequires:  python-itsdangerous
 BuildRequires:  python-decorator
+BuildRequires:  python-webassets
 BuildRequires:  python-flask
 BuildRequires:  python-markdown
 BuildRequires:  python-passlib
@@ -190,6 +187,9 @@ Requires:       python-decorator
 Requires:       python-flask
 Requires:       python-markdown
 Requires:       python-webassets
+Requires:       /usr/bin/lessc
+Requires:       /usr/bin/cssmin
+Requires:       /usr/bin/uglifyjs
 Requires:       python-passlib
 %if %{with_systemd}
 Requires:       systemd-units
@@ -471,6 +471,8 @@ rm -rf %{_var}/lib/beaker/osversion_data
 %attr(-,apache,root) %{_datadir}/bkr/server
 %attr(0660,apache,root) %config(noreplace) %{_sysconfdir}/%{name}/server.cfg
 %dir %{_localstatedir}/log/%{name}
+%dir %{_localstatedir}/cache/%{name}
+%attr(-,apache,root) %dir %{_localstatedir}/cache/%{name}/assets
 %attr(-,apache,root) %dir %{_localstatedir}/www/%{name}/logs
 %attr(-,apache,root) %dir %{_localstatedir}/www/%{name}/rpms
 %attr(-,apache,root) %dir %{_localstatedir}/www/%{name}/repos