Copy CDN app back into main app

app.js

@@ -4,6 +4,8 @@ import createError from 'http-errors';
 import express from 'express';
 import logger from 'morgan';
 import cors from 'cors';
+import cdnRouter from './routes/cdn.js';
+import prepareRouter from './routes/prepare.js';
 import cacheRouter from './routes/cache.js';
 import globalRouter from './routes/global.js';
 import apiRouter from './routes/api.js';
@@ -41,31 +43,9 @@ app.use('/_global/favicon.ico', express.static('static/favicon.ico'));
 app.use('/_global/robots.txt', express.static('static/robots.txt'));
 app.use('/_global/static', express.static('static', {maxAge: '1d'}));
 
-// remove trailing slashes
-app.use((req, res, next) => {
-  if (req.path.slice(-1) === '/' && req.path.length > 1) {
-    const query = req.url.slice(req.path.length)
-    const safepath = req.path.slice(0, -1).replace(/\/+/g, '/')
-    res.redirect(301, safepath + query)
-  } else {
-    next()
-  }
-})
-
-// set pug globals for 'universe' and 'node_env'
-app.use(function(req, res, next){
-  if(process.env.UNIVERSE){
-    req.universe = process.env.UNIVERSE;
-  } else if(req.app.get('env') === 'production'){
-    req.universe = req.hostname.replace('.r-universe.dev', '');
-    res.locals.vhost = req.headers['r-universe-vhost'];
-  }
-  res.locals.universe = req.universe || 'ropensci';
-  res.locals.node_env = req.app.get('env');
-  next();
-});
-
-// check if package/universe exists and handle caching
+//routers
+app.use('/cdn', cdnRouter);
+app.use('/', prepareRouter);
 app.use('/{:package}', cacheRouter);
 app.use('/_global/', globalRouter);
 app.use('/', apiRouter);

routes/cdn.js

@@ -0,0 +1,77 @@
+import express from 'express';
+import gunzip from 'gunzip-maybe';
+import tar from 'tar-stream';
+import createError from 'http-errors';
+import {get_bucket_stream, bucket_find} from '../src/db.js';
+import {index_files_from_stream} from '../src/tools.js';
+
+const router = express.Router();
+
+function send_from_bucket(hash, operation, res){
+  return get_bucket_stream(hash).then(function(pkg){
+    let name = pkg.filename;
+    if(operation == 'send'){
+      let type = name.endsWith('.zip') ? 'application/zip' : 'application/x-gzip';
+      return pkg.stream.pipe(
+        res.type(type).attachment(name).set({
+          'Content-Length': pkg.length,
+          'Cache-Control': 'public, max-age=31557600, immutable',
+          'Last-Modified' : pkg.uploadDate.toUTCString()
+        })
+      );
+    }
+
+    if(operation == 'index'){
+      if(!name.endsWith('gz')){
+        pkg.stream.destroy();
+        throw createError(500, `Unable to index ${name} (only tar.gz files are supported)`);
+      }
+      return index_files_from_stream(pkg.stream).then(function(index){
+        index.files.forEach(function(entry){
+          entry.filename = entry.filename.match(/\/.*/)[0]; //strip pkg root dir
+        });
+        index.gzip = true;
+        res.send(index);
+      });
+    }
+
+    if(operation == 'decompress'){
+      if(!name.endsWith('gz')){
+        pkg.stream.destroy();
+        throw createError(`Unable to decompress ${name} (only tar.gz files are suppored)`);
+      }
+      var tarname = name.replace(/(tar.gz|tgz)/, 'tar');
+      return pkg.stream.pipe(gunzip()).pipe(
+        res.type('application/tar').attachment(tarname).set({
+          'Cache-Control': 'public, max-age=31536000, immutable',
+          'Last-Modified' : pkg.uploadDate.toUTCString()
+        })
+      );
+    }
+
+    throw createError(400, `Unsuppored operation ${operation}`);
+  });
+}
+
+/* Reduce noise from crawlers in log files */
+router.get("/robots.txt", function(req, res, next) {
+  res.type('text/plain').send(`User-agent: *\nDisallow: /\n`);
+});
+
+router.get("/:hash{/:postfix}", function(req, res, next) {
+  let hash = req.params.hash || "";
+  let postfix = req.params.postfix || "send";
+  if(hash.length != 32 && hash.length != 64) //should be md5 or sha256
+    return next(createError(400, "Invalid hash length"));
+  return send_from_bucket(hash, postfix, res);
+});
+
+/* index all the files on the cdn */
+router.get("/", function(req, res, next) {
+  var cursor = bucket_find({}, {sort: {uploadDate: -1}, project: {_id: 1, filename: 1}});
+  cursor.stream({transform: x => `${x._id} ${x.uploadDate.toISOString()} ${x.filename}\n`}).on('error', function(err){
+    next(createError(500, err));
+  }).pipe(res.type('text/plain'));
+});
+
+export default router;

routes/prepare.js

@@ -0,0 +1,19 @@
+export default function(req, res, next){
+  //remove trailing slashes except for root
+  if (req.path.slice(-1) === '/' && req.path.length > 1) {
+    const query = req.url.slice(req.path.length)
+    const safepath = req.path.slice(0, -1).replace(/\/+/g, '/')
+    return res.redirect(301, safepath + query);
+  }
+
+  //set universe globals
+  if(process.env.UNIVERSE){
+    req.universe = process.env.UNIVERSE;
+  } else if(req.app.get('env') === 'production'){
+    req.universe = req.hostname.replace('.r-universe.dev', '');
+    res.locals.vhost = req.headers['r-universe-vhost'];
+  }
+  res.locals.universe = req.universe || 'ropensci';
+  res.locals.node_env = req.app.get('env');
+  next();
+}

src/db.js

@@ -399,6 +399,22 @@ function mongo_package_stream(pkg, universe){
   });
 }
 
+export function get_bucket_stream(hash){
+  if(production){
+    return bucket.find({_id: hash}, {limit:1}).next().then(function(pkg){
+      if(!pkg)
+        throw createError(410, `File ${hash} not available (anymore)`);
+      pkg.stream = bucket.openDownloadStream(hash);
+      pkg.stream.on('error', function(err){
+        throw `Mongo stream error for ${hash}`;
+      });
+      return pkg;
+    });
+  } else {
+    throw "Not implemented for devel";
+  }
+}
+
 export function get_package_info(pkg, universe){
   if(production){
     return mongo_package_info(pkg, universe);
@@ -578,3 +594,11 @@ export function ls_packages(universe){
     throw "Not implemented for devel";
   }
 }
+
+export function bucket_find(query, options = {}){
+  if(production){
+    return bucket.find(query, options);
+  } else {
+    throw "Not implemented for devel";
+  }
+}