Add mechanism to check analysis before it's executed

lib/analysis.js

@@ -9,6 +9,7 @@ var toposort = require('../lib/dag/toposort');
 var validator = require('../lib/dag/validator');
 
 var DatabaseService = require('./service/database');
+var Requirements = require('./service/requirements');
 
 var AnalysisLogger = require('./logging/logger');
 
@@ -60,6 +61,7 @@ AnalysisFactory.prototype.create = function(configuration, definition, callback)
         configuration.batch,
         configuration.limits
     );
+    var requirements = new Requirements(databaseService, configuration.limits);
     var logger = configuration.logger ? new AnalysisLogger(configuration.logger.stream, configuration.user) : undefined;
 
     async.waterfall(
@@ -75,6 +77,11 @@ AnalysisFactory.prototype.create = function(configuration, definition, callback)
 
                 return done(null, new Analysis(rootNode));
             },
+            function analysis$checkLimits(analysis, done) {
+                requirements.checkLimits(analysis, function(err) {
+                    return done(err, analysis);
+                });
+            },
             function analysis$register(analysis, done) {
                 databaseService.registerAnalysisInCatalog(analysis, function(err) {
                     return done(err, analysis);
@@ -100,10 +107,8 @@ AnalysisFactory.prototype.create = function(configuration, definition, callback)
                 if (err && err.message && err.message.match(/permission denied/i)) {
                     err = new Error('Analysis requires authentication with API key: permission denied.');
                 }
-
                 return callback(err);
             }
-
             return callback(null, analysis);
         }
     );

lib/node/node.js

@@ -1,3 +1,4 @@
+
 'use strict';
 
 var util = require('util');
@@ -9,6 +10,7 @@ dot.templateSettings.strip = false;
 var id = require('../util/id');
 var QueryBuilder = require('../filter/query-builder');
 var Validator = require('./validator');
+var NodeRequirements = require('./requirements');
 
 var TYPE = require('./type');
 module.exports.TYPE = TYPE;
@@ -28,7 +30,6 @@ var STATUS = {
 
 module.exports.STATUS = STATUS;
 
-
 var NODE_RESERVED_KEYWORDS = {
     type: 1,
     typeSignature: 1,
@@ -90,6 +91,8 @@ function Node() {
     this.typeSignature = null;
 
     this.version = 0;
+
+    this.requirements = new NodeRequirements(this);
 }
 
 Node.prototype.id = function(skipFilters) {
@@ -468,3 +471,44 @@ function validate(validator, params, expectedParamName) {
 
     return param;
 }
+
+Node.prototype.computeRequirements = function(databaseService, limits, callback) {
+    var default_limit = null; // no limit by default
+
+    // The default comptueRequirements methos computes the number estimated of output rows of the node.
+    // Two possible approaches to do this are:
+    // 1. use the node's query, e.g.
+    //        this.requirements.setEstimatedNumberOfRowsFromQuery(databaseService, default_limit, limits, callback);
+    //    (we could also use setNumberOfRowsFromQuery for the exact number of rows)
+    // 2. reduce overhead by basing the estimation on the previously computed estimations of the input nodes;
+    //    here we would use some simple, common heuristic, such as:
+    //        this.requirements.setMaxInputNumberOfRows(limits, default_limit);
+    //        return callback(null);
+    //    And we'd need to override this for node classes for which this not acceptable
+
+    this.requirements.setMaxInputNumberOfRows(limits, default_limit);
+    return callback(null);
+};
+
+Node.prototype.requirementMessages = function() {
+    var messages = [];
+
+    // Note that any falsy value (null, 0, '', ...) means no limits:
+    if (this.requirements.getLimit('maximumNumberOfRows')) {
+        if (this.requirements.getEstimatedRequirement('numberOfRows') >
+            this.requirements.getLimit('maximumNumberOfRows')) {
+            messages.push('too many result rows');
+        }
+    }
+
+    return messages;
+};
+
+Node.prototype.validateRequirements = function() {
+    var messages = this.requirementMessages();
+    var err = null;
+    if (messages.length > 0) {
+        err = new Error(messages.join('\n'));
+    }
+    return err;
+};

lib/node/nodes/aggregate-intersection.js

@@ -77,3 +77,9 @@ var queryAggregateTemplate = Node.template([
     'WHERE ST_Intersects(_cdb_analysis_source.the_geom, _cdb_analysis_target.the_geom)',
     'GROUP BY {{=it.groupByColumns}}'
 ].join('\n'));
+
+AggregateIntersection.prototype.computeRequirements = function(databaseService, limits, callback) {
+    // we estimate the maximum possible number of rows of the result
+    this.requirements.setProductInputNumberOfRows(limits, 1000000);
+    return callback(null);
+};

lib/node/nodes/buffer.js

@@ -108,3 +108,15 @@ Buffer.prototype.isolinesDissolvedQuery = function() {
         _query: this.source.getQuery()
     });
 };
+
+Buffer.prototype.computeRequirements = function(databaseService, limits, callback) {
+    var default_limit;
+    if (this.dissolved) {
+        default_limit = 10000;
+    }
+    else {
+        default_limit = 1000000;
+    }
+    this.requirements.setSingleInputNumberOfRows('source', limits, default_limit);
+    return callback(null);
+};

lib/node/nodes/georeference-admin-region.js

@@ -56,3 +56,8 @@ var queryTemplate = Node.template([
     '  ) AS the_geom',
     'FROM ({{=it.source}}) AS _camshaft_georeference_admin_region_analysis'
  ].join('\n'));
+
+ GeoreferenceAdminRegion.prototype.computeRequirements = function(databaseService, limits, callback) {
+     this.requirements.setSingleInputNumberOfRows('source', limits, 1000);
+     return callback(null);
+ };

lib/node/nodes/georeference-country.js

@@ -33,3 +33,10 @@ var queryTemplate = Node.template([
     '  cdb_dataservices_client.cdb_geocode_admin0_polygon({{=it.country}}) AS the_geom',
     'FROM ({{=it.source}}) AS _camshaft_georeference_country_analysis'
  ].join('\n'));
+
+ GeoreferenceCountry.prototype.computeRequirements = function(databaseService, limits, callback) {
+     // given that country polygons are large and there are less than 200 countries in the world
+     // we'll set a modest default limit for the number of resulting rows
+     this.requirements.setSingleInputNumberOfRows('source', limits, 500);
+     return callback(null);
+ };

lib/node/nodes/georeference-postal-code.js

@@ -77,3 +77,15 @@ var queryTemplate = Node.template([
     '  ) AS the_geom',
     'FROM ({{=it.source}}) AS _camshaft_georeference_postal_code_analysis'
  ].join('\n'));
+
+ GeoreferencePostalCode.prototype.computeRequirements = function(databaseService, limits, callback) {
+     var default_limit;
+     if (getGecoderFunction(this.output_geometry_type) === 'cdb_geocode_postalcode_polygon') {
+         default_limit = 10000;
+     }
+     else {
+         default_limit = 1000000;
+     }
+     this.requirements.setSingleInputNumberOfRows('source', limits, default_limit);
+     return callback(null);
+ };

lib/node/nodes/line-sequential.js

@@ -1,6 +1,7 @@
 'use strict';
 
 var Node = require('../node');
+var NodeRequirements = require('../requirements');
 var debug = require('../../util/debug')('analysis:line-sequential');
 
 var TYPE = 'line-sequential';
@@ -43,3 +44,33 @@ var routingSequentialQueryTemplate = Node.template([
     '  {{? it.category_column }}GROUP BY {{=it.category_column}}{{?}}',
     ') _cdb_analysis_line_sequential'
 ].join('\n'));
+
+LineSequential.prototype.computeRequirements = function(databaseService, limits, callback) {
+    var inputRows = this.source.requirements.getEstimatedRequirement('numberOfRows');
+    var numCategories = 1;
+    var self = this;
+    var rowsLimit = NodeRequirements.getNodeLimit(limits, TYPE, 'maximumNumberOfRows', 1000000);
+    var pointsLimit = NodeRequirements.getNodeLimit(limits, TYPE, 'maximumNumberOfPointsPerLine', 100000);
+    function storeRequirements(err) {
+        // we make a rough estimate of numCategories result rows and inputRows/numCategories points per line
+        self.requirements.setEstimatedRequirement('numberOfRows', numCategories);
+        self.requirements.setEstimatedRequirement('numberOfPointsPerLine', inputRows/(numCategories || 1));
+        self.requirements.setLimit('maximumNumberOfRows', rowsLimit);
+        self.requirements.setLimit('maximumNumberOfPointsPerLine', pointsLimit);
+        return callback(err);
+    }
+    storeRequirements(null);
+};
+
+LineSequential.prototype.requirementMessages = function() {
+    var messages = [];
+    if (this.requirements.getEstimatedRequirement('numberOfRows') >
+        this.requirements.getLimit('maximumNumberOfRows')) {
+        messages.push('too many result rows');
+    }
+    if (this.requirements.getEstimatedRequirement('numberOfPointsPerLine') >
+        this.requirements.getLimit('maximumNumberOfPointsPerLine')) {
+        messages.push('too many points per line');
+    }
+    return messages;
+};

lib/node/nodes/line-source-to-target.js

@@ -59,3 +59,15 @@ LineSourceToTarget.prototype.sql = function() {
 
     return sql;
 };
+
+LineSourceToTarget.prototype.computeRequirements = function(databaseService, limits, callback) {
+    var default_limit = 1000000;
+    if (this.closest) {
+        this.requirements.setSingleInputNumberOfRows('source', limits, default_limit);
+    } else {
+        // this is the absolute maximum number of lines; it could be greatly reduced
+        // if this.node.source_column && this.node.target_column
+        this.requirements.setProductInputNumberOfRows(limits, default_limit);
+    }
+    return callback(null);
+};

lib/node/nodes/source.js

@@ -30,5 +30,10 @@ Source.prototype.sql = function() {
  * @returns {Node.STATUS}
  */
 Source.prototype.getStatus = function() {
-    return Node.STATUS.READY;
+    return Node.STATUS.READY; // TODO: this ignores the possibility of requirements exceeding the limits
+};
+
+Source.prototype.computeRequirements = function(databaseService, limits, callback) {
+    // By default we don't limit the number of rows of source nodes
+    this.requirements.setEstimatedNumberOfRowsFromQuery(databaseService, null, limits, callback);
 };