passing milestone

bin/validate_live_pixel.mjs

@@ -4,7 +4,6 @@ import JSON5 from 'json5';
 import { ParamsValidator } from '../src/params_validator.mjs';
 import { logErrors } from '../src/error_utils.mjs';
 import { spawnSync } from 'child_process';
-import { exit } from 'process';
 
 const clickhouseHost = process.env.CLICKHOUSE_HOST
 
@@ -16,9 +15,14 @@ if (!clickhouseHost) {
 const args = process.argv.slice(2);
 const mainDir = args[0];
 
-const pixelDefs = JSON5.parse(fs.readFileSync(`${args[1]}`));
-const commonParams = JSON5.parse(fs.readFileSync(`${mainDir}/common_params.json`));
-const commonSuffixes = JSON5.parse(fs.readFileSync(`${mainDir}/common_suffixes.json`));
+const productDef = JSON5.parse(fs.readFileSync(`${mainDir}/product.json`).toString());
+// Whether to force all schemas and pixels to lowercase
+const forceLowerCase = productDef.forceLowerCase;
+
+const pixelDefs = JSON5.parse(getNormalizedCase(fs.readFileSync(`${args[1]}`).toString()));
+const commonParams = JSON5.parse(getNormalizedCase(fs.readFileSync(`${mainDir}/common_params.json`).toString()));
+const ignoreParams = JSON5.parse(getNormalizedCase(fs.readFileSync(`${mainDir}/ignore_params.json`).toString()));
+const commonSuffixes = JSON5.parse(getNormalizedCase(fs.readFileSync(`${mainDir}/common_suffixes.json`).toString()));
 const paramsValidator = new ParamsValidator(commonParams, commonSuffixes);
 
 function main() {
@@ -33,18 +37,22 @@ function main() {
     } else {
         const pixelQueryResults = queryClickhouse(pixelDefs);
         for (const prefix of Object.keys(pixelQueryResults)) {
-            console.log('Validating', prefix, pixelQueryResults)
             validateQueryForPixels(prefix, pixelQueryResults[prefix], paramsValidator)
         }
     }
 }
 
 function queryClickhouse(pixelDefs) {
+    var agents = "'" + productDef.agents.join("','") + "'";
+    const agentString = productDef.agents.length ? `AND agent IN (${agents})` : '';
+
     const pixelQueryResults = {};
     for(const pixel of Object.keys(pixelDefs)) {
         console.log('Querying for', pixel)
-        const pixel_id = pixel.split('.')[0];
-        const clickhouseQuery = spawnSync( 'clickhouse-client', [ '--host',  clickhouseHost, '--query', `SELECT DISTINCT request FROM metrics.pixels WHERE pixel_id = '${pixel_id}' AND date > now() - INTERVAL 30 DAY AND pixel LIKE '${pixel}%' LIMIT 1000` ] );
+        const pixel_id = pixel.split(/[-\.]/)[0];
+        const queryString = `SELECT DISTINCT request FROM metrics.pixels WHERE pixel_id = '${pixel_id}' AND date > now() - INTERVAL 30 DAY AND pixel ILIKE '${pixel}%' ${agentString} LIMIT 1000`;
+        console.log('queryString', queryString)
+        const clickhouseQuery = spawnSync( 'clickhouse-client', [ '--host',  clickhouseHost, '--query', queryString ] );
         const resultString = clickhouseQuery.stdout.toString();
         const resultErr = clickhouseQuery.stderr.toString();
         if (resultErr) {
@@ -53,27 +61,37 @@ function queryClickhouse(pixelDefs) {
             if (resultString) pixelQueryResults[pixel] = resultString;
         }
     }
-    console.log('pixelQueryResults',pixelQueryResults);
 
     return pixelQueryResults;
 }
 
 function validateQueryForPixels(prefix, pixelQuery, paramsValidator) {
+    var minVersion = productDef.target;
 
-        const lines = pixelQuery.split('\n');
-        for (const line of lines) {
-            if (line == '') continue;
-            const pixelRequest = line.split('/')[2];
-            const parts = pixelRequest.split('?');
-            const url = parts[1];
-            const pixelDef = pixelDefs[prefix];
+    const lines = pixelQuery.split('\n');
+    console.log(`Received ${lines.length} results`)
+    for (let line of lines) {
+        if (line == '') continue;
+        line = getNormalizedCase(line);
+        const pixelRequest = line.split('/')[2];
+        const parts = pixelRequest.split('?');
+        const url = parts[1];
+        const pixelDef = pixelDefs[prefix];
 
-            logErrors(`ERRORS for '${pixelRequest}\n`, paramsValidator.validateLivePixels(pixelDef, prefix, line));
+        logErrors(`ERRORS for '${pixelRequest}\n`, paramsValidator.validateLivePixels(pixelDef, ignoreParams, prefix, line, minVersion));
     }
 }
 
 function validateSinglePixel(pixelDefs, prefix, url) {
     logErrors('ERRORS:', paramsValidator.validateLivePixels(pixelDefs[prefix], prefix, url));
 }
 
+function getNormalizedCase(value) {
+    if (forceLowerCase) {
+        return value.toLowerCase();
+    }
+
+    return value;
+
+}
 main();