Merge pull request #4658 from Countly/ar2rsawseen/next

Safe ingestion
Countly · Nov 1, 2023 · f6cc6eb · f6cc6eb
2 parents c8796b1 + 5e62120
commit f6cc6eb
Showing 1 changed file with 5 additions and 10 deletions.
diff --git a/api/utils/requestProcessor.js b/api/utils/requestProcessor.js
@@ -2917,16 +2917,12 @@ const processRequestData = (params, app, done) => {
         var update = {};
         //check if we already processed app users for this request
         if (params.app_user.last_req !== params.request_hash && ob.updates.length) {
-            ob.updates.push({$set: {last_req: params.request_hash, ingested: false}});
             for (let i = 0; i < ob.updates.length; i++) {
                 update = common.mergeQuery(update, ob.updates[i]);
             }
         }
         var newUser = params.app_user.fs ? false : true;
         common.updateAppUser(params, update, function() {
-            if (!plugins.getConfig("api", params.app && params.app.plugins, true).safe && !params.res.finished) {
-                common.returnMessage(params, 200, 'Success');
-            }
             if (params.qstring.begin_session) {
                 plugins.dispatch("/session/retention", {
                     params: params,
@@ -2957,10 +2953,6 @@ const processRequestData = (params, app, done) => {
                         }
                     }
                 }
-                if (!retry && plugins.getConfig("api", params.app && params.app.plugins, true).safe) {
-                    //acknowledge data ingestion
-                    common.updateAppUser(params, {$set: {ingested: true}});
-                }
                 if (!params.res.finished) {
                     if (retry) {
                         common.returnMessage(params, 400, 'Could not ingest data');
@@ -3288,12 +3280,15 @@ const validateAppForWriteAPI = (params, done, try_times) => {
             if (params.req.method.toLowerCase() === 'post') {
                 payload += params.req.body;
             }
-            payload = payload.replace(new RegExp("[?&]?(rr=[^&]+)", "gm"), "");
+            //remove dynamic parameters
+            payload = payload.replace(new RegExp("[?&]?(rr=[^&\n]+)", "gm"), "");
+            payload = payload.replace(new RegExp("[?&]?(checksum=[^&\n]+)", "gm"), "");
+            payload = payload.replace(new RegExp("[?&]?(checksum256=[^&\n]+)", "gm"), "");
             params.request_hash = common.crypto.createHash('sha1').update(payload).digest('hex') + (params.qstring.timestamp || params.time.mstimestamp);
             if (plugins.getConfig("api", params.app && params.app.plugins, true).prevent_duplicate_requests) {
                 //check unique millisecond timestamp, if it is the same as the last request had,
                 //then we are having duplicate request, due to sudden connection termination
-                if (params.app_user.last_req === params.request_hash && (!plugins.getConfig("api", params.app && params.app.plugins, true).safe || params.app_user.ingested)) {
+                if (params.app_user.last_req === params.request_hash) {
                     params.cancelRequest = "Duplicate request";
                 }
             }