chore(shopify): seperate pixel server side events logic from legacy t…

…racker implementation (#3849)

* chore: initial commit, adding transformation and utils

* chore: handle new qparam

* chore: fix tests, qparams and mocks

* chore: add testsx1

* chore: remove redundant id assigns and code

* chore: fixing mocking, temp commit

* chore: refactor and restructure test and transformations

* chore: cleanup redundant testdata

* chore: updates to facilitate utils mocking

* chore: address events, add unit tests for new util functions

* chore: update fxn name

* chore: remove redis related check function in pixel transformation

* chore: remove redundant switch case

---------

Co-authored-by: Sai Sankeerth <[email protected]>

src/v0/sources/shopify/util.js

@@ -2,15 +2,9 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 const { v5 } = require('uuid');
 const sha256 = require('sha256');
-const { TransformationError } = require('@rudderstack/integrations-lib');
+const { TransformationError, isDefinedAndNotNull } = require('@rudderstack/integrations-lib');
 const stats = require('../../../util/stats');
-const {
-  constructPayload,
-  extractCustomFields,
-  flattenJson,
-  generateUUID,
-  isDefinedAndNotNull,
-} = require('../../util');
+const utils = require('../../util');
 const { RedisDB } = require('../../../util/redis/redisConnector');
 const {
   lineItemsMappingJSON,
@@ -92,8 +86,8 @@ const getProductsListFromLineItems = (lineItems) => {
   }
   const products = [];
   lineItems.forEach((lineItem) => {
-    const product = constructPayload(lineItem, lineItemsMappingJSON);
-    extractCustomFields(lineItem, product, 'root', LINE_ITEM_EXCLUSION_FIELDS);
+    const product = utils.constructPayload(lineItem, lineItemsMappingJSON);
+    utils.extractCustomFields(lineItem, product, 'root', LINE_ITEM_EXCLUSION_FIELDS);
     product.variant = getVariantString(lineItem);
     products.push(product);
   });
@@ -103,14 +97,14 @@ const getProductsListFromLineItems = (lineItems) => {
 const createPropertiesForEcomEvent = (message) => {
   const { line_items: lineItems } = message;
   const productsList = getProductsListFromLineItems(lineItems);
-  const mappedPayload = constructPayload(message, productMappingJSON);
-  extractCustomFields(message, mappedPayload, 'root', PRODUCT_MAPPING_EXCLUSION_FIELDS);
+  const mappedPayload = utils.constructPayload(message, productMappingJSON);
+  utils.extractCustomFields(message, mappedPayload, 'root', PRODUCT_MAPPING_EXCLUSION_FIELDS);
   mappedPayload.products = productsList;
   return mappedPayload;
 };
 
 const extractEmailFromPayload = (event) => {
-  const flattenedPayload = flattenJson(event);
+  const flattenedPayload = utils.flattenJson(event);
   let email;
   const regex_email = /\bemail\b/i;
   Object.entries(flattenedPayload).some(([key, value]) => {
@@ -182,7 +176,7 @@ const getAnonymousIdAndSessionId = async (message, metricMetadata, redisData = n
       return { anonymousId, sessionId };
     }
     return {
-      anonymousId: isDefinedAndNotNull(anonymousId) ? anonymousId : generateUUID(),
+      anonymousId: isDefinedAndNotNull(anonymousId) ? anonymousId : utils.generateUUID(),
       sessionId,
     };
   }
@@ -281,4 +275,5 @@ module.exports = {
   checkAndUpdateCartItems,
   getHashLineItems,
   getDataFromRedis,
+  getVariantString,
 };

src/v1/sources/shopify/transform.js

@@ -1,19 +1,27 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-const { processEventFromPixel } = require('./pixelTransform');
+const { processPixelWebEvents } = require('./webpixelTransformations/pixelTransform');
 const { process: processWebhookEvents } = require('../../../v0/sources/shopify/transform');
+const {
+  process: processPixelWebhookEvents,
+} = require('./webhookTransformations/serverSideTransform');
 
 const process = async (inputEvent) => {
   const { event } = inputEvent;
-  // check on the source Config to identify the event is from the tracker-based (legacy)
-  // or the pixel-based (latest) implementation.
+  const { query_parameters } = event;
+  // check identify the event is from the web pixel based on the pixelEventLabel property.
   const { pixelEventLabel: pixelClientEventLabel } = event;
   if (pixelClientEventLabel) {
     // this is a event fired from the web pixel loaded on the browser
     // by the user interactions with the store.
-    const responseV2 = await processEventFromPixel(event);
-    return responseV2;
+    const pixelWebEventResponse = await processPixelWebEvents(event);
+    return pixelWebEventResponse;
   }
-  // this is for common logic for server-side events processing for both pixel and tracker apps.
+  if (query_parameters && query_parameters?.version?.[0] === 'pixel') {
+    // this is a server-side event from the webhook subscription made by the pixel app.
+    const pixelWebhookEventResponse = await processPixelWebhookEvents(event);
+    return pixelWebhookEventResponse;
+  }
+  // this is a server-side event from the webhook subscription made by the legacy tracker-based app.
   const response = await processWebhookEvents(event);
   return response;
 };

src/v1/sources/shopify/webhookTransformations/serverSideTransform.js

@@ -0,0 +1,174 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+const lodash = require('lodash');
+const get = require('get-value');
+// const { RedisError } = require('@rudderstack/integrations-lib');
+const stats = require('../../../../util/stats');
+const {
+  getShopifyTopic,
+  // createPropertiesForEcomEvent,
+  extractEmailFromPayload,
+  getAnonymousIdAndSessionId,
+  // getHashLineItems,
+} = require('../../../../v0/sources/shopify/util');
+// const logger = require('../../../logger');
+const { removeUndefinedAndNullValues, isDefinedAndNotNull } = require('../../../../v0/util');
+// const { RedisDB } = require('../../../util/redis/redisConnector');
+const Message = require('../../../../v0/sources/message');
+const { EventType } = require('../../../../constants');
+const {
+  INTEGERATION,
+  MAPPING_CATEGORIES,
+  IDENTIFY_TOPICS,
+  ECOM_TOPICS,
+  RUDDER_ECOM_MAP,
+  SUPPORTED_TRACK_EVENTS,
+  SHOPIFY_TRACK_MAP,
+  lineItemsMappingJSON,
+} = require('../../../../v0/sources/shopify/config');
+const {
+  createPropertiesForEcomEventFromWebhook,
+  getProductsFromLineItems,
+} = require('./serverSideUtlis');
+
+const NO_OPERATION_SUCCESS = {
+  outputToSource: {
+    body: Buffer.from('OK').toString('base64'),
+    contentType: 'text/plain',
+  },
+  statusCode: 200,
+};
+
+const identifyPayloadBuilder = (event) => {
+  const message = new Message(INTEGERATION);
+  message.setEventType(EventType.IDENTIFY);
+  message.setPropertiesV2(event, MAPPING_CATEGORIES[EventType.IDENTIFY]);
+  if (event.updated_at) {
+    // converting shopify updated_at timestamp to rudder timestamp format
+    message.setTimestamp(new Date(event.updated_at).toISOString());
+  }
+  return message;
+};
+
+const ecomPayloadBuilder = (event, shopifyTopic) => {
+  const message = new Message(INTEGERATION);
+  message.setEventType(EventType.TRACK);
+  message.setEventName(RUDDER_ECOM_MAP[shopifyTopic]);
+
+  const properties = createPropertiesForEcomEventFromWebhook(event);
+  message.properties = removeUndefinedAndNullValues(properties);
+  // Map Customer details if present
+  const customerDetails = get(event, 'customer');
+  if (customerDetails) {
+    message.setPropertiesV2(customerDetails, MAPPING_CATEGORIES[EventType.IDENTIFY]);
+  }
+  if (event.updated_at) {
+    message.setTimestamp(new Date(event.updated_at).toISOString());
+  }
+  if (event.customer) {
+    message.setPropertiesV2(event.customer, MAPPING_CATEGORIES[EventType.IDENTIFY]);
+  }
+  if (event.shipping_address) {
+    message.setProperty('traits.shippingAddress', event.shipping_address);
+  }
+  if (event.billing_address) {
+    message.setProperty('traits.billingAddress', event.billing_address);
+  }
+  if (!message.userId && event.user_id) {
+    message.setProperty('userId', event.user_id);
+  }
+  return message;
+};
+
+const trackPayloadBuilder = (event, shopifyTopic) => {
+  const message = new Message(INTEGERATION);
+  message.setEventType(EventType.TRACK);
+  message.setEventName(SHOPIFY_TRACK_MAP[shopifyTopic]);
+  // eslint-disable-next-line camelcase
+  const { line_items: lineItems } = event;
+  const productsList = getProductsFromLineItems(lineItems, lineItemsMappingJSON);
+  message.setProperty('properties.products', productsList);
+  return message;
+};
+
+const processEvent = async (inputEvent, metricMetadata) => {
+  let message;
+  const event = lodash.cloneDeep(inputEvent);
+  const shopifyTopic = getShopifyTopic(event);
+  delete event.query_parameters;
+  switch (shopifyTopic) {
+    case IDENTIFY_TOPICS.CUSTOMERS_CREATE:
+    case IDENTIFY_TOPICS.CUSTOMERS_UPDATE:
+      message = identifyPayloadBuilder(event);
+      break;
+    case ECOM_TOPICS.ORDERS_CREATE:
+    case ECOM_TOPICS.ORDERS_UPDATE:
+    case ECOM_TOPICS.CHECKOUTS_CREATE:
+    case ECOM_TOPICS.CHECKOUTS_UPDATE:
+      message = ecomPayloadBuilder(event, shopifyTopic);
+      break;
+    default:
+      if (!SUPPORTED_TRACK_EVENTS.includes(shopifyTopic)) {
+        stats.increment('invalid_shopify_event', {
+          writeKey: metricMetadata.writeKey,
+          source: metricMetadata.source,
+          shopifyTopic: metricMetadata.shopifyTopic,
+        });
+        return NO_OPERATION_SUCCESS;
+      }
+      message = trackPayloadBuilder(event, shopifyTopic);
+      break;
+  }
+
+  if (message.userId) {
+    message.userId = String(message.userId);
+  }
+  if (!get(message, 'traits.email')) {
+    const email = extractEmailFromPayload(event);
+    if (email) {
+      message.setProperty('traits.email', email);
+    }
+  }
+  if (message.type !== EventType.IDENTIFY) {
+    const { anonymousId } = await getAnonymousIdAndSessionId(
+      message,
+      { shopifyTopic, ...metricMetadata },
+      null,
+    );
+    if (isDefinedAndNotNull(anonymousId)) {
+      message.setProperty('anonymousId', anonymousId);
+    }
+  }
+  message.setProperty(`integrations.${INTEGERATION}`, true);
+  message.setProperty('context.library', {
+    eventOrigin: 'server',
+    name: 'RudderStack Shopify Cloud',
+    version: '2.0.0',
+  });
+  message.setProperty('context.topic', shopifyTopic);
+  // attaching cart, checkout and order tokens in context object
+  message.setProperty(`context.cart_token`, event.cart_token);
+  message.setProperty(`context.checkout_token`, event.checkout_token);
+  // raw shopify payload passed inside context object under shopifyDetails
+  message.setProperty('context.shopifyDetails', event);
+  if (shopifyTopic === 'orders_updated') {
+    message.setProperty(`context.order_token`, event.token);
+  }
+  message = removeUndefinedAndNullValues(message);
+  return message;
+};
+const process = async (event) => {
+  const metricMetadata = {
+    writeKey: event.query_parameters?.writeKey?.[0],
+    source: 'SHOPIFY',
+  };
+  const response = await processEvent(event, metricMetadata);
+  return response;
+};
+
+module.exports = {
+  process,
+  processEvent,
+  identifyPayloadBuilder,
+  ecomPayloadBuilder,
+  trackPayloadBuilder,
+};

src/v1/sources/shopify/webhookTransformations/serverSideUtils.test.js

@@ -0,0 +1,112 @@
+const {
+  getProductsFromLineItems,
+  createPropertiesForEcomEventFromWebhook,
+} = require('./serverSideUtlis');
+
+const { constructPayload } = require('../../../../v0/util');
+
+const {
+  lineItemsMappingJSON,
+  productMappingJSON,
+} = require('../../../../v0/sources/shopify/config');
+const Message = require('../../../../v0/sources/message');
+jest.mock('../../../../v0/sources/message');
+
+const LINEITEMS = [
+  {
+    id: '41327142600817',
+    grams: 0,
+    presentment_title: 'The Collection Snowboard: Hydrogen',
+    product_id: 7234590408818,
+    quantity: 1,
+    sku: '',
+    taxable: true,
+    title: 'The Collection Snowboard: Hydrogen',
+    variant_id: 41327142600817,
+    variant_title: '',
+    variant_price: '600.00',
+    vendor: 'Hydrogen Vendor',
+    line_price: '600.00',
+    price: '600.00',
+    applied_discounts: [],
+    properties: {},
+  },
+  {
+    id: 14234727743601,
+    gift_card: false,
+    grams: 0,
+    name: 'The Collection Snowboard: Nitrogen',
+    price: '600.00',
+    product_exists: true,
+    product_id: 7234590408817,
+    properties: [],
+    quantity: 1,
+    sku: '',
+    title: 'The Collection Snowboard: Nitrogen',
+    total_discount: '0.00',
+    variant_id: 41327142600817,
+    vendor: 'Hydrogen Vendor',
+  },
+];
+
+describe('serverSideUtils.js', () => {
+  beforeEach(() => {
+    Message.mockClear();
+  });
+
+  describe('Test getProductsFromLineItems function', () => {
+    it('should return empty array if lineItems is empty', () => {
+      const lineItems = [];
+      const result = getProductsFromLineItems(lineItems, lineItemsMappingJSON);
+      expect(result).toEqual([]);
+    });
+
+    it('should return array of products', () => {
+      const mapping = {};
+      const result = getProductsFromLineItems(LINEITEMS, lineItemsMappingJSON);
+      expect(result).toEqual([
+        { brand: 'Hydrogen Vendor', price: '600.00', product_id: 7234590408818, quantity: 1 },
+        {
+          brand: 'Hydrogen Vendor',
+          price: '600.00',
+          product_id: 7234590408817,
+          quantity: 1,
+          title: 'The Collection Snowboard: Nitrogen',
+        },
+      ]);
+    });
+  });
+
+  describe('Test createPropertiesForEcomEventFromWebhook function', () => {
+    it('should return empty array if lineItems is empty', () => {
+      const message = {
+        line_items: [],
+        type: 'track',
+        event: 'checkout created',
+      };
+      const result = createPropertiesForEcomEventFromWebhook(message);
+      expect(result).toEqual([]);
+    });
+
+    it('should return array of products', () => {
+      const message = {
+        line_items: LINEITEMS,
+        type: 'track',
+        event: 'checkout updated',
+      };
+      const result = createPropertiesForEcomEventFromWebhook(message);
+      expect(result).toEqual({
+        products: [
+          { brand: 'Hydrogen Vendor', price: '600.00', product_id: 7234590408818, quantity: 1 },
+          {
+            brand: 'Hydrogen Vendor',
+            price: '600.00',
+            product_id: 7234590408817,
+            quantity: 1,
+            title: 'The Collection Snowboard: Nitrogen',
+          },
+        ],
+      });
+    });
+  });
+});