Feat/blob filter options

BlobViaEventGrid/.eslintignore

@@ -1,2 +1,2 @@
 node_modules
-dist
+dist

BlobViaEventGrid/.eslintrc

@@ -12,4 +12,4 @@
   "rules": {
     "@typescript-eslint/no-explicit-any": "off"
   }
-}
+}

BlobViaEventGrid/.funcignore

@@ -7,4 +7,4 @@
 .eslintignore
 .editorconfig
 local.settings.json
-test
+test

BlobViaEventGrid/ARM/BlobViaEventGrid.json

@@ -66,6 +66,20 @@
         "description": "The pattern that separates the lines in the blob."
       }
     },
+    "PrefixFilter": {
+      "type": "string",
+      "defaultValue": "NoFilter",
+      "metadata": {
+        "description": "The prefix filter to apply to the blob container. Use 'NoFilter' to not filter by prefix. Wildcards are not allowed. Use the following format '/subfolder1/subfolder2/'."
+      }
+    },
+    "SuffixFilter": {
+      "type": "string",
+      "defaultValue": "NoFilter",
+      "metadata": {
+        "description": "The suffix filter to apply to the blob container. Use 'NoFilter' to not filter by suffix. Wildcards are not allowed. Use the following format '.log'."
+      }
+    },
     "FunctionAppServicePlanType": {
       "type": "string",
       "defaultValue": "Consumption",
@@ -89,8 +103,8 @@
     "needEventGrid": "[if(equals(parameters('EventGridSystemTopicName'), 'new'), 'true', 'false')]",
     "eventGridTopicName": "[if(equals(parameters('EventGridSystemTopicName'), 'new'), format('{0}-eg', parameters('storageAccountName')), parameters('EventGridSystemTopicName'))]",
     "packageUri": "https://coralogix-public.s3.eu-west-1.amazonaws.com/azure-functions-repo/BlobViaEventGrid.zip",
-    "sku": "[if(equals(parameters('FunctionAppServicePlanType'), 'Consumption'), 'Y1', 'EP1')]"
-  },
+    "sku": "[if(equals(parameters('FunctionAppServicePlanType'), 'Consumption'), 'Y1', 'EP1')]",
+    "PrefixFilter": "[if(equals(parameters('PrefixFilter'), 'NoFilter'), '', concat('/blobServices/default/containers/', parameters('BlobContainerName'), '/blobs', parameters('PrefixFilter')))]"  },
   "resources": [
     {
       "type": "Microsoft.Storage/storageAccounts",
@@ -249,8 +263,10 @@
                       "includedEventTypes": [
                           "Microsoft.Storage.BlobCreated"
                       ],
-                      "enableAdvancedFilteringOnArrays": true
-                  },
+                      "enableAdvancedFilteringOnArrays": true,
+                      "subjectBeginsWith": "[if(equals(variables('PrefixFilter'), 'NoFilter'), '', variables('PrefixFilter'))]",
+                      "subjectEndsWith": "[if(equals(parameters('SuffixFilter'), 'NoFilter'), '', parameters('SuffixFilter'))]"
+                      },
                   "labels": [
                       "functions-blobviaeventgrid"
                   ],

BlobViaEventGrid/BlobViaEventGrid/function.json

@@ -16,4 +16,4 @@
     }
   ],
   "scriptFile": "../dist/BlobViaEventGrid/index.js"
-}
+}

BlobViaEventGrid/README.md

@@ -40,4 +40,8 @@ The BlobStorage Via Eventgrid trigger integration can be deployed by clicking th
 
 **Newline Pattern** - The newline pattern expected within the blob storage documents
 
+**Prefix Filter** - The prefix filter to apply to the blob container. Use 'NoFilter' to not filter by prefix. Wildcards are not allowed. Use the following format ``/subfolder1/subfolder2/``.
+
+**Suffix Filter** - The suffix filter to apply to the blob container. Use 'NoFilter' to not filter by suffix. Wildcards are not allowed. Use the following format ``.log``.
+
 **Function App Service Plan Type** - The type of the Function App Service Plan. Choose Premium if you need vNet Support.

DiagnosticMetrics/ARM/DiagnosticMetrics.json → DiagnosticData/ARM/DiagnosticData.json

@@ -72,12 +72,12 @@
   },
   "variables": {
     "CoralogixURL": "[if(equals(parameters('CoralogixRegion'), 'Europe'), 'https://ingress.coralogix.com/azure/events/v1', if(equals(parameters('CoralogixRegion'), 'US'), 'https://ingress.coralogix.us/azure/events/v1', if(equals(parameters('CoralogixRegion'), 'Europe2'), 'https://ingress.eu2.coralogix.com/azure/events/v1', if(equals(parameters('CoralogixRegion'), 'India'), 'https://ingress.coralogix.in/azure/events/v1', if(equals(parameters('CoralogixRegion'), 'Singapore'), 'https://ingress.coralogixsg.com/azure/events/v1', 'NULL')))))]",
-    "functionAppName": "[format('DiagnosticMetrics-{0}', uniqueString(concat(resourceGroup().id, parameters('EventhubNamespace'), parameters('EventhubInstanceName'))))]",
+    "functionAppName": "[format('DiagnosticData-{0}', uniqueString(concat(resourceGroup().id, parameters('EventhubNamespace'), parameters('EventhubInstanceName'))))]",
     "hostingPlanName": "[variables('functionAppName')]",
     "applicationInsightsName": "[variables('functionAppName')]",
     "storageAccountName": "[format('azfunctions{0}', uniqueString(resourceGroup().id))]",
     "location": "[resourceGroup().location]",
-    "packageUri": "https://coralogix-public.s3.eu-west-1.amazonaws.com/azure-functions-repo/DiagnosticMetrics.zip",
+    "packageUri": "https://coralogix-public.s3.eu-west-1.amazonaws.com/azure-functions-repo/DiagnosticData.zip",
     "sku": "[if(equals(parameters('FunctionAppServicePlanType'), 'Consumption'), 'Y1', 'EP1')]"
   },
   "resources": [

DiagnosticMetrics/DiagnosticMetrics/function.json → DiagnosticData/DiagnosticData/function.json

@@ -11,6 +11,6 @@
       "dataType": "string"
     }
   ],
-  "scriptFile": "../dist/DiagnosticMetrics/index.js",
+  "scriptFile": "../dist/DiagnosticData/index.js",
   "disabled": false
 }

DiagnosticMetrics/DiagnosticMetrics/index.ts → DiagnosticData/DiagnosticData/index.ts

@@ -20,15 +20,31 @@ import { AzureFunction, Context } from "@azure/functions";
 const eventHubTrigger: AzureFunction = async function (context: Context, eventHubMessages: any): Promise<void> {
     context.log(`JavaScript eventhub trigger function called for message array ${eventHubMessages}`);
 
-    const url = process.env.CORALOGIX_URL || 'https://api.coralogix.com/api/v1/logs'
+    const url = process.env.CORALOGIX_URL || 'https://ingress.coralogix.com/api/v1/logs'
     const key = process.env.CORALOGIX_PRIVATE_KEY || "INVALID_KEY"
     const applicationName = process.env.CORALOGIX_APP_NAME || "NO_APPLICATION"
     const subsystemName = process.env.CORALOGIX_SUB_SYSTEM || "NO_SUBSYSTEM"
-    const response = postData(url, key, applicationName, subsystemName, {'eventHubMessages': eventHubMessages.map(s => JSON.parse(s)) });
+    const response = postData(url, key, applicationName, subsystemName, {'eventHubMessages': eventHubMessages.map(parseJSON) });
     context.log(`Sent messages. Response: ${JSON.stringify(response)}`);
 
 };
 
+// This function is to address an Azure bug where single quotes are used in the diagnostic log messages
+// for Linux consumption plan resulting in invalid JSON. https://github.com/Azure/azure-functions-host/issues/7864
+function parseJSON(data: any): any {
+    try {
+        return JSON.parse(data);
+    } catch (error) {
+        if (error instanceof SyntaxError) {
+            // Replace single quotes with double quotes not preceded by a backslash
+            return JSON.parse(data.replace(/(?<!\\)'/g, '"'))
+        } else {
+            console.error(error);
+            throw error
+        }
+    }
+}
+
 async function postData(url: string, privateKey: string, applicationName: string, subsystemName: string, data: any) {
     if( !privateKey) {
         throw new Error('CORALOGIX_KEY must be defined');

DiagnosticMetrics/README.md → DiagnosticData/README.md

@@ -1,10 +1,8 @@
-# Azure Diagnostic Metrics Function for Coralogix
+# Azure Diagnostic Data Function for Coralogix
 
-Coralogix provides a seamless integration with ``Azure`` cloud so you can send your logs from anywhere and parse them according to your needs.
+Coralogix provides a seamless integration with ``Azure`` cloud so you can send your Logs and Metrics from Azure Resources and parse them according to your needs.
 
-The Azure Diagnostic Metrics integration allows processing of Metrics submitted to an EventHub using the resource "Diagnostic Settings" configuration.
-
-It presently handles only the "allMetrics" metrics format. Support for additional metrics types are being added.
+The Azure Diagnostic Data integration allows processing of Logs and Metrics submitted to an EventHub using Resources' "Diagnostic Settings" configuration.
 
 ## Prerequisites
 
@@ -14,12 +12,12 @@ It presently handles only the "allMetrics" metrics format. Support for additiona
 
 ## Azure Resource Manager Template Deployment
 
-The Diagnostic Metrics integration can be deployed by clicking the link below and signing into your Azure account:
-[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fcoralogix%2Fcoralogix-azure-serverless%2Fmaster%2FDiagnosticMetrics%2FARM%2FDiagnosticMetrics.json)
+The Diagnostic Data integration can be deployed by clicking the link below and signing into your Azure account:
+[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fcoralogix%2Fcoralogix-azure-serverless%2Fmaster%2FDiagnosticData%2FARM%2FDiagnosticData.json)
 
 ## Terraform Deployment
 
-The Diagnostic Metrics integration can also be deployed using our Terraform module found here:
+The Diagnostic Data integration can also be deployed using our Terraform module found here:
 [Terraform Module](https://registry.terraform.io/modules/coralogix/azure/coralogix/latest)
 
 ## Fields

README.md

@@ -6,7 +6,7 @@ We provide several trigger strategies, please select one and follow the inner RE
 - [EventHub](/EventHub)
 - [BlobViaEventGrid](/BlobViaEventGrid)
 - [StorageQueue](/StorageQueue)
-- [DiagnosticMetrics](/DiagnosticMetrics)
+- [DiagnosticData](/DiagnosticData)
 
 ---
 [email protected]