chore: standard formatting

.github/workflows/commitlint.yml

@@ -0,0 +1,36 @@
+name: Commitlint
+
+on: [push]
+
+jobs:
+  commitlint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Print versions
+        run: |
+          git --version
+          node --version
+          npm --version
+          npx commitlint --version
+
+      # Run the commitlint action, considering its own dependencies and yours as well 🚀
+      # `github.workspace` is the path to your repository.
+      - uses: wagoid/commitlint-github-action@v5
+        env:
+          NODE_PATH: ${{ github.workspace }}/node_modules
+        with:
+          commitDepth: 1

.github/workflows/verify.yml

@@ -0,0 +1,58 @@
+name: Verify
+
+on:
+  pull_request:
+
+jobs:
+  py-scripts-lint:
+    name: Check for formatting of python scripts
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          # Make sure the actual branch is checked out when running on pull requests
+          ref: ${{ github.head_ref }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Setting up python libraries
+        run: ./scripts/setup-python.sh
+
+      # Reference: https://black.readthedocs.io/en/stable/integrations/github_actions.html
+      - uses: psf/black@stable
+        with:
+          options: '--check --verbose'
+
+  formatting-lint:
+    name: Check for formatting & lint errors
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          # Make sure the actual branch is checked out when running on pull requests
+          ref: ${{ github.head_ref }}
+
+      - name: Setup Node
+        uses: actions/[email protected]
+        with:
+          node-version-file: .nvmrc
+          cache: 'npm'
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Run Lint Checks
+        run: |
+          npm run lint
+
+      - run: git diff --exit-code
+
+      - name: Error message
+        if: ${{ failure() }}
+        run: |
+          echo 'Eslint check is failing Ensure you have run `npm run lint` and committed the files locally.'

.husky/pre-commit

@@ -2,3 +2,4 @@
 . "$(dirname -- "$0")/_/husky.sh"
 
 npm run pre-commit
+python3 -m black $(dirname -- "$0")

migration/migrate.py

@@ -2,116 +2,126 @@
 import json
 import os
 
-DEST_WEBAPP_FILE_PATH = 'rudder-webapp/src/components/destinations/schemas/'
-DEST_CONFIG_BE_FILE_PATH = 'rudder-config-backend/src/scripts/destinationConfigs/'
-DEST_SCHEMA_FILE_PATH = 'rudder-config-backend/src/scripts/schemaList/destinations/'
+DEST_WEBAPP_FILE_PATH = "rudder-webapp/src/components/destinations/schemas/"
+DEST_CONFIG_BE_FILE_PATH = "rudder-config-backend/src/scripts/destinationConfigs/"
+DEST_SCHEMA_FILE_PATH = "rudder-config-backend/src/scripts/schemaList/destinations/"
 
-SOURCE_WEBAPP_FILE_PATH = 'rudder-webapp/src/components/sources/source/warehouseSource/warehouseSourceList/'
-SOURCE_CONFIG_BE_FILE_PATH = 'rudder-config-backend/src/scripts/sourceConfigs/'
+SOURCE_WEBAPP_FILE_PATH = (
+    "rudder-webapp/src/components/sources/source/warehouseSource/warehouseSourceList/"
+)
+SOURCE_CONFIG_BE_FILE_PATH = "rudder-config-backend/src/scripts/sourceConfigs/"
 # SOURCE_SCHEMA_FILE_PATH = 'rudder-config-backend/src/scripts/schemaList/destinations/'
 
-CONFIGURATIONS_DIR_PATH = '../src/configurations'
-DEST_CONFIG_PATH = f'{CONFIGURATIONS_DIR_PATH}/destinations'
-SRC_CONFIG_PATH = f'{CONFIGURATIONS_DIR_PATH}/sources'
+CONFIGURATIONS_DIR_PATH = "../src/configurations"
+DEST_CONFIG_PATH = f"{CONFIGURATIONS_DIR_PATH}/destinations"
+SRC_CONFIG_PATH = f"{CONFIGURATIONS_DIR_PATH}/sources"
+
 
 def update_destination():
-    dest_list = [d[:-5].lower() for d in os.listdir(f'../../{DEST_CONFIG_BE_FILE_PATH}')]
+    dest_list = [
+        d[:-5].lower() for d in os.listdir(f"../../{DEST_CONFIG_BE_FILE_PATH}")
+    ]
     for dest in dest_list:
         final_data = []
         # read db_config
-        with open(f'../../{DEST_CONFIG_BE_FILE_PATH}{dest.upper()}.json', 'r') as f:
+        with open(f"../../{DEST_CONFIG_BE_FILE_PATH}{dest.upper()}.json", "r") as f:
             db_config = json.loads(f.read())
-        print (db_config)
+        print(db_config)
         final_data.append(db_config)
-        print ("========================")
+        print("========================")
         # read ui_config
-        with open(f'../../{DEST_WEBAPP_FILE_PATH}{dest.upper()}.json', 'r') as f:
+        with open(f"../../{DEST_WEBAPP_FILE_PATH}{dest.upper()}.json", "r") as f:
             ui_config = {"uiConfig": json.loads(f.read())}
-        print (ui_config)
+        print(ui_config)
         final_data.append(ui_config)
-        print ("========================")
+        print("========================")
         # read schema
-        if f'{dest.upper()}.json' in os.listdir(f'../../{DEST_SCHEMA_FILE_PATH}'):
-            with open(f'../../{DEST_SCHEMA_FILE_PATH}{dest.upper()}.json', 'r') as f:
+        if f"{dest.upper()}.json" in os.listdir(f"../../{DEST_SCHEMA_FILE_PATH}"):
+            with open(f"../../{DEST_SCHEMA_FILE_PATH}{dest.upper()}.json", "r") as f:
                 schema = {"schema": json.loads(f.read())}
         else:
             schema = {"schema": None}
-        print (schema)
+        print(schema)
         final_data.append(schema)
-        print ("========================")
+        print("========================")
         # metadata
-        with open('dest_meta_template.json', 'r') as f:
+        with open("dest_meta_template.json", "r") as f:
             metadata = json.loads(f.read())
-        print (metadata)
+        print(metadata)
         final_data.append(metadata)
-        print ("========================")
+        print("========================")
 
         ########################
         ## write new files
         ########################
         if dest not in os.listdir(DEST_CONFIG_PATH):
-            os.system(f'mkdir {DEST_CONFIG_PATH}/{dest}')
-            print (f'created directory for {dest}')
+            os.system(f"mkdir {DEST_CONFIG_PATH}/{dest}")
+            print(f"created directory for {dest}")
 
-        file_names = ['db-config', 'ui-config', 'schema', 'metadata']
+        file_names = ["db-config", "ui-config", "schema", "metadata"]
         for index, f_name in enumerate(file_names):
-            print (f'writing {f_name} for {dest}...')
-            with open(f'{DEST_CONFIG_PATH}/{dest}/{f_name}.json', 'w') as f:
+            print(f"writing {f_name} for {dest}...")
+            with open(f"{DEST_CONFIG_PATH}/{dest}/{f_name}.json", "w") as f:
                 f.write(json.dumps(final_data[index], indent=2))
 
-        print (f'complete for {dest}...')
-        print ('------------------------------------------------')
+        print(f"complete for {dest}...")
+        print("------------------------------------------------")
+
 
 def update_source():
-    source_list = [d[:-5].lower() for d in os.listdir(f'../../{SOURCE_CONFIG_BE_FILE_PATH}')]
+    source_list = [
+        d[:-5].lower() for d in os.listdir(f"../../{SOURCE_CONFIG_BE_FILE_PATH}")
+    ]
     for source in source_list:
         final_data = []
         # read db_config
-        with open(f'../../{SOURCE_CONFIG_BE_FILE_PATH}{source.upper()}.json', 'r') as f:
+        with open(f"../../{SOURCE_CONFIG_BE_FILE_PATH}{source.upper()}.json", "r") as f:
             db_config = json.loads(f.read())
-        print (db_config)
+        print(db_config)
         final_data.append(db_config)
-        print ("========================")
+        print("========================")
         # read ui_config
-        if f'{source.upper()}.json' in os.listdir(f'../../{SOURCE_WEBAPP_FILE_PATH}'):
-            with open(f'../../{SOURCE_WEBAPP_FILE_PATH}{source.upper()}.json', 'r') as f:
+        if f"{source.upper()}.json" in os.listdir(f"../../{SOURCE_WEBAPP_FILE_PATH}"):
+            with open(
+                f"../../{SOURCE_WEBAPP_FILE_PATH}{source.upper()}.json", "r"
+            ) as f:
                 ui_config = {"uiConfig": json.loads(f.read())}
         else:
             ui_config = {"uiConfig": None}
-        print (ui_config)
+        print(ui_config)
         final_data.append(ui_config)
-        print ("========================")
+        print("========================")
         # read schema (we don't need schema for sources)
         # if f'{source.upper()}.json' in os.listdir(f'../../{SOURCE_SCHEMA_FILE_PATH}'):
         #     with open(f'../../{SOURCE_SCHEMA_FILE_PATH}{source.upper()}.json', 'r') as f:
         #         schema = {"schema": json.loads(f.read())}
         # else:
         schema = {"schema": None}
-        print (schema)
+        print(schema)
         final_data.append(schema)
-        print ("========================")
+        print("========================")
         # metadata
-        with open('source_meta_template.json', 'r') as f:
+        with open("source_meta_template.json", "r") as f:
             metadata = json.loads(f.read())
-        print (metadata)
+        print(metadata)
         final_data.append(metadata)
-        print ("========================")
+        print("========================")
 
         ########################
         ## write new files
         ########################
         if source not in os.listdir(SRC_CONFIG_PATH):
-            os.system(f'mkdir {SRC_CONFIG_PATH}/{source}')
-            print (f'created directory for {source}')
+            os.system(f"mkdir {SRC_CONFIG_PATH}/{source}")
+            print(f"created directory for {source}")
 
-        file_names = ['db_config', 'ui_config', 'schema', 'metadata']
+        file_names = ["db_config", "ui_config", "schema", "metadata"]
         for index, f_name in enumerate(file_names):
-            print (f'writing {f_name} for {source}...')
-            with open(f'{SRC_CONFIG_PATH}/{source}/{f_name}.json', 'w') as f:
+            print(f"writing {f_name} for {source}...")
+            with open(f"{SRC_CONFIG_PATH}/{source}/{f_name}.json", "w") as f:
                 f.write(json.dumps(final_data[index], indent=2))
 
-        print (f'complete for {source}...')
-        print ('------------------------------------------------')
+        print(f"complete for {source}...")
+        print("------------------------------------------------")
 
 
 if __name__ == "__main__":

package.json

@@ -23,6 +23,7 @@
     "check:lint": "eslint \"src/**/*.*\" -f json -o reports/eslint.json || exit 0",
     "format": "prettier . --write",
     "lint:fix": "eslint \"src/**/*.*\" --fix",
+    "lint": "npm run format && npm run lint:fix",
     "prepare": "husky install",
     "pre-commit": "npm run test && npx lint-staged",
     "commit-msg": "commitlint --edit",

scripts/configGenerator.py

@@ -4,23 +4,24 @@
 import os
 import sys
 
-ConfigData = TypedDict('ConfigData', {'db_config': str, 'ui_config': str})
+ConfigData = TypedDict("ConfigData", {"db_config": str, "ui_config": str})
+
 
 def generateConfigs(data) -> ConfigData:
     # Read the content of template-db-config.json
-    with open('scripts/template-db-config.json', 'r') as file:
+    with open("scripts/template-db-config.json", "r") as file:
         template_db_config = json.load(file)
 
     # Read the content of template-ui-config.json
-    with open('scripts/template-ui-config.json', 'r') as file:
+    with open("scripts/template-ui-config.json", "r") as file:
         template_ui_config = json.load(file)
 
     # Create db-config object with the same content
     db_config = template_db_config
 
-    db_config['displayName'] = data['displayName']
-    db_config['name'] = data['displayName']
-    formFields = data['formFields']
+    db_config["displayName"] = data["displayName"]
+    db_config["name"] = data["displayName"]
+    formFields = data["formFields"]
 
     # Create db-config object with the same content
     ui_config = template_ui_config
@@ -33,43 +34,41 @@ def appendFieldsInGroups(settings, field):
             if groups:
                 first_group = groups[0]
                 if "fields" in first_group:
-                    del field['required']
+                    del field["required"]
                     first_group["fields"].append(field)
 
-
     def updateUiConfig(field):
         if "uiConfig" in ui_config and "baseTemplate" in ui_config["uiConfig"]:
             base_template = ui_config["uiConfig"]["baseTemplate"]
-            if base_template and field['required'] == True:
+            if base_template and field["required"] == True:
                 connection_settings = base_template[0]
                 appendFieldsInGroups(connection_settings, field)
             elif base_template:
                 configuration_settings = base_template[1]
                 appendFieldsInGroups(configuration_settings, field)
             return ui_config
 
-
     # Iterate over JSON objects in the array
     for obj in formFields:
         # update field in ui-config
         ui_config = updateUiConfig(obj)
         # update db-config
         for key, value in obj.items():
-            if key == 'configKey':
-                db_config['config']['destConfig']['defaultConfig'].append(
-                    value)
-            if key == 'secret' and value == True:
-                db_config['config']['secretKeys'].append(
-                    db_config['config']['destConfig']['defaultConfig'][-1])
+            if key == "configKey":
+                db_config["config"]["destConfig"]["defaultConfig"].append(value)
+            if key == "secret" and value == True:
+                db_config["config"]["secretKeys"].append(
+                    db_config["config"]["destConfig"]["defaultConfig"][-1]
+                )
 
     db_config = json.dumps(db_config)
     ui_config = json.dumps(ui_config)
-    return {'db_config':db_config, 'ui_config': ui_config}
+    return {"db_config": db_config, "ui_config": ui_config}
 
 
-if __name__ == '__main__':
-    file_path = sys.argv[1] if len(sys.argv) > 1 else 'test/configData/inputData.json'
-    with open(file_path, 'r') as file:
+if __name__ == "__main__":
+    file_path = sys.argv[1] if len(sys.argv) > 1 else "test/configData/inputData.json"
+    with open(file_path, "r") as file:
         # Load the JSON data
         data = json.load(file)
 
@@ -79,16 +78,15 @@ def updateUiConfig(field):
     if not os.path.exists(directory):
         os.makedirs(directory)
 
-    with open(file_path, 'w') as file:
+    with open(file_path, "w") as file:
         # Write the new content
-        file.write(configData['db_config'])
-
+        file.write(configData["db_config"])
 
     file_path = f'src/configurations/destinations/{data["displayName"]}/ui-config.json'
     directory = os.path.dirname(file_path)
     if not os.path.exists(directory):
         os.makedirs(directory)
 
-    with open(file_path, 'w') as file:
+    with open(file_path, "w") as file:
         # Write the new content
-        file.write(configData['ui_config'])
+        file.write(configData["ui_config"])