From 4d43918a7893a18c3d9dcc72fcdcbe9ef04bc91b Mon Sep 17 00:00:00 2001 From: Sai Kumar Battinoju <88789928+saikumarrs@users.noreply.github.com> Date: Wed, 21 Feb 2024 20:05:19 +0530 Subject: [PATCH] chore: fix minor issues in tooling and tests (#1231) --- .github/workflows/deploy.yml | 8 +------ .github/workflows/report-code-coverage.yml | 5 +---- .github/workflows/test.yml | 5 +---- .github/workflows/verify.yml | 10 +++------ README.md | 4 ++-- package.json | 6 +++-- scripts/requirements.txt | 1 + scripts/setup-python.sh | 8 ------- test/component_tests/schemaGenerator.test.ts | 23 ++++++++++++++++---- 9 files changed, 32 insertions(+), 38 deletions(-) delete mode 100755 scripts/setup-python.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 24e2d6ab2..6651cbcda 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -41,10 +41,7 @@ jobs: env: HUSKY: 0 run: | - npm ci - - - name: Set up Python - run: scripts/setup-python.sh + npm run setup - name: Display Python Version run: | @@ -57,9 +54,6 @@ jobs: run: | npm run test:ci - - name: Install Python Dependencies - run: pip3 install -r ./scripts/requirements.txt - - name: List Working Directory Files run: | echo current directory diff --git a/.github/workflows/report-code-coverage.yml b/.github/workflows/report-code-coverage.yml index c65c16c76..e9c4f94d2 100644 --- a/.github/workflows/report-code-coverage.yml +++ b/.github/workflows/report-code-coverage.yml @@ -25,10 +25,7 @@ jobs: cache: 'npm' - name: Install Dependencies - run: npm ci - - - name: Set up Python - run: scripts/setup-python.sh + run: npm run setup - name: Run Tests run: npm run test:ci diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e65062640..79b20d244 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,10 +24,7 @@ jobs: cache: 'npm' - name: Install Dependencies - run: npm ci - - - name: Set up Python - run: scripts/setup-python.sh + run: npm run setup - name: Run Tests run: npm run test:ci diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 90ec08d77..f98672904 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -11,12 +11,8 @@ jobs: - name: Checkout uses: actions/checkout@v4.1.1 - - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Setting up python libraries - run: ./scripts/setup-python.sh + - name: Install Dependencies + run: npm run setup:python # Reference: https://black.readthedocs.io/en/stable/integrations/github_actions.html - name: Check formatting for Python files @@ -39,7 +35,7 @@ jobs: cache: 'npm' - name: Install Dependencies - run: npm ci + run: npm run setup:npm - name: Run Lint Checks run: | diff --git a/README.md b/README.md index 5572ef11f..e185d3613 100644 --- a/README.md +++ b/README.md @@ -36,9 +36,9 @@ You need to install Python3. -And then, install Python dependencies: +And then, setup the project dependencies by running below command: -`pip3 install -r ./scripts/requirements.txt` +`npm run setup` Run below command to deploy integrations definitions config to database: diff --git a/package.json b/package.json index 9425018ed..e2ad16a0d 100755 --- a/package.json +++ b/package.json @@ -15,7 +15,9 @@ "url": "https://github.com/rudderlabs/rudder-config-schema.git" }, "scripts": { - "setup": "npm ci", + "setup:python": "pip3 install -r ./scripts/requirements.txt", + "setup:npm": "npm ci", + "setup": "npm run setup:npm && npm run setup:python", "test": "jest --detectOpenHandles --coverage --notify --watchAll=false", "test:ci": "npm run test -- --silent --expand --maxWorkers=2", "test:silent": "npm run test -- --silent", @@ -82,7 +84,7 @@ }, "lint-staged": { "*.{json,js,ts,md}": "prettier --write", - "*.{py}": "python3 -m black" + "*.py": "python3 -m black" }, "config": { "commitizen": { diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 350586c4a..ed926251c 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,3 +1,4 @@ requests jsonschema jsondiff +black diff --git a/scripts/setup-python.sh b/scripts/setup-python.sh deleted file mode 100755 index e5518394d..000000000 --- a/scripts/setup-python.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -sudo apt-get update -sudo apt-get -y install python3-pip -pip3 --version -pip3 install requests -pip3 install jsonschema -pip3 install jsondiff -pip3 install black diff --git a/test/component_tests/schemaGenerator.test.ts b/test/component_tests/schemaGenerator.test.ts index 91a2f546e..b475270c7 100644 --- a/test/component_tests/schemaGenerator.test.ts +++ b/test/component_tests/schemaGenerator.test.ts @@ -2,6 +2,21 @@ import fs from 'fs'; import path from 'path'; import { execSync } from 'child_process'; +function readFile(filePath): string | undefined { + let file; + if (!fs.existsSync(filePath)) { + return file; + } + + try { + file = fs.readFileSync(filePath, 'utf8'); + } catch (e) { + /* empty */ + } + + return file; +} + function readSchemaFile(filePath): string | undefined { let schema; if (!fs.existsSync(filePath)) { @@ -17,8 +32,8 @@ function readSchemaFile(filePath): string | undefined { return schema; } -function writeSchemaFile(filePath, schema) { - fs.writeFileSync(filePath, JSON.stringify(schema, null, 2)); +function writeFile(filePath, data) { + fs.writeFileSync(filePath, data); } describe('Schema Generator', () => { @@ -102,7 +117,7 @@ describe('Schema Generator', () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars it.each(testData)('$description', ({ description, destName, expectedSchemaFile }) => { const schemaFilePath = path.resolve(`${configDir}/destinations/${destName}/schema.json`); - const curSchema = readSchemaFile(schemaFilePath); + const curSchema = readFile(schemaFilePath); const cmd = `CONFIG_DIR=${configDir} npm run update:schema:destination "${destName}"`; @@ -111,7 +126,7 @@ describe('Schema Generator', () => { const schema = readSchemaFile(schemaFilePath); // Restore schema file if (curSchema) { - writeSchemaFile(schemaFilePath, curSchema); + writeFile(schemaFilePath, curSchema); } const expectedSchemaData = readSchemaFile(