From 5f11753ca34be774deda09c5eea33955470634f1 Mon Sep 17 00:00:00 2001 From: Mitry Date: Mon, 4 Mar 2024 13:51:43 -0300 Subject: [PATCH 01/14] feat: workflow --- .github/workflows/tests.yml | 30 ++++++++++++++++++++++++++++++ .gitlab-ci.yml | 18 ------------------ 2 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/tests.yml delete mode 100644 .gitlab-ci.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..1c76fe2 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,30 @@ +name: Tests CI + +on: + push: + branches-ignore: + - main + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Build Images + run: docker-compose -f docker-compose.test.yml build + + - name: Run Tests + run: | + docker-compose -f docker-compose.test.yml up -d --force-recreate + sleep 30 + docker exec iss-test-front npm lint + docker exec iss-tests flake8 + docker exec iss-test-front npm test + # docker exec iss-test-front npm compile + docker exec iss-test-back ./manage.py test + docker exec iss-test-storage python3 src/test.py + # docker exec iss-tests python3 test.py + + - name: Clean up + run: docker-compose -f docker-compose.test.yml down --volumes --remove-orphans diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 17a73fb..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,18 +0,0 @@ -stages: - - test - -build and tests application: - stage: test - script: - - docker-compose -f docker-compose.test.yml build - - docker-compose -f docker-compose.test.yml up -d --force-recreate - - sleep 30 - - docker exec iss-test-front npm lint - - docker exec iss-tests flake8 - - docker exec iss-test-front npm test - - docker exec iss-test-front npm compile - - docker exec iss-test-back ./manage.py test - - docker exec iss-test-storage python3 src/test.py - # - docker exec iss-tests python3 test.py - after_script: - - docker-compose -f docker-compose.test.yml down --volumes --remove-orphans From 566e57477116cc047d23a4748a0a446fdfc11700 Mon Sep 17 00:00:00 2001 From: Mitry Date: Mon, 4 Mar 2024 13:55:24 -0300 Subject: [PATCH 02/14] ref: workflow --- .github/workflows/tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1c76fe2..cf0fba8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,9 +1,10 @@ name: Tests CI on: - push: - branches-ignore: - - main + workflow_dispatch: + # push: + # branches-ignore: + # - main jobs: build: From 41963e94f7098d84c9366d0c94ec2ad489be453c Mon Sep 17 00:00:00 2001 From: Mitry Date: Wed, 6 Mar 2024 10:02:30 -0300 Subject: [PATCH 03/14] ref: workflow --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cf0fba8..e354c3b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,5 +1,7 @@ name: Tests CI +runs-on: self-hosted + on: workflow_dispatch: # push: From 5eefcfe108ac08de96465ea880f2af0663eb66ef Mon Sep 17 00:00:00 2001 From: Mitry Date: Wed, 6 Mar 2024 10:07:19 -0300 Subject: [PATCH 04/14] fix --- .github/workflows/tests.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e354c3b..56a4453 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,5 @@ name: Tests CI -runs-on: self-hosted - on: workflow_dispatch: # push: @@ -9,8 +7,8 @@ on: # - main jobs: - build: - runs-on: ubuntu-latest + tests: + runs-on: self-hosted steps: - uses: actions/checkout@v3 From 03b2090cfaf05e6cb7b74d776f28ced29a24464a Mon Sep 17 00:00:00 2001 From: Mitry Date: Wed, 6 Mar 2024 10:17:40 -0300 Subject: [PATCH 05/14] ref: wkflow --- .github/workflows/tests.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 56a4453..f821921 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -7,14 +7,17 @@ on: # - main jobs: - tests: + build: runs-on: self-hosted - steps: - uses: actions/checkout@v3 - name: Build Images run: docker-compose -f docker-compose.test.yml build + test: + runs-on: self-hosted + needs: build + steps: - name: Run Tests run: | docker-compose -f docker-compose.test.yml up -d --force-recreate @@ -27,5 +30,11 @@ jobs: docker exec iss-test-storage python3 src/test.py # docker exec iss-tests python3 test.py + clean: + runs-on: self-hosted + needs: test + steps: - name: Clean up run: docker-compose -f docker-compose.test.yml down --volumes --remove-orphans + + From 8d8f975859c0ada6c1c1a9f478b59a259f822ac1 Mon Sep 17 00:00:00 2001 From: Mitry Date: Thu, 7 Mar 2024 11:22:28 -0300 Subject: [PATCH 06/14] ref --- .github/workflows/tests.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f821921..8082b29 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,10 +1,9 @@ name: Tests CI on: - workflow_dispatch: - # push: - # branches-ignore: - # - main + push: + branches-ignore: + - main jobs: build: From 6f6698c716b06f1192a4a36e85e97ffa16993aa9 Mon Sep 17 00:00:00 2001 From: Mitry Date: Thu, 7 Mar 2024 11:31:36 -0300 Subject: [PATCH 07/14] ref: test docker version --- docker-compose.test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 9ea371a..0aa29b2 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,4 +1,4 @@ -version: "3.8" +version: "3.7" services: iss-test-db: From c346a8115e73bd22f921fd80288d4ec34be8c9e1 Mon Sep 17 00:00:00 2001 From: Mitry Date: Thu, 7 Mar 2024 11:34:25 -0300 Subject: [PATCH 08/14] fix: test compose env --- docker-compose.test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 0aa29b2..5f932a2 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -35,7 +35,7 @@ services: DB_HOST: iss-test-db DB_NAME: iss_test_app_db CASE: test - DEBUG: true + DEBUG: "true" SERVER_ORIGINS: "iss-test-front,iss-test-back" SECRET_KEY: test-secret-key command: > @@ -60,7 +60,7 @@ services: APP_BACKEND_URL: iss-test-back:8000 DB_USER: testuser DB_PASSWORD: testuserpass - DEBUG: true + DEBUG: "true" CELERY_BROKER_URL: redis://iss-test-broker:6379/0 SECRET_KEY: test-secret-key depends_on: @@ -88,7 +88,7 @@ services: APP_BACKEND_URL: iss-test-back:8000 DB_USER: testuser DB_PASSWORD: testuserpass - DEBUG: true + DEBUG: "true" CELERY_BROKER_URL: redis://iss-test-broker:6379/0 SECRET_KEY: test-secret-key depends_on: @@ -108,7 +108,7 @@ services: - iss-test-back environment: CASE: test - CI: true + CI: "true" command: npm start networks: - iss_test_network From 1c9da41d7dd31408f865eaf98abf106efea6e924 Mon Sep 17 00:00:00 2001 From: Mitry Date: Thu, 7 Mar 2024 11:39:35 -0300 Subject: [PATCH 09/14] fix: test composer --- docker-compose.test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 5f932a2..8d153f8 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -26,6 +26,7 @@ services: image: iss_back_tst build: dockerfile: Dockerfile.backend + context: . restart: always depends_on: - iss-test-db @@ -54,6 +55,7 @@ services: image: iss_storage_tst build: dockerfile: Dockerfile.storage + context: . restart: always environment: DB_HOST: iss-test-storage-db @@ -82,6 +84,7 @@ services: container_name: iss-test-worker build: dockerfile: Dockerfile.storage + context: . restart: always environment: DB_HOST: iss-test-storage-db @@ -103,6 +106,7 @@ services: image: iss_front_tst build: dockerfile: Dockerfile.frontend + context: . restart: always depends_on: - iss-test-back @@ -118,6 +122,7 @@ services: image: iss_tests build: dockerfile: Dockerfile.tests + context: . # depends_on: # - iss-selenium-chrome # - iss-selenium-fox From 8aa321d3b7ba63bbf770fd897d49aeaf66fc1bcc Mon Sep 17 00:00:00 2001 From: Mitry Date: Mon, 11 Mar 2024 09:18:28 -0300 Subject: [PATCH 10/14] ref: test front --- Dockerfile.frtest | 8 ++++++++ docker-compose.test.yml | 2 +- storage-app/src/shared/app_services.py | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 Dockerfile.frtest diff --git a/Dockerfile.frtest b/Dockerfile.frtest new file mode 100644 index 0000000..038757b --- /dev/null +++ b/Dockerfile.frtest @@ -0,0 +1,8 @@ +FROM node:20.4-slim + +WORKDIR /app + +COPY ./frontend-app/ ./ + +RUN npm install + diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 8d153f8..e6e5eeb 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -105,7 +105,7 @@ services: container_name: iss-test-front image: iss_front_tst build: - dockerfile: Dockerfile.frontend + dockerfile: Dockerfile.frtest context: . restart: always depends_on: diff --git a/storage-app/src/shared/app_services.py b/storage-app/src/shared/app_services.py index 5d205d0..6a8ba09 100644 --- a/storage-app/src/shared/app_services.py +++ b/storage-app/src/shared/app_services.py @@ -87,7 +87,7 @@ def _stream_dataset(self) -> StreamingResponse: # TODO: find a way to defer closing maybe i could overpass the iterator async def _iterator(self) -> AsyncGenerator[bytes, None]: self.file.seek(self.chunk_start) - remaining: int = self.chunk_end + remaining: int = self.chunk_length while remaining and (chunk := await self.file.read(CHUNK_SIZE)): remaining -= CHUNK_SIZE if CHUNK_SIZE <= remaining else remaining From ba44ce55393ce3578557154398c16be7c6b07308 Mon Sep 17 00:00:00 2001 From: Mitry Date: Mon, 11 Mar 2024 09:24:01 -0300 Subject: [PATCH 11/14] fix --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8082b29..b4214b3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,10 +21,10 @@ jobs: run: | docker-compose -f docker-compose.test.yml up -d --force-recreate sleep 30 - docker exec iss-test-front npm lint + docker exec iss-test-front npm run lint docker exec iss-tests flake8 - docker exec iss-test-front npm test - # docker exec iss-test-front npm compile + docker exec iss-test-front npm run test + # docker exec iss-test-front npm run compile docker exec iss-test-back ./manage.py test docker exec iss-test-storage python3 src/test.py # docker exec iss-tests python3 test.py From ba8f5a6e385970a1a02dcabcc17c47163cbb669e Mon Sep 17 00:00:00 2001 From: Mitrii Ziuzin Date: Mon, 11 Mar 2024 09:42:00 -0300 Subject: [PATCH 12/14] Update tests.yml --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b4214b3..2927cb5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,7 +23,7 @@ jobs: sleep 30 docker exec iss-test-front npm run lint docker exec iss-tests flake8 - docker exec iss-test-front npm run test + # docker exec iss-test-front npm run test # docker exec iss-test-front npm run compile docker exec iss-test-back ./manage.py test docker exec iss-test-storage python3 src/test.py From b300660635646e18ba1ef1c2b1a8e00c8180f3fb Mon Sep 17 00:00:00 2001 From: Mitrii Ziuzin Date: Mon, 11 Mar 2024 09:59:01 -0300 Subject: [PATCH 13/14] Update tests.yml --- .github/workflows/tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2927cb5..8ed38f8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,7 +23,7 @@ jobs: sleep 30 docker exec iss-test-front npm run lint docker exec iss-tests flake8 - # docker exec iss-test-front npm run test + docker exec iss-test-front npm run test # docker exec iss-test-front npm run compile docker exec iss-test-back ./manage.py test docker exec iss-test-storage python3 src/test.py @@ -34,6 +34,8 @@ jobs: needs: test steps: - name: Clean up - run: docker-compose -f docker-compose.test.yml down --volumes --remove-orphans + run: | + docker-compose -f docker-compose.test.yml down --volumes --remove-orphans + docker system prune --all --volumes --force From a3896789010748eafd981c06b3bd4b5a59fdf107 Mon Sep 17 00:00:00 2001 From: Mitry Date: Mon, 11 Mar 2024 10:36:40 -0300 Subject: [PATCH 14/14] feat-ci --- Dockerfile.frontend | 4 +++- Dockerfile.frtest | 8 -------- docker-compose.test.yml | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) delete mode 100644 Dockerfile.frtest diff --git a/Dockerfile.frontend b/Dockerfile.frontend index 75e2c79..bee89b8 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -2,7 +2,9 @@ FROM node:20.4-slim WORKDIR /app -COPY ./frontend-app/*.json/ ./ +COPY ./frontend-app/package.json ./ +COPY ./frontend-app/tsconfig.json ./ +COPY ./frontend-app/.eslintrc.json ./ RUN npm install diff --git a/Dockerfile.frtest b/Dockerfile.frtest deleted file mode 100644 index 038757b..0000000 --- a/Dockerfile.frtest +++ /dev/null @@ -1,8 +0,0 @@ -FROM node:20.4-slim - -WORKDIR /app - -COPY ./frontend-app/ ./ - -RUN npm install - diff --git a/docker-compose.test.yml b/docker-compose.test.yml index e6e5eeb..8d153f8 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -105,7 +105,7 @@ services: container_name: iss-test-front image: iss_front_tst build: - dockerfile: Dockerfile.frtest + dockerfile: Dockerfile.frontend context: . restart: always depends_on: