How to leverage Flux to preserve the sequence of bulk puts... #2966

Workflow file for this run

	name: CI

	on:
	push:
	pull_request:

	env:
	python-version: '3.11'
	java-version: '11'
	gradle-version: '8.0.2'
	node-version: '18.x'

	jobs:
	workflow-info:
	runs-on: ubuntu-latest
	steps:
	- name: Create JSON Artifact
	run: \|
	echo '{' > workflow-info.json
	echo ' "pr_number": "${{ github.event.pull_request.number }}",' >> workflow-info.json
	echo ' "commit": "${{ github.event.pull_request.head.sha \|\| github.event.after \|\| github.sha }}",' >> workflow-info.json
	echo ' "branch": "${{ github.ref }}",' >> workflow-info.json
	echo ' "commit_parent": "${{ github.event.pull_request.base.sha \|\| github.event.before \|\| github.base.sha }}",' >> workflow-info.json
	echo ' "build_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",' >> workflow-info.json
	echo ' "build": "${{ github.run_id }}"' >> workflow-info.json
	echo '}' >> workflow-info.json
	echo "workflow-info.json created"
	- name: Upload JSON Artifact
	uses: actions/upload-artifact@v4
	with:
	name: workflow-info
	path: workflow-info.json


	style-check:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-java@v4
	with:
	java-version: ${{ env.java-version }}
	distribution: "corretto"
	- uses: gradle/actions/setup-gradle@v4
	with:
	gradle-version: ${{ env.gradle-version }}
	- name: Run Gradle Build
	run: ./gradlew spotlessCheck
	env:
	OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''

	python-lint:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: ${{ env.python-version }}
	- name: Install dependencies
	run: \|
	python3 -m pip install --upgrade pip
	python3 -m pip install flake8
	- name: Analysing the code with flake8
	run: \|
	flake8 $(git ls-files '*.py')

	console-python-tests:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	py-project:
	- lib/console_link
	- cluster_tools
	- console_api
	env:
	WORKING_DIR: ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/${{ matrix.py-project }}
	defaults:
	run:
	working-directory: ${{ env.WORKING_DIR }}
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-python@v5
	with:
	python-version: ${{ env.python-version }}
	- run: \|
	python3 -m pip install --upgrade pipenv
	pipenv install --deploy --dev
	pipenv run test
	pipenv run coverage xml
	- name: Get Sanitized Name
	env:
	PY_PROJECT: ${{ matrix.py-project }}
	run: echo "SANITIZED_PY_PROJECT=${PY_PROJECT/\//-}" >> $GITHUB_ENV
	- name: Upload Coverage Reports
	uses: actions/upload-artifact@v4
	with:
	if-no-files-found: error
	name: coverage-reports-python-tests-${{ env.SANITIZED_PY_PROJECT }}
	path: ${{ env.WORKING_DIR }}/coverage.xml


	gradle-tests:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-java@v4
	with:
	java-version: ${{ env.java-version }}
	distribution: 'corretto'
	- uses: gradle/actions/setup-gradle@v4
	with:
	gradle-version: ${{ env.gradle-version }}
	gradle-home-cache-cleanup: true
	- name: Run Gradle Build
	run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace
	env:
	OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''

	- name: Run Tests with Coverage
	run: ./gradlew mergeJacocoReports -x TrafficCapture:dockerSolution:build -x spotlessCheck --stacktrace
	env:
	OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''

	- uses: actions/upload-artifact@v4
	if: always()
	with:
	name: test-reports-gradle-tests
	path: \|
	**/build/reports/tests/
	**/reports/jacoco/mergedReport/
	- name: Upload Coverage Reports
	uses: actions/upload-artifact@v4
	with:
	if-no-files-found: error
	name: coverage-reports-gradle-tests
	path: ./**/jacocoMergedReport.xml


	python-e2e-tests:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-java@v4
	with:
	java-version: ${{ env.java-version }}
	distribution: 'corretto'
	- uses: gradle/actions/setup-gradle@v4
	with:
	gradle-version: ${{ env.gradle-version }}
	- uses: actions/setup-python@v5
	with:
	python-version: ${{ env.python-version }}
	- name: Generate Cache Key from Dockerfiles
	id: generate_cache_key
	run: \|
	files=$(find . -type f $ -name 'docker-compose.yml' -o -name 'Dockerfile' $)
	file_contents=$(cat $files)
	key=$(echo "${file_contents}" \| sha1sum \| awk '{print $1}')
	echo "key=${key}" >> "$GITHUB_OUTPUT"
	- name: Cache Docker Images
	uses: ScribeMD/[email protected]
	with:
	key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }}
	- name: Start Docker Solution
	run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace
	env:
	OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
	- name: Run E2E test script
	working-directory: TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test
	run: \|
	docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s
	- name: Collect Docker, OpenSearch Benchmark, and Shared Logs
	if: always()
	run: \|
	mkdir -p logs/docker logs/opensearch_benchmark_logs logs/shared_logs_output
	for container in $(docker ps -aq); do
	container_name=$(docker inspect --format '{{.Name}}' $container \| sed 's/\///')
	docker logs $container > logs/docker/${container_name}_logs.txt 2>&1
	done
	docker cp $(docker ps --filter "name=migration-console" -q):/root/.benchmark/logs logs/opensearch_benchmark_logs
	docker cp $(docker ps --filter "name=migration-console" -q):/shared-logs-output logs/shared_logs_output
	- name: Upload Logs
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: e2e-test-logs
	path: \|
	logs/docker
	logs/opensearch_benchmark_logs
	logs/shared_logs_output
	- name: Clean up migrations docker images before caching
	run: \|
	docker stop $(docker ps -q) && docker system prune --volumes -f
	docker image ls --format '{{.Repository}}:{{.Tag}}' \| grep '^migrations/' \| xargs -I {} docker image rm {}

	node-tests:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	npm-project:
	- ./deployment/cdk/opensearch-service-migration
	- ./deployment/migration-assistant-solution
	defaults:
	run:
	working-directory: ${{ matrix.npm-project }}
	steps:
	- uses: actions/checkout@v4
	- uses: actions/setup-node@v4
	with:
	node-version: ${{ env.node-version }}
	- name: Install NPM dependencies
	run: npm ci
	- name: Run CDK Jest Tests (using mocked images)
	run: npm run test

	link-checker:
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4
	- name: lychee Link Checker
	id: lychee
	uses: lycheeverse/lychee-action@v1
	with:
	args: --verbose --accept=200,403,429 "*/.html" "*/.md" "*/.txt" "*/.json"
	--offline
	--exclude "file:///github/workspace/*"
	--exclude "http://localhost*"
	--exclude "https://localhost*"
	--exclude "http://capture-proxy*"
	--exclude "https://capture-proxy*"
	--exclude-mail
	fail: true
	env:
	GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}


	all-ci-checks-pass:
	needs:
	- console-python-tests
	- gradle-tests
	- link-checker
	- node-tests
	- python-e2e-tests
	- python-lint
	- style-check
	if: always()
	runs-on: ubuntu-latest
	steps:
	- if: ${{ contains(needs..result, 'cancelled') \|\| contains(needs..result, 'failure') \|\| contains(needs.*.result, 'skipped') }}
	run: \|
	echo "One or more job cancelled, failed, or skipped" && exit 1
	- run: \|
	echo '## :heavy_check_mark: All continous integration checks pass' >> $GITHUB_STEP_SUMMARY

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

How to leverage Flux to preserve the sequence of bulk puts... #2966

Workflow file

How to leverage Flux to preserve the sequence of bulk puts... #2966

Jobs

Run details

Workflow file for this run