Merge branch 'main' into py312

Samsung · May 24, 2024 · 2f52bbc · 2f52bbc
2 parents 44a7c2b + 951f646
commit 2f52bbc
Show file tree

Hide file tree

Showing 48 changed files with 957 additions and 11,943 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -26,14 +26,18 @@ jobs:
 
       - name: Markup hashing
         run: |
-          for f in $(find meta -type f|sort); do md5sum $f; done >meta.md5
+          md5sum snapshot.yaml >checksums.md5
+          for f in $(find meta -type f|sort); do md5sum $f; done >>checksums.md5
+          for f in $(find . -maxdepth 1 -type f -name "*.py"|sort); do md5sum $f; done >>checksums.md5
+          cat checksums.md5
+          sha256sum checksums.md5
 
       - name: Cache data
         id: cache-data
         uses: actions/cache@v4
         with:
           path: data
-          key: cred-data-${{ hashFiles('meta.md5') }}
+          key: cred-data-${{ hashFiles('checksums.md5') }}
 
       - name: Set up Python 3.8
         if: steps.cache-data.outputs.cache-hit != 'true'
@@ -57,6 +61,8 @@ jobs:
 
   run_benchmark:
 
+    if: ${{ 'pull_request' == github.event_name }}
+
     needs: [ download_data ]
 
     runs-on: ubuntu-latest
@@ -70,14 +76,18 @@ jobs:
 
       - name: Markup hashing
         run: |
-          for f in $(find meta -type f|sort); do md5sum $f; done >meta.md5
+          md5sum snapshot.yaml >checksums.md5
+          for f in $(find meta -type f|sort); do md5sum $f; done >>checksums.md5
+          for f in $(find . -maxdepth 1 -type f -name "*.py"|sort); do md5sum $f; done >>checksums.md5
+          cat checksums.md5
+          sha256sum checksums.md5
 
       - name: Cache data
         id: cache-data
         uses: actions/cache@v4
         with:
           path: data
-          key: cred-data-${{ hashFiles('meta.md5') }}
+          key: cred-data-${{ hashFiles('checksums.md5') }}
 
       - name: Failure in case when cache missed
         if: steps.cache-data.outputs.cache-hit != 'true'
@@ -99,60 +109,53 @@ jobs:
         run: python -m pip install --requirement requirements.txt
 
       - name: Checkout CredSweeper
-        if: ${{ 'pull_request' == github.event_name }}
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           path: temp/CredSweeper
 
       - name: Patch benchmark for PR work
-        if: ${{ 'pull_request' == github.event_name }}
         run: |
           sed -i 's|CREDSWEEPER = "https://github.com/Samsung/CredSweeper.git"|CREDSWEEPER = "dummy://github.com/Samsung/CredSweeper.git"|' benchmark/common/constants.py
           grep --with-filename --line-number 'dummy://github.com/Samsung/CredSweeper.git' benchmark/common/constants.py
 
-      - name: Run Benchmark
+      - name: Install CredSweeper
+        run: |
+          python -m pip install temp/CredSweeper
+          credsweeper_head=
+
+      - name: Run CredSweeper tool
         run: |
-          python -m benchmark --scanner credsweeper | tee credsweeper.log
+          credsweeper --banner --jobs $(nproc) --path data --save-json report.${{ github.event.pull_request.head.sha }}.json | tee credsweeper.${{ github.event.pull_request.head.sha }}.log
 
-      - name: Get only results
+      - name: Run Benchmark
         run: |
-          head -n 235 credsweeper.log | tee benchmark.txt
-          tail -n 15 credsweeper.log | grep -v 'Time Elapsed:' | tee -a benchmark.txt
-          cp -vf ./temp/CredSweeper/output.json report.json
+          python -m benchmark --scanner credsweeper --load report.${{ github.event.pull_request.head.sha }}.json | tee benchmark.${{ github.event.pull_request.head.sha }}.log
+
+      - name: Upload CredSweeper log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: credsweeper
+          path: credsweeper.${{ github.event.pull_request.head.sha }}.log
 
-      - name: Upload artifact
+      - name: Upload CredSweeper report
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: report
-          path: report.json
+          path: report.${{ github.event.pull_request.head.sha }}.json
 
-      - name: Upload artifact
+      - name: Upload benchmark output
         if: always()
         uses: actions/upload-artifact@v4
         with:
           name: benchmark
-          path: benchmark.txt
+          path: benchmark.${{ github.event.pull_request.head.sha }}.log
 
       - name: Verify benchmark scores of the PR
-        if: ${{ 'pull_request' == github.event_name }}
-        # update cicd/benchmark.txt with uploaded artifact if a difference is found
-        run: |
-          diff --ignore-all-space --ignore-blank-lines temp/CredSweeper/cicd/benchmark.txt benchmark.txt
-
-      - name: Checkout CredSweeper on push event
-        if: ${{ 'pull_request' != github.event_name }}
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          path: CredSweeper
-
-      - name: Verify benchmark scores on push event
-        if: ${{ 'pull_request' != github.event_name }}
-        # update cicd/benchmark.txt with uploaded artifact if a difference is found
         run: |
-          diff --ignore-all-space --ignore-blank-lines CredSweeper/cicd/benchmark.txt benchmark.txt
+          diff --ignore-all-space --ignore-blank-lines temp/CredSweeper/cicd/benchmark.txt benchmark.${{ github.event.pull_request.head.sha }}.log
 
   # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
@@ -175,14 +178,18 @@ jobs:
 
       - name: Markup hashing
         run: |
-          for f in $(find meta -type f|sort); do md5sum $f; done >meta.md5
+          md5sum snapshot.yaml >checksums.md5
+          for f in $(find meta -type f|sort); do md5sum $f; done >>checksums.md5
+          for f in $(find . -maxdepth 1 -type f -name "*.py"|sort); do md5sum $f; done >>checksums.md5
+          cat checksums.md5
+          sha256sum checksums.md5
 
       - name: Cache data
         id: cache-data
         uses: actions/cache@v4
         with:
           path: data
-          key: cred-data-${{ hashFiles('meta.md5') }}
+          key: cred-data-${{ hashFiles('checksums.md5') }}
 
       - name: Failure in case when cache missed
         if: steps.cache-data.outputs.cache-hit != 'true'
@@ -346,20 +353,24 @@ jobs:
     steps:
 
       - name: Checkout CredData
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: Samsung/CredData
 
       - name: Markup hashing
         run: |
-          for f in $(find meta -type f|sort); do md5sum $f; done >meta.md5
+          md5sum snapshot.yaml >checksums.md5
+          for f in $(find meta -type f|sort); do md5sum $f; done >>checksums.md5
+          for f in $(find . -maxdepth 1 -type f -name "*.py"|sort); do md5sum $f; done >>checksums.md5
+          cat checksums.md5
+          sha256sum checksums.md5
 
       - name: Cache data
         id: cache-data
         uses: actions/cache@v4
         with:
           path: data
-          key: cred-data-${{ hashFiles('meta.md5') }}
+          key: cred-data-${{ hashFiles('checksums.md5') }}
 
       - name: Failure in case when cache missed
         if: steps.cache-data.outputs.cache-hit != 'true'
@@ -383,7 +394,7 @@ jobs:
         run: python -m pip install --upgrade pip
 
       - name: Checkout current CredSweeper
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           path: CredSweeper.head

diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -150,7 +150,7 @@ jobs:
     - name: Check project style
       if: ${{ always() && steps.setup_credsweeper.conclusion == 'success' }}
       run: |
-        for f in credsweeper tests docs experiment setup.py; do
+        for f in credsweeper tests docs experiment; do
             yapf --style .style.yapf --recursive --in-place --parallel $f
         done
         if [ 0 -ne $(git ls-files -m | wc -l) ]; then

diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml
@@ -12,69 +12,28 @@ on:
     types: [ released ]
 
 jobs:
-  build:
-    uses: samsung/supplychainassurance/.github/workflows/[email protected]
-    with:
-      version: "3.11"
-      upload: ${{ 'release' == github.event_name }}
-
-  slsa_release:
-    needs: [ build ]
-    if: ${{ 'release' == github.event_name }}
-    permissions:
-      id-token: write
-    uses: samsung/supplychainassurance/.github/workflows/[email protected]
-    with:
-      hash: "${{ needs.build.outputs.hash }}"
-      artifact: "${{ needs.build.outputs.artifact }}"
-      build_cmd: "${{ needs.build.outputs.build_command }}"
-    secrets:
-      EXPECTED_REPOSITORY: "${{ secrets.EXPECTED_REPOSITORY }}"
-      ECODETOKEN: "${{ secrets.ECODE_TOKEN }}"
-
-  upload_asset:
-    needs: [ build, slsa_release ]
-    if: ${{ 'release' == github.event_name }}
-    permissions:
-      contents: write
-    runs-on: ubuntu-latest
-    steps:
-    - name: Install hub tool
-      run: |
-        sudo apt-get update && sudo apt-get install -y hub
-    - name: Upload Assets
-      uses: samsung/supplychainassurance/.github/actions/[email protected]
-      env:
-        GITHUBTOKEN: ${{ secrets.GITHUB_TOKEN }}
-      with:
-        artifacts: ${{ needs.slsa_release.outputs.artifacts }}
-
   deploy:
     runs-on: ubuntu-latest
-    needs: [ build, slsa_release, upload_asset ]
     steps:
-    - name: Download Artifacts
-      if: ${{ 'release' == github.event_name }}
-      id: download
-      uses: samsung/supplychainassurance/.github/actions/[email protected]
+    - name: Checkout
+      uses: actions/checkout@v4
       with:
-        hash: ${{ needs.build.outputs.hash }}
+        ref: ${{ github.event.pull_request.head.sha }}
 
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.11"
+        python-version: "3.8"
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install twine
+        pip install -r requirements.txt
+        python -m build
 
     - name: Publish
       if: ${{ 'release' == github.event_name }}
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        cd ${{ steps.download.outputs.outdir }}
-        twine upload ${{ needs.build.outputs.artifact }}
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_PASSWORD }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -35,6 +35,7 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
 
     - name: Upgrade PIP
       run: |

diff --git a/Dockerfile b/Dockerfile
@@ -1,10 +1,10 @@
-FROM python:3.9
+FROM python:3.10
 
 WORKDIR /app
 
 ADD credsweeper /app/credsweeper
 
-COPY setup.py /app/
+COPY pyproject.toml /app/
 COPY README.md /app/
 
 RUN pip install .