Download papers metadata from Airtable #390
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Download papers metadata from Airtable | |
| concurrency: download-airtable-papers-metadata | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| jobs: | |
| downloaddata: | |
| env: | |
| GIT_TERMINAL_PROMPT: 0 | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| node-version: [18.x] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install jq | |
| run: sudo apt-get update && sudo apt-get install -y jq | |
| - run: pip install airtable-export | |
| - name: Run airtable-export with retry | |
| run: | | |
| max_attempts=3 | |
| attempt=1 | |
| while [ $attempt -le $max_attempts ]; do | |
| echo "Attempt $attempt of $max_attempts" | |
| if airtable-export --json export ${{ secrets.AIRTABLE_BASE_ID }} ${{ secrets.AIRTABLE_EXPANDED_TABLE }} --key ${{ secrets.AIRTABLE_KEY }}; then | |
| echo "airtable-export successful" | |
| break | |
| else | |
| echo "airtable-export failed" | |
| if [ $attempt -eq $max_attempts ]; then | |
| echo "All attempts failed. Exiting." | |
| exit 1 | |
| fi | |
| echo "Retrying in 30 seconds..." | |
| sleep 30 | |
| fi | |
| attempt=$((attempt+1)) | |
| done | |
| - name: Check if the table data has changed | |
| id: file-check | |
| run: | | |
| jq '[.[] | select((."Goes online as Top Paper" // false) or (."Goes online in Expanded Papers" // false)) | { "Blog or Video": (."Blog or Video" // null), "Link": (.Link // null), "ML Subfield": (."ML Subfield" // null), "ML Subtopic": (."ML Subtopic" // null), "Category": (."Category" // null), "Safety Topic": (."Safety Topic" // null), "Title": (.Title // null), "Type": (.Type // null), "Twitter": (.Twitter // null), "Supplementary Material": (."Supplementary Material" // null), "Abstract": (.Abstract // null), "Transcripts / Audio / Slides": (."Transcripts / Audio / Slides" // null) } | with_entries(select(.value != null))]' export/${{ secrets.AIRTABLE_EXPANDED_TABLE }}.json > export/paperstmp.json | |
| if cmp -s export/paperstmp.json export/papers.json; then | |
| echo "No changes in papers.json, aborting job." | |
| echo "::set-output name=changed::false" | |
| else | |
| echo "Changes detected in papers.json." | |
| echo "::set-output name=changed::true" | |
| fi | |
| # Conditionally execute the remaining steps | |
| - if: steps.file-check.outputs.changed == 'true' | |
| run: | | |
| rm export/papers.json | |
| mv export/paperstmp.json export/papers.json | |
| git status | |
| - if: steps.file-check.outputs.changed == 'true' | |
| run: | | |
| git config user.name "Michael Keenan" | |
| git config user.email "[email protected]" | |
| - if: steps.file-check.outputs.changed == 'true' | |
| run: | | |
| git add export/papers.json | |
| - if: steps.file-check.outputs.changed == 'true' | |
| run: | | |
| git commit --no-verify -v -m "Update papers.json" | |
| - if: steps.file-check.outputs.changed == 'true' | |
| run: | | |
| git push |