wereldmuseum-collection-archives-create-graph.yaml

name: Update Search Graph with data from dataset 'Collection Archives' in the Knowledge Graph

# Every hour
# on:
#   schedule:
#     - cron: "0 * * * *"

on:
  push:
    branches: ["main"]

# Run a single workflow at a time
concurrency:
  group: search-graph

jobs:
  run:
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - name: Checkout data repository
        uses: actions/checkout@v4
        with:
          path: ./data
          # No 'sparse-checkout': that's actually slower than checking out the full repo
      - name: Checkout code repository
        uses: actions/checkout@v4
        with:
          path: ./code
          repository: colonial-heritage/integration-layer
      - name: Install Node
        uses: actions/setup-node@v4
        with:
          node-version: "20"
      - name: Install app
        run: |
          cd ./code
          npm install --no-progress
          npx turbo run build --filter=@colonial-collections/graph-create
      - name: Create graph
        run: |
          cd ./data
          mkdir -p "$RUNNER_TEMP"
          test -f ./wereldmuseum-collection-archives/data/data.tar.zst && tar --zstd -xf ./wereldmuseum-collection-archives/data/data.tar.zst -C "$RUNNER_TEMP"
          ../code/apps/graph-create/dist/cli.js create \
            --resource-dir ./wereldmuseum-collection-archives/resources \
            --data-file "$RUNNER_TEMP/data.sqlite" \
            --check-endpoint-url "${{ vars.SPARQL_ENDPOINT_URL_KG }}" \
            --check-if-run-must-continue-query-file ./wereldmuseum-collection-archives/queries/check.rq \
            --iterate-endpoint-url "${{ vars.SPARQL_ENDPOINT_URL_KG }}" \
            --iterate-query-file ./wereldmuseum-collection-archives/queries/iterate.rq \
            --generate-endpoint-url "${{ vars.SPARQL_ENDPOINT_URL_KG }}" \
            --generate-query-file ./wereldmuseum-collection-archives/queries/generate.rq \
            --generate-batch-size 300000 \
            --triplydb-instance-url "${{ vars.TRIPLYDB_INSTANCE_URL }}" \
            --triplydb-api-token "${{ secrets.TRIPLYDB_API_TOKEN }}" \
            --triplydb-account "${{ vars.TRIPLYDB_ACCOUNT }}" \
            --triplydb-dataset "${{ vars.TRIPLYDB_DATASET_SG }}" \
            --triplydb-service "search" \
            --triplydb-service-templates-file "../code/packages/triplydb/data/search-graph-templates.json" \
            --graph-name "https://data.colonialcollections.nl/wereldmuseum-collection-archives" \
            --temp-dir "$RUNNER_TEMP"
      - name: Save changes
        run: |
          cd ./data
          mkdir -p ./wereldmuseum-collection-archives/data
          test -f "$RUNNER_TEMP/data.sqlite" && tar --zstd -cf ./wereldmuseum-collection-archives/data/data.tar.zst -C "$RUNNER_TEMP" data.sqlite
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git add .
          git commit --quiet -a -m "Save changes" || true
          git push --force -u origin