rce-colonial-objects-create-graph.yaml

name: Update Search Graph with data from dataset 'Colonial Objects' in the Knowledge Graph

# "At minute 0 past hour 3 and 15"
on:
  schedule:
    - cron: "0 3,15 * * *"

# Run a single workflow at a time
concurrency:
  group: search-graph

jobs:
  run:
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - name: Checkout data repository
        uses: actions/checkout@v4
        with:
          path: ./data
          sparse-checkout: rce-colonial-objects
      - name: Checkout code repository
        uses: actions/checkout@v4
        with:
          path: ./code
          repository: colonial-heritage/integration-layer
      - name: Install Node
        uses: actions/setup-node@v4
        with:
          node-version: "20"
      - name: Install app
        run: |
          cd ./code
          npm install --no-progress
          npx turbo run build --filter=@colonial-collections/graph-create
      - name: Create graph
        run: |
          cd ./data
          mkdir -p "$RUNNER_TEMP"
          test -f ./rce-colonial-objects/data/data.tar.zst && tar --zstd -xf ./rce-colonial-objects/data/data.tar.zst -C "$RUNNER_TEMP"
          ../code/apps/graph-create/dist/cli.js create \
            --resource-dir ./rce-colonial-objects/resources \
            --data-file "$RUNNER_TEMP/data.sqlite" \
            --check-endpoint-url "${{ vars.SPARQL_ENDPOINT_URL_KG }}" \
            --check-if-run-must-continue-query-file ./rce-colonial-objects/queries/check.rq \
            --iterate-endpoint-url "${{ vars.SPARQL_ENDPOINT_URL_KG }}" \
            --iterate-query-file ./rce-colonial-objects/queries/iterate.rq \
            --generate-endpoint-url "${{ vars.SPARQL_ENDPOINT_URL_KG }}" \
            --generate-query-file ./rce-colonial-objects/queries/generate.rq \
            --generate-batch-size 5000000 \
            --triplydb-instance-url "${{ vars.TRIPLYDB_INSTANCE_URL }}" \
            --triplydb-api-token "${{ secrets.TRIPLYDB_API_TOKEN }}" \
            --triplydb-account "${{ vars.TRIPLYDB_ACCOUNT }}" \
            --triplydb-dataset "${{ vars.TRIPLYDB_DATASET_SG }}" \
            --triplydb-service "search" \
            --triplydb-service-templates-file "../code/packages/triplydb/data/search-graph-templates.json" \
            --graph-name "https://data.colonialcollections.nl/rce-colonial-objects" \
            --temp-dir "$RUNNER_TEMP"
      - name: Save changes
        run: |
          cd ./data
          mkdir -p ./rce-colonial-objects/data
          test -f "$RUNNER_TEMP/data.sqlite" && tar --zstd -cf ./rce-colonial-objects/data/data.tar.zst -C "$RUNNER_TEMP" data.sqlite
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git add .
          git commit --quiet -a -m "Save changes" || true
          git push --force -u origin