upgrade to pachyderm_sdk for github action. Add li191r and parQuantum…

…Line

.github/workflow_scripts/put_files/put_files.py

@@ -0,0 +1,58 @@
+from pachyderm_sdk import Client
+from pachyderm_sdk.api import pfs, pps
+import environs
+from urllib.parse import urlparse
+from pathlib import Path
+import os
+
+def setup_client(pachd_address:str, pach_token:str):
+
+    # Example of how to create a robot token with the required permissions 
+    #   pachctl auth get-robot-token testrunner
+    #   pachctl auth set project default repoOwner robot:testrunner
+
+    pach_url = urlparse(pachd_address)
+    host = pach_url.hostname
+    port = pach_url.port
+    if port is None:
+        port = 80
+    if pach_url.scheme == "https" or pach_url.scheme == "grpcs":
+        tls = True
+    else:
+        tls = False
+    return Client(host=host, port=port, tls=tls, auth_token=pach_token) 
+
+
+def main():
+    env = environs.Env()
+    pachd_address = os.environ["PACHD_ADDRESS"] # e.g. "grpcs://pachd.nonprod.gcp.neoninternal.org:443"
+    pach_token = os.environ["PACH_TOKEN"] # auth token (string). Needs repoOwner roles
+    repo_name = os.environ["REPO"] # The Pachyderm repo (e.g. "empty_files_prt")
+    branch_name = os.environ["BRANCH"] # The branch of the pachyderm repo (e.g. "master")
+    in_path = os.environ["IN_PATH"] # The local path to the folder that will be uploaded into pachyderm (e.g. "empty_files/prt"")
+    out_path = os.environ["OUT_PATH"] # The path where the folder will be placed in the pachydemr repo (e.g. "prt")
+
+    # Setup connection to Pachyderm
+    client = setup_client(pachd_address,pach_token)
+
+    # If the repo does not exist, create it and create the desired branch
+    repo = pfs.Repo.from_uri(repo_name)
+    try:
+        client.pfs.create_repo(repo=repo)
+    except:
+        print(f'Did not create repo:',repo_name,' (likely already exists)')# Log the warning
+
+    # Put the updated file(s) into Pachyderm
+    branch = pfs.Branch.from_uri(repo_name+"@"+branch_name)
+    if Path(in_path).is_dir():
+        with client.pfs.commit(branch=branch) as commit:
+            client.pfs.put_files(commit=commit,source=in_path,path=out_path)
+    else:
+        with client.pfs.commit(branch=branch) as commit:
+            with open(in_path, "rb") as source:
+                client.pfs.put_file_from_file(commit=commit,path=out_path,file=source,append=False)
+
+if __name__ == "__main__":
+    main()
+
+

.github/workflow_scripts/put_files/requirements.txt

@@ -0,0 +1,4 @@
+# Python 3.10
+pachyderm_sdk
+certifi
+environs

.github/workflows/li191r_avro_schemas.yml

@@ -0,0 +1,29 @@
+name: li191r-avro-schemas
+on:
+  push:
+    branches:
+      - 'master'
+    paths:
+      - 'avro_schemas/li191r/**'
+  workflow_dispatch: {} # Allows trigger of workflow from web interface
+jobs:
+  put-file:
+    runs-on: arc-neon-gke
+    #runs-on: ubuntu-latest 
+    env:
+      PACHD_ADDRESS: grpcs://pachyderm-dev.transitions-nonprod.gcp.neoninternal.org:443
+      PACH_TOKEN: ${{ secrets.RepoOwnerPachydermDev }}
+      REPO: li191r_avro_schemas # Pachyderm repo
+      BRANCH: master 
+      IN_PATH: 'avro_schemas/li191r' # Single path to local file or directory. If directory, all files in directory will be place in pachyderm at OUT_PATH. 
+      OUT_PATH: '/li191r' # Path to place the file(s) in Pachyderm. If IN_PATH is a file, specify to the file. If IN_PATH is a directory, specify to the directory. 
+    steps:
+      - uses: actions/checkout@v4
+      - run: ls -la
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.10' # Version range or exact version of a Python version to use, using SemVer's version range syntax
+          cache: 'pip'
+      - run: pip install -r .github/workflow_scripts/put_files/requirements.txt
+      - run: python3 -u .github/workflow_scripts/put_files/put_files.py

.github/workflows/li191r_empty_files.yml

@@ -0,0 +1,29 @@
+name: li191r-empty-files
+on:
+  push:
+    branches:
+      - 'master'
+    paths:
+      - 'empty_files/li191r/**'
+  workflow_dispatch: {} # Allows trigger of workflow from web interface
+jobs:
+  put-file:
+    runs-on: arc-neon-gke
+    #runs-on: ubuntu-latest 
+    env:
+      PACHD_ADDRESS: grpcs://pachyderm-dev.transitions-nonprod.gcp.neoninternal.org:443
+      PACH_TOKEN: ${{ secrets.RepoOwnerPachydermDev }}
+      REPO: li191r_empty_files # Pachyderm repo
+      BRANCH: master 
+      IN_PATH: 'empty_files/li191r' # Single path to local file or directory. If directory, all files in directory will be place in pachyderm at OUT_PATH. 
+      OUT_PATH: '/li191r' # Path to place the file(s) in Pachyderm. If IN_PATH is a file, specify to the file. If IN_PATH is a directory, specify to the directory. 
+    steps:
+      - uses: actions/checkout@v4
+      - run: ls -la
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.10' # Version range or exact version of a Python version to use, using SemVer's version range syntax
+          cache: 'pip'
+      - run: pip install -r .github/workflow_scripts/put_files/requirements.txt
+      - run: python3 -u .github/workflow_scripts/put_files/put_files.py

.github/workflows/parQuantumLine_avro_schemas.yml

@@ -0,0 +1,29 @@
+name: parQuantumLine-avro-schemas
+on:
+  push:
+    branches:
+      - 'master'
+    paths:
+      - 'avro_schemas/parQuantumLine/**'
+  workflow_dispatch: {} # Allows trigger of workflow from web interface
+jobs:
+  put-file:
+    runs-on: arc-neon-gke
+    #runs-on: ubuntu-latest 
+    env:
+      PACHD_ADDRESS: grpcs://pachyderm-dev.transitions-nonprod.gcp.neoninternal.org:443
+      PACH_TOKEN: ${{ secrets.RepoOwnerPachydermDev }}
+      REPO: parQuantumLine_avro_schemas # Pachyderm repo
+      BRANCH: master 
+      IN_PATH: 'avro_schemas/parQuantumLine' # Single path to local file or directory. If directory, all files in directory will be place in pachyderm at OUT_PATH. 
+      OUT_PATH: '/parQuantumLine' # Path to place the file(s) in Pachyderm. If IN_PATH is a file, specify to the file. If IN_PATH is a directory, specify to the directory. 
+    steps:
+      - uses: actions/checkout@v4
+      - run: ls -la
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.10' # Version range or exact version of a Python version to use, using SemVer's version range syntax
+          cache: 'pip'
+      - run: pip install -r .github/workflow_scripts/put_files/requirements.txt
+      - run: python3 -u .github/workflow_scripts/put_files/put_files.py