[ADD] cloud function that triggers on newly uploaded files

cloud_analysis/README.md

@@ -16,7 +16,7 @@ Then you need to change the permission for **Storage Object Viewer** so that the
 
 Then create a **Key** that will act as the `GOOGLE_APPLICATION_CREDENTIALS`, an environment variable that authentificate a user for accessing the Google Cloud Bucket (see `main.py/fetch_audio_data`).
 
-Copy/Paste the `.json` file created from the key and copy it in a file called `g_application_credentials.json`. This will be access in the `main.py/fetch_audio_data`:
+Copy/Paste the `.json` file created from the key and copy it in a file called `key-file.json`. This will be access in the `main.py/fetch_audio_data`:
 
 ```
     credentials = service_account.Credentials.from_service_account_file(
@@ -123,9 +123,57 @@ docker push gcr.io/snoskuter-akustikk/snowmobile_cloud:main
 gcloud run deploy \
     model \
     --image gcr.io/snoskuter-akustikk/snowmobile_cloud:main \
-    --memory 1Gi \
+    --memory 4Gi \
     --region europe-north1 \
     --platform managed
 ```
 
 Be sure to allocate enough memory for the service to be able to process the files and run the model.
+
+5- Test that the service is running on Google Cloud
+
+You can make a POST request using:
+
+```bash
+cd cloud_analysis
+./test_cloud.sh
+```
+
+## Run the service on newly uploaded file
+
+We use **Cloud functions** for this. A function that trigger the service (`trigger_audio_analysis`) on newly uploaded file can be found in the script `cloud_function_directory.py`.
+
+Cloud Function (trigger_audio_analysis) is in a file named `main.py` and has to be the correct directory when you're trying to deploy it. Moreover, a `requirement.txt` of the dependancies necessary to make the script run has to be specified. Your directory should look something like this:
+
+```
+--> cloud_function_directory
+    --> main.py
+    --> requirements.txt
+```
+
+Because of the structure of the `main.py`, the `key-file.json` that has been created in [Create service account with access to the Google Cloud bucket](#create-service-account-with-access-to-the-google-cloud-bucket) need to be uploaded in the Bucket where the function will be trigered.
+
+Once this is done you can deploy the function using:
+
+```bash
+gcloud functions deploy trigger_audio_analysis \ 
+    --runtime python310 \
+    --trigger-resource snoskuter-detector-test \
+    --trigger-event google.storage.object.finalize \
+    --entry-point trigger_audio_analysis \
+    --source .
+```
+
+Explanation of the command:
+
+`gcloud functions deploy trigger_audio_analysis`: Deploys a function named trigger_audio_analysis.
+
+`--runtime python310`: Specifies that the runtime environment for the function is Python 3.10.
+
+`--trigger-resource snoskuter-detector-test`: Specifies that the function should be triggered by events related to the snoskuter-detector-test resource, which should be a Cloud Storage bucket in this context.
+
+-`-trigger-event google.storage.object.finalize`: Specifies that the function should be triggered when a new object (file) is created (or an existing object is overwritten, and a new generation of the object is created) in the bucket.
+
+`--entry-point trigger_audio_analysis`: Specifies that the function to execute is named trigger_audio_analysis within your source code.
+
+`--source .`: Specifies that the source code that should be deployed is in the current directory.

cloud_analysis/main.py

@@ -89,7 +89,7 @@ def fetch_audio_data(bucket_name, blob_name):
     import google.auth
 
     credentials = service_account.Credentials.from_service_account_file(
-        '/app/cloud_analysis/g_application_credentials.json'
+        '/app/cloud_analysis/key-file.json'
 )
 
     storage_client = storage.Client(credentials=credentials)

cloud_analysis/test_cloud.sh

@@ -3,7 +3,7 @@ ID_TOKEN=$(gcloud auth print-identity-token)
 curl -X POST \
      -H "Content-Type: application/json" \
      -H "Authorization: Bearer ${ID_TOKEN}" \
-     -d '{"audio_file_path": "/app/example/example_audio.mp3", "audio_id": "test-id", "audio_rec": {"location": {"latitude": 0, "longitude": 0}}}' \
+     -d '{"audio_id": "test-id", "audio_rec": {"location": {"latitude": 0, "longitude": 0}}, "bucket_name": "snoskuter-detector-test", "blob_name": "example_audio.mp3"}' \
      https://model-4uhtnq5xla-lz.a.run.app/process-audio
 
 

cloud_analysis/trigger_audio_analysis.py

@@ -10,7 +10,7 @@ def trigger_audio_analysis(data, context):
     bucket_name = data['bucket']
 
     # URL of your Cloud Run service
-    cloud_run_url = "https://model-4uhtnq5xla-lz.a.run.app/process-audio-gcs"
+    cloud_run_url = "https://model-4uhtnq5xla-lz.a.run.app/process-audio"
 
     # ID Token for authentication
     id_token = os.environ.get("ID_TOKEN")

cloud_function_directory/main.py

@@ -0,0 +1,91 @@
+import requests
+import google.auth
+from google.auth.transport.requests import Request
+from google.oauth2 import service_account
+from google.cloud import storage
+import io
+import json
+
+from google.oauth2 import id_token
+from google.auth.transport.requests import AuthorizedSession
+
+def get_id_token(service_account_info):
+    """
+    Obtain an ID token for authentication from the service account info.
+
+    Parameters:
+        service_account_info (dict): The service account info in dictionary format.
+
+    Returns:
+        str: An ID token for authentication.
+    """
+    # Create credentials object from service account info
+    credentials = service_account.Credentials.from_service_account_info(service_account_info)
+
+    # Create a Request object
+    request_obj = Request()
+
+    # Obtain an ID token to use for authentication
+    id_token_info = id_token.fetch_id_token(request_obj, "https://model-4uhtnq5xla-lz.a.run.app")
+
+    return id_token_info
+
+def fetch_service_account_key(bucket_name, blob_name):
+    """
+    Fetch the service account key file from Google Cloud Storage.
+
+    Parameters:
+        bucket_name (str): The name of the GCS bucket.
+        blob_name (str): The name of the blob (file) in the GCS bucket.
+
+    Returns:
+        dict: The service account key file as a dictionary.
+    """
+    storage_client = storage.Client()
+    bucket = storage_client.get_bucket(bucket_name)
+    blob = bucket.blob(blob_name)
+
+    # Download the JSON key file content
+    key_file_content = blob.download_as_text()
+
+    # Convert the JSON content to a dictionary
+    key_file_dict = json.loads(key_file_content)
+
+    return key_file_dict
+
+def trigger_audio_analysis(data, context):
+    file_name = data['name']
+    bucket_name = data['bucket']
+
+    # URL of your Cloud Run service
+    cloud_run_url = "https://model-4uhtnq5xla-lz.a.run.app/process-audio"
+
+    # Payload to send to Cloud Run
+    payload = {
+        "bucket_name": bucket_name,
+        "blob_name": file_name,
+        "audio_id": "example-audio-id",
+        "audio_rec": {"location": {"latitude": 0, "longitude": 0}}
+    }
+
+    # Specify the GCS bucket and blob name of the service account key file
+    key_file_bucket = "snoskuter-detector-test"
+    key_file_blob = "key-file.json"
+
+    # Fetch the service account key file from GCS
+    service_account_info = fetch_service_account_key(key_file_bucket, key_file_blob)
+
+    # Obtain the identity token
+    id_token = get_id_token(service_account_info)
+
+    # Headers for the request
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {id_token}"
+    }
+
+    # Send a POST request to the Cloud Run service
+    response = requests.post(cloud_run_url, json=payload, headers=headers)
+
+    # Log the response
+    print(f"Response from Cloud Run: {response.text}")

cloud_function_directory/requirements.txt

@@ -0,0 +1,3 @@
+google-cloud-storage==1.42.3
+requests==2.26.0
+google-auth==2.3.3