Skip to content

Commit

Permalink
Load DAGS into airflow (#198)
Browse files Browse the repository at this point in the history
  • Loading branch information
danielscholl authored Sep 16, 2024
1 parent 0fdc1db commit 7dbab67
Show file tree
Hide file tree
Showing 7 changed files with 267 additions and 241 deletions.
34 changes: 30 additions & 4 deletions bicep/modules/blade_common.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -173,10 +173,6 @@ var vaultSecrets = [
secretName: 'airflow-db-connection'
secretValue: 'postgresql://dbuser:${substring(uniqueString('dbuser', resourceGroup().id, bladeConfig.sectionName), 0, 8)}@airflow-cluster-rw.postgresql.svc.cluster.local:5432/airflow-db'
}
{
secretName: 'airflow-admin-username'
secretValue: 'airflow'
}
{
secretName: 'airflow-admin-password'
secretValue: substring(uniqueString('airflow', resourceGroup().id, bladeConfig.sectionName), 0, 8)
Expand Down Expand Up @@ -422,6 +418,36 @@ module conversionShareUpload './script-share-upload/main.bicep' = {
}
}

module manifestDagShareUpload './script-share-upload/main.bicep' = {
name: '${bladeConfig.sectionName}-storage-dag-upload-manifest'
params: {
storageAccountName: configStorage.outputs.name
location: location
shareName: 'airflow-dags'
filename: 'osdu-ingest-r3.py'
fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/raw/master/src/osdu_dags/osdu-ingest-r3.py'
useExistingManagedIdentity: true
managedIdentityName: deploymentScriptIdentity
existingManagedIdentitySubId: subscription().subscriptionId
existingManagedIdentityResourceGroupName:resourceGroup().name
}
}

module manifestRefDagShareUpload './script-share-upload/main.bicep' = {
name: '${bladeConfig.sectionName}-storage-dag-upload-manifest-by-ref'
params: {
storageAccountName: configStorage.outputs.name
location: location
shareName: 'airflow-dags'
filename: 'osdu-ingest-r3-by-reference.py'
fileurl: 'https://community.opengroup.org/osdu/platform/data-flow/ingestion/ingestion-dags/-/raw/master/src/osdu_dags/osdu-ingest-r3-by-reference.py'
useExistingManagedIdentity: true
managedIdentityName: deploymentScriptIdentity
existingManagedIdentitySubId: subscription().subscriptionId
existingManagedIdentityResourceGroupName:resourceGroup().name
}
}

/*
_______ .______ ___ .______ __ __
/ _____|| _ \ / \ | _ \ | | | |
Expand Down
1 change: 1 addition & 0 deletions bicep/modules/blade_service.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,7 @@ module helmAppConfigProvider './aks-run-command/main.bicep' = {
params: {
aksName: cluster.outputs.aksClusterName
location: location
initialScriptDelay: '130s'

newOrExistingManagedIdentity: 'existing'
managedIdentityName: managedIdentityName
Expand Down
37 changes: 0 additions & 37 deletions dags/test_bash_operator.py

This file was deleted.

36 changes: 36 additions & 0 deletions dags/test_pip_packages.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
from airflow import DAG
from airflow.operators.python_operator import PythonOperator
from datetime import datetime
import subprocess

# Define a function to get the list of installed pip packages
def list_installed_packages():
# Run the pip freeze command to get installed packages
installed_packages = subprocess.check_output(['pip', 'freeze']).decode('utf-8')

# Log the output to Airflow logs
print("Installed pip packages:")
print(installed_packages)

# Define the DAG
default_args = {
'owner': 'airflow',
'start_date': datetime(2023, 1, 1),
'retries': 1
}

with DAG(
dag_id='list_pip_packages',
default_args=default_args,
schedule_interval=None,
catchup=False
) as dag:

# Define the task using PythonOperator
check_pip_packages = PythonOperator(
task_id='check_pip_packages',
python_callable=list_installed_packages
)

# Set task dependencies (if needed)
check_pip_packages
Loading

0 comments on commit 7dbab67

Please sign in to comment.