Skip to content

Commit

Permalink
Update prod config and ci
Browse files Browse the repository at this point in the history
  • Loading branch information
sydneynotthecity committed Apr 18, 2024
1 parent b17ac16 commit 4eaf764
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 95 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ jobs:
- id: "get-credentials"
uses: "google-github-actions/get-gke-credentials@v2"
with:
cluster_name: "us-central1-hubble-1pt5-dev-7db0e004-gke"
location: "us-central1-c"
cluster_name: "us-central1-hubble-14c4ca64-gke"
location: "us-central1"

- name: Pytest
run: pytest dags/
Expand Down Expand Up @@ -75,17 +75,17 @@ jobs:
run: python dags/stellar_etl_airflow/add_files_to_composer.py --bucket $BUCKET --env prod
env:
GOOGLE_CLOUD_PROJECT: hubble-261722
BUCKET: us-central1-hubble-2-d948d67b-bucket
BUCKET: us-central1-hubble-14c4ca64-bucket

- name: Update Airflow variables
uses: actions-hub/gcloud@master
env:
PROJECT_ID: hubble-261722
APPLICATION_CREDENTIALS: "${{ secrets.CREDS_TEST_HUBBLE }}"
COMPOSER_ENVIRONMENT: hubble-2
COMPOSER_ENVIRONMENT: hubble
LOCATION: us-central1
with:
args: >
components install kubectl && gcloud composer environments run
$COMPOSER_ENVIRONMENT --location $LOCATION variables import
-- gcsfuse/variables.json
-- gcsfuse/actual_mount_path/variables.json
94 changes: 4 additions & 90 deletions airflow_variables_prod.json
Original file line number Diff line number Diff line change
@@ -1,91 +1,5 @@
{
"affinity": {
"default": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "cloud.google.com/gke-nodepool",
"operator": "In",
"values": ["default-pool"]
}
]
}
]
}
}
},
"backfill": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "cloud.google.com/gke-nodepool",
"operator": "In",
"values": ["hubble-etl"]
}
]
}
]
}
}
},
"cc": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "cloud.google.com/gke-nodepool",
"operator": "In",
"values": ["hubble-etl"]
}
]
}
]
}
}
},
"wocc": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "cloud.google.com/gke-nodepool",
"operator": "In",
"values": ["default-pool"]
}
]
}
]
}
}
},
"state": {
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [
{
"matchExpressions": [
{
"key": "cloud.google.com/gke-nodepool",
"operator": "In",
"values": ["state-table"]
}
]
}
]
}
}
}
},
"affinity": {},
"api_key_path": "/home/airflow/gcs/data/apiKey.json",
"bq_dataset": "crypto_stellar_internal_2",
"bq_project": "hubble-261722",
Expand Down Expand Up @@ -143,17 +57,17 @@
"partnership_assets__asset_activity_fact": false,
"trade_agg": false
},
"dbt_image_name": "stellar/stellar-dbt:b621192",
"dbt_image_name": "stellar/stellar-dbt:8ae3438",
"dbt_job_execution_timeout_seconds": 1800,
"dbt_job_retries": 1,
"dbt_mart_dataset": "sdf_marts",
"dbt_maximum_bytes_billed": 100000000000000,
"dbt_project": "hubble-261722",
"dbt_target": "prod",
"dbt_threads": 12,
"gcs_exported_data_bucket_name": "us-central1-hubble-2-d948d67b-bucket",
"gcs_exported_data_bucket_name": "us-central1-hubble-14c4ca64-bucket",
"gcs_exported_object_prefix": "dag-exported",
"image_name": "stellar/stellar-etl:75c9a9c",
"image_name": "stellar/stellar-etl:395f1f8",
"image_output_path": "/etl/exported_data/",
"image_pull_policy": "IfNotPresent",
"kube_config_location": "",
Expand Down

0 comments on commit 4eaf764

Please sign in to comment.