generate_nhp_data.yaml

resources:
  jobs:
    Generate_NHP_Data:
      name: Generate NHP Data
      email_notifications:
        no_alert_for_skipped_runs: true
      webhook_notifications:
        on_success:
          - id: 11be34cb-f1f0-42ca-99d8-e7b3e75e20ca
        on_failure:
          - id: 11be34cb-f1f0-42ca-99d8-e7b3e75e20ca
      notification_settings:
        no_alert_for_skipped_runs: true
        no_alert_for_canceled_runs: true
      tasks:
        - task_key: Generate_Provider_Successors
          notebook_task:
            notebook_path: generate_provider_successors
            source: GIT
          job_cluster_key: generate_nhp_apc
        - task_key: Main_ICB
          depends_on:
            - task_key: Generate_Provider_Successors
          notebook_task:
            notebook_path: generate_main_icb
            source: GIT
          job_cluster_key: generate_nhp_apc
        - task_key: NHP_APC
          depends_on:
            - task_key: Main_ICB
          notebook_task:
            notebook_path: generate_inpatients
            source: GIT
          job_cluster_key: generate_nhp_apc
        - task_key: NHP_APC_Mitigators
          depends_on:
            - task_key: NHP_APC
            - task_key: NHP_ECDS
            - task_key: NHP_OPA
          notebook_task:
            notebook_path: generate_inpatients_mitigators
            source: GIT
          job_cluster_key: generate_nhp_apc
        - task_key: NHP_APC_Mitigators_Check
          depends_on:
            - task_key: NHP_APC_Mitigators
          notebook_task:
            notebook_path: check_inpatients_mitigators
            source: GIT
          job_cluster_key: generate_nhp_apc
        - task_key: NHP_ECDS
          depends_on:
            - task_key: Main_ICB
          notebook_task:
            notebook_path: generate_ecds
            source: GIT
          job_cluster_key: generate_nhp_apc
        - task_key: NHP_OPA
          depends_on:
            - task_key: Main_ICB
          notebook_task:
            notebook_path: generate_outpatients
            source: GIT
          job_cluster_key: generate_nhp_apc
      job_clusters:
        - job_cluster_key: generate_nhp_apc
          new_cluster:
            cluster_name: ""
            spark_version: 14.3.x-scala2.12
            azure_attributes:
              first_on_demand: 1
              availability: SPOT_WITH_FALLBACK_AZURE
              spot_bid_max_price: -1
            node_type_id: Standard_D8ads_v5
            enable_elastic_disk: true
            data_security_mode: SINGLE_USER
            runtime_engine: STANDARD
            autoscale:
              min_workers: 2
              max_workers: 8
      git_source:
        git_url: https://github.com/The-Strategy-Unit/nhp_data
        git_provider: gitHub
        git_branch: main
      queue:
        enabled: true