.pipeline.yml.aws.template

pipeline:
  create_cluster:
    image: banzaicloud/plugin-pipeline-client:0.3.0
    cluster_name: "[[your-cluster-name]]"
    cluster_provider: amazon
    amazon_node_instance_type: m4.xlarge
    amazon_node_spot_price: 0.3
    amazon_node_min_count: 2
    amazon_node_max_count: 3

    secrets: [ plugin_endpoint, plugin_token ]

  install_monitoring:
    image: banzaicloud/plugin-pipeline-client:0.3.0
    deployment_name: "banzaicloud-stable/pipeline-cluster-monitor"
    deployment_release_name: "monitor"

    secrets: [plugin_endpoint, plugin_token]

  install_zeppelin:
    image: banzaicloud/plugin-pipeline-client:0.3.0
    deployment_name: "banzaicloud-stable/zeppelin-spark"
    deployment_release_name: "release-1"
    deployment_state: "created"
    deployment_values:
      zeppelin:
        sparkSubmitOptions:
          eventLogDirectory: "s3a://[[your-s3-bucket]]/"
      historyServer:
        enabled: true
      spark:
        spark-hs:
          app:
            logDirectory: "s3a://[[your-s3-bucket]]/"
    secrets: [plugin_endpoint, plugin_token]

  remote_checkout:
    image: banzaicloud/plugin-k8s-proxy:0.3.0
    original_image: plugins/git

  run:
    image: banzaicloud/plugin-k8s-proxy:0.3.0
    original_image: banzaicloud/zeppelin-client:0.3.0
    notebook_name: "sf-police-incidents"
    notebook_file_path: "sf-police-incidents-aws.json"
    notebook_state: "running"
    zeppelin_endpoint: "http://release-1-zeppelin:8080/zeppelin"