diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 18d708c..43385f8 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -102,12 +102,10 @@ jobs: steps: - name: checkout uses: actions/checkout@v2 - - name: Install Dependencies - run: sudo apt-get install -y jq - name: drain-staging-space uses: cloud-gov/cg-cli-tools@main with: - command: cf install-plugin -f -r CF-Community "drains" && ./create-space-drain.sh + command: apt-get install -y jq && cf install-plugin -f -r CF-Community "drains" && mkdir -p /root/.cf/ && touch /root/.cf/config.json && ./create-space-drain.sh cf_org: gsa-datagov cf_space: staging cf_username: ${{secrets.CF_SERVICE_USER}} @@ -122,12 +120,10 @@ jobs: steps: - name: checkout uses: actions/checkout@v2 - - name: Install Dependencies - run: sudo apt-get install -y jq - name: drain-prod-space uses: cloud-gov/cg-cli-tools@main with: - command: cf install-plugin -f -r CF-Community "drains" && ./create-space-drain.sh + command: apt-get install -y jq && cf install-plugin -f -r CF-Community "drains" && mkdir -p /root/.cf/ && touch /root/.cf/config.json && ./create-space-drain.sh cf_org: gsa-datagov cf_space: prod cf_username: ${{secrets.CF_SERVICE_USER}} diff --git a/Makefile b/Makefile index 2c3ef7c..fe81752 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ logstash-installation: curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o logstash/awscliv2.zip - curl https://artifacts.elastic.co/downloads/logstash/logstash-oss-7.16.3-linux-x86_64.tar.gz -o logstash/logstash-oss-7.16.3-linux-x86_64.tar.gz - docker run -v $${PWD}:$${PWD} -w $${PWD} --rm docker.elastic.co/logstash/logstash-oss:7.16.3 bin/prepare-offline-logstash + curl https://artifacts.elastic.co/downloads/logstash/logstash-oss-7.17.3-linux-x86_64.tar.gz -o logstash/logstash-oss-7.17.3-linux-x86_64.tar.gz + docker run -v $${PWD}:$${PWD} -w $${PWD} --rm docker.elastic.co/logstash/logstash-oss:7.17.3 bin/prepare-offline-logstash diff --git a/logstash/.profile b/logstash/.profile index f9ae827..f4b7877 100755 --- a/logstash/.profile +++ b/logstash/.profile @@ -17,9 +17,9 @@ function parse_vcap_services () { parse_vcap_services echo "Unpacking logstash..." - tar xzvf logstash-oss-7.16.3-linux-x86_64.tar.gz > /dev/null 2>&1 && \ - rm logstash-oss-7.16.3-linux-x86_64.tar.gz -export LS_HOME="$PWD/logstash-7.16.3" + tar xzvf logstash-oss-7.17.3-linux-x86_64.tar.gz > /dev/null 2>&1 && \ + rm logstash-oss-7.17.3-linux-x86_64.tar.gz +export LS_HOME="$PWD/logstash-7.17.3" echo "Installing logstash plugins..." "$LS_HOME"/bin/logstash-plugin install file://"$PWD"/plugins.zip diff --git a/logstash/logstash.conf b/logstash/logstash.conf index b6b022a..f8d3fb0 100644 --- a/logstash/logstash.conf +++ b/logstash/logstash.conf @@ -63,16 +63,15 @@ filter{ # } # mutate { add_field => { "[@metadata][target_index]" => "index-%{+YYYY.MM}" } } - mutate { - add_field => { "[@metadata][NEWRELIC_KEY]" => "${NEWRELIC_LICENSE_KEY:notpresent}" } - add_field => { "[@metadata][AWS_S3_PROXY]" => "${AWS_S3_PROXY:notpresent}" } - } + # mutate { + # add_field => { "[@metadata][AWS_S3_PROXY]" => "${AWS_S3_PROXY:notpresent}" } + # } } output { - - # Explicit support for environment variable expansion in conditionals is not yet supported, though it's very close: + + # Explicit support for environment variable expansion in conditionals is not yet supported, though it's very close: # https://github.com/elastic/logstash/pull/13370 # # There's a clever workaround described here... @@ -80,33 +79,33 @@ output { # https://github.com/elastic/logstash/issues/5115#issuecomment-279552849 # # See the mutate filter above that sets this value - if [@metadata][NEWRELIC_KEY] != "notpresent" { + if "${NEWRELIC_LICENSE_KEY}" != "" { newrelic { - base_uri => "https://log-api.newrelic.com/log/v1" - license_key => "[@metadata][NEWRELIC_KEY]" + base_uri => "${NEWRELIC_LOG_URI}" + license_key => "${NEWRELIC_LICENSE_KEY}" } } - if [@metadata][AWS_S3_PROXY] != "notpresent" { - s3 { - access_key_id => "${AWS_ACCESS_KEY_ID:yourawsaccesskeyid}" - secret_access_key => "${AWS_SECRET_ACCESS_KEY:yourawsaccesskeysecret}" - region => "${AWS_REGION:yourawsregion}" - bucket => "${AWS_BUCKET:yourawsbucket}" - endpoint => "${AWS_ENDPOINT:https://yourawsendpoint}" - proxy_uri => "[@metadata][AWS_S3_PROXY]" - validate_credentials_on_root_bucket => "false" - additional_settings => { "force_path_style" => true } - } - } else { - s3 { - access_key_id => "${AWS_ACCESS_KEY_ID:yourawsaccesskeyid}" - secret_access_key => "${AWS_SECRET_ACCESS_KEY:yourawsaccesskeysecret}" - region => "${AWS_REGION:yourawsregion}" - bucket => "${AWS_BUCKET:yourawsbucket}" - endpoint => "${AWS_ENDPOINT:https://yourawsendpoint}" - validate_credentials_on_root_bucket => "false" - additional_settings => { "force_path_style" => true } - } - } + # if [@metadata][AWS_S3_PROXY] != "notpresent" { + # s3 { + # access_key_id => "${AWS_ACCESS_KEY_ID:yourawsaccesskeyid}" + # secret_access_key => "${AWS_SECRET_ACCESS_KEY:yourawsaccesskeysecret}" + # region => "${AWS_REGION:yourawsregion}" + # bucket => "${AWS_BUCKET:yourawsbucket}" + # endpoint => "${AWS_ENDPOINT:https://yourawsendpoint}" + # proxy_uri => "[@metadata][AWS_S3_PROXY]" + # validate_credentials_on_root_bucket => "false" + # additional_settings => { "force_path_style" => true } + # } + # } else { + # s3 { + # access_key_id => "${AWS_ACCESS_KEY_ID:yourawsaccesskeyid}" + # secret_access_key => "${AWS_SECRET_ACCESS_KEY:yourawsaccesskeysecret}" + # region => "${AWS_REGION:yourawsregion}" + # bucket => "${AWS_BUCKET:yourawsbucket}" + # endpoint => "${AWS_ENDPOINT:https://yourawsendpoint}" + # validate_credentials_on_root_bucket => "false" + # additional_settings => { "force_path_style" => true } + # } + # } } diff --git a/manifest.yml b/manifest.yml index 63f5e49..f1d39ab 100644 --- a/manifest.yml +++ b/manifest.yml @@ -19,4 +19,5 @@ applications: - ((app_name))-s3 - ((app_name))-secrets env: - NEWRELIC_LICENSE_KEY: ((logstash_newrelic_license_key)) \ No newline at end of file + NEWRELIC_LICENSE_KEY: ((logstash_newrelic_license_key)) + NEWRELIC_LOG_URI: ((logstash_newrelic_log_uri)) diff --git a/vars.management-staging.yml b/vars.management-staging.yml index 961be1e..560f1e9 100644 --- a/vars.management-staging.yml +++ b/vars.management-staging.yml @@ -17,3 +17,4 @@ https_proxy: # Optional license key for delivering logs to New Relic # See https://github.com/newrelic/logstash-output-plugin logstash_newrelic_license_key: +logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1 diff --git a/vars.management.yml b/vars.management.yml index c483e45..733c357 100644 --- a/vars.management.yml +++ b/vars.management.yml @@ -17,3 +17,4 @@ https_proxy: # Optional license key for delivering logs to New Relic # See https://github.com/newrelic/logstash-output-plugin logstash_newrelic_license_key: +logstash_newrelic_log_uri: https://gov-log-api.newrelic.com/log/v1