diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c112539..44937ff 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,8 +1,9 @@ # NHS Notify Code Owners -* @NHSDigital/nhs-notify-amet +# Notify default owners +* @rossbugginsnhs @m-houston @aidenvaines-bjss @timireland -# Default protection for codeowners, must be last in file. +# Codeowners must be final check /.github/CODEOWNERS @NHSDigital/nhs-notify-code-owners /CODEOWNERS @NHSDigital/nhs-notify-code-owners diff --git a/.github/ISSUE_TEMPLATE/1_support_request.yaml b/.github/ISSUE_TEMPLATE/1_support_request.yaml new file mode 100644 index 0000000..eb24401 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1_support_request.yaml @@ -0,0 +1,52 @@ +# See: +# - https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/configuring-issue-templates-for-your-repository +# - https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms +# - https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/common-validation-errors-when-creating-issue-forms + +name: 🔧 Support Request +description: Get help +labels: ["support"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to file a support request. Please fill out this form as completely as possible. + - type: textarea + attributes: + label: What exactly are you trying to do? + description: Describe in as much detail as possible. + validations: + required: true + - type: textarea + attributes: + label: What have you tried so far? + description: Describe what you have tried so far. + validations: + required: true + - type: textarea + attributes: + label: Output of any commands you have tried + description: Please copy and paste any relevant output. This will be automatically formatted into codeblock. + render: Shell + validations: + required: false + - type: textarea + attributes: + label: Additional context + description: Add any other context about the problem here. + validations: + required: false + - type: checkboxes + attributes: + label: Code of Conduct + description: By submitting this issue you agree to follow our [Code of Conduct](../../docs/CODE_OF_CONDUCT.md) + options: + - label: I agree to follow this project's Code of Conduct + required: true + - type: checkboxes + attributes: + label: Sensitive Information Declaration + description: To ensure the utmost confidentiality and protect your privacy, we kindly ask you to NOT including [PII (Personal Identifiable Information) / PID (Personal Identifiable Data)](https://digital.nhs.uk/data-and-information/keeping-data-safe-and-benefitting-the-public) or any other sensitive data in this form. We appreciate your cooperation in maintaining the security of your information. + options: + - label: I confirm that neither PII/PID nor sensitive data are included in this form + required: true diff --git a/.github/ISSUE_TEMPLATE/2_feature_request.yaml b/.github/ISSUE_TEMPLATE/2_feature_request.yaml new file mode 100644 index 0000000..705d083 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2_feature_request.yaml @@ -0,0 +1,42 @@ +# See: +# - https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/configuring-issue-templates-for-your-repository +# - https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms +# - https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/common-validation-errors-when-creating-issue-forms + +name: 🚀 Feature Request +description: Suggest an idea for this project +labels: ["feature request"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to file a feature request. Please fill out this form as completely as possible. + - type: textarea + attributes: + label: What is the problem this feature will solve? + description: Tell us why this change is needed or helpful and what problems it may help solve. + validations: + required: true + - type: textarea + attributes: + label: What is the feature that you are proposing to solve the problem? + description: Provide detailed information for what we should add. + validations: + required: true + - type: textarea + attributes: + label: What alternatives have you considered? + - type: checkboxes + attributes: + label: Code of Conduct + description: By submitting this issue you agree to follow our [Code of Conduct](../../docs/CODE_OF_CONDUCT.md) + options: + - label: I agree to follow this project's Code of Conduct + required: true + - type: checkboxes + attributes: + label: Sensitive Information Declaration + description: To ensure the utmost confidentiality and protect your privacy, we kindly ask you to NOT including [PII (Personal Identifiable Information) / PID (Personal Identifiable Data)](https://digital.nhs.uk/data-and-information/keeping-data-safe-and-benefitting-the-public) or any other sensitive data in this form. We appreciate your cooperation in maintaining the security of your information. + options: + - label: I confirm that neither PII/PID nor sensitive data are included in this form + required: true diff --git a/.github/ISSUE_TEMPLATE/3_bug_report.yaml b/.github/ISSUE_TEMPLATE/3_bug_report.yaml new file mode 100644 index 0000000..12a8c6e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/3_bug_report.yaml @@ -0,0 +1,63 @@ +# See: +# - https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/configuring-issue-templates-for-your-repository +# - https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/syntax-for-issue-forms +# - https://docs.github.com/en/communities/using-templates-to-encourage-useful-issues-and-pull-requests/common-validation-errors-when-creating-issue-forms + +name: 🐞 Bug Report +description: File a bug report +labels: ["bug", "triage"] +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to file a bug report. Please fill out this form as completely as possible. + - type: checkboxes + attributes: + label: Is there an existing issue for this? + description: Please, search the Issues to see if an issue already exists for the bug you have encountered. + options: + - label: I have searched the existing Issues + required: true + - type: textarea + attributes: + label: Current Behavior + description: A concise description of what you are experiencing. + validations: + required: true + - type: textarea + attributes: + label: Expected Behavior + description: A concise description of what you expect to happen. + validations: + required: true + - type: textarea + attributes: + label: Steps To Reproduce + description: Steps to reproduce the behavior + placeholder: | + 1. In this environment... + 2. With this config... + 3. Run `...` + validations: + required: false + - type: textarea + attributes: + label: Output + description: Please copy and paste any relevant output. This will be automatically formatted into codeblock. + render: Shell + validations: + required: false + - type: checkboxes + attributes: + label: Code of Conduct + description: By submitting this issue you agree to follow our [Code of Conduct](../../docs/CODE_OF_CONDUCT.md) + options: + - label: I agree to follow this project's Code of Conduct + required: true + - type: checkboxes + attributes: + label: Sensitive Information Declaration + description: To ensure the utmost confidentiality and protect your privacy, we kindly ask you to NOT including [PII (Personal Identifiable Information) / PID (Personal Identifiable Data)](https://digital.nhs.uk/data-and-information/keeping-data-safe-and-benefitting-the-public) or any other sensitive data in this form. We appreciate your cooperation in maintaining the security of your information. + options: + - label: I confirm that neither PII/PID nor sensitive data are included in this form + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6df34a6..c00ff41 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,7 +15,6 @@ - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would change existing functionality) - [ ] Bug fix (non-breaking change which fixes an issue) -- [ ] Documentation ## Checklist diff --git a/.github/actions/lint-terraform/action.yaml b/.github/actions/lint-terraform/action.yaml index d5dfe35..28d990c 100644 --- a/.github/actions/lint-terraform/action.yaml +++ b/.github/actions/lint-terraform/action.yaml @@ -16,5 +16,6 @@ runs: run: | stacks=${{ inputs.root-modules }} for dir in $(find infrastructure/environments -maxdepth 1 -mindepth 1 -type d; echo ${stacks//,/$'\n'}); do + dir=$dir opts='-backend=false' make terraform-init dir=$dir make terraform-validate done diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml new file mode 100644 index 0000000..bd57a9a --- /dev/null +++ b/.github/actions/setup/action.yaml @@ -0,0 +1,10 @@ +name: Make Config Action +description: Install dependencies and execute make config + +runs: + using: composite + steps: + - name: Install dependencies and execute make config + shell: bash + run: | + scripts/setup/setup.sh diff --git a/.github/actions/tfsec/action.yaml b/.github/actions/tfsec/action.yaml new file mode 100644 index 0000000..e63b99c --- /dev/null +++ b/.github/actions/tfsec/action.yaml @@ -0,0 +1,17 @@ +name: "TFSec Scan" +description: "Scan HCL using TFSec" +runs: + using: "composite" + steps: + - name: "TFSec Scan - Components" + shell: bash + run: | + for component in $(find infrastructure/terraform/components -mindepth 1 -type d); do + scripts/terraform/tfsec.sh $component + done + - name: "TFSec Scan - Modules" + shell: bash + run: | + for module in $(find infrastructure/terraform/modules -mindepth 1 -type d); do + scripts/terraform/tfsec.sh $module + done diff --git a/.gitignore b/.gitignore index a8ea15a..a0ad8fe 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,3 @@ version.json !project.code-workspace # Please, add your custom content below! - -!nhs-notify.code-workspace diff --git a/.gitleaksignore b/.gitleaksignore index cceb449..1c92293 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -1,3 +1,5 @@ # SEE: https://github.com/gitleaks/gitleaks/blob/master/README.md#gitleaksignore cd9c0efec38c5d63053dd865e5d4e207c0760d91:docs/guides/Perform_static_analysis.md:generic-api-key:37 +96096685ab3d6876671e2bc9a6ff4d48fc56e521:src/helloworld/helloworld.sln:ipv4:4 +4f4e8c15629b2cb09356a7fed4d72953590227ce:docs/Gemfile.lock:ipv4:4 diff --git a/infrastructure/terraform/.gitignore b/infrastructure/terraform/.gitignore new file mode 100644 index 0000000..579b641 --- /dev/null +++ b/infrastructure/terraform/.gitignore @@ -0,0 +1,67 @@ +### Terraform ### + +# Transient backends +components/**/backend_tfscaffold.tf + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Compiled files +**/*.tfstate +**/*.tfplan +**/*.tfstate.backup +**/.terraform +**/.terraform.lock.hcl +**/.terraform/* +**/build/* +**/work/* +**/*tfstate.lock.info + +# Scaffold Plugin Cache +plugin-cache/* + +# PyCache +**/__pycache__ + +### OSX ### +**/.DS_Store +**/.AppleDouble +**/.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +*.swp +.nyc_output + +# VS Code +.vscode + +# IntelliJ Idea +.idea +**/*.iml + +# js +node_modules diff --git a/infrastructure/terraform/README b/infrastructure/terraform/README new file mode 100644 index 0000000..838d177 --- /dev/null +++ b/infrastructure/terraform/README @@ -0,0 +1,3 @@ +This is an implementation of https://github.com/tfutils/tfscaffold for NHS Notify + +Update the `etc/global.tfvars` file according to your NHS Notify Domain, and follow https://github.com/tfutils/tfscaffold?tab=readme-ov-file#bootstrapping to get your tfstate s3 bucket set up diff --git a/infrastructure/terraform/bin/terraform.sh b/infrastructure/terraform/bin/terraform.sh new file mode 100755 index 0000000..756b4ef --- /dev/null +++ b/infrastructure/terraform/bin/terraform.sh @@ -0,0 +1,807 @@ +#!/bin/bash +# Terraform Scaffold +# +# A wrapper for running terraform projects +# - handles remote state +# - uses consistent .tfvars files for each environment + +## +# Set Script Version +## +readonly script_ver="1.8.1"; + +## +# Standardised failure function +## +function error_and_die { + echo -e "ERROR: ${1}" >&2; + exit 1; +}; + +## +# Print Script Version +## +function version() { + echo "${script_ver}"; +} + +## +# Print Usage Text +## +function usage() { + +cat < + +action: + - Special actions: + * plan / plan-destroy + * apply / destroy + * graph + * taint / untaint + * shell + - Generic actions: + * See https://www.terraform.io/docs/commands/ + +bucket_prefix (optional): + Defaults to: "\${project_name}-tfscaffold" + - myproject-terraform + - terraform-yourproject + - my-first-tfscaffold-project + +build_id (optional): + - testing + - \$BUILD_ID (jenkins) + +component_name: + - the name of the terraform component module in the components directory + +environment: + - dev + - test + - prod + - management + +group: + - dev + - live + - mytestgroup + +project: + - The name of the project being deployed + +region (optional): + Defaults to value of \$AWS_DEFAULT_REGION + - the AWS region name unique to all components and terraform processes + +detailed-exitcode (optional): + When not provided, false. + Changes the plan operation to exit 0 only when there are no changes. + Will be ignored for actions other than plan. + +no-color (optional): + Append -no-color to all terraform calls + +compact-warnings (optional): + Append -compact-warnings to all terraform calls + +lockfile: + Append -lockfile=MODE to calls to terraform init + +additional arguments: + Any arguments provided after "--" will be passed directly to terraform as its own arguments +EOF +}; + +## +# Test for GNU getopt +## +getopt_out=$(getopt -T) +if (( $? != 4 )) && [[ -n $getopt_out ]]; then + error_and_die "Non GNU getopt detected. If you're using a Mac then try \"brew install gnu-getopt\""; +fi + +## +# Execute getopt and process script arguments +## +readonly raw_arguments="${*}"; +ARGS=$(getopt \ + -o dhnvwa:b:c:e:g:i:l:p:r: \ + -l "help,version,bootstrap,action:,bucket-prefix:,build-id:,component:,environment:,group:,project:,region:,lockfile:,detailed-exitcode,no-color,compact-warnings" \ + -n "${0}" \ + -- \ + "$@"); + +#Bad arguments +if [ $? -ne 0 ]; then + usage; + error_and_die "command line argument parse failure"; +fi; + +eval set -- "${ARGS}"; + +declare bootstrap="false"; +declare component_arg; +declare region_arg; +declare environment_arg; +declare group; +declare action; +declare bucket_prefix; +declare build_id; +declare project; +declare detailed_exitcode; +declare no_color; +declare compact_warnings; +declare lockfile; + +while true; do + case "${1}" in + -h|--help) + usage; + exit 0; + ;; + -v|--version) + version; + exit 0; + ;; + -c|--component) + shift; + if [ -n "${1}" ]; then + component_arg="${1}"; + shift; + fi; + ;; + -r|--region) + shift; + if [ -n "${1}" ]; then + region_arg="${1}"; + shift; + fi; + ;; + -e|--environment) + shift; + if [ -n "${1}" ]; then + environment_arg="${1}"; + shift; + fi; + ;; + -g|--group) + shift; + if [ -n "${1}" ]; then + group="${1}"; + shift; + fi; + ;; + -a|--action) + shift; + if [ -n "${1}" ]; then + action="${1}"; + shift; + fi; + ;; + -b|--bucket-prefix) + shift; + if [ -n "${1}" ]; then + bucket_prefix="${1}"; + shift; + fi; + ;; + -i|--build-id) + shift; + if [ -n "${1}" ]; then + build_id="${1}"; + shift; + fi; + ;; + -l|--lockfile) + shift; + if [ -n "${1}" ]; then + lockfile="-lockfile=${1}"; + shift; + fi; + ;; + -p|--project) + shift; + if [ -n "${1}" ]; then + project="${1}"; + shift; + fi; + ;; + --bootstrap) + shift; + bootstrap="true"; + ;; + -d|--detailed-exitcode) + shift; + detailed_exitcode="true"; + ;; + -n|--no-color) + shift; + no_color="-no-color"; + ;; + -w|--compact-warnings) + shift; + compact_warnings="-compact-warnings"; + ;; + --) + shift; + break; + ;; + esac; +done; + +declare extra_args="${@} ${no_color} ${compact_warnings}"; # All arguments supplied after "--" + +## +# Script Set-Up +## + +# Determine where I am and from that derive basepath and project name +script_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; +base_path="${script_path%%\/bin}"; +project_name_default="${base_path##*\/}"; + +status=0; + +echo "Args ${raw_arguments}"; + +# Ensure script console output is separated by blank line at top and bottom to improve readability +trap echo EXIT; +echo; + +## +# Munge Params +## + +# Set Region from args or environment. Exit if unset. +readonly region="${region_arg:-${AWS_DEFAULT_REGION}}"; +[ -n "${region}" ] \ + || error_and_die "No AWS region specified. No -r/--region argument supplied and AWS_DEFAULT_REGION undefined"; + +[ -n "${project}" ] \ + || error_and_die "Required argument -p/--project not specified"; + +# Bootstrapping is special +if [ "${bootstrap}" == "true" ]; then + [ -n "${component_arg}" ] \ + && error_and_die "The --bootstrap parameter and the -c/--component parameter are mutually exclusive"; + [ -n "${build_id}" ] \ + && error_and_die "The --bootstrap parameter and the -i/--build-id parameter are mutually exclusive. We do not currently support plan files for bootstrap"; + [ -n "${environment_arg}" ] && readonly environment="${environment_arg}"; +else + # Validate component to work with + [ -n "${component_arg}" ] \ + || error_and_die "Required argument missing: -c/--component"; + readonly component="${component_arg}"; + + # Validate environment to work with + [ -n "${environment_arg}" ] \ + || error_and_die "Required argument missing: -e/--environment"; + readonly environment="${environment_arg}"; +fi; + +[ -n "${action}" ] \ + || error_and_die "Required argument missing: -a/--action"; + +# Validate AWS Credentials Available +iam_iron_man="$(aws sts get-caller-identity --query 'Arn' --output text)"; +if [ -n "${iam_iron_man}" ]; then + echo -e "AWS Credentials Found. Using ARN '${iam_iron_man}'"; +else + error_and_die "No AWS Credentials Found. \"aws sts get-caller-identity --query 'Arn' --output text\" responded with ARN '${iam_iron_man}'"; +fi; + +# Query canonical AWS Account ID +aws_account_id="$(aws sts get-caller-identity --query 'Account' --output text)"; +if [ -n "${aws_account_id}" ]; then + echo -e "AWS Account ID: ${aws_account_id}"; +else + error_and_die "Couldn't determine AWS Account ID. \"aws sts get-caller-identity --query 'Account' --output text\" provided no output"; +fi; + +# Validate S3 bucket. Set default if undefined +if [ -n "${bucket_prefix}" ]; then + readonly bucket="${bucket_prefix}-${aws_account_id}-${region}" + echo -e "Using S3 bucket s3://${bucket}"; +else + readonly bucket="${project}-tfscaffold-${aws_account_id}-${region}"; + echo -e "No bucket prefix specified. Using S3 bucket s3://${bucket}"; +fi; + +declare component_path; +if [ "${bootstrap}" == "true" ]; then + component_path="${base_path}/bootstrap"; +else + component_path="${base_path}/components/${component}"; +fi; + +# Get the absolute path to the component +if [[ "${component_path}" != /* ]]; then + component_path="$(cd "$(pwd)/${component_path}" && pwd)"; +else + component_path="$(cd "${component_path}" && pwd)"; +fi; + +[ -d "${component_path}" ] || error_and_die "Component path ${component_path} does not exist"; + +## Debug +#echo $component_path; + +## +# Begin parameter-dependent logic +## + +case "${action}" in + apply) + refresh="-refresh=true"; + ;; + destroy) + destroy='-destroy'; + refresh="-refresh=true"; + ;; + plan) + refresh="-refresh=true"; + ;; + plan-destroy) + action="plan"; + destroy="-destroy"; + refresh="-refresh=true"; + ;; + *) + ;; +esac; + +# Tell terraform to moderate its output to be a little +# more friendly to automation wrappers +# Value is irrelavant, just needs to be non-null +export TF_IN_AUTOMATION="true"; + +for rc_path in "${base_path}" "${base_path}/etc" "${component_path}"; do + if [ -f "${rc_path}/.terraformrc" ]; then + echo "Found .terraformrc at ${rc_path}/.terraformrc. Overriding."; + export TF_CLI_CONFIG_FILE="${rc_path}/.terraformrc"; + fi; +done; + +# Configure the plugin-cache location so plugins are not +# downloaded to individual components +declare default_plugin_cache_dir="$(pwd)/plugin-cache"; +export TF_PLUGIN_CACHE_DIR="${TF_PLUGIN_CACHE_DIR:-${default_plugin_cache_dir}}" +mkdir -p "${TF_PLUGIN_CACHE_DIR}" \ + || error_and_die "Failed to created the plugin-cache directory (${TF_PLUGIN_CACHE_DIR})"; +[ -w "${TF_PLUGIN_CACHE_DIR}" ] \ + || error_and_die "plugin-cache directory (${TF_PLUGIN_CACHE_DIR}) not writable"; + +# Clear cache, safe enough as we enforce plugin cache +rm -rf ${component_path}/.terraform; + +# Run global pre.sh +if [ -f "pre.sh" ]; then + source pre.sh "${region}" "${environment}" "${action}" \ + || error_and_die "Global pre script execution failed with exit code ${?}"; +fi; + +# Make sure we're running in the component directory +pushd "${component_path}"; +readonly component_name=$(basename ${component_path}); + +# install terraform +# verify terraform version matches .tool-versions +echo ${PWD} +tool_version=$(grep "terraform " .tool-versions | cut -d ' ' -f 2) +asdf plugin-add terraform && asdf install terraform "${tool_version}" +current_version=$(terraform --version | head -n 1 | cut -d 'v' -f 2) + +if [ -z "${current_version}" ] || [ "${current_version}" != "${tool_version}" ]; then + error_and_die "Terraform version mismatch. Expected: ${tool_version}, Actual: ${current_version}" +fi + +# Regardless of bootstrapping or not, we'll be using this string. +# If bootstrapping, we will fill it with variables, +# if not we will fill it with variable file parameters +declare tf_var_params; + +if [ "${bootstrap}" == "true" ]; then + if [ "${action}" == "destroy" ]; then + error_and_die "You cannot destroy a bootstrap bucket using tfscaffold, it's just too dangerous. If you're absolutely certain that you want to delete the bucket and all contents, including any possible state files environments and components within this project, then you will need to do it from the AWS Console. Note you cannot do this from the CLI because the bootstrap bucket is versioned, and even the --force CLI parameter will not empty the bucket of versions"; + fi; + + # Bootstrap requires this parameter as explicit as it is constructed here + # for multiple uses, so we cannot just depend on it being set in tfvars + tf_var_params+=" -var bucket_name=${bucket}"; +fi; + +# Run pre.sh +if [ -f "pre.sh" ]; then + source pre.sh "${region}" "${environment}" "${action}" \ + || error_and_die "Component pre script execution failed with exit code ${?}"; +fi; + +# Pull down secret TFVAR file from S3 +# Anti-pattern and security warning: This secrets mechanism provides very little additional security. +# It permits you to inject secrets directly into terraform without storing them in source control or unencrypted in S3. +# Secrets will still be stored in all copies of your state file - which will be stored on disk wherever this script is run and in S3. +# This script does not currently support encryption of state files. +# Use this feature only if you're sure it's the right pattern for your use case. +declare -a secrets=(); +readonly secrets_file_name="secret.tfvars.enc"; +readonly secrets_file_path="build/${secrets_file_name}"; +aws s3 ls s3://${bucket}/${project}/${aws_account_id}/${region}/${environment}/${secrets_file_name} >/dev/null 2>&1; +if [ $? -eq 0 ]; then + mkdir -p build; + aws s3 cp s3://${bucket}/${project}/${aws_account_id}/${region}/${environment}/${secrets_file_name} ${secrets_file_path} \ + || error_and_die "S3 secrets file is present, but inaccessible. Ensure you have permission to read s3://${bucket}/${project}/${aws_account_id}/${region}/${environment}/${secrets_file_name}"; + if [ -f "${secrets_file_path}" ]; then + secrets=($(aws kms decrypt --ciphertext-blob fileb://${secrets_file_path} --output text --query Plaintext | base64 --decode)); + fi; +fi; + +if [ -n "${secrets[0]}" ]; then + secret_regex='^[A-Za-z0-9_-]+=.+$'; + secret_count=1; + for secret_line in "${secrets[@]}"; do + if [[ "${secret_line}" =~ ${secret_regex} ]]; then + var_key="${secret_line%=*}"; + var_val="${secret_line##*=}"; + export TF_VAR_${var_key}="${var_val}"; + ((secret_count++)); + else + echo "Malformed secret on line ${secret_count} - ignoring"; + fi; + done; +fi; + +# Pull down additional dynamic plaintext tfvars file from S3 +# Anti-pattern warning: Your variables should almost always be in source control. +# There are a very few use cases where you need constant variability in input variables, +# and even in those cases you should probably pass additional -var parameters to this script +# from your automation mechanism. +# Use this feature only if you're sure it's the right pattern for your use case. +readonly dynamic_file_name="dynamic.tfvars"; +readonly dynamic_file_path="build/${dynamic_file_name}"; +aws s3 ls s3://${bucket}/${project}/${aws_account_id}/${region}/${environment}/${dynamic_file_name} >/dev/null 2>&1; +if [ $? -eq 0 ]; then + aws s3 cp s3://${bucket}/${project}/${aws_account_id}/${region}/${environment}/${dynamic_file_name} ${dynamic_file_path} \ + || error_and_die "S3 tfvars file is present, but inaccessible. Ensure you have permission to read s3://${bucket}/${project}/${aws_account_id}/${region}/${environment}/${dynamic_file_name}"; +fi; + +# Use versions TFVAR files if exists +readonly versions_file_name="versions_${region}_${environment}.tfvars"; +readonly versions_file_path="${base_path}/etc/${versions_file_name}"; + +# Check for presence of an environment variables file, and use it if readable +if [ -n "${environment}" ]; then + readonly env_file_path="${base_path}/etc/env_${region}_${environment}.tfvars"; +fi; + +# Check for presence of a global variables file, and use it if readable +readonly global_vars_file_name="global.tfvars"; +readonly global_vars_file_path="${base_path}/etc/${global_vars_file_name}"; + +# Check for presence of a region variables file, and use it if readable +readonly region_vars_file_name="${region}.tfvars"; +readonly region_vars_file_path="${base_path}/etc/${region_vars_file_name}"; + +# Check for presence of a group variables file if specified, and use it if readable +if [ -n "${group}" ]; then + readonly group_vars_file_name="group_${group}.tfvars"; + readonly group_vars_file_path="${base_path}/etc/${group_vars_file_name}"; +fi; + +# Collect the paths of the variables files to use +declare -a tf_var_file_paths; + +# Use Global and Region first, to allow potential for terraform to do the +# honourable thing and override global and region settings with environment +# specific ones; however we do not officially support the same variable +# being declared in multiple locations, and we warn when we find any duplicates +[ -f "${global_vars_file_path}" ] && tf_var_file_paths+=("${global_vars_file_path}"); +[ -f "${region_vars_file_path}" ] && tf_var_file_paths+=("${region_vars_file_path}"); + +# If a group has been specified, load the vars for the group. If we are to assume +# terraform correctly handles override-ordering (which to be fair we don't hence +# the warning about duplicate variables below) we add this to the list after +# global and region-global variables, but before the environment variables +# so that the environment can explicitly override variables defined in the group. +if [ -n "${group}" ]; then + if [ -f "${group_vars_file_path}" ]; then + tf_var_file_paths+=("${group_vars_file_path}"); + else + echo -e "[WARNING] Group \"${group}\" has been specified, but no group variables file is available at ${group_vars_file_path}"; + fi; +fi; + +# Environment is normally expected, but in bootstrapping it may not be provided +if [ -n "${environment}" ]; then + if [ -f "${env_file_path}" ]; then + tf_var_file_paths+=("${env_file_path}"); + else + echo -e "[WARNING] Environment \"${environment}\" has been specified, but no environment variables file is available at ${env_file_path}"; + fi; +fi; + +# If present and readable, use versions and dynamic variables too +[ -f "${versions_file_path}" ] && tf_var_file_paths+=("${versions_file_path}"); +[ -f "${dynamic_file_path}" ] && tf_var_file_paths+=("${dynamic_file_path}"); + +# Warn on duplication +duplicate_variables="$(cat "${tf_var_file_paths[@]}" | sed -n -e 's/\(^[a-zA-Z0-9_\-]\+\)\s*=.*$/\1/p' | sort | uniq -d)"; +[ -n "${duplicate_variables}" ] \ + && echo -e " +################################################################### +# WARNING WARNING WARNING WARNING WARNING WARNING WARNING WARNING # +################################################################### +The following input variables appear to be duplicated: + +${duplicate_variables} + +This could lead to unexpected behaviour. Overriding of variables +has previously been unpredictable and is not currently supported, +but it may work. + +Recent changes to terraform might give you useful overriding and +map-merging functionality, please use with caution and report back +on your successes & failures. +###################################################################"; + +# Build up the tfvars arguments for terraform command line +for file_path in "${tf_var_file_paths[@]}"; do + tf_var_params+=" -var-file=${file_path}"; +done; + +## +# Start Doing Real Things +## + +# Really Hashicorp? Really?! +# +# In order to work with terraform >=0.9.2 (I say 0.9.2 because 0.9 prior +# to 0.9.2 is barely usable due to key bugs and missing features) +# we now need to do some ugly things to our terraform remote backend configuration. +# The long term hope is that they will fix this, and maybe remove the need for it +# altogether by supporting interpolation in the backend config stanza. +# +# For now we're left with this garbage, and no more support for <0.9.0. +if [ -f backend_tfscaffold.tf ]; then + echo -e "WARNING: backend_tfscaffold.tf exists and will be overwritten!" >&2; +fi; + +declare backend_prefix; +declare backend_filename; + +if [ "${bootstrap}" == "true" ]; then + backend_prefix="${project}/${aws_account_id}/${region}/bootstrap"; + backend_filename="bootstrap.tfstate"; +else + backend_prefix="${project}/${aws_account_id}/${region}/${environment}"; + backend_filename="${component_name}.tfstate"; +fi; + +readonly backend_key="${backend_prefix}/${backend_filename}"; +readonly backend_config="terraform { + backend \"s3\" { + region = \"${region}\" + bucket = \"${bucket}\" + key = \"${backend_key}\" + dynamodb_table = \"${bucket}\" + } +}"; + +# We're now all ready to go. All that's left is to: +# * Write the backend config +# * terraform init +# * terraform ${action} +# +# But if we're dealing with the special bootstrap component +# we can't remotely store the backend until we've bootstrapped it +# +# So IF the S3 bucket already exists, we will continue as normal +# because we want to be able to manage changes to an existing +# bootstrap bucket. But if it *doesn't* exist, then we need to be +# able to plan and apply it with a local state, and *then* configure +# the remote state. + +# In default operations we assume we are already bootstrapped +declare bootstrapped="true"; + +# If we are in bootstrap mode, we need to know if we have already bootstrapped +# or we are working with or modifying an existing bootstrap bucket +if [ "${bootstrap}" == "true" ]; then + # For this exist check we could do many things, but we explicitly perform + # an ls against the key we will be working with so as to not require + # permissions to, for example, list all buckets, or the bucket root keyspace + aws s3 ls s3://${bucket}/${backend_prefix}/${backend_filename} >/dev/null 2>&1; + [ $? -eq 0 ] || bootstrapped="false"; +fi; + +if [ "${bootstrapped}" == "true" ]; then + echo -e "${backend_config}" > backend_tfscaffold.tf \ + || error_and_die "Failed to write backend config to $(pwd)/backend_tfscaffold.tf"; + + # Nix the horrible hack on exit + trap "rm -f $(pwd)/backend_tfscaffold.tf" EXIT; + + declare lockfile_or_upgrade; + [ -n ${lockfile} ] && lockfile_or_upgrade='-upgrade' || lockfile_or_upgrade="${lockfile}"; + + # Configure remote state storage + echo "Setting up S3 remote state from s3://${bucket}/${backend_key}"; + terraform init ${no_color} ${compact_warnings} ${lockfile_or_upgrade} \ + || error_and_die "Terraform init failed"; +else + # We are bootstrapping. Download the providers, skip the backend config. + terraform init \ + -backend=false \ + ${no_color} \ + ${compact_warnings} \ + ${lockfile} \ + || error_and_die "Terraform init failed"; +fi; + +case "${action}" in + 'plan') + if [ -n "${build_id}" ]; then + mkdir -p build; + + plan_file_name="${component_name}_${build_id}.tfplan"; + plan_file_remote_key="${backend_prefix}/plans/${plan_file_name}"; + + out="-out=build/${plan_file_name}"; + fi; + + if [ "${detailed_exitcode}" == "true" ]; then + detailed="-detailed-exitcode"; + fi; + + terraform "${action}" \ + -input=false \ + ${refresh} \ + ${tf_var_params} \ + ${extra_args} \ + ${destroy} \ + ${out} \ + ${detailed} \ + -parallelism=300; + + status="${?}"; + + # Even when detailed exitcode is set, a 1 is still a fail, + # so exit + # (detailed exit codes are 0 and 2) + if [ "${status}" -eq 1 ]; then + error_and_die "Terraform plan failed"; + fi; + + if [ -n "${build_id}" ]; then + aws s3 cp build/${plan_file_name} s3://${bucket}/${plan_file_remote_key} \ + || error_and_die "Plan file upload to S3 failed (s3://${bucket}/${plan_file_remote_key})"; + fi; + + exit ${status}; + ;; + 'graph') + mkdir -p build || error_and_die "Failed to create output directory '$(pwd)/build'"; + terraform graph ${extra_args} -draw-cycles | dot -Tpng > build/${project}-${aws_account_id}-${region}-${environment}.png \ + || error_and_die "Terraform simple graph generation failed"; + terraform graph ${extra_args} -draw-cycles -verbose | dot -Tpng > build/${project}-${aws_account_id}-${region}-${environment}-verbose.png \ + || error_and_die "Terraform verbose graph generation failed"; + exit 0; + ;; + 'apply'|'destroy'|'refresh') + + # Support for terraform <0.10 is now deprecated + if [ "${action}" == "apply" ]; then + echo "Compatibility: Adding to terraform arguments: -auto-approve=true"; + extra_args+=" -auto-approve=true"; + else # action is `destroy` + # Check terraform version - if pre-0.15, need to add `-force`; 0.15 and above instead use `-auto-approve` + if [ $(terraform version | head -n1 | cut -d" " -f2 | cut -d"." -f1) == "v0" ] && [ $(terraform version | head -n1 | cut -d" " -f2 | cut -d"." -f2) -lt 15 ]; then + echo "Compatibility: Adding to terraform arguments: -force"; + force='-force'; + elif [ "${action}" != "refresh" ]; then + extra_args+=" -auto-approve"; + fi; + fi; + + if [ -n "${build_id}" ]; then + mkdir -p build; + plan_file_name="${component_name}_${build_id}.tfplan"; + plan_file_remote_key="${backend_prefix}/plans/${plan_file_name}"; + + aws s3 cp s3://${bucket}/${plan_file_remote_key} build/${plan_file_name} \ + || error_and_die "Plan file download from S3 failed (s3://${bucket}/${plan_file_remote_key})"; + + apply_plan="build/${plan_file_name}"; + + terraform "${action}" \ + -input=false \ + ${refresh} \ + -parallelism=300 \ + ${extra_args} \ + ${force} \ + ${apply_plan}; + exit_code=$?; + else + terraform "${action}" \ + -input=false \ + ${refresh} \ + ${tf_var_params} \ + -parallelism=300 \ + ${extra_args} \ + ${force}; + exit_code=$?; + + if [ "${bootstrapped}" == "false" ]; then + # If we are here, and we are in bootstrap mode, and not already bootstrapped, + # Then we have just bootstrapped for the first time! Congratulations. + # Now we need to copy our state file into the bootstrap bucket + echo -e "${backend_config}" > backend_tfscaffold.tf \ + || error_and_die "Failed to write backend config to $(pwd)/backend_tfscaffold.tf"; + + # Nix the horrible hack on exit + trap "rm -f $(pwd)/backend_tfscaffold.tf" EXIT; + + # Push Terraform Remote State to S3 + # TODO: Add -upgrade to init when we drop support for <0.10 + echo "yes" | terraform init ${lockfile} || error_and_die "Terraform init failed"; + + # Hard cleanup + rm -f backend_tfscaffold.tf; + rm -f terraform.tfstate # Prime not the backup + rm -rf .terraform; + + # This doesn't mean anything here, we're just celebrating! + bootstrapped="true"; + fi; + + fi; + + if [ ${exit_code} -ne 0 ]; then + error_and_die "Terraform ${action} failed with exit code ${exit_code}"; + fi; + + if [ -f "post.sh" ]; then + source post.sh "${region}" "${environment}" "${action}" \ + || error_and_die "Component post script execution failed with exit code ${?}"; + fi; + ;; + '*taint') + terraform "${action}" ${extra_args} || error_and_die "Terraform ${action} failed."; + ;; + 'import') + terraform "${action}" ${tf_var_params} ${extra_args} || error_and_die "Terraform ${action} failed."; + ;; + 'shell') + echo -e "Here's a shell for the ${component} component.\nIf you want to run terraform actions specific to the ${environment}, pass the following options:\n\n${tf_var_params} ${extra_args}\n\n'exit 0' / 'Ctrl-D' to continue, other exit codes will abort tfscaffold with the same code."; + bash -l || exit "${?}"; + ;; + *) + echo -e "Generic action case invoked. Only the additional arguments will be passed to terraform, you break it you fix it:"; + echo -e "\tterraform ${action} ${extra_args}"; + terraform "${action}" ${extra_args} \ + || error_and_die "Terraform ${action} failed."; + ;; +esac; + +popd + +if [ -f "post.sh" ]; then + source post.sh "${region}" "${environment}" "${action}" \ + || error_and_die "Global post script execution failed with exit code ${?}"; +fi; + +exit 0; diff --git a/infrastructure/terraform/modules/.gitkeep b/infrastructure/terraform/modules/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/config/pre-commit.yaml b/scripts/config/pre-commit.yaml index cf82a65..5a5ba09 100644 --- a/scripts/config/pre-commit.yaml +++ b/scripts/config/pre-commit.yaml @@ -14,7 +14,6 @@ repos: - id: mixed-line-ending - id: pretty-format-json args: ['--autofix'] - exclude: \.vscode|devcontainer.json # - id: ... - repo: local hooks: diff --git a/scripts/config/vale/styles/config/vocabularies/words/accept.txt b/scripts/config/vale/styles/config/vocabularies/words/accept.txt index 01db323..3b07d50 100644 --- a/scripts/config/vale/styles/config/vocabularies/words/accept.txt +++ b/scripts/config/vale/styles/config/vocabularies/words/accept.txt @@ -4,25 +4,23 @@ bot Cognito Cyber Dependabot +draw.io drawio -enablement +endcapture endfor endraw GitHub Gitleaks Grype -hotfix idempotence -io Jira OAuth Octokit onboarding -[Oo]nboarding Podman Python -relative_url -repo +rawContent +sed Syft Terraform toolchain diff --git a/scripts/docker/examples/python/assets/hello_world/requirements.txt b/scripts/docker/examples/python/assets/hello_world/requirements.txt index a38fca7..a3611c8 100644 --- a/scripts/docker/examples/python/assets/hello_world/requirements.txt +++ b/scripts/docker/examples/python/assets/hello_world/requirements.txt @@ -3,10 +3,10 @@ click==8.1.7 Flask-WTF==1.2.0 Flask==2.3.3 itsdangerous==2.1.2 -Jinja2==3.1.3 +Jinja2==3.1.4 MarkupSafe==2.1.3 pip==23.3 setuptools==65.5.1 -Werkzeug==3.0.1 +Werkzeug==3.0.3 wheel==0.41.1 WTForms==3.0.1