From 4f0dc792482898f8b9418a6350acc2918d870ca0 Mon Sep 17 00:00:00 2001 From: Javier Date: Wed, 20 Nov 2024 17:09:33 +0000 Subject: [PATCH 1/4] add: `terraform` gitignore file --- ops/tf-modules/.gitignore | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 ops/tf-modules/.gitignore diff --git a/ops/tf-modules/.gitignore b/ops/tf-modules/.gitignore new file mode 100644 index 000000000..2faf43d0a --- /dev/null +++ b/ops/tf-modules/.gitignore @@ -0,0 +1,37 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +# Exclude all .tfvars files, which are likely to contain sensitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +*.tfvars.json + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Ignore transient lock info files created by terraform apply +.terraform.tfstate.lock.info + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc From 270d22278458b22a72e4bdd635530bac3536fd15 Mon Sep 17 00:00:00 2001 From: Javier Date: Wed, 20 Nov 2024 17:09:51 +0000 Subject: [PATCH 2/4] add: `gce` vm terraform module --- ops/tf-modules/gce-dev-vm/main.tf | 92 ++++++++++++++ ops/tf-modules/gce-dev-vm/outputs.tf | 9 ++ ops/tf-modules/gce-dev-vm/variables.tf | 158 +++++++++++++++++++++++++ 3 files changed, 259 insertions(+) create mode 100644 ops/tf-modules/gce-dev-vm/main.tf create mode 100644 ops/tf-modules/gce-dev-vm/outputs.tf create mode 100644 ops/tf-modules/gce-dev-vm/variables.tf diff --git a/ops/tf-modules/gce-dev-vm/main.tf b/ops/tf-modules/gce-dev-vm/main.tf new file mode 100644 index 000000000..55ae20741 --- /dev/null +++ b/ops/tf-modules/gce-dev-vm/main.tf @@ -0,0 +1,92 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} + +provider "google" { + project = var.project_id + credentials = file(var.credentials_file) + region = var.region + zone = var.zone +} + +resource "google_service_account" "iam_service_account" { + account_id = var.service_account_id + display_name = var.service_account_display_name +} + +resource "google_project_iam_member" "bigquery_admin" { + project = var.project_id + role = var.bigquery_admin_role + member = "serviceAccount:${google_service_account.iam_service_account.email}" +} + +resource "google_project_iam_member" "iam_service_account_admin" { + project = var.project_id + role = var.iam_service_account_admin_role + member = "serviceAccount:${google_service_account.iam_service_account.email}" +} + +resource "google_project_iam_member" "cloud_platform" { + project = var.project_id + role = var.cloud_platform_role + member = "serviceAccount:${google_service_account.iam_service_account.email}" +} + +resource "google_compute_network" "dev_network" { + name = var.network_name +} + +resource "google_compute_firewall" "ssh_firewall" { + name = var.firewall_name + network = google_compute_network.dev_network.name + + allow { + protocol = "tcp" + ports = var.firewall_ports + } + + target_tags = var.firewall_target_tags + source_ranges = var.firewall_source_ranges +} + +resource "google_compute_instance" "dev_vm" { + name = var.machine_name + machine_type = var.machine_type + hostname = var.machine_hostname + allow_stopping_for_update = true + tags = var.vm_tags + service_account { + email = google_service_account.iam_service_account.email + scopes = var.service_account_scopes + } + + boot_disk { + device_name = var.boot_disk_device_name + + initialize_params { + image = var.image + type = var.disk_type + size = var.disk_size + } + } + + scheduling { + preemptible = var.preemptible + } + + network_interface { + network = google_compute_network.dev_network.id + access_config { + + } + } + + metadata = { + ssh-keys = "${var.ssh_user}:${file(var.public_key_path)}" + startup-script = file(var.startup_script_path) + } +} diff --git a/ops/tf-modules/gce-dev-vm/outputs.tf b/ops/tf-modules/gce-dev-vm/outputs.tf new file mode 100644 index 000000000..6ff8f5d1d --- /dev/null +++ b/ops/tf-modules/gce-dev-vm/outputs.tf @@ -0,0 +1,9 @@ +output "vm_public_ip" { + description = "Public IP of the VM" + value = google_compute_instance.dev_vm.network_interface.0.access_config.0.nat_ip +} + +output "service_account_email" { + description = "Email address of the IAM service account" + value = google_service_account.iam_service_account.email +} diff --git a/ops/tf-modules/gce-dev-vm/variables.tf b/ops/tf-modules/gce-dev-vm/variables.tf new file mode 100644 index 000000000..b18261b13 --- /dev/null +++ b/ops/tf-modules/gce-dev-vm/variables.tf @@ -0,0 +1,158 @@ +variable "project_id" { + description = "Your GCP Project ID" + type = string +} + +variable "credentials_file" { + description = "Path to your GCP service account credentials file" + type = string +} + +variable "region" { + description = "GCP Region" + type = string +} + +variable "zone" { + description = "GCP Zone within the region" + type = string +} + +variable "service_account_id" { + description = "Service account ID" + type = string + default = "oso-service-account" +} + +variable "service_account_display_name" { + description = "Service account display name" + type = string + default = "OSO Service Account" +} + +variable "bigquery_admin_role" { + description = "BigQuery Admin role" + type = string + default = "roles/bigquery.admin" +} + +variable "iam_service_account_admin_role" { + description = "IAM Service Account Admin role" + type = string + default = "roles/iam.serviceAccountAdmin" +} + +variable "cloud_platform_role" { + description = "Cloud Platform role" + type = string + default = "roles/editor" +} + +variable "network_name" { + description = "Name of the network" + type = string + default = "dev-network" +} + +variable "firewall_name" { + description = "Name of the firewall rule" + type = string + default = "ssh-firewall" +} + +variable "firewall_ports" { + description = "Ports to open in the firewall rule" + type = list(string) + default = ["22"] +} + +variable "firewall_target_tags" { + description = "Target tags for the firewall rule" + type = list(string) + default = ["dev-vm"] +} + +variable "firewall_source_ranges" { + description = "Source ranges for the firewall rule" + type = list(string) + default = ["0.0.0.0/0"] +} + +variable "machine_name" { + description = "Name of the VM instance" + type = string + default = "oso-polar" +} + +variable "machine_type" { + description = "Machine type for the VM" + type = string + default = "n2d-custom-8-16384" +} + +variable "machine_hostname" { + description = "Hostname for the VM instance" + type = string + default = "oso.polar" +} + +variable "vm_tags" { + description = "Tags for the VM instance" + type = list(string) + default = ["dev-vm"] +} + +variable "service_account_scopes" { + description = "Service account scopes" + type = list(string) + default = [ + "https://www.googleapis.com/auth/cloud-platform", + "https://www.googleapis.com/auth/bigquery", + ] +} + +variable "boot_disk_device_name" { + description = "Boot disk device name" + type = string + default = "oso-polar-disk" +} + +variable "disk_type" { + description = "Boot disk type for the VM" + type = string + default = "pd-standard" +} + +variable "disk_size" { + description = "Boot disk size in GB" + type = number + default = 64 +} + +variable "preemptible" { + description = "Whether the VM is preemptible" + type = bool + default = false +} + +variable "image" { + description = "OS Image for the VM instance" + type = string + default = "debian-cloud/debian-11" +} + +variable "public_key_path" { + description = "Path to your public SSH key file" + type = string +} + +variable "ssh_user" { + description = "SSH user for the VM instance" + type = string +} + +variable "startup_script_path" { + description = "Path to your startup script file" + type = string + default = "./scripts/startup.sh" +} From b0d98641218c6cedd31a43968458112566e4b04d Mon Sep 17 00:00:00 2001 From: Javier Date: Wed, 20 Nov 2024 17:10:04 +0000 Subject: [PATCH 3/4] add: convenience `scripts` --- ops/tf-modules/gce-dev-vm/scripts/connect.sh | 110 +++++++++++++++++++ ops/tf-modules/gce-dev-vm/scripts/startup.sh | 37 +++++++ 2 files changed, 147 insertions(+) create mode 100755 ops/tf-modules/gce-dev-vm/scripts/connect.sh create mode 100755 ops/tf-modules/gce-dev-vm/scripts/startup.sh diff --git a/ops/tf-modules/gce-dev-vm/scripts/connect.sh b/ops/tf-modules/gce-dev-vm/scripts/connect.sh new file mode 100755 index 000000000..7f930cfb3 --- /dev/null +++ b/ops/tf-modules/gce-dev-vm/scripts/connect.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +ZONE="us-central1-a" +USER="$USER" + +parse_arguments() { + while [[ $# -gt 0 ]]; do + case "$1" in + start | stop) + ACTION="$1" + shift + ;; + --project | -p) + PROJECT_ID="$2" + shift 2 + ;; + --instance | -i) + INSTANCE_NAME="$2" + shift 2 + ;; + --zone | -z) + ZONE="$2" + shift 2 + ;; + --user | -u) + USER="$2" + shift 2 + ;; + --help | -h) + help + exit 0 + ;; + *) + echo "Unknown option: $1" + help + exit 1 + ;; + esac + done + + if [[ -z "$ACTION" || -z "$PROJECT_ID" || -z "$INSTANCE_NAME" ]]; then + echo "Error: Missing required arguments." + help + exit 1 + fi +} + +start_instance() { + config_file="$HOME/.ssh/config" + + [[ ! -f "$config_file" ]] && touch "$config_file" + + gcloud compute instances start "$INSTANCE_NAME" --zone "$ZONE" --project "$PROJECT_ID" + + external_ip=$( + gcloud compute instances describe "$INSTANCE_NAME" --zone "$ZONE" --project "$PROJECT_ID" | grep natIP | awk '{print $2}' + ) + + if [[ ! -f "$HOME/.ssh/google_compute_engine" ]]; then + echo "No SSH key found. Please select one from the list below:" + select key in $(ls $HOME/.ssh/*.pub); do + echo "Using $key, creating symlink to $HOME/.ssh/google_compute_engine" + ln -s "$key" $HOME/.ssh/google_compute_engine + break + done + fi + + if grep -q "Host $INSTANCE_NAME:$PROJECT_ID:$ZONE" "$config_file"; then + sed -i "" "s/HostName .*/HostName $external_ip/" "$config_file" + else + [[ $(od -An -tc -N1 "$config_file") != $'\n' ]] && echo >>"$config_file" + echo "Host $INSTANCE_NAME:$PROJECT_ID:$ZONE" >>"$config_file" + echo " HostName $external_ip" >>"$config_file" + echo " User $USER" >>"$config_file" + echo " IdentityFile $HOME/.ssh/google_compute_engine" >>"$config_file" + fi +} + +stop_instance() { + config_file="$HOME/.ssh/config" + + if grep -q "Host $INSTANCE_NAME:$PROJECT_ID:$ZONE" "$config_file"; then + sed -i "" "/Host $INSTANCE_NAME:$PROJECT_ID:$ZONE/,+3d" "$config_file" + sed -i "" -e :a -e '/^\n*$/{$d;N;ba' -e '}' "$config_file" + fi + + gcloud compute instances stop "$INSTANCE_NAME" --zone "$ZONE" --project "$PROJECT_ID" +} + +help() { + echo "Usage: $0 start|stop --project --instance [--zone ] [--user ]" + echo + echo "Options:" + echo " start|stop Action to perform (start or stop the instance)" + echo " --project, -p Google Cloud project ID" + echo " --instance, -i Name of the compute instance" + echo " --zone, -z Compute instance zone (default: us-central1-a)" + echo " --user, -u SSH user (default: current system user)" + echo " --help, -h Display this help message" +} + +parse_arguments "$@" + +if [[ "$ACTION" == "start" ]]; then + start_instance +elif [[ "$ACTION" == "stop" ]]; then + stop_instance +else + help +fi diff --git a/ops/tf-modules/gce-dev-vm/scripts/startup.sh b/ops/tf-modules/gce-dev-vm/scripts/startup.sh new file mode 100755 index 000000000..6164d4ce3 --- /dev/null +++ b/ops/tf-modules/gce-dev-vm/scripts/startup.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +sudo apt-get update +sudo apt install -y wget \ + build-essential \ + libssl-dev \ + zlib1g-dev \ + libncurses5-dev \ + libgdbm-dev \ + libnss3-dev \ + libreadline-dev \ + libffi-dev \ + curl \ + libbz2-dev \ + git \ + pkg-config + +cd /tmp/ +wget https://www.python.org/ftp/python/3.12.0/Python-3.12.0.tgz +tar -xf Python-3.12.0.tgz +cd Python-3.12.0 + +./configure --enable-optimizations +make -j $(nproc) +sudo make altinstall + +cd /tmp/ +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash + +echo 'export NVM_DIR="$HOME/.nvm"' >>~/.bashrc +echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >>~/.bashrc +echo '[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"' >>~/.bashrc + +. ~/.bashrc + +nvm install node +nvm use node From 790dc22f3b48b585da4ff3767aec3107fb85644c Mon Sep 17 00:00:00 2001 From: Javier Date: Wed, 20 Nov 2024 17:10:28 +0000 Subject: [PATCH 4/4] add: `readme` with instructions --- ops/tf-modules/gce-dev-vm/README.md | 96 +++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 ops/tf-modules/gce-dev-vm/README.md diff --git a/ops/tf-modules/gce-dev-vm/README.md b/ops/tf-modules/gce-dev-vm/README.md new file mode 100644 index 000000000..f8b656cbd --- /dev/null +++ b/ops/tf-modules/gce-dev-vm/README.md @@ -0,0 +1,96 @@ +# gce-dev-vm + +Sets up a GCE vm suitable for remote development. + +# usage + +> [!WARNING] +> Make sure you are logged in to the correct GCP project before running the +> following commands. + +Create a `terraform.tfvars` file with the following content. To pick your zone +and region, the rule of thumb is to pick the region closest to you. Use this +[link](https://googlecloudplatform.github.io/region-picker/) to find the region +and zone closest to you. + +```hcl +project_id = "your-project-id" +credentials_file = "/path/to/application_default_credentials.json" +public_key_path = "/path/to/ssh_key.pub" +ssh_user = "user" +zone = "your-zone1" +region = "your-zone1-a" +``` + +Now, initialize the terraform module, and plan the changes. + +```bash +terraform init +terraform plan +``` + +Check that everything looks good, and apply the changes. + +```bash +terraform apply +``` + +After the apply is complete, you should see the external IP of the VM in the +output. You can now ssh into the VM using the following command. + +```bash +ssh -i /path/to/ssh_key user@external-ip +``` + +# features + +Installs development tools, `python3.12` from source and `nvm` for managing +node. + +## post-install + +After the VM is created, you should change the password for the user. You can +run the following command to change the password. + +```bash +sudo passwd user +``` + +You should also add ssh keys to the newly created VM. You can either copy your +current ssh keys to the VM or generate new ones. To copy your current ssh keys +to the VM, you can run the following command. + +```bash +ssh-copy-id -i /path/to/ssh_key user@external-ip +``` + +To generate new ssh keys, you can run the following command. + +```bash +ssh-keygen -t rsa -b 4096 -C "user@email.com" +``` + +After generating the keys, follow +[GitHub's guide](https://docs.github.com/en/github/authenticating-to-github/adding-a-new-ssh-key-to-your-github-account) +to add the ssh key to your GitHub account. + +You should also set `GPG` keys for signing commits. You can follow +[GitHub's guide](https://docs.github.com/en/github/authenticating-to-github/managing-commit-signature-verification) +as well. + +## convenience scripts + +We also provide [`scrips/connect.sh`](./scripts/connect.sh), which you can use +to automatically add the machine `ip` to your `~/.ssh/config` file. This will +allow you to ssh into the machine using the `ssh user@name:project:region` +command. + +With this script, we can create two aliases in our `~/.bashrc` file, to start +and stop the VM. These will add and remove the machine `ip` from the +`~/.ssh/config` file respectively, as well as start and stop the VM, to save +costs. + +```bash +alias osostart="bash /path/to/gce-dev-vm/scripts/connect.sh start --project $PROJECT --instance $NAME -z $ZONE -u $USER" +alias osostop="bash /path/to/gce-dev-vm/scripts/connect.sh stop --project $PROJECT --instance $NAME --zone $ZONE" +```