provision fcrepo/solr/mariadb servers

PROVISION.md

@@ -0,0 +1,13 @@
+# To create a new fcrepo / mariadb / solr server
+
+1) Decrypt the secrets with `./bin/decrypt-secrets`
+
+2) Create or edit the .env.$ENVIRONMENT file, changin the namespace to be unique for each network set you want to make.
+
+3) Make sure the AWS profile in your ~/.aws/config and ~/.aws/credentials files match the AWS account you want to deploy to.
+
+4) Run `./bin/tf workspace new $ENVIRONMENT`
+
+5) Run `./bin/tf $ENVIRONMENT init`
+
+6) Run `./bin/tf $ENVIRONMENT apply`

bin/decrypt-secrets

@@ -8,18 +8,14 @@ Dir.chdir(File.join(parent_dir))
   # TODO: Troubleshoot local env encrypt/decrypt
   # ".env",
   # ".env.*",
-  "chart/*-values.yaml",
-  "ops/kube_config.yml",
-  "ops/.backend",
-  "ops/*-deploy.tmpl.yaml",
-  "ops/k8s/*-values.yaml"
+  "ops/provision/.backend",
+  "ops/provision/.env.*"
 ].each do |files|
   Dir.glob(files).each do |file|
-    if file.match(/enc/)
-      next unless File.exists?(file)
-      cmd = "sops --decrypt #{file} > #{file.gsub(/.enc$/, '')}"
+    if File.exists?(file + ".enc")
+      cmd = "sops --decrypt #{file}.enc > #{file}"
       puts cmd
       `#{cmd}`
     end
   end
-end
+end

bin/encrypt-secrets

@@ -5,18 +5,14 @@
 parent_dir = File.dirname(__dir__)
 [
   # TODO: Troubleshoot local env encrypt/decrypt
-  # ".env",
   # ".env.*",
-  "chart/*-values.yaml",
-  "ops/kube_config.yml",
-  "ops/.backend",
-  "ops/*-deploy.tmpl.yaml",
-  "ops/k8s/*-values.yaml"
+  "ops/provision/.backend",
+  "ops/provision/.env.*"
 ].each do |files|
   Dir.glob(files).each do |file|
     next if /enc/.match?(file)
     cmd = "sops --encrypt #{file} > #{file}.enc"
     puts cmd
     `#{cmd}`
   end
-end
+end

bin/tf

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+# require 'byebug'
+
+dir = File.expand_path(File.join(__FILE__, '../../ops/provision'))
+env_file = File.expand_path("#{dir}/.env.#{ARGV[0]}")
+workspace = "#{ARGV[0]}"
+
+# workspace commands can not have TF_WORKSPACE set
+cmd = if(ARGV[0].match(/workspace/) && ARGV[1].match(/new/))
+        %Q{cd #{dir} && unset TF_WORKSPACE && terraform workspace #{ARGV[1..-1].join(' ')}}
+      elsif ARGV[0].match(/workspace/)
+        %Q{cd #{dir} && TF_WORKSPACE=default terraform workspace #{ARGV[1..-1].join(' ')}}
+      else
+        %Q{cd #{dir} && TF_WORKSPACE=#{workspace} dotenv -f #{env_file} "terraform #{ARGV[1..-1].join(' ')}"}
+      end
+
+if ARGV[1].match(/init/)
+ cmd[0..-2] += " -backend-config=./.backend "
+end
+
+puts cmd
+exec cmd

ops/provision/.backend.enc

@@ -0,0 +1,21 @@
+{
+	"data": "ENC[AES256_GCM,data:Ewgr1+G9pWtEmcRIB3TnfR7g4FUXaoBhCO+ftg3LQy4pHN8ij2A9EjdpISALdTzk7FBm39H2+zTA++lt4oqfyXtabNvCi8fRTHJLFPgQ8L5NdWklvpsSAT3UwU3Fopcl7Pvc2etx+kYfPjw/x207Tlk=,iv:Tsxdt13gBH+Ps/XV4XPSNCi0OXXOby6nvSA2Cq5G3qQ=,tag:ECjzHHi4diP1/+TdXxiC1w==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": null,
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2022-09-01T23:58:01Z",
+		"mac": "ENC[AES256_GCM,data:MU/kWWPao2iTy1eBpLIu1aD6p1+3+J/ljZyUuJHdugqJ9u+Xu76AiRkFXXMdJEX9rNvq1p0Ivip0uXoRh28UdJGYl+81CZefAhOtTDRdgvFPIBDzWIyNO3h1prVNelrDb7uheqXe6JinNTGaGTzSe7oK8feGS/zVhHHkSyqzMqw=,iv:KF1v2eINhc2qz+L35eskQTjENt7xgHC1QP/lKraFW90=,tag:wKev4XqUspW6BaB0yB51VQ==,type:str]",
+		"pgp": [
+			{
+				"created_at": "2022-09-01T23:58:00Z",
+				"enc": "-----BEGIN PGP MESSAGE-----\n\nwcFMAx1u4ocvSXxJARAAobSuTLCtx17k5BKXmXXXQrsgLBnNNvyNp63HsGUH1STL\nGMk493GBOQpSXuwlKpEH5V9WFwHyxs/1sWizVc3ZANWomYAtxK0XshpaNQV3wicK\n6xxBfx+M7K+hOo+Cqd5yJuPsmG+sXuOTlciN4mQ32NhMA61QQitt7omPspoZa160\novjSEaNuzRGlFClwS76N1wqhwCCcSNGMDFvHM2BuHmlnbLQFr3nuD9TMM2NrTCCI\nW9yYUEPUTW+Wvkg0hmbLE0gTsRyPBTbg41Tu4IZGs0NZyAAfem2W/AslSH0Hk/Me\n8nItE5xt/yqd4MdT11yZSe0HPwuy9ecuP4SCV+aGwZzJG3s/d9aSoJMnTCHAN1HS\n9c+8CcDLSRaQhVLqIeNBuY/sg477JBhD/QvFkLyGFInBcRZg99bIhWS6JL6nblCN\neOx3N1SfFkFUIsnTMyqIZeR5fVgddxF20Jjyl76fbhHzS+/pIyUIWr58uk/F7Wlb\nw5dGV0yRgRP2hUItiS/D1gFV9UvZ4M9j2S7+jDAVzOVqWYdnnTu0YKBw6gyW/fgF\nenQAOxunuAZntD23AcSwZJzg5odtzQ6qtBI1wHw9i9cMmZa4uHVdfm/lAJrbLWbk\nuBhG3hgqtvQ850fjJn81OIeJ/hhMMqRy1/CPyO0nP3InrgR2J2dqaNQFgjCMki3S\n5gEsFKIDL8QRiHlWgAEqym7twau5L8oFGkmj5vpEs8DESs9k90AtxRetQJKfW8Am\nsemLliFEahXvuks4IPM9+Y7k2YIQPIsz6OwA7wY5ur2LyeJMwmpYAA==\n=ClwK\n-----END PGP MESSAGE-----",
+				"fp": "B6125B16B0DD59F34D6975FBF885927FDA9C48E2"
+			}
+		],
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.2"
+	}
+}

ops/provision/.env.notch8.enc

@@ -0,0 +1,21 @@
+{
+	"data": "ENC[AES256_GCM,data:CxEUPydRgaLZ+QZZfqbHtx6sWtXbhA2oyJQITrV8pgctLgp9w/klq2VuXWjK5y1xa9zc+y7Jr8T9OMSrJAuLGh6H6BgaYHIaGgllOWZCGmUq52RhJSz7K4RMSvJvF7VBdL9StNHRoV/K+x+gYTd7eSvXo3xkH+MbPV7kYIemxOC/8rRJFoLToe08DWLbK2s2aDqDhzaQZ7lBTxAfLDJZQp58fmo9I/nV6umfMmYW+5JfTTAo4DFdGGJ9Up0H4OOS0jcrPa+v6UQUAlrjfgyek3vuZC3i2Q2hzL/A/gqiRFzWqrId2jYzD8gCSPN5EOGpSxkoGqc8dE4tBzssM5uFH06TspdBupkNlRuMBINlKgqe0SCfFewWNbe9bvqFX25QHJ71PmB312sbvM5b5szLbAS8MuSVW9z7CcSNk8oJ4OtEAt7bGARnhiPnWQ6kj6AgKC1iARqB02MUNQb/2q3z3RgBS4x5s47C0fiWFw==,iv:EpzdZ36fkczA2dvHhzeGTbJQJpV9ojy4VotBMmRYLEQ=,tag:OkymqCpCf3yF0Zwk0vwuBA==,type:str]",
+	"sops": {
+		"kms": null,
+		"gcp_kms": null,
+		"azure_kv": null,
+		"hc_vault": null,
+		"age": null,
+		"lastmodified": "2022-09-01T23:58:02Z",
+		"mac": "ENC[AES256_GCM,data:fX8+ojw8IiqR/xWQ3tHdZvuNRs6YbL0Reu8H04PF2XmDZzTWPIXQ7+PGAEgVRpubC+P1G80V/CPn9qnqK4u35X3+mUss1BT+lPykeHU3ifbuQ/jsFnd91f4uyV3ji+ZMQ5j6Yqq0BEfcoe/0/Te/iqp42+ye5Tv7Tn38WrF9UZY=,iv:AWIgVAa4vh6zSsF8+GIcZPYBgFsFC2DT43DOK5pWHko=,tag:iKPSKezCH+uas5CXbPQtkA==,type:str]",
+		"pgp": [
+			{
+				"created_at": "2022-09-01T23:58:01Z",
+				"enc": "-----BEGIN PGP MESSAGE-----\n\nwcFMAx1u4ocvSXxJARAAUamwpQNhcrX2D01OEINsRoly4v6TgVLitmLd6CA93EDS\nUKq4AHPTUr5W+GSgXAikgSbablQ73GBj004li1kRGrvXbpycOhMQ8hf3qay8k6Zp\nwZeUHi8I1y2spxd75ux6IAqZOps8akFrG6qz4fMVotRuWay1kAKC6NhtdZN0pq4Y\nWWDqidxtnakTAdQ0XMLUOPDwhKjOjZqeca+QNcPXIAjFuEHgrQ2g3jsgXo3wpxaX\n/mFz2qGO1CV4K9CRe2bwSnbkzGcjsBjwW19y/68aXkKbZ8yjZR4S7mFNrq/sQXmC\n1vqPaV54g31eceEGPzv3JCwIWOhrx/EoHR9rMuJp0lBQR8x/sf4WBzqgkY45Iplj\n6X7+YXQP5qREYyKlM6O2KuBKdc1r9JXpjlwYhVxJh90fPf+3w5k6y76Dumfa42ca\nZZjxYwyZbx1/c7alOJjgfiS0Eowxj51Wd205Zwjy7WJo+dbTWSa4Gm+LGlSk7hhl\nHIjxobZAFZ7YSiejvpZyy0uXdS3qMwhrUwSlpUxbGbqKd5CX/jT5lYX/LH7jGgoJ\n/9Efvn7B9pVwwrXrMXqNScoF27HMoyHDA9jFVtW9Iy41/VnbLS8wONULnWTt1xY6\nyMHKKyK+L7Xwf27EFmOrI1FqGJxQzLh5YvJ/KofHWqegn47wdc4lE/C4HBUPCsXS\n5gE/AsK7tQzgobUw4Z1PkAmBMUAOsGz+bRrFafiFQ03VJ/cAN2AUZOgGAJt9c9kv\nU4z3exjgruwjNVB6l0Lilt3kJ0adeMuuv7wuYIaQ0xi0DeIJoZdLAA==\n=d98t\n-----END PGP MESSAGE-----",
+				"fp": "B6125B16B0DD59F34D6975FBF885927FDA9C48E2"
+			}
+		],
+		"unencrypted_suffix": "_unencrypted",
+		"version": "3.7.2"
+	}
+}

ops/provision/.gitignore

@@ -0,0 +1,36 @@
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+
+# Ignore any .tfvars files that are generated automatically for each Terraform run. Most
+# .tfvars files are managed as part of configuration and so should be included in
+# version control.
+#
+# example.tfvars
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+#
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+.backend
+.env.*
+!.env.*.enc
+*-values.yaml
+kube_config.*.yml
+*.pem
+.terraform.lock.hcl

ops/provision/main.tf

@@ -0,0 +1,26 @@
+module "networking" {
+  source    = "./modules/networking"
+  namespace = var.namespace
+}
+
+module "ssh" {
+  source    = "./modules/ssh"
+  namespace = var.namespace
+}
+
+module "ec2" {
+  source             = "./modules/ec2"
+  namespace          = var.namespace
+  vpc                = module.networking.vpc
+  sg_pub_id          = module.networking.sg_pub_id
+  key_name           = module.ssh.key_name
+  keypair            = module.ssh.ssh_keypair
+  fcrepo_instance    = var.fcrepo_instance
+  fcrepo_snapshot    = var.fcrepo_snapshot
+  fcrepo_db_hostname = var.fcrepo_db_hostname
+  fcrepo_db_username = var.fcrepo_db_username
+  fcrepo_db_password = var.fcrepo_db_password
+  solr_collection    = var.solr_collection
+  site24x7_key       = var.site24x7_key
+  site24x7_group     = var.site24x7_group
+}

ops/provision/modules/ec2/iam.tf

@@ -0,0 +1,44 @@
+resource "aws_iam_role" "instance_role" {
+  name = "${var.namespace}_instance_role"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "ec2.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_instance_profile" "instance_profile" {
+  name = "${var.namespace}_profile"
+  role = "${aws_iam_role.instance_role.name}"
+}
+
+resource "aws_iam_role_policy" "instance_policy" {
+  name = "${var.namespace}_policy"
+  role = "${aws_iam_role.instance_role.id}"
+
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "s3:*"
+      ],
+      "Effect": "Allow",
+      "Resource": "*"
+    }
+  ]
+}
+EOF
+}

ops/provision/modules/ec2/main.tf

@@ -0,0 +1,74 @@
+data "aws_ami" "amazon-linux-2" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["amzn2-ami-hvm*"]
+  }
+}
+
+# data "aws_ami" "ubuntu" {
+#   most_recent = true
+#   filter {
+#     name   = "name"
+#     values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
+#   }
+
+#   filter {
+#     name = "virtualization-type"
+#     values = ["hvm"]
+#   }
+
+#   owners = ["099720109477"]
+# }
+
+resource "aws_instance" "fcrepo" {
+  for_each = {
+    prod = "prod"
+    demo = "demo"
+  }
+
+  ami                         = data.aws_ami.amazon-linux-2.id
+  associate_public_ip_address = true
+  instance_type               = var.fcrepo_instance
+  key_name                    = var.key_name
+  subnet_id                   = var.vpc.public_subnets[0]
+  vpc_security_group_ids      = [var.sg_pub_id]
+  iam_instance_profile        = "${aws_iam_instance_profile.instance_profile.name}"
+
+  tags = {
+    "Name" = "${var.namespace}-fcrepo-${each.value}"
+    "Role" = "fcrepo"
+  }
+
+  root_block_device {
+    volume_type = "gp2"
+    volume_size = 60
+  }
+
+  ebs_block_device {
+    device_name = "/dev/sdf"
+    volume_type = "gp2"
+    volume_size = 60
+    snapshot_id = var.fcrepo_snapshot != "" ? var.fcrepo_snapshot : null
+    encrypted = true
+    delete_on_termination = false
+  }
+
+  # Mounts isnt working so we write the fstab the old school way for now
+  # echo "n" keeps the fs from being overwritten if it already exists
+  user_data          =  templatefile("./modules/ec2/user_data.fcrepo.yaml", {
+    var = {
+      hostname = "${var.namespace}-fcrepo-${each.value}"
+      keypair = var.keypair
+      key_name = var.key_name
+      fcrepo_db_hostname = var.fcrepo_db_hostname
+      fcrepo_db_username = var.fcrepo_db_username
+      fcrepo_db_password = var.fcrepo_db_password
+      solr_collection = var.solr_collection
+      site24x7_key = var.site24x7_key
+      site24x7_group = var.site24x7_group
+    }
+  })
+}

ops/provision/modules/ec2/output.tf

@@ -0,0 +1,5 @@
+output "fcrepo_ips" {
+  value = {
+    for k, v in aws_instance.fcrepo : k => v.public_ip
+  }
+}

ops/provision/modules/ec2/user_data.fcrepo.yaml

@@ -0,0 +1,55 @@
+#cloud-config
+bootcmd:
+  - mkdir -p /mnt/sdf
+
+runcmd:
+  - echo "n" | mkfs.ext4 /dev/sdf
+  - echo "/dev/sdf    /mnt/sdf    auto   defaults,nofail 0  0" >> /etc/fstab
+  - mount -a
+  - echo ${base64encode(var.keypair)} | base64 -d >> /home/ec2-user/.ssh/${var.key_name}.pem
+  - chmod 400 /home/ec2-user/.ssh/${var.key_name}.pem
+  - hostnamectl set-hostname ${var.hostname}
+  - echo 'Install Java 8 and Mariadb'
+  - yum -y remove java-1.7.0-openjdk
+  - yum -y install java-1.8.0
+  - echo 'Instal Mariadb'
+  - test ! -f /mnt/sdf/complete && mkdir -p /mnt/sdf/mysql-data
+  - ln -sf /mnt/sdf/mysql-data /var/lib/mysql
+  - yum -y install mysql-devel mariadb-server
+  - service mariadb restart
+  - test ! -f /mnt/sdf/complete && mysql -e "CREATE USER '${var.fcrepo_db_username}'@'localhost' IDENTIFIED BY '${var.fcrepo_db_password}';"
+  - test ! -f /mnt/sdf/complete && mysql -e "GRANT ALL PRIVILEGES ON *.* TO '${var.fcrepo_db_username}'@'localhost';"
+  - echo 'Install Fedora'
+  - yum -y install tomcat
+  - echo 'JAVA_OPTS=\"$${JAVA_OPTS} -Dfcrepo.home=/mnt/sdf/fedora-data\"' >> /etc/sysconfig/tomcat7
+  - test ! -f /mnt/sdf/complete && mkdir -p /mnt/sdf/fedora-data
+  - test ! -f /mnt/sdf/complete && chown tomcat:tomcat /mnt/sdf/fedora-data/
+  - cd /tmp
+  - wget https://github.com/fcrepo4/fcrepo4/releases/download/fcrepo-4.7.5/fcrepo-webapp-4.7.5.war
+  - cp fcrepo-webapp-4.7.5.war /var/lib/tomcat/webapps/
+  - echo 'fcrepo.home=/mnt/sdf/fedora-data' >> /etc/tomcat/catalina.properties
+  - echo 'fcrepo.mysql.host=${var.fcrepo_db_hostname}' >> /etc/tomcat/catalina.properties
+  - echo 'fcrepo.mysql.username=${var.fcrepo_db_username}' >> /etc/tomcat/catalina.properties
+  - echo 'fcrepo.mysql.password=${var.fcrepo_db_password}' >> /etc/tomcat/catalina.properties
+  - echo 'fcrepo.modeshape.configuration=file:/var/lib/tomcat/webapps/fcrepo-webapp-4.7.5/WEB-INF/classes/config/jdbc-mysql/repository.json' >> /etc/tomcat/catalina.properties
+  - service tomcat restart
+  - echo 'Install Solr'
+  - cd /tmp
+  - wget http://archive.apache.org/dist/lucene/solr/7.7.3/solr-7.7.3.tgz
+  - tar xzf solr-7.7.3.tgz solr-7.7.3/bin/install_solr_service.sh --strip-components=2
+  - ./install_solr_service.sh solr-7.7.3.tgz -d /mnt/sdf/solr-data
+  - mkdir -p /tmp/hyrax-config/
+  - aws s3 sync s3://hyrax-install-assets/solr-config/ /tmp/hyrax-config/
+  - test ! -f /mnt/sdf/complete && sudo -u solr /opt/solr/bin/solr create -c ${var.solr_collection} -d /tmp/hyrax-config
+  - /opt/solr/bin/init.d/solr restart
+  - chkconfig mariadb on
+  - chkconfig tomcat on
+  - chkconfig solr on
+  - echo 'Install site24x7'
+  - wget https://staticdownloads.site24x7.com/server/Site24x7InstallScript.sh
+  - bash Site24x7InstallScript.sh -i -key=${var.site24x7_key} -gn=${var.site24x7_group} -tp="Default Threshold - SERVER" -np="Main"
+  - echo 'complete' >> /status
+  - touch /mnt/sdf/complete
+
+# debug logging
+output : { all : '| tee -a /var/log/cloud-init-output.log' }