Initial commit

.circleci/config.yml

@@ -0,0 +1,183 @@
+version: 2
+jobs:
+  verify:
+    docker:
+      - image: circleci/golang:latest
+    steps:
+      - checkout
+      - run:
+          name: Download and Install Terraform
+          command: |
+            wget -O terraform.zip https://releases.hashicorp.com/terraform/0.12.2/terraform_0.12.2_linux_amd64.zip
+            unzip terraform.zip
+            sudo mv terraform /usr/local/bin/terraform
+
+      - run: for dir in $(find . -name \*.tf -exec dirname {} \; | sort | uniq); do echo ${dir} && terraform init ${dir} && terraform validate ${dir}; done
+      - run: terraform fmt -check
+
+  hold:
+    machine: true
+    steps:
+      - run: echo "This build requires approval to continue..."
+  build:
+    machine: true
+    working_directory: ~/repo
+    steps:
+      - checkout
+      - run:
+          name: Create directory for artifacts
+          command: |
+            # Create artifacts directory
+            if [ ! -d "/tmp/artifacts" ]; then
+              mkdir /tmp/artifacts
+            fi
+
+      - run:
+          name: Creating a Packet Server
+          command: |
+            ## Provision a Type1 baremetal Packet.net server
+            echo "[$(date +%H:%M:%S)]: Provisioning a server on Packet.net"
+            DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "sjc1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
+            echo $DEVICE_ID > /tmp/device_id
+            # Make sure the device ID is sane.
+            # TODO: maybe make this a regex
+            if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
+              echo "[$(date +%H:%M:%S)]: Server may have failed provisionining. Device ID is set to: $DEVICE_ID"
+              echo "[$(date +%H:%M:%S)]: This usually happens if there are no servers available in the selected datacenter."
+              echo "[$(date +%H:%M:%S)]: Attempting to retry in another datacenter..."
+              export DEVICE_ID=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{ "facility": "ewr1", "plan": "baremetal_1", "hostname": "detectionlab", "description": "testing", "billing_cycle": "hourly", "operating_system": "ubuntu_16_04", "userdata": "", "locked": "false", "project_ssh_keys": ["315a9565-d5b1-41b6-913d-fcf022bb89a6", "755b134a-f63c-4fc5-9103-c1b63e65fdfc"] }' 'https://api.packet.net/projects/0b3f4f2e-ff05-41a8-899d-7923f620ca85/devices' | jq ."id" | tr -d '"')
+              if [ "$(echo -n $DEVICE_ID | wc -c)" -ne 36 ]; then
+                echo "[$(date +%H:%M:%S)]: This script was still unable to successfully provision a server. Exiting."
+                exit 1
+              fi
+            fi
+            echo "[$(date +%H:%M:%S)]: Server successfully created with ID: $DEVICE_ID"
+
+      - run:
+          name: Waiting for Packet server provisioning to complete
+          command: |
+            DEVICE_ID=$(cat /tmp/device_id)
+            echo "[$(date +%H:%M:%S)]: Waiting for server to finish provisioning..."
+            # Continue to poll the API until the state of the host is "active"
+            export STATE="provisioning"
+            while [ "$STATE" != "active" ]; do
+              sleep 10
+              echo "[$(date +%H:%M:%S)]: Sleeping for 10 seconds. Server is still $STATE."
+              export STATE="$(curl -s --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID" | jq .state | tr -d '"')"
+            done
+            echo "[$(date +%H:%M:%S)]: Device with ID $DEVICE_ID has finished provisioning! Onto the build process..."
+
+      - run:
+          name: Mount external storage
+          command: |
+            DEVICE_ID=$(cat /tmp/device_id)
+            ## Mount external storage containing Vagrant boxes
+            echo "[$(date +%H:%M:%S)]: Attempting to mount external storage to this server..."
+            MOUNT_STATUS=$(curl -s -X POST --header 'Accept: application/json' --header 'Content-Type: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" -d '{"device_id": "'"$DEVICE_ID"'"}' 'https://api.packet.net/storage/fed37d73-6719-451a-9160-df8b0addc915/attachments' | jq .id | wc -c)
+            # Stupid check to make sure MOUNT_STATUS contains a UUID
+            if [ "$MOUNT_STATUS" != "39" ]; then
+              echo "[$(date +%H:%M:%S)]: Mounting may have failed. ID is $MOUNT_STATUS"
+            else
+              echo "[$(date +%H:%M:%S)]: External storage successfully mounted!"
+            fi
+
+      - run:
+          name: Record the IP address of the Packet server
+          command: |
+            ## Recording the IP address of the newly provisioned Packet server
+            DEVICE_ID=$(cat /tmp/device_id)
+            IP_ADDRESS=$(curl -s -X GET --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" "https://api.packet.net/devices/$DEVICE_ID/ips" | jq ."ip_addresses[0].address" | tr -d '"')
+            echo $IP_ADDRESS > /tmp/ip_address
+
+      - run:
+          name: SCP Repo to Packet Server
+          command: |
+            # Copy repo to Packet server
+            # TODO: Tar up the repo and expand it remotely
+            IP_ADDRESS=$(cat /tmp/ip_address)
+            cd ~/repo && rsync -Paq -e "ssh -i ~/.ssh/id_rsa" ~/repo/ root@"$IP_ADDRESS":/opt/DetectionLab
+
+      - run:
+          name: Run the build machine bootstrap script
+          command: |
+            IP_ADDRESS=$(cat /tmp/ip_address)
+            ssh -i ~/.ssh/id_rsa root@"$IP_ADDRESS" 'bash -s' -- < ci/build_machine_bootstrap.sh
+
+      - run:
+          name: Wait for build results
+          command: |
+            export MINUTES_PAST=0
+            IP_ADDRESS=$(cat /tmp/ip_address)
+            DEVICE_ID=$(cat /tmp/device_id)
+            while [ "$MINUTES_PAST" -le 240 ]; do
+              export STATUS=$(curl -s $IP_ADDRESS)
+              if [ "$STATUS" == "building" ]; then
+                echo "[$(date +%H:%M:%S)]: $STATUS"
+                scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present"
+                sleep 300
+                ((MINUTES_PAST += 5))
+              else
+                scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/ || echo "Vagrant log not yet present"
+                echo "$STATUS" > /tmp/status
+                break
+              fi
+              if [ "$MINUTES_PAST" -gt 240 ]; then
+                echo "[$(date +%H:%M:%S)]: Serer timed out. Uptime: $MINUTES_PAST minutes."
+                scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/
+                curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"'?force_delete=true'
+                exit 1
+              fi
+            done
+
+      - run:
+          name: Post the build results
+          command: |
+            ## Recording the build results
+            STATUS=$(cat /tmp/status)
+            IP_ADDRESS=$(cat /tmp/ip_address)
+            DEVICE_ID=$(cat /tmp/device_id)
+            echo "[$(date +%H:%M:%S)]: $STATUS"
+            if [ "$STATUS" != "success" ]; then
+              scp -q -i ~/.ssh/id_rsa root@"$IP_ADDRESS":/opt/DetectionLab/Vagrant/vagrant_up_*.log /tmp/artifacts/
+              echo "Build failed. Cleaning up server with ID $DEVICE_ID"
+              curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"'?force_delete=true'
+              exit 1
+            fi
+            echo "[$(date +%H:%M:%S)]: Build was successful. Cleaning up server with ID $DEVICE_ID"
+            curl -s -X DELETE --header 'Accept: application/json' --header 'X-Auth-Token: '"$PACKET_API_TOKEN" 'https://api.packet.net/devices/'"$DEVICE_ID"'?force_delete=true'
+            exit 0
+
+      - store_artifacts:
+          path: /tmp/artifacts
+
+workflows:
+  version: 2
+  workflow:
+    jobs:
+      - verify
+      - hold:
+          requires:
+            - verify
+          type: approval
+          filters:
+            branches:
+              # Don't rebuild the PR after it has been merged into master
+              # Users cannot push directly to master as it is a protected branch
+              ignore: master
+      - build:
+          requires:
+            - hold
+          filters:
+            branches:
+              # Don't rebuild the PR after it has been merged into master
+              ignore: master
+  weekly-build:
+    jobs:
+      - build
+    triggers:
+      - schedule:
+          cron: "0 0 * * 0"
+          filters:
+            branches:
+              # Automatically re-build the contents of master once per week
+              only: master

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: clong

.github/issue_template.md

@@ -0,0 +1,25 @@
+* Operating System Version:
+* Provider (VirtualBox/VMWare):
+* Vagrant Version:
+* Packer Version:
+* Are you using stock boxes (downloaded) or were they built from scratch using Packer?
+* Is the issue reproducible or intermittent?
+
+Please verify that you are building from an updated Master branch before filing an issue.
+
+### Description of the issue:
+<!--
+For example:
+
+While building the logger host, I'm running into the following error message that causes the build to stop:
+```
+Error message goes here
+```
+-->
+
+### Link to Gist Containing Build Logs:
+<!--
+If you're having issues provisioning a host or building a Packer image, please paste the build logs into a [Gist](https://gist.github.com/) and add a link to it here.
+
+Find your Vagrant logs in `/DetectionLab/Vagrant/vagrant_up_$host.log`
+-->

.gitignore

@@ -0,0 +1,17 @@
+Vagrant/.vagrant/*
+Vagrant/vagrant*.log
+Packer/packer_cache/*
+Packer/packer_build.log
+Packer/output-*
+Boxes/*
+.DS_Store
+Terraform/*/*.tfstate
+Terraform/*/.terraform
+Terraform/*/*.tfvars
+Terraform/*/*.lock.info
+Terraform/*/*.backup
+Terraform/*.tfstate
+Terraform/*.terraform
+Terraform/*.tfvars
+Terraform/*.lock.info
+Terraform/*.backup

Boxes/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

CONTRIBUTING.md

@@ -0,0 +1,41 @@
+# Contributing to DetectionLab
+
+Please feel free to make contributions to DetectionLab that fit into any of the following categories:
+
+* Bug fixes
+* Improvements to existing configurations
+* Feature additions/enhancements
+* Tooling additions/improvements
+
+The following types of changes should be maintained on a personal fork and should **not** submitted as a PR:
+
+* [Switching out existing tooling based on personal taste](https://github.com/clong/DetectionLab/issues/43) (e.g. replacing Splunk with ELK)
+* [Adding additional Boxes/VMs](https://github.com/clong/DetectionLab/issues/125)
+* Any changes that result in drastically longer build times
+* Any configurations that are not portable
+
+
+### Pull requests
+
+All contributions are submitted via pull requests open against the
+[master](https://github.com/clong/DetectionLab/tree/master) branch. Pull requests are all reviewed and must pass continuous integration tests before being merged.
+
+If you're unfamiliar with GitHub or how pull requests work, GitHub has a very easy to follow guide
+that teaches you how to fork the project and submit your first PR. You can follow it
+[here](https://guides.github.com/activities/forking/).
+
+Once you submit your PR, it will be held for approval until someone manually approves the CI test on CircleCI.
+
+If the test fails or the reviewer requests changes, please submit those changes by **appending new
+commits** to your feature branch.
+
+Once your pull request is approved and the CircleCI build passes, the PR is ready to merge. A maintainer will merge your PR into master at this point in time.
+
+### Branches and tags
+
+The DetectionLab repo contains only the [master](https://github.com/clong/DetectionLab/tree/master) branch. I don't keep feature or release branches.
+
+## License
+
+By contributing to DetectionLab you agree that your contributions will be licensed as defined on the
+[LICENSE](LICENSE) file.

LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Chris Long
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.