Merge branch 'dev' into USAGOV-2050-Cypress-Github-Actions

.sqlfluff

@@ -0,0 +1,19 @@
+[sqlfluff]
+# Supported dialects https://docs.sqlfluff.com/en/stable/perma/dialects.html
+# Or run 'sqlfluff dialects'
+dialect = mysql
+# One of [raw|jinja|python|placeholder]
+templater = raw
+# Very large files can make the parser effectively hang.
+# The more efficient check is the _byte_ limit check which
+# is enabled by default. The previous _character_ limit check
+# is still present for backward compatibility. This will be
+# removed in a future version.
+# Set either to 0 to disable.
+large_file_skip_char_limit = 0
+large_file_skip_byte_limit = 210000000
+# CPU processes to use while linting.
+# If positive, just implies number of processes.
+# If negative or zero, implies number_of_cpus - specified_number.
+# e.g. -1 means use all processors but one. 0  means all cpus.
+processes = 0

bin/cloudgov/audit/db-integrity

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+#
+# Requires sqlfluff, run brew[or pip] install sqlfluff if this doesn't work.
+#
+
+# we might be running in circleci
+if [ -f /home/circleci/project/env.local ]; then
+  . /home/circleci/project/env.local
+fi
+# we might be running from a local dev machine
+SCRIPT_DIR="$(dirname "$0")"
+if [ -f $SCRIPT_DIR/env.local ]; then
+  . $SCRIPT_DIR/env.local
+fi
+if [ -f ./env.local ]; then
+  . ./env.local
+fi
+if [ -f $SCRIPT_DIR/../../deploy/includes ]; then
+  . $SCRIPT_DIR/../../deploy/includes
+else
+   echo Cannot find $SCRIPT_DIR/../../deploy/includes
+   exit 1
+fi
+
+SPACE=${1:-please-provide-space-name-as-first-argument}
+SPACE=$(echo "$SPACE" | tr '[:upper:]' '[:lower:]')
+assertCurSpace $SPACE
+shift
+
+echo "Testing static site integrity."
+
+echo "Getting s3 access."
+source bin/cloudgov/get-s3-access storage >/dev/null 2>&1
+
+echo "Getting three random backup tags from s3."
+DEPLOY_TAGS=$(aws s3 ls s3://"$S3_BUCKET"/db-backup/ | awk '{ print $4 }' | sort -R | tail -n 3 )
+
+echo "Tags:"
+echo "$DEPLOY_TAGS"
+
+echo "----------------------------------------------------------------------"
+echo "$DEPLOY_TAGS" | while read -r DEPLOY_TAG ; do
+  echo "Downloading db: $DEPLOY_TAG"
+  aws s3 cp --only-show-errors s3://"$S3_BUCKET"/db-backup/"$DEPLOY_TAG" "$DEPLOY_TAG"
+  gunzip "$DEPLOY_TAG"
+  sleep 5
+  SQL_FILE=$(echo "$DEPLOY_TAG" | sed 's/.gz$//')
+
+  echo "Testing: $SQL_FILE"
+
+  size=$(du -hs "$SQL_FILE")
+  echo "SQL file size: $(echo "$size" | awk '{print $1}')"
+
+  echo "Splitting and Validating SQL. This will take a while. (Split function will output \"csplit: ^CREATE TABLE .\*: no match\"; this is normal, it just means it can't find anymore matches because it's done.)"
+
+  csplit -n 4 -k $SQL_FILE '/^CREATE TABLE .*/' '{800}' > db_log.log
+
+  # Read the first line, extract table name and rename the file
+  ERROR=""
+  for f in $(ls xx*); do
+    table_name=$(head -1 $f | awk '{ sub(/.*CREATE TABLE /, ""); sub(/ .*/, ""); print }')
+    echo "Validating table $table_name (split from $SQL_FILE)" >> db_log.log
+
+    validate=$(sqlfluff lint "$f")
+
+    if [ "$validate" != "All Finished!" ]; then
+      ERROR+=$validate
+    fi
+    rm $f
+  done;
+  if [ -n "$ERROR" ]; then
+    echo -e "\033[0;31mSQL file is invalid. SQL file will be retained for further inspection.  Error(s):"
+    echo -e "$validate\033[0m"
+  else
+    echo "SQL valid."
+    rm $SQL_FILE
+  fi
+echo "----------------------------------------------------------------------"
+
+done
+echo "Automated testing complete."

bin/cloudgov/events/README.md

@@ -0,0 +1,12 @@
+# Cloud Foundry Event API - events available from the API
+
+
+## There are two files in this directory, containing info on the events available from the API
+
+1. all-event-names.txt
+
+    This is just a text file containing all the event names available.  This was used to when upgrading the `get-events` from API v2 to v3. (compared the deprecated event names in the script against the new event names in order to implement and changes/additions/deletions).
+
+1. v3-events.json
+
+    This is a formatted JSON file, with the category names, as well as the event names. This could theoretically be used to build event queries programmatically, rather than have them hardcoded into the `get-events` (or any other) script.

bin/cloudgov/events/all-event-names.txt

@@ -0,0 +1,118 @@
+app.crash
+audit.app.apply_manifest
+audit.app.build.create
+audit.app.copy-bits
+audit.app.create
+audit.app.delete-request
+audit.app.deployment.cancel
+audit.app.deployment.continue
+audit.app.deployment.create
+audit.app.droplet.create
+audit.app.droplet.delete
+audit.app.droplet.download
+audit.app.droplet.mapped
+audit.app.droplet.upload
+audit.app.environment.show
+audit.app.environment_variables.show
+audit.app.map-route
+audit.app.package.create
+audit.app.package.delete
+audit.app.package.download
+audit.app.package.upload
+audit.app.process.crash
+audit.app.process.create
+audit.app.process.delete
+audit.app.process.not-ready
+audit.app.process.ready
+audit.app.process.rescheduling
+audit.app.process.scale
+audit.app.process.terminate_instance
+audit.app.process.update
+audit.app.restage
+audit.app.restart
+audit.app.revision.create
+audit.app.revision.environment_variables.show
+audit.app.ssh-authorized
+audit.app.ssh-unauthorized
+audit.app.start
+audit.app.stop
+audit.app.task.cancel
+audit.app.task.create
+audit.app.unmap-route
+audit.app.update
+audit.app.upload-bits
+audit.organization.create
+audit.organization.delete-request
+audit.organization.update
+audit.route.create
+audit.route.delete-request
+audit.route.share
+audit.route.transfer-owner
+audit.route.unshare
+audit.route.update
+audit.service.create
+audit.service.delete
+audit.service_binding.create
+audit.service_binding.delete
+audit.service_binding.show
+audit.service_binding.start_create
+audit.service_binding.start_delete
+audit.service_binding.update
+audit.service_broker.create
+audit.service_broker.delete
+audit.service_broker.update
+audit.service_dashboard_client.create
+audit.service_dashboard_client.delete
+audit.service_instance.bind_route
+audit.service_instance.create
+audit.service_instance.delete
+audit.service_instance.purge
+audit.service_instance.share
+audit.service_instance.show
+audit.service_instance.start_create
+audit.service_instance.start_delete
+audit.service_instance.start_update
+audit.service_instance.unbind_route
+audit.service_instance.unshare
+audit.service_instance.update
+audit.service_key.create
+audit.service_key.delete
+audit.service_key.show
+audit.service_key.start_create
+audit.service_key.start_delete
+audit.service_key.update
+audit.service_plan.create
+audit.service_plan.delete
+audit.service_plan.update
+audit.service_plan_visibility.create
+audit.service_plan_visibility.delete
+audit.service_plan_visibility.update
+audit.service_route_binding.create
+audit.service_route_binding.delete
+audit.service_route_binding.start_create
+audit.service_route_binding.start_delete
+audit.service_route_binding.update
+audit.space.create
+audit.space.delete-request
+audit.space.update
+audit.user.organization_auditor_add
+audit.user.organization_auditor_remove
+audit.user.organization_billing_manager_add
+audit.user.organization_billing_manager_remove
+audit.user.organization_manager_add
+audit.user.organization_manager_remove
+audit.user.organization_user_add
+audit.user.organization_user_remove
+audit.user.space_auditor_add
+audit.user.space_auditor_remove
+audit.user.space_developer_add
+audit.user.space_developer_remove
+audit.user.space_manager_add
+audit.user.space_manager_remove
+audit.user.space_supporter_add
+audit.user.space_supporter_remove
+audit.user_provided_service_instance.create
+audit.user_provided_service_instance.delete
+audit.user_provided_service_instance.show
+audit.user_provided_service_instance.update
+blob.remove_orphan

bin/cloudgov/events/get-events

@@ -87,38 +87,28 @@ if [ -z "$LAST_EVENT_TIME" ]; then
 fi
 
 declare -rA AUDIT_TYPES=(
-    ["app"]="audit.app.create,audit.app.start,audit.app.restage,audit.app.stop,audit.app.delete"
-    ["user"]="audit.user.space_developer_add,audit.user.space_developer_remove,audit.user.space_auditor_add,audit.user.space_auditor_remove,audit.user.space_manager_add,audit.user.space_manager_remove"
-    ["route"]="audit.route.create,audit.route.delete-request,audit.route.update"
-    ["service"]="audit.service.create,audit.service.delete,audit.service.update,audit.service_binding.create,audit.service_binding.delete,service_instance.bind_route,audit.service_instance.create,audit.service_instance.delete,audit.service_instance.unbind_route,audit.service_instance.update"
-    ["service_instance"]="audit.service_instance.create,audit.service_instance.bind_route,audit.service_instance.update,audit.service_instance.unbind_route,audit.service_instance.delete"
+  ["all_events"]="audit.app.create,audit.app.delete-request,audit.app.restage,audit.app.restart,audit.app.start,audit.app.stop,audit.app.task.cancel,audit.app.task.create,audit.app.ssh-authorized,audit.app.ssh-unauthorized,audit.app.map-route,audit.app.unmap-route,audit.route.create,audit.route.delete-request,audit.route.update,audit.service_instance.bind_route,audit.service_instance.unbind_route,audit.user.organization_manager_add,audit.user.organization_manager_remove,audit.user.space_auditor_add,audit.user.space_auditor_remove,audit.user.space_developer_add,audit.user.space_developer_remove,audit.user.space_manager_add,audit.user.space_manager_remove,audit.service.create,audit.service.delete,audit.service_binding.create,audit.service_binding.delete,audit.service_binding.update,audit.service_instance.create,audit.service_instance.delete,audit.service_instance.update,audit.service_key.create,audit.service_key.delete,audit.service_key.update,audit.space.create,audit.space.delete-request,audit.space.update,audit.user_provided_service_instance.create,audit.user_provided_service_instance.delete,audit.user_provided_service_instance.update,audit.app.deployment.create,audit.app.deployment.cancel,audit.app.droplet.create,audit.app.droplet.delete,audit.app.process.create,audit.app.process.crash,audit.app.process.update,audit.app.process.terminate_instance"
 )
 
 ### These are the same currently, but may change. Therefore I've got them in variables for now
 declare -rA AUDIT_FIELDS=(
-                 ["app"]=".metadata.guid, .entity.timestamp, .entity.type, .entity.actee_name, .entity.space_guid, .entity.actor_type, .entity.actor_name"
-                ["user"]=".metadata.guid, .entity.timestamp, .entity.type, .entity.actee_name, .entity.space_guid, .entity.actor_type, .entity.actor_name"
-               ["route"]=".metadata.guid, .entity.timestamp, .entity.type, .entity.actee_name, .entity.space_guid, .entity.actor_type, .entity.actor_name"
-             ["service"]=".metadata.guid, .entity.timestamp, .entity.type, .entity.actee_name, .entity.space_guid, .entity.actor_type, .entity.actor_name"
-    ["service_instance"]=".metadata.guid, .entity.timestamp, .entity.type, .entity.actee_name, .entity.space_guid, .entity.actor_type, .entity.actor_name"
+  ["all_events"]=".guid, .created_at, .type, .target.name, .space.guid, .actor.type, .actor.name"
 )
 
 ### Reformat event outoput into key:value pairs that can be more easily ingested by NewRelic (1)
 match='"\(.*\)","\(.*\)","\(.*\)","\(.*\)","\(.*\)","\(.*\)","\(.*\)"'
-replace='{"cfevent.metadata.guid":"\1","timestamp":"\2","cfevent.entity.type":"\3","cfevent.entity.actee_name":"\4","cfevent.entity.space_name":"\5","cfevent.entity.actor_type":"\6","cfevent.entity.actor_name":"\7","mobomo.logrev.cfevent":"2"}'
+replace='{"cfevent.metadata.guid":"\1","timestamp":"\2","cfevent.entity.type":"\3","cfevent.entity.actee_name":"\4","cfevent.entity.space_name":"\5","cfevent.entity.actor_type":"\6","cfevent.entity.actor_name":"\7","mobomo.logrev.cfevent":"3"}'
 
 # For each category of audit events, perform a (possibly multi-page) query 
 for resource_type in ${!AUDIT_TYPES[@]}; do
     types=${AUDIT_TYPES[$resource_type]}
     fields=${AUDIT_FIELDS[$resource_type]}
 
     # Manually create the query to fetch the first page of results
-    NEXT="/v2/events?&q=type+IN+${types}&q=organization_guid:${ORG_GUID}&q=space_guid:${SPACE_GUID}&q=timestamp>${LAST_EVENT_TIME}&order-by:timestamp"
-
-    while [ -n $NEXT -a ! $NEXT == 'null' ]; do
+    NEXT="/v3/audit_events?types=${types}&organization_guids=${ORG_GUID}&space_guids=${SPACE_GUID}&created_ats[gt]=${LAST_EVENT_TIME}&order_by=created_at"
+    while [ -n "$NEXT" -a ! "$NEXT" == 'null' ]; do
 
         EVENTS=$(cf curl $NEXT)
-
         ## Reformat event outoput into key:value pairs that can be more easily ingested by NewRelic (2)
         captured_events=$(echo "$EVENTS" | jq -r ".resources[] | [$fields] | @csv" | sed "s/$match/$replace/g")
 
@@ -133,6 +123,6 @@ for resource_type in ${!AUDIT_TYPES[@]}; do
         fi
 
         # Each query provides the URL for the next page of results.  Use it to loop through pages
-        NEXT=$(echo $EVENTS | jq -r '.next_url')
+        NEXT=$(echo $EVENTS | jq -r '.pagination.next.href' | sed 's/\\//g' | sed 's/https:\/\/api.fr.cloud.gov//')
     done
 done