feat: add initial module code

README.md

@@ -32,3 +32,46 @@ Terraform linter for finding possible errors, old syntax, unused declarations et
 # Recommended Repo configuration
 
 We recommend protecting the `main` branch and to allow new code pushes only via Pull Requests. This way it's ensured that all tests pass before a new release is pushed.
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.7.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.40.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.40.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_athena_database.cloudfront](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_database) | resource |
+| [aws_athena_database.cloudtrail](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_database) | resource |
+| [aws_athena_database.ses](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_database) | resource |
+| [aws_athena_named_query.cloudfront](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_named_query) | resource |
+| [aws_athena_named_query.cloudtrail](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_named_query) | resource |
+| [aws_athena_named_query.ses](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_named_query) | resource |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_cloudfront"></a> [cloudfront](#input\_cloudfront) | The name of the s3 bucket containing the cloudfront logs. Creates a db and saved cloudfront queries if set. | `string` | `null` | no |
+| <a name="input_cloudtrail"></a> [cloudtrail](#input\_cloudtrail) | Configuration for cloudtrail. Creates a db and saved cloudfront queries if bucket\_name is set. Only set prefix if you configured one in your cloudtrail | <pre>object({<br/>    bucket_name = optional(string)<br/>    prefix      = optional(string)<br/>  })</pre> | `{}` | no |
+| <a name="input_query_bucket_name"></a> [query\_bucket\_name](#input\_query\_bucket\_name) | The name of the bucket to save the query into. | `string` | n/a | yes |
+| <a name="input_ses"></a> [ses](#input\_ses) | The name of the s3 bucket containing the ses logs. Creates a db and saved ses queries if set | `string` | `null` | no |
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

main.tf

@@ -0,0 +1,93 @@
+locals {
+  named_queries = {
+    cloudfront = var.cloudfront != null ? {
+      cloudfront_create_log_table               = "Create the table for CloudFront logs",
+      cloudfront_log_yesterday_today            = "Show requests to CloudFront yesterday and today",
+      cloudfront_log_yesterday_today_for_ip     = "Show requests to CloudFront from a given IP yesterday and today"
+      cloudfront_logs_for_specific_distribution = "Shows requests to a given cloudfront distribution, with optional filter for date, status code or ssl_protocol"
+    } : {}
+    cloudtrail = var.cloudtrail.bucket_name != null ? {
+      cloudtrail_create_log_table = "Create the table for CloudTrail logs",
+      cloudtrail_console_login    = "Displays console login events, ordered by date"
+    } : {}
+    ses = var.ses != null ? {
+      ses_create_log_table             = "Create the table for SES logs"
+      ses_bounce_ratio                 = "Displays bounced email as % of send emails"
+      ses_formatted_mails              = "Displays only the most relevant information per email, optional filter for event type"
+      ses_daily_not_suppressed_bounces = "Displays amount of bounced emails per day, filtering out bounces caused by addresses being on the suppression list"
+    } : {}
+  }
+
+}
+
+data "aws_caller_identity" "current" {}
+
+resource "aws_athena_database" "cloudfront" {
+  count = var.cloudfront != null ? 1 : 0
+
+  name   = "cloudfront"
+  bucket = var.query_bucket_name
+  encryption_configuration {
+    encryption_option = "SSE_S3"
+  }
+}
+
+resource "aws_athena_database" "cloudtrail" {
+  count = var.cloudtrail.bucket_name != null ? 1 : 0
+
+  name   = "cloudtrail"
+  bucket = var.query_bucket_name
+  encryption_configuration {
+    encryption_option = "SSE_S3"
+  }
+}
+
+resource "aws_athena_database" "ses" {
+  count = var.ses != null ? 1 : 0
+
+  name   = "ses"
+  bucket = var.query_bucket_name
+  encryption_configuration {
+    encryption_option = "SSE_S3"
+  }
+}
+
+resource "aws_athena_named_query" "cloudfront" {
+  for_each    = local.named_queries.cloudfront
+  name        = replace(each.key, "_", "-")
+  database    = aws_athena_database.cloudfront[0].name
+  description = each.value
+  query = templatefile("${path.module}/queries/${each.key}.sql.tftpl",
+    {
+      "bucket"  = var.cloudfront,
+      "account" = data.aws_caller_identity.current.account_id,
+    }
+  )
+}
+
+resource "aws_athena_named_query" "cloudtrail" {
+  for_each    = local.named_queries.cloudtrail
+  name        = replace(each.key, "_", "-")
+  database    = aws_athena_database.cloudtrail[0].name
+  description = each.value
+  query = templatefile("${path.module}/queries/${each.key}.sql.tftpl",
+    {
+      "bucket"  = var.cloudtrail.bucket_name,
+      "account" = data.aws_caller_identity.current.account_id,
+      "prefix"  = var.cloudtrail.prefix != null ? "${var.cloudtrail.prefix}/" : ""
+    }
+  )
+}
+
+resource "aws_athena_named_query" "ses" {
+  for_each    = local.named_queries.ses
+  name        = replace(each.key, "_", "-")
+  database    = aws_athena_database.ses[0].name
+  description = each.value
+  query = templatefile("${path.module}/queries/${each.key}.sql.tftpl",
+    {
+      "bucket"  = var.ses,
+      "account" = data.aws_caller_identity.current.account_id,
+    }
+  )
+}

queries/cloudfront_create_log_table.sql.tftpl

@@ -0,0 +1,40 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS cloudfront_logs (
+    `date` DATE,
+    time STRING,
+    location STRING,
+    bytes BIGINT,
+    request_ip STRING,
+    method STRING,
+    host STRING,
+    uri STRING,
+    status INT,
+    referrer STRING,
+    user_agent STRING,
+    query_string STRING,
+    cookie STRING,
+    result_type STRING,
+    request_id STRING,
+    host_header STRING,
+    request_protocol STRING,
+    request_bytes BIGINT,
+    time_taken FLOAT,
+    xforwarded_for STRING,
+    ssl_protocol STRING,
+    ssl_cipher STRING,
+    response_result_type STRING,
+    http_version STRING,
+    fle_status STRING,
+    fle_encrypted_fields INT,
+    c_port INT,
+    time_to_first_byte FLOAT,
+    x_edge_detailed_result_type STRING,
+    sc_content_type STRING,
+    sc_content_len BIGINT,
+    sc_range_start BIGINT,
+    sc_range_end BIGINT
+)
+    ROW FORMAT DELIMITED
+    FIELDS TERMINATED BY '\t'
+    LOCATION 's3://${ bucket }/'
+    TBLPROPERTIES ( 'skip.header.line.count'='2' )
+;

queries/cloudfront_log_yesterday_today.sql.tftpl

@@ -0,0 +1,11 @@
+SELECT
+    CONCAT(CAST(date AS VARCHAR), 'T', CAST(time AS VARCHAR)) AS timestamp,
+    request_ip as "client_ip",
+    status as "cloudfront_status_code",
+    method as "method",
+    uri as "uri",
+    user_agent as "ua"
+FROM cloudfront.cloudfront_logs
+WHERE
+  date BETWEEN date_add('day', -1, current_timestamp) and current_timestamp
+;

queries/cloudfront_log_yesterday_today_for_ip.sql.tftpl

@@ -0,0 +1,13 @@
+SELECT
+    CONCAT(CAST(date AS VARCHAR), 'T', CAST(time AS VARCHAR)) AS timestamp,
+    request_ip as "client_ip",
+    status as "cloudfront_status_code",
+    method as "method",
+    uri as "uri",
+    user_agent as "ua"
+FROM cloudfront.cloudfront_logs
+WHERE
+    date BETWEEN date_add('day', -1, current_timestamp) and current_timestamp
+AND
+    request_ip = '$REPLACE_ME'
+;

queries/cloudfront_logs_for_specific_distribution.sql.tftpl

@@ -0,0 +1,6 @@
+    SELECT * FROM cloudfront.cloudfront_logs
+    WHERE host='https://$CLOUDFRONT_NAME.cloudfront.net'
+    -- AND "date" BETWEEN DATE '2023-08-10' AND DATE '2023-08-11' --date is a reserved word and needs to be doublequoted
+    -- AND status=200
+    -- AND ssl_protocol=TLSv1.2
+    LIMIT 100;

queries/cloudtrail_console_login.sql.tftpl

@@ -0,0 +1,4 @@
+select useridentity.username, sourceipaddress, eventtime, additionaleventdata
+from cloudtrail.cloudtrail_logs
+where eventname = 'ConsoleLogin'
+order by date(from_iso8601_timestamp(eventtime)) desc

queries/cloudtrail_create_log_table.sql.tftpl

@@ -0,0 +1,56 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS cloudtrail_logs (
+    eventVersion STRING,
+    userIdentity STRUCT<
+        type: STRING,
+        principalId: STRING,
+        arn: STRING,
+        accountId: STRING,
+        invokedBy: STRING,
+        accessKeyId: STRING,
+        userName: STRING,
+        sessionContext: STRUCT<
+            attributes: STRUCT<
+                mfaAuthenticated: STRING,
+                creationDate: STRING>,
+            sessionIssuer: STRUCT<
+                type: STRING,
+                principalId: STRING,
+                arn: STRING,
+                accountId: STRING,
+                username: STRING>,
+            ec2RoleDelivery: STRING,
+            webIdFederationData: MAP<STRING,STRING>>>,
+    eventTime STRING,
+    eventSource STRING,
+    eventName STRING,
+    awsRegion STRING,
+    sourceIpAddress STRING,
+    userAgent STRING,
+    errorCode STRING,
+    errorMessage STRING,
+    requestParameters STRING,
+    responseElements STRING,
+    additionalEventData STRING,
+    requestId STRING,
+    eventId STRING,
+    resources ARRAY<STRUCT<
+        arn: STRING,
+        accountId: STRING,
+        type: STRING>>,
+    eventType STRING,
+    apiVersion STRING,
+    readOnly STRING,
+    recipientAccountId STRING,
+    serviceEventDetails STRING,
+    sharedEventID STRING,
+    vpcEndpointId STRING,
+    tlsDetails STRUCT<
+        tlsVersion: STRING,
+        cipherSuite: STRING,
+        clientProvidedHostHeader: STRING>
+)
+ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'
+STORED AS INPUTFORMAT 'com.amazon.emr.cloudtrail.CloudTrailInputFormat'
+OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
+LOCATION 's3://${ bucket }/${ prefix }AWSLogs/${ account }/CloudTrail/'
+TBLPROPERTIES ('classification'='cloudtrail');

queries/ses_bounce_ratio.sql.tftpl

@@ -0,0 +1,12 @@
+select bounces.date, bounces, send, ((bounces * 1.0)/(send * 1.0) * 100 ) as "% bounces"
+    from
+        ( select
+            date(from_iso8601_timestamp(mail.timestamp)) as date, count(*) as bounces from ses.ses_bounce_logs where  eventtype='Bounce'and bounce.bouncedrecipients[1].diagnosticcode not like '%Amazon SES did not send the message to this address%'
+        GROUP BY date(from_iso8601_timestamp(mail.timestamp)) order by date(from_iso8601_timestamp(mail.timestamp)) desc) bounces
+
+    inner join
+        ( select
+            date(from_iso8601_timestamp(mail.timestamp)) as date, count(*) as send from ses.ses_bounce_logs where  eventtype='Send'
+        GROUP BY date(from_iso8601_timestamp(mail.timestamp)) order by date(from_iso8601_timestamp(mail.timestamp)) desc) send
+    ON
+     bounces.date = send.date

queries/ses_create_log_table.sql.tftpl

@@ -0,0 +1,63 @@
+CREATE EXTERNAL TABLE IF NOT EXISTS ses (
+eventType string,
+complaint struct < arrivaldate: string,
+complainedrecipients: array < struct < emailaddress: string >>,
+complaintfeedbacktype: string,
+feedbackid: string,
+`timestamp`: string,
+useragent: string >,
+bounce struct < bouncedrecipients: array < struct < action: string,
+diagnosticcode: string,
+emailaddress: string,
+status: string >>,
+bouncesubtype: string,
+bouncetype: string,
+feedbackid: string,
+reportingmta: string,
+`timestamp`: string >,
+mail struct < timestamp: string,
+source: string,
+sourcearn: string,
+sendingaccountid: string,
+messageid: string,
+destination: string,
+headerstruncated: boolean,
+headers: array < struct < name: string,
+value: string >>,
+commonheaders: struct < `from`: array < string >,
+to: array < string >,
+messageid: string,
+subject: string >,
+tags: struct < ses_source_tls_version: string,
+ses_operation: string,
+ses_configurationset: string,
+ses_source_ip: string,
+ses_outgoing_ip: string,
+ses_from_domain: string,
+ses_caller_identity: string >>,
+send string,
+delivery struct < processingtimemillis: int,
+recipients: array < string >,
+reportingmta: string,
+smtpresponse: string,
+`timestamp`: string >,
+open struct < ipaddress: string,
+`timestamp`: string,
+userAgent: string >,
+reject struct < reason: string >,
+click struct < ipAddress: string,
+`timestamp`: string,
+userAgent: string,
+link: string >
+)
+ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
+WITH SERDEPROPERTIES (
+"mapping.ses_caller_identity" = "ses:caller-identity",
+"mapping.ses_configurationset" = "ses:configuration-set",
+"mapping.ses_from_domain" = "ses:from-domain",
+"mapping.ses_operation" = "ses:opeation",
+"mapping.ses_outgoing_ip" = "ses:outgoing-ip",
+"mapping.ses_source_ip" = "ses:source-ip",
+"mapping.ses_source_tls_version" = "ses:source-tls-version"
+)
+LOCATION 's3://${ bucket }/'

queries/ses_daily_not_suppressed_bounces.sql.tftpl

@@ -0,0 +1,3 @@
+select
+    date(from_iso8601_timestamp(mail.timestamp)) as date, count(*) as bounces from ses.ses_bounce_logs where  eventtype='Bounce'and bounce.bouncedrecipients[1].diagnosticcode not like '%Amazon SES did not send the message to this address%'
+GROUP BY date(from_iso8601_timestamp(mail.timestamp)) order by date(from_iso8601_timestamp(mail.timestamp)) desc

queries/ses_formatted_mails.sql.tftpl

@@ -0,0 +1,2 @@
+select mail.timestamp, mail.source, mail.destination, bounce.bouncedrecipients[1].diagnosticcode as diagnosticcode from ses.ses_bounce_logs
+-- where  eventtype='Bounce' -- or any other event type

variables.tf

@@ -0,0 +1,25 @@
+variable "query_bucket_name" {
+  description = "The name of the bucket to save the query into."
+  type        = string
+}
+
+variable "cloudfront" {
+  description = "The name of the s3 bucket containing the cloudfront logs. Creates a db and saved cloudfront queries if set."
+  type        = string
+  default     = null
+}
+
+variable "cloudtrail" {
+  description = "Configuration for cloudtrail. Creates a db and saved cloudfront queries if bucket_name is set. Only set prefix if you configured one in your cloudtrail"
+  type = object({
+    bucket_name = optional(string)
+    prefix      = optional(string)
+  })
+  default = {}
+}
+
+variable "ses" {
+  description = "The name of the s3 bucket containing the ses logs. Creates a db and saved ses queries if set"
+  type        = string
+  default     = null
+}

versions.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_version = ">= 1.7.0"
+  required_providers {
+    aws = {
+      version = ">= 5.40.0"
+    }
+  }
+}