Building aws athena query federation using container_env

This is intended to be run on linux with docker installed (a cloud9 linux instance will suffice)

Setup the docker image and container_env script

Just copy paste this entire block into your terminal and hit enter:

mkdir -p ~/docker_images/athena-federation-dev/

cd ~/docker_images/athena-federation-dev/

docker pull amazonlinux:2

# -----------------------

# Dockerfile
cat << "EOF" > Dockerfile
FROM amazonlinux:2

RUN amazon-linux-extras install epel -y

RUN yum -y install yum-utils && yum-config-manager --enable epel && yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo

RUN yum -y update && yum -y install java-11-amazon-corretto krb5-workstation krb5-libs krb5-auth-dialog libkadm5 json-c-devel which tar gzip unzip wget less groff glibc vim rpm rpm-build.x86_64 git python3 python3-pip jq.x86_64 libxml2.x86_64 curl zip git-lfs gh

RUN pip3 install lxml beautifulsoup4

ENV LATEST_MAVEN_VERSION="$(curl -s https://dlcdn.apache.org/maven/maven-3/ | grep DIR | tail -1 | sed 's/.*>\(.*\)\/<.*/\1/g')"

RUN bash -c "\
wget https://dlcdn.apache.org/maven/maven-3/${LATEST_MAVEN_VERSION}/binaries/apache-maven-${LATEST_MAVEN_VERSION}-bin.tar.gz -O /tmp/apache-maven-${LATEST_MAVEN_VERSION}-bin.tar.gz; \
tar xzf /tmp/apache-maven-${LATEST_MAVEN_VERSION}-bin.tar.gz -C /opt; \
mv /opt/apache-maven-${LATEST_MAVEN_VERSION} /opt/apache-maven-latest; \
"

ENV M2_HOME=/opt/apache-maven-latest

ENV PATH="${M2_HOME}/bin:${PATH}"

RUN wget https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O /tmp/awscli-exe-linux-x86_64.zip && \
    unzip /tmp/awscli-exe-linux-x86_64.zip -d /tmp/awscli && \
    /tmp/awscli/aws/install

RUN wget https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip -O /tmp/aws-sam-cli-linux-x86_64.zip && \
    unzip /tmp/aws-sam-cli-linux-x86_64.zip -d /tmp/sam-installation && \
    /tmp/sam-installation/install
EOF

# -----------------------

# build.sh
cat << "EOF" > build.sh
#!/bin/bash
SCRIPT_PATH=$(dirname $(readlink -f $0))

# cd into where the Dockerfile is (which is our SCRIPT_PATH) to build the image
cd $SCRIPT_PATH

docker image build -t athena-federation-dev .
EOF

# -----------------------

# Build the athena-federation-dev container
chmod 0755 build.sh
./build.sh

cd ~/docker_images/
# Grab the env.sh script and remove the gpus flag since we don't require gpus for this situation at all
curl -s https://raw.githubusercontent.com/henrymai/container_env/master/env.sh | sed 's#.*--gpus.*# \\#g' > env.sh
chmod 0755 env.sh


# --------------
# Usage looks like (you can run this from anywhere)
# IMAGE=athena-federation-dev ~/docker_images/env.sh bash

How to build the Athena Federation SDK and Connectors

mkdir -p ~/workspace

cd ~/workspace

git clone [email protected]:awslabs/aws-athena-query-federation.git

cd aws-athena-query-federation

IMAGE=athena-federation-dev ~/docker_images/env.sh bash

mvn clean install -T 2C