Enable S3 tests

Move dependencies into tests/functional/common.sh.
Skip build and install if SKIP_BUILD is set.
Rename configuration file to container_hierarchy

tests/functional/simple.cfg → conf/s3_container_hierarchy.cfg

@@ -1,11 +1,11 @@
 [DEFAULT]
 bind_port = 5000
 workers = 1
-user = murlock
+user = USER
 log_facility = LOG_LOCAL0
 log_level = DEBUG
 eventlet_debug = true
-sds_default_account = murlock
+sds_default_account = AUTH_demo
 
 [pipeline:main]
 #pipeline = catch_errors cache swift3 tempauth proxy-logging slo versioned_writes proxy-server
@@ -19,7 +19,6 @@ allow_account_management = true
 account_autocreate = true
 sds_namespace = OPENIO
 sds_proxy_url = http://127.0.0.1:6000
-user = murlock
 # object_post_as_copy = false
 object_post_as_copy = true
 log_name = OIO,NS,oioswift,1
@@ -101,3 +100,4 @@ account_first = true
 # Create empty objects to mark subdirectories. This enables
 # recursive object listings.
 create_dir_placeholders = true
+recursive_placeholders = true

test-requirements.txt

@@ -4,3 +4,4 @@ oio>=3.3.0
 coverage
 codecov
 awscli
+lxml

tests/functional/common.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+function install_deps() {
+  if [ -n "${SKIP_BUILD}" ]; then
+    return
+  fi
+  sudo apt-get install -y --force-yes \
+    apache2 apache2-dev libapache2-mod-wsgi \
+    beanstalkd \
+    bison \
+    curl libcurl4-gnutls-dev \
+    flex \
+    libapreq2-dev \
+    libattr1-dev \
+    liberasurecode-dev \
+    libevent-dev \
+    libglib2.0-dev \
+    libjson-c-dev \
+    libleveldb1 libleveldb-dev \
+    liblzo2-dev \
+    libsqlite3-dev \
+    libzmq3-dev \
+    libzookeeper-mt-dev \
+    openio-gridinit openio-asn1c \
+    python-all-dev python-virtualenv
+}
+
+function compile_sds() {
+  if [ -n "${SKIP_BUILD}" ]; then
+    return
+  fi
+  cd third_party/oio-sds || return
+  cmake \
+    -DCMAKE_INSTALL_PREFIX="/tmp/oio" \
+    -DLD_LIBDIR="lib" \
+    -DCMAKE_BUILD_TYPE="Debug" \
+    -DSTACK_PROTECTOR=1 \
+    -DZK_LIBDIR="/usr/lib" \
+    -DZK_INCDIR="/usr/include/zookeeper" \
+    -DAPACHE2_LIBDIR="/usr/lib/apache2" \
+    -DAPACHE2_INCDIR="/usr/include/apache2" \
+    -DAPACHE2_MODDIR="/tmp/oio/lib/apache2/module" \
+    .
+  make all install
+  cd ../.. || return
+}

tests/functional/run_ns_wide_versioning_tests.sh

@@ -1,43 +1,6 @@
 #!/bin/bash
 
-function install_deps() {
-  sudo apt-get install -y --force-yes \
-    apache2 apache2-dev libapache2-mod-wsgi \
-    beanstalkd \
-    bison \
-    curl libcurl4-gnutls-dev \
-    flex \
-    libapreq2-dev \
-    libattr1-dev \
-    liberasurecode-dev \
-    libevent-dev \
-    libglib2.0-dev \
-    libjson-c-dev \
-    libleveldb1 libleveldb-dev \
-    liblzo2-dev \
-    libsqlite3-dev \
-    libzmq3-dev \
-    libzookeeper-mt-dev \
-    openio-gridinit openio-asn1c \
-    python-all-dev python-virtualenv
-}
-
-function compile_sds() {
-  cd third_party/oio-sds || return
-  cmake \
-    -DCMAKE_INSTALL_PREFIX="/tmp/oio" \
-    -DLD_LIBDIR="lib" \
-    -DCMAKE_BUILD_TYPE="Debug" \
-    -DSTACK_PROTECTOR=1 \
-    -DZK_LIBDIR="/usr/lib" \
-    -DZK_INCDIR="/usr/include/zookeeper" \
-    -DAPACHE2_LIBDIR="/usr/lib/apache2" \
-    -DAPACHE2_INCDIR="/usr/include/apache2" \
-    -DAPACHE2_MODDIR="/tmp/oio/lib/apache2/module" \
-    .
-  make all install
-  cd ../.. || return
-}
+source tests/functional/common.sh
 
 function run_sds() {
   export G_DEBUG_LEVEL=D PATH="$PATH:/tmp/oio/bin" LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/tmp/oio/lib"

tests/functional/run_s3_tests.sh

@@ -1,16 +1,23 @@
 #!/bin/bash
 
-CURDIR="$( cd "$(dirname "$0")" ; pwd -P )"
-
-# CREATE AWS CONFIGURATION
-mkdir -p $HOME/.aws
-cat <<EOF >$HOME/.aws/credentials
+source tests/functional/common.sh
+
+function run_sds() {
+  export G_DEBUG_LEVEL=D PATH="$PATH:/tmp/oio/bin" LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/tmp/oio/lib"
+  oio-reset.sh -v -v -N "$OIO_NS" \
+    -f third_party/oio-sds/etc/bootstrap-preset-SINGLE.yml
+}
+
+function configure_aws() {
+  # CREATE AWS CONFIGURATION
+  mkdir -p $HOME/.aws
+  cat <<EOF >$HOME/.aws/credentials
 [default]
 aws_access_key_id=demo:demo
 aws_secret_access_key=DEMO_PASS
 EOF
 
-cat <<EOF >$HOME/.aws/config
+  cat <<EOF >$HOME/.aws/config
 [default]
 s3 =
     signature_version = s3
@@ -19,22 +26,29 @@ s3 =
     multipart_threshold = 15MB
     multipart_chunksize = 5MB
 EOF
+}
+
+function configure_oioswift() {
+    sed -i "s/USER/$(id -un)/g" "$1"
+}
+
 
-coverage run -a runserver.py ${CURDIR}/simple.cfg -v &
+export OIO_NS="OPENIO" OIO_ACCOUNT="test_account" OIO_USER=USER-$RANDOM OIO_PATH=PATH-$RANDOM
+install_deps
+compile_sds
+run_sds
+configure_aws
+configure_oioswift conf/s3_container_hierarchy.cfg
+
+coverage run -p runserver.py conf/s3_container_hierarchy.cfg -v &
 sleep 1
 PID=$(jobs -p)
 
-for i in $CURDIR/s3_*.sh; do
-    bash $i
-    if [ $? -ne 0 ]; then
-        echo "Exiting with error"
-        break
-    fi
-done
+bash tests/functional/s3_container_hierarchy_v2.sh
 
 for pid in $PID; do
     kill $pid
     wait $pid
 done
 
-
+# TODO(FVE): gridinit_cmd stop

tests/functional/s3_container_hierarchy_v2.sh

@@ -42,7 +42,6 @@ echo ${OUT} | grep root
 echo ${OUT} | grep dir1/dir2/object
 
 
-S3CMD="/home/murlock/openio/resources/s3cmd"
 ${AWS} s3 cp bigfile s3://${BUCKET}/subdir/bigfile
 
 ${AWS} s3 cp s3://${BUCKET}/subdir/bigfile testfile