Run more functional tests

.gitmodules

@@ -5,7 +5,7 @@
 [submodule "third_party/oio-sds"]
 	path = third_party/oio-sds
 	url = https://github.com/open-io/oio-sds.git
-	branch = 4.x
+	branch = 4.1.x
 [submodule "third_party/swift3"]
 	path = third_party/swift3
 	url = https://github.com/open-io/swift3

.travis.yml

@@ -5,8 +5,15 @@ language: python
 services:
   - docker
 
+env:
+  matrix:
+    - TEST_SUITE=unit
+    - TEST_SUITE=ns-wide-versioning
+    # - TEST_SUITE=s3
+
 install:
   - sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe multiverse"
+  - sudo apt-add-repository "deb http://mirror.openio.io/pub/repo/openio/sds/17.04/ubuntu/trusty ./"
   - sudo apt-get update -qq
   - sudo apt-get install -y --force-yes python-virtualenv liberasurecode-dev libssl-dev libattr1-dev libleveldb1 libleveldb-dev
   - virtualenv $HOME/venv
@@ -25,11 +32,8 @@ install:
   # - sleep 20
 
 script:
-  - coverage run --source=oioswift -a $(which nosetests) -v tests/unit/controllers
-  - coverage run --source=oioswift -a $(which nosetests) -v tests/unit/common/middleware/test_versioned_writes.py:OioVersionedWritesTestCase
   - flake8 oioswift
-  - ./oio-check-version.sh
-  # - ./tests/functional/run_tests.sh
+  - ./tests/run_tests.sh "$TEST_SUITE"
 
 after_success:
   - bash <(curl -s https://codecov.io/bash) -f cmake_coverage.output

conf/hashed-containers.cfg

@@ -0,0 +1,92 @@
+[DEFAULT]
+bind_port = 5000
+workers = 1
+user = travis
+log_facility = LOG_LOCAL0
+log_level = INFO
+eventlet_debug = true
+sds_namespace = OPENIO
+sds_default_account = test_account
+sds_proxy_url = http://127.0.0.1:6000
+swift_dir = /tmp
+
+[pipeline:main]
+pipeline = catch_errors proxy-logging cache proxy-logging bulk dlo slo proxy-logging hashedcontainer proxy-logging proxy-server
+
+[app:proxy-server]
+use = egg:oioswift#main
+allow_account_management = true
+account_autocreate = true
+sds_namespace = OPENIO
+sds_proxy_url = http://127.0.0.1:6000
+object_post_as_copy = false
+log_name = OIO,OPENIO,oioswift,0
+sds_connection_timeout=5.0
+sds_read_timeout=35.0
+sds_write_timeout=25.0
+sds_max_retries=0
+oio_storage_policies=SINGLE,EC,THREECOPIES
+#auto_storage_policies=EC,THREECOPIES:0,EC:1048576
+
+[filter:hashedcontainer]
+use = egg:oioswift#hashedcontainer
+
+# Should the server strip the first "v1" token if matched?
+# If used in conjunction with 'account_first', 'strip_v1' is applied BEFORE
+# false by default
+#strip_v1 = true
+
+# Is the OpenIO-SDS account expected as the first token of the URL?
+# If used in conjunction with 'strip_v1', 'account_first' is applied AFTER
+# false by default
+#account_first = true
+
+[filter:bulk]
+use = egg:swift#bulk
+
+[filter:gatekeeper]
+use = egg:swift#gatekeeper
+
+[filter:proxy-logging]
+use = egg:swift#proxy_logging
+
+[filter:catch_errors]
+use = egg:swift#catch_errors
+
+[filter:ratelimit]
+use = egg:swift#ratelimit
+
+[filter:healthcheck]
+use = egg:swift#healthcheck
+
+[filter:cache]
+use = egg:swift#memcache
+memcache_servers = 127.0.0.1:11211
+memcache_max_connections = 100
+
+[filter:copy]
+use = egg:swift#copy
+object_post_as_copy = false
+
+[filter:dlo]
+use = egg:swift#dlo
+
+[filter:slo]
+use = egg:swift#slo
+
+[filter:staticweb]
+use = egg:swift#staticweb
+
+[filter:tempurl]
+use = egg:swift#tempurl
+
+[filter:versioned_writes]
+#use = egg:swift#versioned_writes
+use = egg:oioswift#versioned_writes
+allow_versioned_writes = true
+
+[filter:account-quotas]
+use = egg:swift#account_quotas
+
+[filter:container-quotas]
+use = egg:swift#container_quotas

oioswift/__init__.py

@@ -1 +1 @@
-__version__ = '1.2.8'
+__version__ = '1.2.10'

tests/functional/ns_wide_versioning_tests.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# This script expects a namespace with unlimited versioning,
+# and a swift gateway with hashedcontainer middleware.
+
+export OIO_NS="${1:-OPENIO}"
+export OIO_ACCOUNT="${2:-test_account}"
+
+ETAG_REGEX='s/(Etag: )([[:alnum:]]*)(.*)/\2/p'
+CONTENT_TYPE_REGEX='s/(Content-Type: )([[:print:]]*)(.*)/\2/p'
+HTTP_STATUS_REGEX='s/(HTTP\/1.1 )([[:digit:]]*)(.*)/\2/p'
+
+GW_NETLOC="http://127.0.0.1:5000/"
+OBJ_SEED=$(mktemp -tu test-vers-XXXXXX)
+OBJ_1_SRC="${OBJ_SEED}-1.dat"
+OBJ_2_SRC="${OBJ_SEED}-2.dat"
+OBJ_3_SRC="${OBJ_SEED}-3.dat"
+
+set -e
+
+echo "Preparing fixtures $OBJ_1_SRC $OBJ_2_SRC $OBJ_3_SRC"
+cp "/etc/resolv.conf" "$OBJ_1_SRC"
+cp "/etc/fstab" "$OBJ_2_SRC"
+dd if=/dev/zero of="$OBJ_3_SRC" bs=1k count=2042
+
+echo "Uploading one object"
+curl -XPUT "${GW_NETLOC}${OBJ_1_SRC}" --data-binary "@${OBJ_1_SRC}"
+
+echo "Finding it with openio CLI, checking MD5"
+openio object show --auto "${OBJ_1_SRC}"
+OBJ_1_CONTAINER=$(openio object show --auto "${OBJ_1_SRC}" -f value -c container)
+OBJ_1_HASH=$(openio object show --auto "${OBJ_1_SRC}" -f value -c hash)
+echo "$OBJ_1_HASH $OBJ_1_SRC" | md5sum -c
+
+echo "Doing HEAD request, checking MD5"
+OBJ_1_HASH=$(curl -s -I "${GW_NETLOC}${OBJ_1_SRC}" | sed -n -E -e "${ETAG_REGEX}")
+echo "$OBJ_1_HASH $OBJ_1_SRC" | md5sum -c
+
+echo "Overwriting it twice"
+curl -XPUT "${GW_NETLOC}${OBJ_1_SRC}" --data-binary "@${OBJ_2_SRC}"
+curl -XPUT "${GW_NETLOC}${OBJ_1_SRC}" --data-binary "@${OBJ_3_SRC}"
+
+echo "Finding it with openio CLI, checking MD5"
+openio object show --auto "${OBJ_1_SRC}"
+OBJ_3_HASH=$(openio object show --auto "${OBJ_1_SRC}" -f value -c hash)
+echo "$OBJ_3_HASH $OBJ_3_SRC" | md5sum -c
+
+echo "Doing HEAD request, checking MD5"
+OBJ_3_HASH=$(curl -s -I "${GW_NETLOC}${OBJ_1_SRC}" | sed -n -E -e "${ETAG_REGEX}")
+echo "$OBJ_3_HASH $OBJ_3_SRC" | md5sum -c
+
+echo "Checking number of versions"
+OBJ_VERSIONS=$(openio object list --auto --versions --prefix "${OBJ_1_SRC}" -f value | wc -l)
+[ "$OBJ_VERSIONS" -eq 3 ]
+
+echo "Doing DELETE request"
+curl -XDELETE "${GW_NETLOC}${OBJ_1_SRC}"
+
+echo "Doing HEAD request, checking Content-Type"
+OBJ_4_CT=$(curl -s -I "${GW_NETLOC}${OBJ_1_SRC}" | sed -n -E -e "${CONTENT_TYPE_REGEX}")
+echo "$OBJ_4_CT" | grep -q "application/x-deleted"
+
+echo "Doing DELETE request again"
+curl -XDELETE "${GW_NETLOC}${OBJ_1_SRC}"
+
+echo "Doing HEAD request, checking Content-Type"
+OBJ_3_CT=$(curl -s -I "${GW_NETLOC}${OBJ_1_SRC}" | sed -n -E -e "${CONTENT_TYPE_REGEX}")
+echo "$OBJ_3_CT" | grep -qv "application/x-deleted"
+
+echo "Purging container $OBJ_1_CONTAINER, keeping 3 versions (should be noop)"
+openio container purge "$OBJ_1_CONTAINER" --max-versions 3
+
+echo "Checking number of versions"
+OBJ_VERSIONS=$(openio object list --auto --versions --prefix "${OBJ_1_SRC}" -f value | wc -l)
+[ "$OBJ_VERSIONS" -eq 3 ] && echo "OK"
+
+echo "Purging container $OBJ_1_CONTAINER, keeping 2 versions"
+openio container purge "$OBJ_1_CONTAINER" --max-versions 2
+
+echo "Checking number of versions"
+OBJ_VERSIONS=$(openio object list --auto --versions --prefix "${OBJ_1_SRC}" -f value | wc -l)
+[ "$OBJ_VERSIONS" -eq 2 ] && echo "OK"
+
+echo "Purging container $OBJ_1_CONTAINER, keeping 1 version"
+openio container purge "$OBJ_1_CONTAINER" --max-versions 1
+
+echo "Checking number of versions"
+OBJ_VERSIONS=$(openio object list --auto --versions --prefix "${OBJ_1_SRC}" -f value | wc -l)
+[ "$OBJ_VERSIONS" -eq 1 ] && echo "OK"
+
+echo "Deleting last object version (explicitly, with openio CLI)"
+OBJ_3_VERS=$(openio object show --auto "${OBJ_1_SRC}" -f value -c version)
+openio object delete --auto --object-version "$OBJ_3_VERS" "${OBJ_1_SRC}"
+
+echo "Doing HEAD request (expect 404)"
+OBJ_STATUS=$(curl -s -I "${GW_NETLOC}${OBJ_1_SRC}" | sed -n -E -e "${HTTP_STATUS_REGEX}")
+[ "$OBJ_STATUS" -eq "404" ] && echo "OK"
+
+echo "Removing fixtures"
+rm -f "$OBJ_1_SRC" "$OBJ_2_SRC" "$OBJ_3_SRC"

tests/functional/run_ns_wide_versioning_tests.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+function install_deps() {
+  sudo apt-get install -y --force-yes \
+    apache2 apache2-dev libapache2-mod-wsgi \
+    beanstalkd \
+    bison \
+    curl libcurl4-gnutls-dev \
+    flex \
+    libapreq2-dev \
+    libattr1-dev \
+    liberasurecode-dev \
+    libevent-dev \
+    libglib2.0-dev \
+    libjson-c-dev \
+    libleveldb1 libleveldb-dev \
+    liblzo2-dev \
+    libsqlite3-dev \
+    libzmq3-dev \
+    libzookeeper-mt-dev \
+    openio-gridinit openio-asn1c \
+    python-all-dev python-virtualenv
+}
+
+function compile_sds() {
+  cd third_party/oio-sds || return
+  cmake \
+    -DCMAKE_INSTALL_PREFIX="/tmp/oio" \
+    -DLD_LIBDIR="lib" \
+    -DCMAKE_BUILD_TYPE="Debug" \
+    -DSTACK_PROTECTOR=1 \
+    -DZK_LIBDIR="/usr/lib" \
+    -DZK_INCDIR="/usr/include/zookeeper" \
+    -DAPACHE2_LIBDIR="/usr/lib/apache2" \
+    -DAPACHE2_INCDIR="/usr/include/apache2" \
+    -DAPACHE2_MODDIR="/tmp/oio/lib/apache2/module" \
+    .
+  make all install
+  cd ../.. || return
+}
+
+function run_sds() {
+  export G_DEBUG_LEVEL=D PATH="$PATH:/tmp/oio/bin" LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/tmp/oio/lib"
+  if [ ! -f third_party/oio-sds/etc/bootstrap-option-versioning.yml ]
+  then
+    echo "config:" >> third_party/oio-sds/etc/bootstrap-option-versioning.yml
+    echo "  meta2.max_versions: -1" >> third_party/oio-sds/etc/bootstrap-option-versioning.yml
+  fi
+  oio-reset.sh -v -v -N "$OIO_NS" \
+    -f third_party/oio-sds/etc/bootstrap-preset-SINGLE.yml \
+    -f third_party/oio-sds/etc/bootstrap-meta1-1digits.yml \
+    -f third_party/oio-sds/etc/bootstrap-option-cache.yml \
+    -f third_party/oio-sds/etc/bootstrap-option-versioning.yml
+}
+
+export OIO_NS="OPENIO" OIO_ACCOUNT="test_account" OIO_USER=USER-$RANDOM OIO_PATH=PATH-$RANDOM
+install_deps
+compile_sds
+run_sds
+
+coverage run -a runserver.py conf/hashed-containers.cfg -v &
+sleep 1
+PID=$(jobs -p)
+
+bash tests/functional/ns_wide_versioning_tests.sh "$OIO_NS" "$OIO_ACCOUNT"
+
+for pid in $PID; do
+    kill $pid
+    wait $pid
+done
+
+# TODO(FVE): gridinit_cmd stop

tests/run_tests.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+export TEST_SUITE="${TEST_SUITE:-$1}"
+
+if [ "$TEST_SUITE" = "unit" ]
+then
+  tests/unit/run_unit_tests.sh
+elif [ "$TEST_SUITE" = "ns-wide-versioning" ]
+then
+  tests/functional/run_ns_wide_versioning_tests.sh $*
+elif [ "$TEST_SUITE" = "s3" ]
+then
+  tests/functional/run_s3_tests.sh $*
+else
+  echo "Test suite '$TEST_SUITE' not implemented"
+fi

tests/unit/run_unit_tests.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+coverage run --source=oioswift -a $(which nosetests) -v tests/unit/controllers
+coverage run --source=oioswift -a $(which nosetests) -v tests/unit/common/middleware/test_versioned_writes.py:OioVersionedWritesTestCase
+
+./oio-check-version.sh