diff --git a/.gitmodules b/.gitmodules index 65e1edc6..037be737 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,7 +5,7 @@ [submodule "third_party/oio-sds"] path = third_party/oio-sds url = https://github.com/open-io/oio-sds.git - branch = 4.x + branch = 4.1.x [submodule "third_party/swift3"] path = third_party/swift3 url = https://github.com/open-io/swift3 diff --git a/.travis.yml b/.travis.yml index 98c3b85c..dfde44f2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,15 @@ language: python services: - docker +env: + matrix: + - TEST_SUITE=unit + - TEST_SUITE=ns-wide-versioning + # - TEST_SUITE=s3 + install: - sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe multiverse" + - sudo apt-add-repository "deb http://mirror.openio.io/pub/repo/openio/sds/17.04/ubuntu/trusty ./" - sudo apt-get update -qq - sudo apt-get install -y --force-yes python-virtualenv liberasurecode-dev libssl-dev libattr1-dev libleveldb1 libleveldb-dev - virtualenv $HOME/venv @@ -25,11 +32,8 @@ install: # - sleep 20 script: - - coverage run --source=oioswift -a $(which nosetests) -v tests/unit/controllers - - coverage run --source=oioswift -a $(which nosetests) -v tests/unit/common/middleware/test_versioned_writes.py:OioVersionedWritesTestCase - flake8 oioswift - - ./oio-check-version.sh - # - ./tests/functional/run_tests.sh + - ./tests/run_tests.sh "$TEST_SUITE" after_success: - bash <(curl -s https://codecov.io/bash) -f cmake_coverage.output diff --git a/conf/hashed-containers.cfg b/conf/hashed-containers.cfg new file mode 100644 index 00000000..1d896495 --- /dev/null +++ b/conf/hashed-containers.cfg @@ -0,0 +1,92 @@ +[DEFAULT] +bind_port = 5000 +workers = 1 +user = travis +log_facility = LOG_LOCAL0 +log_level = INFO +eventlet_debug = true +sds_namespace = OPENIO +sds_default_account = test_account +sds_proxy_url = http://127.0.0.1:6000 +swift_dir = /tmp + +[pipeline:main] +pipeline = catch_errors proxy-logging cache proxy-logging bulk dlo slo proxy-logging hashedcontainer proxy-logging proxy-server + +[app:proxy-server] +use = egg:oioswift#main +allow_account_management = true +account_autocreate = true +sds_namespace = OPENIO +sds_proxy_url = http://127.0.0.1:6000 +object_post_as_copy = false +log_name = OIO,OPENIO,oioswift,0 +sds_connection_timeout=5.0 +sds_read_timeout=35.0 +sds_write_timeout=25.0 +sds_max_retries=0 +oio_storage_policies=SINGLE,EC,THREECOPIES +#auto_storage_policies=EC,THREECOPIES:0,EC:1048576 + +[filter:hashedcontainer] +use = egg:oioswift#hashedcontainer + +# Should the server strip the first "v1" token if matched? +# If used in conjunction with 'account_first', 'strip_v1' is applied BEFORE +# false by default +#strip_v1 = true + +# Is the OpenIO-SDS account expected as the first token of the URL? +# If used in conjunction with 'strip_v1', 'account_first' is applied AFTER +# false by default +#account_first = true + +[filter:bulk] +use = egg:swift#bulk + +[filter:gatekeeper] +use = egg:swift#gatekeeper + +[filter:proxy-logging] +use = egg:swift#proxy_logging + +[filter:catch_errors] +use = egg:swift#catch_errors + +[filter:ratelimit] +use = egg:swift#ratelimit + +[filter:healthcheck] +use = egg:swift#healthcheck + +[filter:cache] +use = egg:swift#memcache +memcache_servers = 127.0.0.1:11211 +memcache_max_connections = 100 + +[filter:copy] +use = egg:swift#copy +object_post_as_copy = false + +[filter:dlo] +use = egg:swift#dlo + +[filter:slo] +use = egg:swift#slo + +[filter:staticweb] +use = egg:swift#staticweb + +[filter:tempurl] +use = egg:swift#tempurl + +[filter:versioned_writes] +#use = egg:swift#versioned_writes +use = egg:oioswift#versioned_writes +allow_versioned_writes = true + +[filter:account-quotas] +use = egg:swift#account_quotas + +[filter:container-quotas] +use = egg:swift#container_quotas diff --git a/oioswift/__init__.py b/oioswift/__init__.py index 6a3fa6ed..c7ac77f9 100644 --- a/oioswift/__init__.py +++ b/oioswift/__init__.py @@ -1 +1 @@ -__version__ = '1.2.8' +__version__ = '1.2.10' diff --git a/tests/functional/ns_wide_versioning_tests.sh b/tests/functional/ns_wide_versioning_tests.sh new file mode 100644 index 00000000..a42f00b0 --- /dev/null +++ b/tests/functional/ns_wide_versioning_tests.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +# This script expects a namespace with unlimited versioning, +# and a swift gateway with hashedcontainer middleware. + +export OIO_NS="${1:-OPENIO}" +export OIO_ACCOUNT="${2:-test_account}" + +ETAG_REGEX='s/(Etag: )([[:alnum:]]*)(.*)/\2/p' +CONTENT_TYPE_REGEX='s/(Content-Type: )([[:print:]]*)(.*)/\2/p' +HTTP_STATUS_REGEX='s/(HTTP\/1.1 )([[:digit:]]*)(.*)/\2/p' + +GW_NETLOC="http://127.0.0.1:5000/" +OBJ_SEED=$(mktemp -tu test-vers-XXXXXX) +OBJ_1_SRC="${OBJ_SEED}-1.dat" +OBJ_2_SRC="${OBJ_SEED}-2.dat" +OBJ_3_SRC="${OBJ_SEED}-3.dat" + +set -e + +echo "Preparing fixtures $OBJ_1_SRC $OBJ_2_SRC $OBJ_3_SRC" +cp "/etc/resolv.conf" "$OBJ_1_SRC" +cp "/etc/fstab" "$OBJ_2_SRC" +dd if=/dev/zero of="$OBJ_3_SRC" bs=1k count=2042 + +echo "Uploading one object" +curl -XPUT "${GW_NETLOC}${OBJ_1_SRC}" --data-binary "@${OBJ_1_SRC}" + +echo "Finding it with openio CLI, checking MD5" +openio object show --auto "${OBJ_1_SRC}" +OBJ_1_CONTAINER=$(openio object show --auto "${OBJ_1_SRC}" -f value -c container) +OBJ_1_HASH=$(openio object show --auto "${OBJ_1_SRC}" -f value -c hash) +echo "$OBJ_1_HASH $OBJ_1_SRC" | md5sum -c + +echo "Doing HEAD request, checking MD5" +OBJ_1_HASH=$(curl -s -I "${GW_NETLOC}${OBJ_1_SRC}" | sed -n -E -e "${ETAG_REGEX}") +echo "$OBJ_1_HASH $OBJ_1_SRC" | md5sum -c + +echo "Overwriting it twice" +curl -XPUT "${GW_NETLOC}${OBJ_1_SRC}" --data-binary "@${OBJ_2_SRC}" +curl -XPUT "${GW_NETLOC}${OBJ_1_SRC}" --data-binary "@${OBJ_3_SRC}" + +echo "Finding it with openio CLI, checking MD5" +openio object show --auto "${OBJ_1_SRC}" +OBJ_3_HASH=$(openio object show --auto "${OBJ_1_SRC}" -f value -c hash) +echo "$OBJ_3_HASH $OBJ_3_SRC" | md5sum -c + +echo "Doing HEAD request, checking MD5" +OBJ_3_HASH=$(curl -s -I "${GW_NETLOC}${OBJ_1_SRC}" | sed -n -E -e "${ETAG_REGEX}") +echo "$OBJ_3_HASH $OBJ_3_SRC" | md5sum -c + +echo "Checking number of versions" +OBJ_VERSIONS=$(openio object list --auto --versions --prefix "${OBJ_1_SRC}" -f value | wc -l) +[ "$OBJ_VERSIONS" -eq 3 ] + +echo "Doing DELETE request" +curl -XDELETE "${GW_NETLOC}${OBJ_1_SRC}" + +echo "Doing HEAD request, checking Content-Type" +OBJ_4_CT=$(curl -s -I "${GW_NETLOC}${OBJ_1_SRC}" | sed -n -E -e "${CONTENT_TYPE_REGEX}") +echo "$OBJ_4_CT" | grep -q "application/x-deleted" + +echo "Doing DELETE request again" +curl -XDELETE "${GW_NETLOC}${OBJ_1_SRC}" + +echo "Doing HEAD request, checking Content-Type" +OBJ_3_CT=$(curl -s -I "${GW_NETLOC}${OBJ_1_SRC}" | sed -n -E -e "${CONTENT_TYPE_REGEX}") +echo "$OBJ_3_CT" | grep -qv "application/x-deleted" + +echo "Purging container $OBJ_1_CONTAINER, keeping 3 versions (should be noop)" +openio container purge "$OBJ_1_CONTAINER" --max-versions 3 + +echo "Checking number of versions" +OBJ_VERSIONS=$(openio object list --auto --versions --prefix "${OBJ_1_SRC}" -f value | wc -l) +[ "$OBJ_VERSIONS" -eq 3 ] && echo "OK" + +echo "Purging container $OBJ_1_CONTAINER, keeping 2 versions" +openio container purge "$OBJ_1_CONTAINER" --max-versions 2 + +echo "Checking number of versions" +OBJ_VERSIONS=$(openio object list --auto --versions --prefix "${OBJ_1_SRC}" -f value | wc -l) +[ "$OBJ_VERSIONS" -eq 2 ] && echo "OK" + +echo "Purging container $OBJ_1_CONTAINER, keeping 1 version" +openio container purge "$OBJ_1_CONTAINER" --max-versions 1 + +echo "Checking number of versions" +OBJ_VERSIONS=$(openio object list --auto --versions --prefix "${OBJ_1_SRC}" -f value | wc -l) +[ "$OBJ_VERSIONS" -eq 1 ] && echo "OK" + +echo "Deleting last object version (explicitly, with openio CLI)" +OBJ_3_VERS=$(openio object show --auto "${OBJ_1_SRC}" -f value -c version) +openio object delete --auto --object-version "$OBJ_3_VERS" "${OBJ_1_SRC}" + +echo "Doing HEAD request (expect 404)" +OBJ_STATUS=$(curl -s -I "${GW_NETLOC}${OBJ_1_SRC}" | sed -n -E -e "${HTTP_STATUS_REGEX}") +[ "$OBJ_STATUS" -eq "404" ] && echo "OK" + +echo "Removing fixtures" +rm -f "$OBJ_1_SRC" "$OBJ_2_SRC" "$OBJ_3_SRC" diff --git a/tests/functional/run_ns_wide_versioning_tests.sh b/tests/functional/run_ns_wide_versioning_tests.sh new file mode 100755 index 00000000..bd517073 --- /dev/null +++ b/tests/functional/run_ns_wide_versioning_tests.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +function install_deps() { + sudo apt-get install -y --force-yes \ + apache2 apache2-dev libapache2-mod-wsgi \ + beanstalkd \ + bison \ + curl libcurl4-gnutls-dev \ + flex \ + libapreq2-dev \ + libattr1-dev \ + liberasurecode-dev \ + libevent-dev \ + libglib2.0-dev \ + libjson-c-dev \ + libleveldb1 libleveldb-dev \ + liblzo2-dev \ + libsqlite3-dev \ + libzmq3-dev \ + libzookeeper-mt-dev \ + openio-gridinit openio-asn1c \ + python-all-dev python-virtualenv +} + +function compile_sds() { + cd third_party/oio-sds || return + cmake \ + -DCMAKE_INSTALL_PREFIX="/tmp/oio" \ + -DLD_LIBDIR="lib" \ + -DCMAKE_BUILD_TYPE="Debug" \ + -DSTACK_PROTECTOR=1 \ + -DZK_LIBDIR="/usr/lib" \ + -DZK_INCDIR="/usr/include/zookeeper" \ + -DAPACHE2_LIBDIR="/usr/lib/apache2" \ + -DAPACHE2_INCDIR="/usr/include/apache2" \ + -DAPACHE2_MODDIR="/tmp/oio/lib/apache2/module" \ + . + make all install + cd ../.. || return +} + +function run_sds() { + export G_DEBUG_LEVEL=D PATH="$PATH:/tmp/oio/bin" LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/tmp/oio/lib" + if [ ! -f third_party/oio-sds/etc/bootstrap-option-versioning.yml ] + then + echo "config:" >> third_party/oio-sds/etc/bootstrap-option-versioning.yml + echo " meta2.max_versions: -1" >> third_party/oio-sds/etc/bootstrap-option-versioning.yml + fi + oio-reset.sh -v -v -N "$OIO_NS" \ + -f third_party/oio-sds/etc/bootstrap-preset-SINGLE.yml \ + -f third_party/oio-sds/etc/bootstrap-meta1-1digits.yml \ + -f third_party/oio-sds/etc/bootstrap-option-cache.yml \ + -f third_party/oio-sds/etc/bootstrap-option-versioning.yml +} + +export OIO_NS="OPENIO" OIO_ACCOUNT="test_account" OIO_USER=USER-$RANDOM OIO_PATH=PATH-$RANDOM +install_deps +compile_sds +run_sds + +coverage run -a runserver.py conf/hashed-containers.cfg -v & +sleep 1 +PID=$(jobs -p) + +bash tests/functional/ns_wide_versioning_tests.sh "$OIO_NS" "$OIO_ACCOUNT" + +for pid in $PID; do + kill $pid + wait $pid +done + +# TODO(FVE): gridinit_cmd stop diff --git a/tests/functional/run_tests.sh b/tests/functional/run_s3_tests.sh similarity index 100% rename from tests/functional/run_tests.sh rename to tests/functional/run_s3_tests.sh diff --git a/tests/run_tests.sh b/tests/run_tests.sh new file mode 100755 index 00000000..52dd0e57 --- /dev/null +++ b/tests/run_tests.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +export TEST_SUITE="${TEST_SUITE:-$1}" + +if [ "$TEST_SUITE" = "unit" ] +then + tests/unit/run_unit_tests.sh +elif [ "$TEST_SUITE" = "ns-wide-versioning" ] +then + tests/functional/run_ns_wide_versioning_tests.sh $* +elif [ "$TEST_SUITE" = "s3" ] +then + tests/functional/run_s3_tests.sh $* +else + echo "Test suite '$TEST_SUITE' not implemented" +fi diff --git a/tests/unit/run_unit_tests.sh b/tests/unit/run_unit_tests.sh new file mode 100755 index 00000000..ea423465 --- /dev/null +++ b/tests/unit/run_unit_tests.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +coverage run --source=oioswift -a $(which nosetests) -v tests/unit/controllers +coverage run --source=oioswift -a $(which nosetests) -v tests/unit/common/middleware/test_versioned_writes.py:OioVersionedWritesTestCase + +./oio-check-version.sh