From 8ae5c96987e80428a86748187c0049166e7ea43e Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Tue, 26 Jan 2021 21:12:05 -0800 Subject: [PATCH 001/160] Adding TLS support to proxyfsd (#572) * Add etcdclient pkg to make calls to etcd. Also add TLS * Use same path as kubepsray * Pass tlsconfig to etcdclient.New() --- etcdclient/Makefile | 3 +++ etcdclient/api.go | 57 ++++++++++++++++++++++++++++++++++++++++++++ headhunter/config.go | 15 ++++++++---- mkproxyfs/api.go | 16 ++++++++----- pfs-fsck/main.go | 15 ++++++++---- 5 files changed, 90 insertions(+), 16 deletions(-) create mode 100644 etcdclient/Makefile create mode 100644 etcdclient/api.go diff --git a/etcdclient/Makefile b/etcdclient/Makefile new file mode 100644 index 000000000..ca85948cb --- /dev/null +++ b/etcdclient/Makefile @@ -0,0 +1,3 @@ +gosubdir := github.com/swiftstack/ProxyFS/etcdclient + +include ../GoMakefile diff --git a/etcdclient/api.go b/etcdclient/api.go new file mode 100644 index 000000000..cc834efe6 --- /dev/null +++ b/etcdclient/api.go @@ -0,0 +1,57 @@ +package etcdclient + +import ( + "log" + "os" + "time" + + etcd "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/pkg/transport" +) + +const ( + certPath = "/etc/ssl/etcd/ssl/" + trustedCAFile = certPath + "ca.pem" +) + +// New initializes etcd config structures and returns an etcd client +func New(tlsInfo *transport.TLSInfo, endPoints []string, autoSyncInterval time.Duration, dialTimeout time.Duration) (etcdClient *etcd.Client, err error) { + tlsConfig, etcdErr := tlsInfo.ClientConfig() + if etcdErr != nil { + log.Fatal(etcdErr) + } + + etcdClient, err = etcd.New(etcd.Config{ + Endpoints: endPoints, + AutoSyncInterval: autoSyncInterval, + DialTimeout: dialTimeout, + TLS: tlsConfig, + }) + return +} + +// GetCertFile returns the name of the cert file for the local node +func GetCertFile() string { + h, _ := os.Hostname() + return certPath + "node-" + h + ".pem" +} + +// GetKeyFile returns the name of the key file for the local node +func GetKeyFile() string { + h, _ := os.Hostname() + return certPath + "node-" + h + "-key.pem" +} + +// GetCA returns the name of the certificate authority for the local node +func GetCA() string { + var ( + caFile string + ) + + _, statErr := os.Stat(trustedCAFile) + if os.IsExist(statErr) { + caFile = trustedCAFile + } + + return caFile +} diff --git a/headhunter/config.go b/headhunter/config.go index dc6d22bd4..63c86c0be 100644 --- a/headhunter/config.go +++ b/headhunter/config.go @@ -12,12 +12,14 @@ import ( "time" etcd "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/pkg/transport" "github.com/swiftstack/cstruct" "github.com/swiftstack/sortedmap" "github.com/swiftstack/ProxyFS/bucketstats" "github.com/swiftstack/ProxyFS/conf" + "github.com/swiftstack/ProxyFS/etcdclient" "github.com/swiftstack/ProxyFS/logger" "github.com/swiftstack/ProxyFS/swiftclient" "github.com/swiftstack/ProxyFS/trackedlock" @@ -469,11 +471,14 @@ func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { // Initialize etcd Client & KV objects - globals.etcdClient, err = etcd.New(etcd.Config{ - Endpoints: globals.etcdEndpoints, - AutoSyncInterval: globals.etcdAutoSyncInterval, - DialTimeout: globals.etcdDialTimeout, - }) + tlsInfo := transport.TLSInfo{ + CertFile: etcdclient.GetCertFile(), + KeyFile: etcdclient.GetKeyFile(), + TrustedCAFile: etcdclient.GetCA(), + } + + globals.etcdClient, err = etcdclient.New(&tlsInfo, globals.etcdEndpoints, + globals.etcdAutoSyncInterval, globals.etcdDialTimeout) if nil != err { return } diff --git a/mkproxyfs/api.go b/mkproxyfs/api.go index 31b4b1a86..68842bacd 100644 --- a/mkproxyfs/api.go +++ b/mkproxyfs/api.go @@ -12,9 +12,11 @@ import ( "time" etcd "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/pkg/transport" "github.com/swiftstack/ProxyFS/blunder" "github.com/swiftstack/ProxyFS/conf" + "github.com/swiftstack/ProxyFS/etcdclient" "github.com/swiftstack/ProxyFS/headhunter" "github.com/swiftstack/ProxyFS/logger" "github.com/swiftstack/ProxyFS/swiftclient" @@ -131,13 +133,15 @@ func Format(mode Mode, volumeNameToFormat string, confFile string, confStrings [ return } - // Initialize etcd Client & KV objects + tlsInfo := transport.TLSInfo{ + CertFile: etcdclient.GetCertFile(), + KeyFile: etcdclient.GetKeyFile(), + TrustedCAFile: etcdclient.GetCA(), + } - etcdClient, err = etcd.New(etcd.Config{ - Endpoints: etcdEndpoints, - AutoSyncInterval: etcdAutoSyncInterval, - DialTimeout: etcdDialTimeout, - }) + // Initialize etcd Client & KV objects + etcdClient, err = etcdclient.New(&tlsInfo, etcdEndpoints, + etcdAutoSyncInterval, etcdDialTimeout) if nil != err { return } diff --git a/pfs-fsck/main.go b/pfs-fsck/main.go index 1eecbb9e8..d2b6be1b3 100644 --- a/pfs-fsck/main.go +++ b/pfs-fsck/main.go @@ -17,11 +17,13 @@ import ( "time" etcd "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/pkg/transport" "github.com/swiftstack/cstruct" "github.com/swiftstack/sortedmap" "github.com/swiftstack/ProxyFS/conf" + "github.com/swiftstack/ProxyFS/etcdclient" "github.com/swiftstack/ProxyFS/headhunter" ) @@ -376,11 +378,14 @@ func setup() { log.Fatal(err) } - globals.etcdClient, err = etcd.New(etcd.Config{ - Endpoints: globals.etcdEndpoints, - AutoSyncInterval: globals.etcdAutoSyncInterval, - DialTimeout: globals.etcdDialTimeout, - }) + tlsInfo := transport.TLSInfo{ + CertFile: etcdclient.GetCertFile(), + KeyFile: etcdclient.GetKeyFile(), + TrustedCAFile: etcdclient.GetCA(), + } + + globals.etcdClient, err = etcdclient.New(&tlsInfo, globals.etcdEndpoints, + globals.etcdAutoSyncInterval, globals.etcdDialTimeout) if nil != err { log.Fatalf("unable to create etcdClient: %v\n", err) } From 8298d4a45718654028939abb0509ce044df65256 Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Tue, 26 Jan 2021 21:47:31 -0800 Subject: [PATCH 002/160] Add missing copyright files (#573) * Add missing copyright files * Build etcdclient --- Makefile | 1 + etcdclient/Makefile | 3 +++ etcdclient/api.go | 3 +++ 3 files changed, 7 insertions(+) diff --git a/Makefile b/Makefile index d0722cf20..9412bb060 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ gopkgsubdirs = \ conf \ confgen \ dlm \ + etcdclient \ evtlog \ fs \ fuse \ diff --git a/etcdclient/Makefile b/etcdclient/Makefile index ca85948cb..41aba670b 100644 --- a/etcdclient/Makefile +++ b/etcdclient/Makefile @@ -1,3 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + gosubdir := github.com/swiftstack/ProxyFS/etcdclient include ../GoMakefile diff --git a/etcdclient/api.go b/etcdclient/api.go index cc834efe6..528c8efb2 100644 --- a/etcdclient/api.go +++ b/etcdclient/api.go @@ -1,3 +1,6 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + package etcdclient import ( From 2cb6ea33bb91fe314e95bba2f39887ff5936a636 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 22 Jan 2021 09:59:45 -0800 Subject: [PATCH 003/160] Removed jrpcclient & vfs submodules --- .gitmodules | 8 -------- jrpcclient | 1 - vfs | 1 - 3 files changed, 10 deletions(-) delete mode 100644 .gitmodules delete mode 160000 jrpcclient delete mode 160000 vfs diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index b9caa4ce5..000000000 --- a/.gitmodules +++ /dev/null @@ -1,8 +0,0 @@ -[submodule "proxyfs-jrpc-client"] - path = jrpcclient - url = git@github.com:swiftstack/proxyfs-jrpc-client.git - branch = master -[submodule "proxyfs-vfs"] - path = vfs - url = git@github.com:swiftstack/proxyfs-vfs.git - branch = master diff --git a/jrpcclient b/jrpcclient deleted file mode 160000 index 7ce4e2d66..000000000 --- a/jrpcclient +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7ce4e2d66252ac880df63c78cd7d95894dc9df7f diff --git a/vfs b/vfs deleted file mode 160000 index 768b89e6e..000000000 --- a/vfs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 768b89e6e67666020efd0689fd4ee2fb94e93488 From 2403e448276ca4549b7e13b3515e1d7b396e0473 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 22 Jan 2021 10:50:00 -0800 Subject: [PATCH 004/160] Updates following submodule deprecation --- DEBUG.md | 3 - Makefile | 39 +----- README.md | 8 +- regression_test.py | 304 --------------------------------------------- 4 files changed, 6 insertions(+), 348 deletions(-) delete mode 100755 regression_test.py diff --git a/DEBUG.md b/DEBUG.md index 860a071ee..05819f0c0 100644 --- a/DEBUG.md +++ b/DEBUG.md @@ -3,7 +3,6 @@ ## Logging Logs from proxyfsd are output to `proxyfsd.log`. - ## Kill with trace data Find the daemon's pid: `ps -eaf | grep proxyfsd` @@ -14,9 +13,7 @@ Find the daemon's pid: `ps -eaf | grep proxyfsd` Tell gdb to attach to that pid: `gdb -p ` - ## Generating a core file If you want to generate a core file from within gdb: `(gdb) gcore` If you want to generate a core file without using gdb: `gcore ` - diff --git a/Makefile b/Makefile index 9412bb060..3fad56c34 100644 --- a/Makefile +++ b/Makefile @@ -81,13 +81,13 @@ ifeq ($(uname),Linux) else distro := $(shell python -c "import platform; print platform.linux_distribution()[0]") - all: version fmt pre-generate generate install test python-test c-clean c-build c-install c-test + all: version fmt pre-generate generate install test python-test - ci: version fmt pre-generate generate install test cover python-test c-clean c-build c-install c-test + ci: version fmt pre-generate generate install test cover python-test - all-deb-builder: version fmt pre-generate generate install c-clean c-build c-install-deb-builder + all-deb-builder: version fmt pre-generate generate install - minimal: pre-generate generate install c-clean c-build c-install + minimal: pre-generate generate install endif else all: version fmt pre-generate generate install test @@ -99,7 +99,7 @@ endif pfsagent: pre-generate generate pfsagent-install -.PHONY: all all-deb-builder bench c-build c-clean c-install c-install-deb-builder c-test ci clean cover fmt generate install pfsagent pfsagent-install pre-generate python-test test version +.PHONY: all all-deb-builder bench ci clean cover fmt generate install pfsagent pfsagent-install pre-generate python-test test version bench: @set -e; \ @@ -110,35 +110,6 @@ bench: $(MAKE) --no-print-directory -C $$gosubdir bench; \ done -c-build: - $(MAKE) -w -C jrpcclient all - $(MAKE) -w -C vfs - -c-clean: - $(MAKE) -w -C jrpcclient clean - $(MAKE) -w -C vfs clean - -c-install: -ifeq ($(distro),CentOS Linux) - sudo -E $(MAKE) -w -C jrpcclient installcentos - sudo -E $(MAKE) -w -C vfs installcentos -else - sudo -E $(MAKE) -w -C jrpcclient install - sudo -E $(MAKE) -w -C vfs install -endif - -c-install-deb-builder: -ifeq ($(distro),CentOS Linux) - $(MAKE) -w -C jrpcclient installcentos - $(MAKE) -w -C vfs installcentos -else - $(MAKE) -w -C jrpcclient install - $(MAKE) -w -C vfs install -endif - -c-test: - cd jrpcclient ; ./regression_test.py --just-test-libs - clean: @set -e; \ rm -f $(GOPATH)/bin/stringer; \ diff --git a/README.md b/README.md index 5aef2dfa7..3ee4da5dc 100644 --- a/README.md +++ b/README.md @@ -48,19 +48,13 @@ one. * cd src/github.com/swiftstack * git clone git@github.com:swiftstack/ProxyFS.git * cd ProxyFS -* git submodule update --init --recursive ## How to run unit tests (in your Development Environment) * Install/update to at least Go 1.8.3 (if not using Runway) * Ensure $GOPATH/bin is in your $PATH * cd $GOPATH/src/github.com/swiftstack/ProxyFS -* ./regression_test.py - -## Commercial Deployment - -ProxyFS powers File Access for the SwiftStack product. -See: http://swiftstack.com +* make ## License diff --git a/regression_test.py b/regression_test.py deleted file mode 100755 index 06bde89bc..000000000 --- a/regression_test.py +++ /dev/null @@ -1,304 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -""" -A script to lint and test ProxyFS project code. -""" - -from __future__ import print_function, unicode_literals -from threading import Timer - -import os -import argparse -import functools -import logging -import platform -import contextlib -import subprocess -import shutil -import sys -import tempfile -import time - - -PACKAGES = ["blunder", - "cleanproxyfs", - "conf", - "dlm", - "fs", - "fsworkout", - "fuse", - "headhunter", - "httpserver", - "inode", - "inodeworkout", - "jrpcfs", - "logger", - "mkproxyfs", "mkproxyfs/mkproxyfs", - "pfs-stress", - "pfsconfjson", "pfsconfjsonpacked", - "pfsworkout", - "platform", - "proxyfsd", "proxyfsd/proxyfsd", - "ramswift", "ramswift/ramswift", - "stats", - "statslogger", - "swiftclient", - "utils"] - - -COLORS = {"bright red": '1;31', "bright green": '1;32'} - - -@contextlib.contextmanager -def return_to_wd(): - curdir = os.getcwd() - try: - yield - finally: - os.chdir(curdir) - - -@contextlib.contextmanager -def self_cleaning_tempdir(*args, **kwargs): - our_tempdir = tempfile.mkdtemp(*args, **kwargs) - try: - yield our_tempdir - finally: - shutil.rmtree(our_tempdir, ignore_errors=True) - - -def proxyfs_binary_path(binary): - try: - gopath = os.environ["GOPATH"] - except KeyError: - color_print("$GOPATH must be set", 'bright red') - os.exit(1) - return os.path.join(gopath, "bin", binary) - - -def color_print(content, color=None): - print("\x1b[{color}m{content}\x1b[0m".format(content=content, - color=COLORS[color])) - - -def proxyfs_package_path(package): - try: - gopath = os.environ["GOPATH"] - except KeyError: - color_print("$GOPATH must be set", 'bright red') - os.exit(1) - return os.path.join(gopath, "src/github.com/swiftstack/ProxyFS", package) - - -def color_print(content, color=None): - print("\x1b[{color}m{content}\x1b[0m".format(content=content, - color=COLORS[color])) - - -def report(task, success=False): - printer = color_print if sys.stdout.isatty() else lambda *a, **kw: print(*a) - if success: - printer("{} {}".format(task, "succeeded!"), color="bright green") - else: - printer("{} {}".format(task, "failed!"), color="bright red") - - -class GoCommand(object): - def __init__(self, command, project_path, - options=None, report_as=None, skip=False): - self.command = command - self.project_path = project_path - self.options = options or [] - self.report_as = report_as - self.skip = skip - - def execute(self, package): - if self.skip: - return None - package_path = "{}{}".format(self.project_path, package) - command_line = ["go", self.command] + self.options + [package_path] - logging.info(' '.join("'{}'".format(s) if ' ' in s else s - for s in command_line)) - success = not bool(subprocess.call(command_line)) - return success - - -def get_go_commands(options, project_path, skip_tests=False): - commands = [ - GoCommand('fmt', project_path), - GoCommand('get', project_path, ['-t', '-u'], skip=not options.get), - # Source code has to be `go install`ed before `stringer` can run on - # it, so we install before generate, which mysteriously does work? - # see https://github.com/golang/go/issues/10249 - GoCommand('install', project_path, ['-gcflags', '-N -l'], report_as="`go install`"), - GoCommand('generate', project_path, report_as="`go generate`"), - ] - if not options.deb_builder: - commands.append( - GoCommand('test', project_path, - filter(lambda x: x, [options.bench, options.cover]), - report_as="`go test`", - skip=skip_tests) - ) - commands.append(GoCommand('vet', project_path, report_as="`go vet`")) - return commands - - -def execute_go_commands(commands, packages): - reports = [] - failures = 0 - for command in commands: - successes = filter(lambda success: success is not None, - [command.execute(package) for package in packages]) - success = all(successes) - failures += not success - if command.report_as is not None: - reports.append(functools.partial(report, command.report_as, - success=success)) - for reporter in reports: - reporter() - return failures - - -def test_pfs_middleware(options): - failures = 0 - with return_to_wd(): - proxyfs_dir = os.path.dirname(os.path.abspath(__file__)) - pfs_middleware_dir = os.path.join(proxyfs_dir, "pfs_middleware") - os.chdir(pfs_middleware_dir) - failures += bool(subprocess.call((['python', 'setup.py', 'test']))) - report("test_pfs_middleware()", not failures) - return failures - - -def build_proxyfs(options): - commands = get_go_commands(options, "github.com/swiftstack/ProxyFS/") - if options.packages is None: - selected_packages = PACKAGES - else: - selected_packages = options.packages - return execute_go_commands(commands, selected_packages) - - -def build_dependencies(options): - failures = 0 - proxyfs_dir = os.path.dirname(os.path.abspath(__file__)) - stringer_path = 'golang.org/x/tools/cmd/stringer' - full_stringer_path = os.path.join(proxyfs_dir, "vendor", stringer_path) - print("Building " + stringer_path) - install_cmd = ("go", "install") - with return_to_wd(): - os.chdir(full_stringer_path) - success = not(bool(subprocess.call(install_cmd))) - failures += not success - report("build_dependencies()", not failures) - return failures - - -def build_jrpcclient(options): - proxyfs_dir = os.path.dirname(os.path.abspath(__file__)) - jrpcclient_dir = os.path.join(proxyfs_dir, "jrpcclient") - command = ['./regression_test.py'] - if options.no_install: - command.append('--no-install') - if options.deb_builder: - command.append('--deb-builder') - if options.just_build_libs: - command.append('--just-build-libs') - return bool(subprocess.call(command, cwd=jrpcclient_dir)) - - -def build_vfs(options): - proxyfs_dir = os.path.dirname(os.path.abspath(__file__)) - vfs_dir = os.path.join(proxyfs_dir, "vfs") - failures = 0 - clean_cmd = ('make', 'clean') - failures += bool(subprocess.call(clean_cmd, cwd=vfs_dir)) - if failures: - return failures - failures += bool(subprocess.call('make', cwd=vfs_dir)) - if failures: - return failures - distro = platform.linux_distribution()[0] - if not options.no_install: - if options.deb_builder: - if 'centos' in distro.lower(): - install_cmd = ('make', 'installcentos') - else: - install_cmd = ('make', 'install') - else: - if 'centos' in distro.lower(): - install_cmd = ('sudo', '-E', 'make', 'installcentos') - else: - install_cmd = ('sudo', '-E', 'make', 'install') - failures += bool(subprocess.call(install_cmd, cwd=vfs_dir)) - return failures - - -def main(options): - failures = 0 - - if not options.just_libs and not options.just_build_libs: - - go_version = subprocess.check_output((['go', 'version'])) - color_print(go_version[:-1], "bright green") - - if not options.quiet: - logging.basicConfig(format="%(message)s", level=logging.INFO) - - failures += build_dependencies(options) - if failures: - return failures - - failures += build_proxyfs(options) - if failures: - return failures - - if platform.system() != "Darwin": - - if not options.no_libs: - - failures += build_jrpcclient(options) - if failures: - return failures - - failures += build_vfs(options) - if failures: - return failures - - return failures - - -if __name__ == "__main__": - arg_parser = argparse.ArgumentParser(description=__doc__) - arg_parser.add_argument('--bench', '-bench', - action='store_const', const='-bench=.', - help="include benchmark measurements in test output") - arg_parser.add_argument('--cover', '-cover', - action='store_const', const='-cover', - help="include coverage statistics in test output") - arg_parser.add_argument('--get', '-get', action='store_true', - help="invoke `go get` to retrieve new dependencies") - arg_parser.add_argument('--packages', '-p', action='store', nargs='*', - help="specific packages to process") - arg_parser.add_argument('--no-install', action='store_true', - help="When building C libraries, do not attempt " - "to install resulting objects") - arg_parser.add_argument('--deb-builder', action='store_true', - help="Modify commands to run inside " - "swift-deb-builder") - arg_parser.add_argument('--quiet', '-q', action='store_true', - help="suppress printing of what commands are being run") - libs_group = arg_parser.add_mutually_exclusive_group() - libs_group.add_argument('--no-libs', action='store_true', - help="don't build C libraries or run C tests") - libs_group.add_argument('--just-build-libs', action='store_true', - help="only build C libraries") - libs_group.add_argument('--just-libs', action='store_true', - help="only build C libraries and run C tests") - options = arg_parser.parse_args() - - exit(main(options)) From 43f0e7dc4025a75221d7796f6fa13412063c3b29 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 22 Jan 2021 11:38:55 -0800 Subject: [PATCH 005/160] Removed references to Samba --- saio/bin/provision_samba_source | 62 ------- saio/bin/start_and_mount_pfs | 290 -------------------------------- saio/vagrant_provision.sh | 150 +---------------- sait/vagrant_provision.sh | 54 +----- 4 files changed, 2 insertions(+), 554 deletions(-) delete mode 100755 saio/bin/provision_samba_source delete mode 100755 saio/bin/start_and_mount_pfs diff --git a/saio/bin/provision_samba_source b/saio/bin/provision_samba_source deleted file mode 100755 index fef07fafe..000000000 --- a/saio/bin/provision_samba_source +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -# Setup Samba Source - -cd /vagrant/src/github.com/swiftstack/ProxyFS/vfs -if [[ -d samba4-6-centos ]] -then - if [[ -L samba4-6-centos ]] - then - echo "non-directory symlink \$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba4-6-centos cannot pre-exist" - exit 1 - else - echo "\$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba4-6-centos assumed to be as desired" - fi -else - if [[ -L samba4-6-centos ]] - then - echo "non-directory symlink \$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba4-6-centos cannot pre-exist" - exit 1 - else - git clone -b v4-6-stable --single-branch --depth 1 https://github.com/samba-team/samba.git samba4-6-centos - fi -fi -if [[ -L samba ]] -then - samba_symlink_target=`readlink "samba"` - if [[ "samba4-6-centos" == "$samba_symlink_target" ]] - then - echo "symlink samba -> samba4-6-centos already" - else - echo "redirecting samba -> samba4-6-centos" - rm samba - ln -s samba4-6-centos samba - fi -else - if [[ -e samba ]] - then - echo "non-symlink \$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba cannot pre-exist" - exit 1 - else - echo "establishing samba -> samba4-6-centos" - ln -s samba4-6-centos samba - fi -fi -cd samba -if [[ -d bin ]] -then - echo "./configure has already been run" -else - ./configure - make clean -fi -if [[ -f bin/default/librpc/gen_ndr/ndr_smb_acl.h ]] -then - echo "make GEN_NDR_TABLES has already been run" -else - make clean - make GEN_NDR_TABLES -fi diff --git a/saio/bin/start_and_mount_pfs b/saio/bin/start_and_mount_pfs deleted file mode 100755 index e14eb097e..000000000 --- a/saio/bin/start_and_mount_pfs +++ /dev/null @@ -1,290 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -# A simple script to start the services and mount the sample mount point -# The PATH, etc should already be setup by systemctl environment - -function await_proxyfsd_startup { - /usr/bin/systemctl -q is-active proxyfsd - if [ $? -ne 0 ]; then - echo "ProxyFS failed to start. Exiting..." - exit 1 - fi - while true; do - curl http://127.0.0.1:15346/ 2>/dev/null >/dev/null - if [ $? -eq 0 ]; then - break - fi - echo "Waiting for ProxyFS to be started..." - sleep 1 - done -} - -function await_pfsagentd_startup { - /usr/bin/systemctl -q is-active pfsagentd - if [ $? -ne 0 ]; then - echo "PFSAgent failed to start. Exiting..." - exit 1 - fi - while true; do - curl http://127.0.0.1:9090/version 2>/dev/null >/dev/null - if [ $? -eq 0 ]; then - # Service is active and curl to the HTTP server succeeded. - # We're go to go. - break - fi - echo "Waiting for PFSAgent to be started..." - sleep 1 - done -} - -function await_swift_startup { - while true - do - curl http://127.0.0.1:8090/info 2>/dev/null >/dev/null - if [ $? -eq 0 ] - then - break - fi - echo "Waiting for Swift to be started..." - sleep 1 - done -} - -function format_volume_if_necessary { - sudo /vagrant/bin/mkproxyfs -I $1 /vagrant/src/github.com/swiftstack/ProxyFS/saio/proxyfs.conf SwiftClient.RetryLimit=1 - if [ $? -ne 0 ] - then - echo "Could not pre-format $1" - exit 1 - fi -} - -function containsElement () { - local e match="$1" - shift - for e; do [[ "$e" == "$match" ]] && return 0; done - return 1 -} - -help() { - echo "Usage: $0 [mount type]" - echo "Mount type options:" - echo " all: NFS v3 and SMB v3.0 (default option)" - echo " keepmounts: don't umount/mount any mountpoints" - echo " nfs: NFS v3" - echo " smb: SMB v3.0" - echo " smb1: SMB v1.0" - echo " smb2: SMB v2.1" - echo " smb3: SMB v3.0" - echo " pfsa: PFSAgent" -} - -MOUNT_OPTIONS=() -if [ $# -gt 0 ]; then - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -h|--help) - help - exit 0 - ;; - all|keepmounts|smb|smb1|smb2|smb3|nfs|pfsa) - if containsElement "$key" "${MOUNT_OPTIONS[@]}"; then - echo "Error: duplicated option '${key}'." - echo - help - exit 1 - fi - MOUNT_OPTIONS+=("$1") - shift - ;; - *) - echo "Invalid argument '$key'." - echo - help - exit 1 - esac - done - - # Now that we allow more than one mount option to be passed, we have to check - # for incompatible options. - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "all" "${MOUNT_OPTIONS[@]}"; then - echo "Error: no other options are allowed while using 'all'." - echo - help - exit 1 - fi - - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "keepmounts" "${MOUNT_OPTIONS[@]}"; then - echo "Error: no other options are allowed while using 'keepmounts'." - echo - help - exit 1 - fi - - # I know the next 4 'if' clauses are gross, but I don't have the patience to - # find out the proper way to do it in bash. - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "smb" "${MOUNT_OPTIONS[@]}"; then - if containsElement "smb1" "${MOUNT_OPTIONS[@]}" || containsElement "smb2" "${MOUNT_OPTIONS[@]}" || containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - echo "Error: only one SMB option can be specified at a time." - echo - help - exit 1 - fi - fi - - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "smb1" "${MOUNT_OPTIONS[@]}"; then - if containsElement "smb" "${MOUNT_OPTIONS[@]}" || containsElement "smb2" "${MOUNT_OPTIONS[@]}" || containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - echo "Error: only one SMB option can be specified at a time." - echo - help - exit 1 - fi - fi - - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "smb2" "${MOUNT_OPTIONS[@]}"; then - if containsElement "smb1" "${MOUNT_OPTIONS[@]}" || containsElement "smb" "${MOUNT_OPTIONS[@]}" || containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - echo "Error: only one SMB option can be specified at a time." - echo - help - exit 1 - fi - fi - - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - if containsElement "smb1" "${MOUNT_OPTIONS[@]}" || containsElement "smb2" "${MOUNT_OPTIONS[@]}" || containsElement "smb" "${MOUNT_OPTIONS[@]}"; then - echo "Error: only one SMB option can be specified at a time." - echo - help - exit 1 - fi - fi -else - MOUNT_OPTIONS+=("all") -fi - -if containsElement "all" "${MOUNT_OPTIONS[@]}"; then - NFS_VERS=3 - SMB_VERS=3.0 -else - if containsElement "nfs" "${MOUNT_OPTIONS[@]}"; then - NFS_VERS=3 - fi - - if containsElement "smb1" "${MOUNT_OPTIONS[@]}"; then - SMB_VERS=1.0 - elif containsElement "smb2" "${MOUNT_OPTIONS[@]}"; then - SMB_VERS=2.1 - elif containsElement "smb" "${MOUNT_OPTIONS[@]}" || containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - SMB_VERS=3.0 - fi -fi - -MOUNT_POINT_NFS=127.0.0.1:/CommonMountPoint -SHARE_NFS=/mnt/nfs_proxyfs_mount/ - -MOUNT_POINT_SMB=//127.0.0.1/proxyfs -SHARE_SMB=/mnt/smb_proxyfs_mount/ - -UID_SMB=`id -u` -GID_SMB=`id -g` - -sudo mount -a - -# "keepmounts" means "keep everything as it was", but PFSAgent works differently, -# so just preserving the mount points is not enough. What we'll do is check if -# it's running before trying to stop it, and if it is, save it as a "MOUNT_OPTION" -if [ -f /usr/bin/systemctl ] && containsElement "keepmounts" "${MOUNT_OPTIONS[@]}"; then - /usr/bin/systemctl is-active --quiet pfsagentd && MOUNT_OPTIONS+=("pfsa") -fi - -if containsElement "keepmounts" "${MOUNT_OPTIONS[@]}"; then - echo "Shutting down services..." - /vagrant/src/github.com/swiftstack/ProxyFS/saio/bin/unmount_and_stop_pfs keepmounts -else - echo "Shutting down services and mount points..." - /vagrant/src/github.com/swiftstack/ProxyFS/saio/bin/unmount_and_stop_pfs -fi -echo - -echo "Bringing up services..." -if [ -f /usr/bin/systemctl ]; then - # Centos - MOUNT=/usr/bin/mount - sudo /usr/bin/systemctl start memcached - sudo /usr/bin/swift-init main start - await_swift_startup - format_volume_if_necessary CommonVolume - sudo /usr/bin/systemctl start proxyfsd - await_proxyfsd_startup - echo "ProxyFS successfully started" - sudo /usr/bin/systemctl start smb - sudo /usr/bin/systemctl start rpcbind - sudo /usr/bin/systemctl start nfs-server - sudo /usr/bin/systemctl start nfs-lock - sudo /usr/bin/systemctl start nfs-idmap - if containsElement "all" "${MOUNT_OPTIONS[@]}" || containsElement "pfsa" "${MOUNT_OPTIONS[@]}"; then - echo "Starting PFSAgent..." - sudo /usr/bin/systemctl start pfsagentd - await_pfsagentd_startup - fi -else - # Ubuntu (not tested!) - MOUNT=/bin/mount - sudo /usr/sbin/service memcached start - sudo /usr/bin/swift-init main start - await_swift_startup - format_volume_if_necessary CommonVolume - sudo /usr/sbin/service proxyfsd start - await_proxyfsd_startup - echo "ProxyFS successfully started" - sudo /usr/sbin/service smbd start - sudo /usr/sbin/service rpcbind start - sudo /usr/sbin/service nfs-server start - sudo /usr/sbin/service nfs-lock start - sudo /usr/sbin/service nfs-idmap start - # Here we should start pfsagentd (if 'all' or 'pfsa' are present in - # $MOUNT_OPTIONS), but we don't support Ubuntu -fi -echo - -if containsElement "all" "${MOUNT_OPTIONS[@]}" || containsElement "nfs" "${MOUNT_OPTIONS[@]}"; then - mountpoint -q $SHARE_NFS - if [ $? -ne 0 ]; then - for i in $(seq 5); do - sleep 5 - sudo $MOUNT -t nfs -o vers=$NFS_VERS $MOUNT_POINT_NFS $SHARE_NFS && break - echo "Mount of $SHARE_NFS failed. Retrying..." - done - fi - - mountpoint -q $SHARE_NFS - if [ $? -ne 0 ]; then - echo "ERROR: Could not mount $SHARE_NFS." - exit 1 - else - echo "$SHARE_NFS successfully mounted" - fi -fi - -if containsElement "all" "${MOUNT_OPTIONS[@]}" || containsElement "smb" "${MOUNT_OPTIONS[@]}" || containsElement "smb1" "${MOUNT_OPTIONS[@]}" || containsElement "smb2" "${MOUNT_OPTIONS[@]}" || containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - mountpoint -q $SHARE_SMB - if [ $? -ne 0 ]; then - for i in $(seq 5); do - sleep 5 - sudo $MOUNT -t cifs -o user=swift,password=swift,uid=$UID_SMB,gid=$GID_SMB,vers=$SMB_VERS,iocharset=utf8,actimeo=0 $MOUNT_POINT_SMB $SHARE_SMB && break - echo "Mount of $SHARE_SMB failed. Retrying..." - done - fi - - mountpoint -q $SHARE_SMB - if [ $? -ne 0 ]; then - echo "ERROR: Could not mount $SHARE_SMB." - exit 1 - else - echo "$SHARE_SMB successfully mounted" - fi -fi diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index a97c429c2..76b672ccc 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -117,77 +117,6 @@ echo "alias goclean=\"go clean;go clean --cache;go clean --testcache\"" >> ~vagr echo "alias gogetdlv=\"go get -u github.com/go-delve/delve/cmd/dlv\"" >> ~vagrant/.bash_profile echo "user_allow_other" >> /etc/fuse.conf -# Setup Samba - -yum -y install gcc-c++-4.8.5-16.el7_4.2 \ - python-devel-2.7.5-58.el7 \ - gnutls-devel-3.3.26-9.el7 \ - libacl-devel-2.2.51-12.el7 \ - openldap-devel-2.4.44-5.el7 \ - samba-4.6.2-12.el7_4 \ - samba-client-4.6.2-12.el7_4 \ - cifs-utils-6.2-10.el7 -cd /vagrant/src/github.com/swiftstack/ProxyFS/vfs -OS_DISTRO=centos -OS_DISTRO_VERSION=7.4 -SAMBA_VERSION=4.6.12 -SAMBA_DIR=build-samba-`echo $SAMBA_VERSION | tr . -`-${OS_DISTRO}-`echo $OS_DISTRO_VERSION | tr . -` -if [[ -d $SAMBA_DIR ]] -then - if [[ -L $SAMBA_DIR ]] - then - echo "directory symlink \$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/$SAMBA_DIR cannot pre-exist" - exit 1 - else - echo "\$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/$SAMBA_DIR assumed to be as desired" - fi -else - if [[ -L $SAMBA_DIR ]] - then - echo "non-directory symlink \$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/$SAMBA_DIR cannot pre-exist" - exit 1 - else - git clone -b samba-$SAMBA_VERSION --single-branch --depth 1 https://github.com/samba-team/samba.git $SAMBA_DIR - fi -fi -if [[ -L samba ]] -then - samba_symlink_target=`readlink "samba"` - if [[ "$SAMBA_DIR" == "$samba_symlink_target" ]] - then - echo "symlink samba -> $SAMBA_DIR already" - else - echo "redirecting samba -> $SAMBA_DIR" - rm samba - ln -s $SAMBA_DIR samba - fi -else - if [[ -e samba ]] - then - echo "non-symlink \$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba cannot pre-exist" - exit 1 - else - echo "establishing samba -> $SAMBA_DIR" - ln -s $SAMBA_DIR samba - fi -fi -cd samba -if [[ -d bin ]] -then - echo "./configure has already been run" -else - ./configure - make clean -fi -if [[ -f bin/default/librpc/gen_ndr/ndr_smb_acl.h ]] -then - echo "make GEN_NDR_TABLES has already been run" -else - make clean - make GEN_NDR_TABLES -fi -echo "export SAMBA_SOURCE=\$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba" >> ~vagrant/.bash_profile - # Install Python tox pip install tox==3.5.3 @@ -392,82 +321,12 @@ chmod 777 /var/log chmod 777 /var/log/proxyfsd chmod 666 /var/log/proxyfsd/proxyfsd.log -# Create Mount Points for ProxyFS (FUSE, NFS, & SMB) +# Create Mount Points for ProxyFS (embedded FUSE) rm -rf /CommonMountPoint mkdir /CommonMountPoint chmod 777 /CommonMountPoint -rm -rf /AgentMountPoint -mkdir /AgentMountPoint -chmod 777 /AgentMountPoint - -rm -rf /mnt/nfs_proxyfs_mount -mkdir /mnt/nfs_proxyfs_mount -chmod 777 /mnt/nfs_proxyfs_mount - -rm -rf /mnt/smb_proxyfs_mount -mkdir /mnt/smb_proxyfs_mount -chmod 777 /mnt/smb_proxyfs_mount - -# Configure exports (NFS) / shares (SMB) - -cp /vagrant/src/github.com/swiftstack/ProxyFS/saio/etc/exports /etc/exports -cp /vagrant/src/github.com/swiftstack/ProxyFS/saio/etc/samba/smb.conf /etc/samba/smb.conf -echo -e "swift\nswift" | smbpasswd -a swift - -# Install Kerberos Client to SDOM{1|2|3|4}.LOCAL hosted by sdc{1|2|3|4}.sdom{1|2|3|4}.local - -yum -y install krb5-workstation - -cat >> /etc/hosts << EOF -172.28.128.11 sdc1 sdc1.sdom1.local -172.28.128.12 sdc2 sdc2.sdom2.local -172.28.128.13 sdc3 sdc3.sdom3.local -172.28.128.14 sdc4 sdc4.sdom4.local -172.28.128.21 saio1 saio1.sdom1.local -172.28.128.22 saio2 saio2.sdom2.local -172.28.128.23 saio3 saio3.sdom3.local -172.28.128.24 saio4 saio4.sdom4.local -EOF - -cat > /etc/krb5.conf.d/SambaDCs << EOF -[libdefaults] -dns_lookup_kdc = false - -[realms] -SDOM1.LOCAL = { - admin_server = sdc1.sdom1.local - kdc = sdc1.sdom1.local - default_domain = SDOM1 -} -SDOM2.LOCAL = { - admin_server = sdc2.sdom2.local - kdc=sdc2.sdom2.local - default_domain = SDOM2 -} -SDOM3.LOCAL = { - admin_server = sdc3.sdom3.local - kdc=sdc3.sdom3.local - default_domain = SDOM3 -} -SDOM4.LOCAL = { - admin_server = sdc4.sdom4.local - kdc=sdc4.sdom4.local - default_domain = SDOM4 -} - -[domain_realm] -.sdom1.local = SDOM1.LOCAL -sdom1.local = SDOM1.LOCAL -.sdom2.local = SDOM2.LOCAL -sdom2.local = SDOM2.LOCAL -.sdom3.local = SDOM3.LOCAL -sdom3.local = SDOM3.LOCAL -.sdom4.local = SDOM4.LOCAL -sdom4.local = SDOM4.LOCAL -EOF - # Install systemd .service files for ProxyFS cp /vagrant/src/github.com/swiftstack/ProxyFS/saio/usr/lib/systemd/system/proxyfsd.service /usr/lib/systemd/system/. @@ -550,13 +409,6 @@ EOF systemctl daemon-reload -# Add some VIPs - -ip addr add dev enp0s8 172.28.128.21/24 -ip addr add dev enp0s8 172.28.128.22/24 -ip addr add dev enp0s8 172.28.128.23/24 -ip addr add dev enp0s8 172.28.128.24/24 - # All done echo "SAIO for ProxyFS provisioned" diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index 3705f1672..e369f8293 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -324,7 +324,7 @@ chmod 777 /var/log chmod 777 /var/log/proxyfsd chmod 666 /var/log/proxyfsd/proxyfsd.log -# Create Mount Points for ProxyFS (FUSE, NFS, & SMB) +# Create Mount Points for ProxyFS (embedded FUSE) if [ "$SAIT_INSTANCE" == "1" ] then @@ -333,58 +333,6 @@ then chmod 777 /CommonMountPoint fi -# Install Kerberos Client to SDOM{1|2|3|4}.LOCAL hosted by sdc{1|2|3|4}.sdom{1|2|3|4}.local - -yum -y install krb5-workstation - -cat >> /etc/hosts << EOF -172.28.128.11 sdc1 sdc1.sdom1.local -172.28.128.12 sdc2 sdc2.sdom2.local -172.28.128.13 sdc3 sdc3.sdom3.local -172.28.128.14 sdc4 sdc4.sdom4.local -172.28.128.21 saio1 saio1.sdom1.local -172.28.128.22 saio2 saio2.sdom2.local -172.28.128.23 saio3 saio3.sdom3.local -172.28.128.24 saio4 saio4.sdom4.local -EOF - -cat > /etc/krb5.conf.d/SambaDCs << EOF -[libdefaults] -dns_lookup_kdc = false - -[realms] -SDOM1.LOCAL = { - admin_server = sdc1.sdom1.local - kdc = sdc1.sdom1.local - default_domain = SDOM1 -} -SDOM2.LOCAL = { - admin_server = sdc2.sdom2.local - kdc=sdc2.sdom2.local - default_domain = SDOM2 -} -SDOM3.LOCAL = { - admin_server = sdc3.sdom3.local - kdc=sdc3.sdom3.local - default_domain = SDOM3 -} -SDOM4.LOCAL = { - admin_server = sdc4.sdom4.local - kdc=sdc4.sdom4.local - default_domain = SDOM4 -} - -[domain_realm] -.sdom1.local = SDOM1.LOCAL -sdom1.local = SDOM1.LOCAL -.sdom2.local = SDOM2.LOCAL -sdom2.local = SDOM2.LOCAL -.sdom3.local = SDOM3.LOCAL -sdom3.local = SDOM3.LOCAL -.sdom4.local = SDOM4.LOCAL -sdom4.local = SDOM4.LOCAL -EOF - # Install systemd .service files for ProxyFS cp /vagrant/src/github.com/swiftstack/ProxyFS/sait/sait$SAIT_INSTANCE/usr/lib/systemd/system/proxyfsd.service /usr/lib/systemd/system/. From fa0568b8adb68a9ac046da14ddda1efc0260f9e1 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 25 Jan 2021 12:28:22 -0800 Subject: [PATCH 006/160] Initial check-in of IMGR... the replacement for ProxyFS (proxyfsd) --- imgr/Makefile | 6 + imgr/dummy_test.go | 11 + imgr/globals.go | 233 +++++++++++++++++++++ imgr/http_server.go_HIDE | 431 +++++++++++++++++++++++++++++++++++++++ imgr/imgr.conf | 31 +++ imgr/log.go | 75 +++++++ imgr/main.go | 80 ++++++++ 7 files changed, 867 insertions(+) create mode 100644 imgr/Makefile create mode 100644 imgr/dummy_test.go create mode 100644 imgr/globals.go create mode 100644 imgr/http_server.go_HIDE create mode 100644 imgr/imgr.conf create mode 100644 imgr/log.go create mode 100644 imgr/main.go diff --git a/imgr/Makefile b/imgr/Makefile new file mode 100644 index 000000000..f699a7896 --- /dev/null +++ b/imgr/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/swiftstack/ProxyFS/imgr + +include ../GoMakefile diff --git a/imgr/dummy_test.go b/imgr/dummy_test.go new file mode 100644 index 000000000..d42991728 --- /dev/null +++ b/imgr/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/imgr/globals.go b/imgr/globals.go new file mode 100644 index 000000000..2f7955d82 --- /dev/null +++ b/imgr/globals.go @@ -0,0 +1,233 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "container/list" + "os" + "sync" + "time" + + "github.com/swiftstack/ProxyFS/bucketstats" + "github.com/swiftstack/ProxyFS/conf" +) + +type configStruct struct { + PrivateIPAddr string + PublicIPAddr string + JSONRPCPort uint16 // To be served only on PrivateIPAddr via TCP + RetryRPCPort uint16 // To be served only on PublicIPAddr via TLS + HTTPServerPort uint16 // To be served only on PrivateIPAddr via TCP + + RetryRPCTTLCompleted time.Duration + RetryRPCAckTrim time.Duration + RetryRPCDeadlineIO time.Duration + RetryRPCKeepAlivePeriod time.Duration + + MinLeaseDuration time.Duration + LeaseInterruptInterval time.Duration + LeaseInterruptLimit uint32 + + SwiftNoAuthIPAddr string + SwiftNoAuthTCPPort uint16 + + SwiftRetryDelay time.Duration + SwiftRetryExpBackoff float64 + SwiftRetryLimit uint32 + + SwiftConnectionPoolSize uint32 + + LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled + LogToConsole bool + TraceEnabled bool +} + +type chunkedPutContextStruct struct { + sync.WaitGroup // Used to await completion of performChunkedPut goroutine + containerName string // + objectName string // + buf []byte // + chunkedPutListElement *list.Element // FIFO Element of fileInodeStruct.chunkedPutList + state uint8 // One of chunkedPutContextState{Open|Closing|Closed} + pos int // ObjectOffset just after last sent chunk + sendChan chan struct{} // Single element buffered chan to wake up *chunkedPutContextStruct.sendDaemon() + // will be closed to indicate a flush is requested + wakeChan chan struct{} // Single element buffered chan to wake up *chunkedPutContextStruct.Read() + // will be closed to indicate a flush is requested + inRead bool // Set when in Read() as a hint to Close() to help Read() cleanly exit + flushRequested bool // Set to remember that a flush has been requested of *chunkedPutContextStruct.Read() +} + +type statsStruct struct { + SharedLeaseRequests bucketstats.BucketLog2Round + PromoteLeaseRequests bucketstats.BucketLog2Round + ExclusiveLeaseRequests bucketstats.BucketLog2Round + DemoteLeaseRequests bucketstats.BucketLog2Round + ReleaseLeaseRequests bucketstats.BucketLog2Round + + DemoteLeaseInterrupts bucketstats.Totaler + RevokeLeaseInterrupts bucketstats.Totaler + + InodeGets bucketstats.BucketLog2Round + InodePuts bucketstats.BucketLog2Round +} + +type globalsStruct struct { + sync.RWMutex + config configStruct + logFile *os.File // == nil if config.LogFilePath == "" + stats *statsStruct +} + +var globals globalsStruct + +func initializeGlobals(confMap conf.ConfMap) { + var ( + err error + ) + + // Default logging related globals + + globals.config.LogFilePath = "" + globals.config.LogToConsole = true + globals.logFile = nil + + // Process resultant confMap + + globals.config.PublicIPAddr, err = confMap.FetchOptionValueString("IMGR", "PublicIPAddr") + if nil != err { + logFatal(err) + } + globals.config.PrivateIPAddr, err = confMap.FetchOptionValueString("IMGR", "PrivateIPAddr") + if nil != err { + logFatal(err) + } + globals.config.JSONRPCPort, err = confMap.FetchOptionValueUint16("IMGR", "JSONRPCPort") + if nil != err { + logFatal(err) + } + globals.config.RetryRPCPort, err = confMap.FetchOptionValueUint16("IMGR", "RetryRPCPort") + if nil != err { + logFatal(err) + } + globals.config.HTTPServerPort, err = confMap.FetchOptionValueUint16("IMGR", "HTTPServerPort") + if nil != err { + logFatal(err) + } + globals.config.HTTPServerPort, err = confMap.FetchOptionValueUint16("IMGR", "HTTPServerPort") + if nil != err { + logFatal(err) + } + + globals.config.RetryRPCTTLCompleted, err = confMap.FetchOptionValueDuration("IMGR", "RetryRPCTTLCompleted") + if nil != err { + logFatal(err) + } + globals.config.RetryRPCAckTrim, err = confMap.FetchOptionValueDuration("IMGR", "RetryRPCAckTrim") + if nil != err { + logFatal(err) + } + globals.config.RetryRPCDeadlineIO, err = confMap.FetchOptionValueDuration("IMGR", "RetryRPCDeadlineIO") + if nil != err { + logFatal(err) + } + globals.config.RetryRPCKeepAlivePeriod, err = confMap.FetchOptionValueDuration("IMGR", "RetryRPCKeepAlivePeriod") + if nil != err { + logFatal(err) + } + + globals.config.MinLeaseDuration, err = confMap.FetchOptionValueDuration("IMGR", "MinLeaseDuration") + if nil != err { + logFatal(err) + } + globals.config.LeaseInterruptInterval, err = confMap.FetchOptionValueDuration("IMGR", "LeaseInterruptInterval") + if nil != err { + logFatal(err) + } + globals.config.LeaseInterruptLimit, err = confMap.FetchOptionValueUint32("IMGR", "LeaseInterruptLimit") + if nil != err { + logFatal(err) + } + + globals.config.SwiftNoAuthIPAddr, err = confMap.FetchOptionValueString("IMGR", "SwiftNoAuthIPAddr") + if nil != err { + logFatal(err) + } + globals.config.SwiftNoAuthTCPPort, err = confMap.FetchOptionValueUint16("IMGR", "SwiftNoAuthTCPPort") + if nil != err { + logFatal(err) + } + + globals.config.SwiftRetryDelay, err = confMap.FetchOptionValueDuration("IMGR", "SwiftRetryDelay") + if nil != err { + logFatal(err) + } + globals.config.SwiftRetryExpBackoff, err = confMap.FetchOptionValueFloat64("IMGR", "SwiftRetryExpBackoff") + if nil != err { + logFatal(err) + } + globals.config.SwiftRetryLimit, err = confMap.FetchOptionValueUint32("IMGR", "SwiftRetryLimit") + if nil != err { + logFatal(err) + } + + globals.config.SwiftConnectionPoolSize, err = confMap.FetchOptionValueUint32("IMGR", "SwiftConnectionPoolSize") + if nil != err { + logFatal(err) + } + + globals.config.LogFilePath, err = confMap.FetchOptionValueString("IMGR", "LogFilePath") + if nil != err { + err = confMap.VerifyOptionValueIsEmpty("IMGR", "LogFilePath") + if nil == err { + globals.config.LogFilePath = "" + } else { + logFatalf("[IMGR]LogFilePath must either be a valid string or empty]") + } + } + globals.config.LogToConsole, err = confMap.FetchOptionValueBool("IMGR", "LogToConsole") + if nil != err { + logFatal(err) + } + globals.config.TraceEnabled, err = confMap.FetchOptionValueBool("IMGR", "TraceEnabled") + if nil != err { + logFatal(err) + } + + globals.stats = &statsStruct{} + + bucketstats.Register("IMGR", "", globals.stats) +} + +func uninitializeGlobals() { + globals.config.PrivateIPAddr = "" + globals.config.PublicIPAddr = "" + globals.config.JSONRPCPort = 0 + globals.config.RetryRPCPort = 0 + globals.config.HTTPServerPort = 0 + + globals.config.RetryRPCTTLCompleted = time.Duration(0) + globals.config.RetryRPCAckTrim = time.Duration(0) + globals.config.RetryRPCDeadlineIO = time.Duration(0) + globals.config.RetryRPCKeepAlivePeriod = time.Duration(0) + + globals.config.MinLeaseDuration = time.Duration(0) + globals.config.LeaseInterruptInterval = time.Duration(0) + globals.config.LeaseInterruptLimit = 0 + + globals.config.SwiftNoAuthIPAddr = "" + globals.config.SwiftNoAuthTCPPort = 0 + + globals.config.SwiftRetryDelay = time.Duration(0) + globals.config.SwiftRetryExpBackoff = 0.0 + globals.config.SwiftRetryLimit = 0 + + globals.config.SwiftConnectionPoolSize = 0 + + globals.config.LogFilePath = "" + globals.config.LogToConsole = false + globals.config.TraceEnabled = false + + bucketstats.UnRegister("IMGR", "") +} diff --git a/imgr/http_server.go_HIDE b/imgr/http_server.go_HIDE new file mode 100644 index 000000000..486d54b13 --- /dev/null +++ b/imgr/http_server.go_HIDE @@ -0,0 +1,431 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "container/list" + "context" + "encoding/json" + "fmt" + "log" + "net" + "net/http" + "net/http/pprof" + "reflect" + "runtime" + "strconv" + "strings" + "sync/atomic" + + "github.com/swiftstack/sortedmap" + + "github.com/swiftstack/ProxyFS/bucketstats" + "github.com/swiftstack/ProxyFS/version" +) + +type leaseReportStruct struct { + MountID string + None []string // inode.InodeNumber in 16-digit Hex (no leading "0x") + SharedRequested []string + SharedGranted []string + SharedPromoting []string + SharedReleasing []string + ExclusiveRequested []string + ExclusiveGranted []string + ExclusiveDemoting []string + ExclusiveReleasing []string +} + +func serveHTTP() { + var ( + ipAddrTCPPort string + ) + + ipAddrTCPPort = net.JoinHostPort(globals.config.HTTPServerIPAddr, strconv.Itoa(int(globals.config.HTTPServerTCPPort))) + + globals.httpServer = &http.Server{ + Addr: ipAddrTCPPort, + Handler: &globals, + } + + globals.httpServerWG.Add(1) + + go func() { + var ( + err error + ) + + err = globals.httpServer.ListenAndServe() + if http.ErrServerClosed != err { + log.Fatalf("httpServer.ListenAndServe() exited unexpectedly: %v", err) + } + + globals.httpServerWG.Done() + }() +} + +func unserveHTTP() { + var ( + err error + ) + + err = globals.httpServer.Shutdown(context.TODO()) + if nil != err { + log.Fatalf("httpServer.Shutdown() returned with an error: %v", err) + } + + globals.httpServerWG.Wait() +} + +func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { + switch request.Method { + case http.MethodGet: + serveGet(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func serveGet(responseWriter http.ResponseWriter, request *http.Request) { + var ( + path string + ) + + path = strings.TrimRight(request.URL.Path, "/") + + switch { + case "" == path: + serveGetOfIndexDotHTML(responseWriter, request) + case "/config" == path: + serveGetOfConfig(responseWriter, request) + case "/debug/pprof/cmdline" == path: + pprof.Cmdline(responseWriter, request) + case "/debug/pprof/profile" == path: + pprof.Profile(responseWriter, request) + case "/debug/pprof/symbol" == path: + pprof.Symbol(responseWriter, request) + case "/debug/pprof/trace" == path: + pprof.Trace(responseWriter, request) + case strings.HasPrefix(path, "/debug/pprof"): + pprof.Index(responseWriter, request) + case "index.html" == path: + serveGetOfIndexDotHTML(responseWriter, request) + case "/leases" == path: + serveGetOfLeases(responseWriter, request) + case "/metrics" == path: + serveGetOfMetrics(responseWriter, request) + case "/stats" == path: + serveGetOfStats(responseWriter, request) + case "/version" == path: + serveGetOfVersion(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusNotFound) + } +} + +func serveGetOfConfig(responseWriter http.ResponseWriter, request *http.Request) { + var ( + confMapJSON bytes.Buffer + confMapJSONPacked []byte + ok bool + paramList []string + sendPackedConfig bool + ) + + paramList, ok = request.URL.Query()["compact"] + if ok { + if 0 == len(paramList) { + sendPackedConfig = false + } else { + sendPackedConfig = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) + } + } else { + sendPackedConfig = false + } + + confMapJSONPacked, _ = json.Marshal(globals.config) + + responseWriter.Header().Set("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + + if sendPackedConfig { + _, _ = responseWriter.Write(confMapJSONPacked) + } else { + json.Indent(&confMapJSON, confMapJSONPacked, "", "\t") + _, _ = responseWriter.Write(confMapJSON.Bytes()) + _, _ = responseWriter.Write([]byte("\n")) + } +} + +func serveGetOfIndexDotHTML(responseWriter http.ResponseWriter, request *http.Request) { + responseWriter.Header().Set("Content-Type", "text/html") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, net.JoinHostPort(globals.config.HTTPServerIPAddr, strconv.Itoa(int(globals.config.HTTPServerTCPPort)))))) +} + +func serveGetOfLeases(responseWriter http.ResponseWriter, request *http.Request) { + var ( + fileInode *fileInodeStruct + leaseListElement *list.Element + leaseReport *leaseReportStruct + leaseReportJSON bytes.Buffer + leaseReportJSONPacked []byte + ok bool + paramList []string + sendPackedLeaseReport bool + ) + + leaseReport = &leaseReportStruct{ + MountID: fmt.Sprintf("%s", globals.mountID), + None: make([]string, 0), + SharedRequested: make([]string, 0), + SharedGranted: make([]string, 0), + SharedPromoting: make([]string, 0), + SharedReleasing: make([]string, 0), + ExclusiveRequested: make([]string, 0), + ExclusiveGranted: make([]string, 0), + ExclusiveDemoting: make([]string, 0), + ExclusiveReleasing: make([]string, 0), + } + + globals.Lock() + + for leaseListElement = globals.unleasedFileInodeCacheLRU.Front(); leaseListElement != nil; leaseListElement = leaseListElement.Next() { + fileInode = leaseListElement.Value.(*fileInodeStruct) + switch fileInode.leaseState { + case fileInodeLeaseStateNone: + leaseReport.None = append(leaseReport.None, fmt.Sprintf("%016X", fileInode.InodeNumber)) + case fileInodeLeaseStateSharedReleasing: + leaseReport.SharedReleasing = append(leaseReport.SharedReleasing, fmt.Sprintf("%016X", fileInode.InodeNumber)) + case fileInodeLeaseStateExclusiveReleasing: + leaseReport.ExclusiveReleasing = append(leaseReport.ExclusiveReleasing, fmt.Sprintf("%016X", fileInode.InodeNumber)) + default: + logFatalf("serveGetOfLeases() found unexpected fileInode.leaseState %v on globals.unleasedFileInodeCacheLRU", fileInode.leaseState) + } + } + + for leaseListElement = globals.sharedLeaseFileInodeCacheLRU.Front(); leaseListElement != nil; leaseListElement = leaseListElement.Next() { + fileInode = leaseListElement.Value.(*fileInodeStruct) + switch fileInode.leaseState { + case fileInodeLeaseStateSharedRequested: + leaseReport.SharedRequested = append(leaseReport.SharedRequested, fmt.Sprintf("%016X", fileInode.InodeNumber)) + case fileInodeLeaseStateSharedGranted: + leaseReport.SharedGranted = append(leaseReport.SharedGranted, fmt.Sprintf("%016X", fileInode.InodeNumber)) + case fileInodeLeaseStateExclusiveDemoting: + leaseReport.ExclusiveDemoting = append(leaseReport.ExclusiveDemoting, fmt.Sprintf("%016X", fileInode.InodeNumber)) + default: + logFatalf("serveGetOfLeases() found unexpected fileInode.leaseState %v on globals.sharedLeaseFileInodeCacheLRU", fileInode.leaseState) + } + } + + for leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.Front(); leaseListElement != nil; leaseListElement = leaseListElement.Next() { + fileInode = leaseListElement.Value.(*fileInodeStruct) + switch fileInode.leaseState { + case fileInodeLeaseStateSharedPromoting: + leaseReport.SharedPromoting = append(leaseReport.SharedPromoting, fmt.Sprintf("%016X", fileInode.InodeNumber)) + case fileInodeLeaseStateExclusiveRequested: + leaseReport.ExclusiveRequested = append(leaseReport.ExclusiveRequested, fmt.Sprintf("%016X", fileInode.InodeNumber)) + case fileInodeLeaseStateExclusiveGranted: + leaseReport.ExclusiveGranted = append(leaseReport.ExclusiveGranted, fmt.Sprintf("%016X", fileInode.InodeNumber)) + default: + logFatalf("serveGetOfLeases() found unexpected fileInode.leaseState %v on globals.exclusiveLeaseFileInodeCacheLRU", fileInode.leaseState) + } + } + + globals.Unlock() + + paramList, ok = request.URL.Query()["compact"] + if ok { + if 0 == len(paramList) { + sendPackedLeaseReport = false + } else { + sendPackedLeaseReport = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) + } + } else { + sendPackedLeaseReport = false + } + + leaseReportJSONPacked, _ = json.Marshal(leaseReport) + + responseWriter.Header().Set("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + + if sendPackedLeaseReport { + _, _ = responseWriter.Write(leaseReportJSONPacked) + } else { + json.Indent(&leaseReportJSON, leaseReportJSONPacked, "", "\t") + _, _ = responseWriter.Write(leaseReportJSON.Bytes()) + _, _ = responseWriter.Write([]byte("\n")) + } +} + +func serveGetOfMetrics(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + format string + i int + keyAsKey sortedmap.Key + keyAsString string + line string + longestKeyAsString int + longestValueAsString int + memStats runtime.MemStats + metricsFieldName string + metricsFieldValuePtr *uint64 + metricsLLRB sortedmap.LLRBTree + metricsLLRBLen int + metricsStructValue reflect.Value + metricsValue reflect.Value + ok bool + pauseNsAccumulator uint64 + valueAsString string + valueAsValue sortedmap.Value + ) + + runtime.ReadMemStats(&memStats) + + metricsLLRB = sortedmap.NewLLRBTree(sortedmap.CompareString, nil) + + // General statistics. + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Alloc", memStats.Alloc) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_TotalAlloc", memStats.TotalAlloc) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Sys", memStats.Sys) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Lookups", memStats.Lookups) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Mallocs", memStats.Mallocs) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Frees", memStats.Frees) + + // Main allocation heap statistics. + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapAlloc", memStats.HeapAlloc) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapSys", memStats.HeapSys) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapIdle", memStats.HeapIdle) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapInuse", memStats.HeapInuse) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapReleased", memStats.HeapReleased) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapObjects", memStats.HeapObjects) + + // Low-level fixed-size structure allocator statistics. + // Inuse is bytes used now. + // Sys is bytes obtained from system. + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_StackInuse", memStats.StackInuse) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_StackSys", memStats.StackSys) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MSpanInuse", memStats.MSpanInuse) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MSpanSys", memStats.MSpanSys) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MCacheInuse", memStats.MCacheInuse) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MCacheSys", memStats.MCacheSys) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_BuckHashSys", memStats.BuckHashSys) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_GCSys", memStats.GCSys) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_OtherSys", memStats.OtherSys) + + // Garbage collector statistics (fixed portion). + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_LastGC", memStats.LastGC) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseTotalNs", memStats.PauseTotalNs) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_NumGC", uint64(memStats.NumGC)) + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_GCCPUPercentage", uint64(100.0*memStats.GCCPUFraction)) + + // Garbage collector statistics (go_runtime_MemStats_PauseAverageNs). + if 0 == memStats.NumGC { + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseAverageNs", 0) + } else { + pauseNsAccumulator = 0 + if memStats.NumGC < 255 { + for i = 0; i < int(memStats.NumGC); i++ { + pauseNsAccumulator += memStats.PauseNs[i] + } + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseAverageNs", pauseNsAccumulator/uint64(memStats.NumGC)) + } else { + for i = 0; i < 256; i++ { + pauseNsAccumulator += memStats.PauseNs[i] + } + insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseAverageNs", pauseNsAccumulator/256) + } + } + + // Add in locally generated metrics + + metricsStructValue = reflect.Indirect(reflect.ValueOf(globals.metrics)) + metricsValue = reflect.ValueOf(globals.metrics).Elem() + + for i = 0; i < metricsStructValue.NumField(); i++ { + metricsFieldName = metricsStructValue.Type().Field(i).Name + metricsFieldValuePtr = metricsValue.Field(i).Addr().Interface().(*uint64) + insertInMetricsLLRB(metricsLLRB, metricsFieldName, atomic.LoadUint64(metricsFieldValuePtr)) + } + + // Produce sorted and column-aligned response + + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + + metricsLLRBLen, err = metricsLLRB.Len() + if nil != err { + logFatalf("metricsLLRB.Len() failed: %v", err) + } + + longestKeyAsString = 0 + longestValueAsString = 0 + + for i = 0; i < metricsLLRBLen; i++ { + keyAsKey, valueAsValue, ok, err = metricsLLRB.GetByIndex(i) + if nil != err { + logFatalf("llrb.GetByIndex(%v) failed: %v", i, err) + } + if !ok { + logFatalf("llrb.GetByIndex(%v) returned ok == false", i) + } + keyAsString = keyAsKey.(string) + valueAsString = valueAsValue.(string) + if len(keyAsString) > longestKeyAsString { + longestKeyAsString = len(keyAsString) + } + if len(valueAsString) > longestValueAsString { + longestValueAsString = len(valueAsString) + } + } + + format = fmt.Sprintf("%%-%vs %%%vs\n", longestKeyAsString, longestValueAsString) + + for i = 0; i < metricsLLRBLen; i++ { + keyAsKey, valueAsValue, ok, err = metricsLLRB.GetByIndex(i) + if nil != err { + logFatalf("llrb.GetByIndex(%v) failed: %v", i, err) + } + if !ok { + logFatalf("llrb.GetByIndex(%v) returned ok == false", i) + } + keyAsString = keyAsKey.(string) + valueAsString = valueAsValue.(string) + line = fmt.Sprintf(format, keyAsString, valueAsString) + _, _ = responseWriter.Write([]byte(line)) + } +} + +func serveGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(bucketstats.SprintStats(bucketstats.StatFormatParsable1, "*", "*"))) +} + +func insertInMetricsLLRB(metricsLLRB sortedmap.LLRBTree, metricKey string, metricValueAsUint64 uint64) { + var ( + err error + metricValueAsString string + ok bool + ) + + metricValueAsString = fmt.Sprintf("%v", metricValueAsUint64) + + ok, err = metricsLLRB.Put(metricKey, metricValueAsString) + if nil != err { + logFatalf("metricsLLRB.Put(%v, %v) failed: %v", metricKey, metricValueAsString, err) + } + if !ok { + logFatalf("metricsLLRB.Put(%v, %v) returned ok == false", metricKey, metricValueAsString) + } +} + +func serveGetOfVersion(responseWriter http.ResponseWriter, request *http.Request) { + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write([]byte(version.ProxyFSVersion)) +} diff --git a/imgr/imgr.conf b/imgr/imgr.conf new file mode 100644 index 000000000..d471c93b3 --- /dev/null +++ b/imgr/imgr.conf @@ -0,0 +1,31 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +[IMGR] +PublicIPAddr: 0.0.0.0 +PrivateIPAddr: 0.0.0.0 +JSONRPCPort: 12345 +RetryRPCPort: 32356 +HTTPServerPort: 15346 + +RetryRPCTTLCompleted: 10m +RetryRPCAckTrim: 100ms +RetryRPCDeadlineIO: 60s +RetryRPCKeepAlivePeriod: 60s + +MinLeaseDuration: 250ms +LeaseInterruptInterval: 250ms +LeaseInterruptLimit: 20 + +SwiftNoAuthIPAddr: 127.0.0.1 +SwiftNoAuthTCPPort: 8090 + +SwiftRetryDelay: 1s +SwiftRetryExpBackoff: 1.5 +SwiftRetryLimit: 11 + +SwiftConnectionPoolSize: 128 + +LogFilePath: # imgr.log +LogToConsole: true # false +TraceEnabled: false diff --git a/imgr/log.go b/imgr/log.go new file mode 100644 index 000000000..3736ab567 --- /dev/null +++ b/imgr/log.go @@ -0,0 +1,75 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "time" +) + +func logFatal(err error) { + logf("FATAL", "%v", err) + os.Exit(1) +} + +func logFatalf(format string, args ...interface{}) { + logf("FATAL", format, args...) + os.Exit(1) +} + +func logErrorf(format string, args ...interface{}) { + logf("ERROR", format, args...) +} + +func logWarnf(format string, args ...interface{}) { + logf("WARN", format, args...) +} + +func logInfof(format string, args ...interface{}) { + logf("INFO", format, args...) +} + +func logTracef(format string, args ...interface{}) { + if globals.config.TraceEnabled { + logf("TRACE", format, args...) + } +} + +func logf(level string, format string, args ...interface{}) { + var ( + enhancedArgs []interface{} + enhancedFormat string + err error + logMsg string + ) + + enhancedFormat = "[%s][%s] " + format + enhancedArgs = append([]interface{}{time.Now().Format(time.RFC3339Nano), level}, args...) + + logMsg = fmt.Sprintf(enhancedFormat, enhancedArgs[:]...) + + if nil == globals.logFile { + if "" != globals.config.LogFilePath { + globals.logFile, err = os.OpenFile(globals.config.LogFilePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666) + if nil == err { + _, _ = globals.logFile.WriteString(logMsg + "\n") + } else { + globals.logFile = nil + } + } + } else { + globals.logFile.WriteString(logMsg + "\n") + } + if globals.config.LogToConsole { + fmt.Fprintln(os.Stderr, logMsg) + } +} + +func logSIGHUP() { + if nil != globals.logFile { + _ = globals.logFile.Close() + globals.logFile = nil + } +} diff --git a/imgr/main.go b/imgr/main.go new file mode 100644 index 000000000..9a86ffee9 --- /dev/null +++ b/imgr/main.go @@ -0,0 +1,80 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "log" + "os" + "os/signal" + + "golang.org/x/sys/unix" + + "github.com/swiftstack/ProxyFS/conf" +) + +func main() { + var ( + confMap conf.ConfMap + err error + signalChan chan os.Signal + signalReceived os.Signal + ) + + if len(os.Args) < 2 { + log.Fatalf("no .conf file specified") + } + + // Parse arguments (at this point, logging goes only to the console) + + globals.logFile = nil + globals.config.LogFilePath = "" + globals.config.LogToConsole = true + + confMap, err = conf.MakeConfMapFromFile(os.Args[1]) + if nil != err { + log.Fatalf("failed to load config: %v", err) + } + + err = confMap.UpdateFromStrings(os.Args[2:]) + if nil != err { + log.Fatalf("failed to apply config overrides: %v", err) + } + + // Arm signal handler used to indicate termination and wait on it + // + // Note: signalled chan must be buffered to avoid race with window between + // arming handler and blocking on the chan read + + signalChan = make(chan os.Signal, 1) + + signal.Notify(signalChan, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) + + // Initialize globals + + initializeGlobals(confMap) + + // Indicate we are UP + + logInfof("UP") + + // Await any of specified signals or fission exit + + for { + signalReceived = <-signalChan + logInfof("Received signal: %v", signalReceived) + if unix.SIGHUP == signalReceived { + logSIGHUP() + } else { + break + } + } + + // Indicate we are DOWN + + logInfof("DOWN") + + // Uninitialize globals + + uninitializeGlobals() +} From 4218b30cdc2b098e1a7b863860ee63935f8bc23b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 25 Jan 2021 12:30:51 -0800 Subject: [PATCH 007/160] Disable "before_install" Travis doing a `git submodule update` --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 82bafdb6b..e64ba31f8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,5 @@ notifications: on_failure: always on_pull_requests: true -before_install: -- sed -i 's/git@github.com:/https:\/\/github.com\//' .gitmodules -- git submodule update --init --recursive - script: - docker run -e "COVERALLS_TOKEN=$COVERALLS_TOKEN" -e "TRAVIS_BRANCH=$TRAVIS_BRANCH" --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/swiftstack/ProxyFS swiftstack/proxyfs_unit_tests From 06ca1cbf1cb906a2046888187ed6b847c1478301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Mon, 25 Jan 2021 13:25:08 -0800 Subject: [PATCH 008/160] Remove SMB/VFS stuff from TravisCI --- test/container/launch.sh | 64 +++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/test/container/launch.sh b/test/container/launch.sh index 09ff8dd8a..3dee2a1ec 100755 --- a/test/container/launch.sh +++ b/test/container/launch.sh @@ -6,37 +6,39 @@ set -e set -x -# Setup Samba -# Ideally, these 3 vars should be populated with info from the system. -# For now, it's just hardcoded. -OS_DISTRO=centos -OS_DISTRO_VERSION=7.4 -SAMBA_VERSION=4.6.12 - -SAMBA_DIR=build-samba-`echo $SAMBA_VERSION | tr . -`-${OS_DISTRO}-`echo $OS_DISTRO_VERSION | tr . -` - -cd $GOPATH/src/github.com/swiftstack/ProxyFS/vfs - -if [[ -L samba ]]; then - rm samba -else - if [[ -e samba ]]; then - echo "non-symlink \$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba cannot pre-exist" - exit 1 - fi -fi - -if [[ -d "${SAMBA_DIR}" ]]; then - ln -s $SAMBA_DIR samba -else - git clone -b samba-$SAMBA_VERSION --single-branch --depth 1 https://github.com/samba-team/samba.git $SAMBA_DIR - ln -s $SAMBA_DIR samba - cd samba - ./configure - make clean - make GEN_NDR_TABLES -fi -export SAMBA_SOURCE=$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba +# TODO: We don't need this anymore, do we? Delete when we're 100% sure! +## Setup Samba +## Ideally, these 3 vars should be populated with info from the system. +## For now, it's just hardcoded. +#OS_DISTRO=centos +#OS_DISTRO_VERSION=7.4 +#SAMBA_VERSION=4.6.12 +# +#SAMBA_DIR=build-samba-`echo $SAMBA_VERSION | tr . -`-${OS_DISTRO}-`echo $OS_DISTRO_VERSION | tr . -` +# +#cd $GOPATH/src/github.com/swiftstack/ProxyFS/vfs +# +#if [[ -L samba ]]; then +# rm samba +#else +# if [[ -e samba ]]; then +# echo "non-symlink \$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba cannot pre-exist" +# exit 1 +# fi +#fi +# +#if [[ -d "${SAMBA_DIR}" ]]; then +# ln -s $SAMBA_DIR samba +#else +# git clone -b samba-$SAMBA_VERSION --single-branch --depth 1 https://github.com/samba-team/samba.git $SAMBA_DIR +# ln -s $SAMBA_DIR samba +# cd samba +# ./configure +# make clean +# make GEN_NDR_TABLES +#fi +#export SAMBA_SOURCE=$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba +# TODO: End of to-do # Build ProxyFS and run tests cd $GOPATH/src/github.com/swiftstack/ProxyFS From 22c6008fd5766228aebba7cbdd992d851cf53f90 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 25 Jan 2021 14:01:27 -0800 Subject: [PATCH 009/160] Went ahead and removed the commented out lines referring to Samba in test/container/launch.sh --- test/container/launch.sh | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/test/container/launch.sh b/test/container/launch.sh index 3dee2a1ec..4df25c0d8 100755 --- a/test/container/launch.sh +++ b/test/container/launch.sh @@ -6,40 +6,6 @@ set -e set -x -# TODO: We don't need this anymore, do we? Delete when we're 100% sure! -## Setup Samba -## Ideally, these 3 vars should be populated with info from the system. -## For now, it's just hardcoded. -#OS_DISTRO=centos -#OS_DISTRO_VERSION=7.4 -#SAMBA_VERSION=4.6.12 -# -#SAMBA_DIR=build-samba-`echo $SAMBA_VERSION | tr . -`-${OS_DISTRO}-`echo $OS_DISTRO_VERSION | tr . -` -# -#cd $GOPATH/src/github.com/swiftstack/ProxyFS/vfs -# -#if [[ -L samba ]]; then -# rm samba -#else -# if [[ -e samba ]]; then -# echo "non-symlink \$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba cannot pre-exist" -# exit 1 -# fi -#fi -# -#if [[ -d "${SAMBA_DIR}" ]]; then -# ln -s $SAMBA_DIR samba -#else -# git clone -b samba-$SAMBA_VERSION --single-branch --depth 1 https://github.com/samba-team/samba.git $SAMBA_DIR -# ln -s $SAMBA_DIR samba -# cd samba -# ./configure -# make clean -# make GEN_NDR_TABLES -#fi -#export SAMBA_SOURCE=$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba -# TODO: End of to-do - # Build ProxyFS and run tests cd $GOPATH/src/github.com/swiftstack/ProxyFS make ci From 9c6388c19dbaad193224f712fdf4601ca9c84b8f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 25 Jan 2021 14:05:19 -0800 Subject: [PATCH 010/160] Added imgr to top-level Makefile --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 3fad56c34..8922c0d25 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ gopkgsubdirs = \ gobinsubdirs = \ cleanproxyfs \ fsworkout \ + imgr \ inodeworkout \ pfs-crash \ pfs-fsck \ @@ -59,6 +60,7 @@ gobinsubdirs = \ ramswift/ramswift gobinsubdirsforci = \ + imgr \ pfsconfjson \ pfsconfjsonpacked \ confgen/confgen \ From 540cb7adb22362e203fbb07c1fb6024dbf62bdc8 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 25 Jan 2021 17:53:28 -0800 Subject: [PATCH 011/160] cp - a lot of recasting of ramswift -> emswift to capture --- Makefile | 6 +- emswift/Makefile | 6 + emswift/dummy_test.go | 11 + emswift/emswift.conf | 15 + emswift/emswiftpkg/Makefile | 6 + emswift/emswiftpkg/api.go | 23 + emswift/emswiftpkg/dummy_test.go | 11 + emswift/emswiftpkg/impl.go | 1542 +++++++++++++++++ .../emswiftpkg/setup_teardown_test.go_HIDE | 369 ++++ .../swift_proxy_emulator_test.go_HIDE | 411 +++++ emswift/main.go | 69 + imgr/imgrpkg/Makefile | 6 + imgr/imgrpkg/dummy.go | 4 + imgr/imgrpkg/dummy_test.go | 11 + 14 files changed, 2489 insertions(+), 1 deletion(-) create mode 100644 emswift/Makefile create mode 100644 emswift/dummy_test.go create mode 100644 emswift/emswift.conf create mode 100644 emswift/emswiftpkg/Makefile create mode 100644 emswift/emswiftpkg/api.go create mode 100644 emswift/emswiftpkg/dummy_test.go create mode 100644 emswift/emswiftpkg/impl.go create mode 100644 emswift/emswiftpkg/setup_teardown_test.go_HIDE create mode 100644 emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE create mode 100644 emswift/main.go create mode 100644 imgr/imgrpkg/Makefile create mode 100644 imgr/imgrpkg/dummy.go create mode 100644 imgr/imgrpkg/dummy_test.go diff --git a/Makefile b/Makefile index 8922c0d25..e78d746ad 100644 --- a/Makefile +++ b/Makefile @@ -32,10 +32,13 @@ gopkgsubdirs = \ transitions \ trackedlock \ utils \ - version + version \ + emswift/emswiftpkg \ + imgr/imgrpkg gobinsubdirs = \ cleanproxyfs \ + emswift \ fsworkout \ imgr \ inodeworkout \ @@ -60,6 +63,7 @@ gobinsubdirs = \ ramswift/ramswift gobinsubdirsforci = \ + emswift \ imgr \ pfsconfjson \ pfsconfjsonpacked \ diff --git a/emswift/Makefile b/emswift/Makefile new file mode 100644 index 000000000..74b2d5323 --- /dev/null +++ b/emswift/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/swiftstack/ProxyFS/emswift + +include ../GoMakefile diff --git a/emswift/dummy_test.go b/emswift/dummy_test.go new file mode 100644 index 000000000..d42991728 --- /dev/null +++ b/emswift/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/emswift/emswift.conf b/emswift/emswift.conf new file mode 100644 index 000000000..7bbdf9f28 --- /dev/null +++ b/emswift/emswift.conf @@ -0,0 +1,15 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +[EMSWIFT] +AuthIPAddr: # 0.0.0.0 +AuthTCPPort: # 8080 + +NoAuthIPAddr: 127.0.0.1 +NoAuthTCPPort: 8090 + +MaxAccountNameLength: 256 +MaxContainerNameLength: 256 +MaxObjectNameLength: 1024 +AccountListingLimit: 10000 +ContainerListingLimit: 10000 diff --git a/emswift/emswiftpkg/Makefile b/emswift/emswiftpkg/Makefile new file mode 100644 index 000000000..e40813d02 --- /dev/null +++ b/emswift/emswiftpkg/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/swiftstack/ProxyFS/emswift/emswiftpkg + +include ../../GoMakefile diff --git a/emswift/emswiftpkg/api.go b/emswift/emswiftpkg/api.go new file mode 100644 index 000000000..080d66b3b --- /dev/null +++ b/emswift/emswiftpkg/api.go @@ -0,0 +1,23 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package emswiftpkg + +import ( + "github.com/swiftstack/ProxyFS/conf" +) + +// Start is called to start serving the NoAuth Swift Proxy Port and, +// optionally, the Auth Swift Proxy Port +// +func Start(confMap conf.ConfMap) (err error) { + err = start(confMap) + return +} + +// Stop is called to stop serving +// +func Stop() (err error) { + err = stop() + return +} diff --git a/emswift/emswiftpkg/dummy_test.go b/emswift/emswiftpkg/dummy_test.go new file mode 100644 index 000000000..7c7334abd --- /dev/null +++ b/emswift/emswiftpkg/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package emswiftpkg + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/emswift/emswiftpkg/impl.go b/emswift/emswiftpkg/impl.go new file mode 100644 index 000000000..3ab3ea7d8 --- /dev/null +++ b/emswift/emswiftpkg/impl.go @@ -0,0 +1,1542 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package emswiftpkg + +import ( + "fmt" + "io/ioutil" + "math/rand" + "net" + "net/http" + "strconv" + "strings" + "sync" + "syscall" + + "golang.org/x/sys/unix" + + "github.com/swiftstack/sortedmap" + + "github.com/swiftstack/ProxyFS/conf" + "github.com/swiftstack/ProxyFS/utils" +) + +type swiftAccountStruct struct { + name string + headers http.Header + swiftContainerTree sortedmap.LLRBTree // key is swiftContainerStruct.name; value is *swiftContainerStruct +} + +type swiftContainerStruct struct { + name string + swiftAccount *swiftAccountStruct // back-reference to swiftAccountStruct + headers http.Header + swiftObjectTree sortedmap.LLRBTree // key is swiftObjectStruct.name; value is *swiftObjectStruct +} + +type swiftObjectStruct struct { + name string + swiftContainer *swiftContainerStruct // back-reference to swiftContainerStruct + headers http.Header + contents []byte +} + +type configStruct struct { + AuthIPAddr string // Only required if Auth Swift Proxy enabled + AuthTCPPort uint16 // Only required if Auth Swift Proxy enabled + + NoAuthIPAddr string + NoAuthTCPPort uint16 + + MaxAccountNameLength uint64 + MaxContainerNameLength uint64 + MaxObjectNameLength uint64 + AccountListingLimit uint64 + ContainerListingLimit uint64 +} + +type authEmulatorStruct struct { + httpServer *http.Server + wg sync.WaitGroup +} + +type noAuthEmulatorStruct struct { + httpServer *http.Server + wg sync.WaitGroup +} + +type globalsStruct struct { + sync.Mutex + config configStruct + authEmulator *authEmulatorStruct + noAuthEmulator *noAuthEmulatorStruct + swiftAccountMap map[string]*swiftAccountStruct // key is swiftAccountStruct.name; value is *swiftAccountStruct +} + +var globals globalsStruct + +type httpRequestHandler struct{} + +type rangeStruct struct { + startOffset uint64 + stopOffset uint64 +} + +type stringSet map[string]bool + +var headerNameIgnoreSet = stringSet{"Accept": true, "Accept-Encoding": true, "User-Agent": true, "Content-Length": true} + +func start(confMap conf.ConfMap) (err error) { + err = initializeGlobals(confMap) + if nil != err { + return + } + + err = startNoAuth() + if nil != err { + return + } + + err = startAuthIfRequested() + if nil != err { + return + } + + return +} + +func stop() (err error) { + err = stopAuthIfRequested() + if nil != err { + return + } + + err = stopNoAuth() + if nil != err { + return + } + + uninitializeGlobals() + + err = nil + return +} + +func initializeGlobals(confMap conf.ConfMap) (err error) { + globals.config.AuthIPAddr, err = confMap.FetchOptionValueString("EMSWIFT", "AuthIPAddr") + if nil == err { + globals.config.AuthTCPPort, err = confMap.FetchOptionValueUint16("EMSWIFT", "AuthTCPPort") + if nil != err { + return + } + } else { + err = nil + globals.config.AuthIPAddr = "" + globals.config.AuthTCPPort = 0 + } + + globals.config.NoAuthIPAddr, err = confMap.FetchOptionValueString("EMSWIFT", "NoAuthIPAddr") + if nil != err { + return + } + globals.config.NoAuthTCPPort, err = confMap.FetchOptionValueUint16("EMSWIFT", "NoAuthTCPPort") + if nil != err { + return + } + + globals.config.MaxAccountNameLength, err = confMap.FetchOptionValueUint64("EMSWIFT", "MaxAccountNameLength") + if nil != err { + return + } + globals.config.MaxContainerNameLength, err = confMap.FetchOptionValueUint64("EMSWIFT", "MaxContainerNameLength") + if nil != err { + return + } + globals.config.MaxObjectNameLength, err = confMap.FetchOptionValueUint64("EMSWIFT", "MaxObjectNameLength") + if nil != err { + return + } + globals.config.AccountListingLimit, err = confMap.FetchOptionValueUint64("EMSWIFT", "AccountListingLimit") + if nil != err { + return + } + globals.config.ContainerListingLimit, err = confMap.FetchOptionValueUint64("EMSWIFT", "ContainerListingLimit") + if nil != err { + return + } + + globals.authEmulator = nil + globals.noAuthEmulator = nil + + globals.swiftAccountMap = make(map[string]*swiftAccountStruct) + + return +} + +func uninitializeGlobals() { + globals.config.AuthIPAddr = "" + globals.config.AuthTCPPort = 0 + + globals.config.NoAuthIPAddr = "" + globals.config.NoAuthTCPPort = 0 + + globals.config.MaxAccountNameLength = 0 + globals.config.MaxContainerNameLength = 0 + globals.config.MaxObjectNameLength = 0 + globals.config.AccountListingLimit = 0 + globals.config.ContainerListingLimit = 0 + + globals.authEmulator = nil + globals.noAuthEmulator = nil + + globals.swiftAccountMap = make(map[string]*swiftAccountStruct) +} + +func startAuthIfRequested() (err error) { + var ( + authEmulator *authEmulatorStruct + ) + + if "" == globals.config.AuthIPAddr { + globals.authEmulator = nil + err = nil + return + } + + authEmulator = &authEmulatorStruct{ + httpServer: &http.Server{ + Addr: net.JoinHostPort(globals.config.AuthIPAddr, fmt.Sprintf("%d", globals.config.AuthTCPPort)), + }, + } + authEmulator.httpServer.Handler = authEmulator + + authEmulator.wg.Add(1) + + globals.authEmulator = authEmulator + + go func() { + _ = globals.authEmulator.httpServer.ListenAndServe() + globals.authEmulator.wg.Done() + }() + + err = nil + return +} + +func stopAuthIfRequested() (err error) { + if nil == globals.authEmulator { + err = nil + return + } + + err = globals.authEmulator.httpServer.Close() + if nil != err { + return + } + + globals.authEmulator.wg.Wait() + + globals.authEmulator = nil + + return +} + +func startNoAuth() (err error) { + var ( + noAuthEmulator *noAuthEmulatorStruct + ) + + noAuthEmulator = &noAuthEmulatorStruct{ + httpServer: &http.Server{ + Addr: net.JoinHostPort(globals.config.NoAuthIPAddr, fmt.Sprintf("%d", globals.config.NoAuthTCPPort)), + }, + } + noAuthEmulator.httpServer.Handler = noAuthEmulator + + noAuthEmulator.wg.Add(1) + + globals.noAuthEmulator = noAuthEmulator + + go func() { + _ = globals.noAuthEmulator.httpServer.ListenAndServe() + globals.noAuthEmulator.wg.Done() + }() + + err = nil + return +} + +func stopNoAuth() (err error) { + err = globals.noAuthEmulator.httpServer.Close() + if nil != err { + return + } + + globals.noAuthEmulator.wg.Wait() + + globals.noAuthEmulator = nil + + return +} + +func (dummy *authEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { + // TODO +} + +func (dummy *noAuthEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { + // TODO +} + +func (dummy *globalsStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { + keyAsString = fmt.Sprintf("%v", key) + err = nil + return +} + +func (dummy *globalsStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { + valueAsString = fmt.Sprintf("%v", value) + err = nil + return +} + +func parsePath(request *http.Request) (infoOnly bool, swiftAccountName string, swiftContainerName string, swiftObjectName string) { + var ( + pathSplit []string + ) + + infoOnly = false + swiftAccountName = "" + swiftContainerName = "" + swiftObjectName = "" + + if "/info" == request.URL.Path { + infoOnly = true + return + } + + if strings.HasPrefix(request.URL.Path, "/v1/") { + pathSplit = strings.SplitN(request.URL.Path[4:], "/", 3) + swiftAccountName = pathSplit[0] + if 1 == len(pathSplit) { + swiftContainerName = "" + swiftObjectName = "" + } else { + swiftContainerName = pathSplit[1] + if 2 == len(pathSplit) { + swiftObjectName = "" + } else { + swiftObjectName = pathSplit[2] + } + } + } + + return +} + +func parseRangeHeader(request *http.Request, objectLen int) (ranges []rangeStruct, err error) { + var ( + off int + rangeHeaderValue string + rangeHeaderValueSuffix string + rangeString string + rangeStringSlice []string + rangesStrings []string + rangesStringsIndex int + startOffset int64 + stopOffset int64 + ) + + rangeHeaderValue = request.Header.Get("Range") + if "" == rangeHeaderValue { + ranges = make([]rangeStruct, 0) + err = nil + return + } + + if !strings.HasPrefix(rangeHeaderValue, "bytes=") { + err = fmt.Errorf("rangeHeaderValue (%v) does not start with expected \"bytes=\"", rangeHeaderValue) + return + } + + rangeHeaderValueSuffix = rangeHeaderValue[len("bytes="):] + + rangesStrings = strings.SplitN(rangeHeaderValueSuffix, ",", 2) + + ranges = make([]rangeStruct, len(rangesStrings)) + + for rangesStringsIndex, rangeString = range rangesStrings { + rangeStringSlice = strings.SplitN(rangeString, "-", 2) + if 2 != len(rangeStringSlice) { + err = fmt.Errorf("rangeHeaderValue (%v) malformed", rangeHeaderValue) + return + } + if "" == rangeStringSlice[0] { + startOffset = int64(-1) + } else { + off, err = strconv.Atoi(rangeStringSlice[0]) + if nil != err { + err = fmt.Errorf("rangeHeaderValue (%v) malformed (strconv.Atoi() failure: %v)", rangeHeaderValue, err) + return + } + startOffset = int64(off) + } + + if "" == rangeStringSlice[1] { + stopOffset = int64(-1) + } else { + off, err = strconv.Atoi(rangeStringSlice[1]) + if nil != err { + err = fmt.Errorf("rangeHeaderValue (%v) malformed (strconv.Atoi() failure: %v)", rangeHeaderValue, err) + return + } + stopOffset = int64(off) + } + + if ((0 > startOffset) && (0 > stopOffset)) || (startOffset > stopOffset) { + err = fmt.Errorf("rangeHeaderValue (%v) malformed", rangeHeaderValue) + return + } + + if startOffset < 0 { + startOffset = int64(objectLen) - stopOffset + if startOffset < 0 { + err = fmt.Errorf("rangeHeaderValue (%v) malformed...computed startOffset negative", rangeHeaderValue) + return + } + stopOffset = int64(objectLen - 1) + } else if stopOffset < 0 { + stopOffset = int64(objectLen - 1) + } else { + if stopOffset > int64(objectLen-1) { + stopOffset = int64(objectLen - 1) + } + } + + ranges[rangesStringsIndex].startOffset = uint64(startOffset) + ranges[rangesStringsIndex].stopOffset = uint64(stopOffset) + } + + err = nil + return +} + +func locateSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, errno syscall.Errno) { + var ( + ok bool + ) + + swiftAccount, ok = globals.swiftAccountMap[swiftAccountName] + if !ok { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func createSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, errno syscall.Errno) { + var ( + ok bool + ) + + _, ok = globals.swiftAccountMap[swiftAccountName] + if ok { + errno = unix.EEXIST + return + } + swiftAccount = &swiftAccountStruct{ + name: swiftAccountName, + headers: make(http.Header), + swiftContainerTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), + } + globals.swiftAccountMap[swiftAccountName] = swiftAccount + errno = 0 + return +} + +func createOrLocateSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, wasCreated bool) { + var ( + ok bool + ) + + swiftAccount, ok = globals.swiftAccountMap[swiftAccountName] + if ok { + wasCreated = false + } else { + swiftAccount = &swiftAccountStruct{ + name: swiftAccountName, + headers: make(http.Header), + swiftContainerTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), + } + globals.swiftAccountMap[swiftAccountName] = swiftAccount + wasCreated = true + } + return +} + +func deleteSwiftAccount(swiftAccountName string, force bool) (errno syscall.Errno) { + var ( + err error + ok bool + swiftAccount *swiftAccountStruct + swiftswiftAccountContainerCount int + ) + + swiftAccount, ok = globals.swiftAccountMap[swiftAccountName] + if ok { + if force { + // ok if account contains data... we'll forget it + } else { + swiftswiftAccountContainerCount, err = swiftAccount.swiftContainerTree.Len() + if nil != err { + panic(err) + } + if 0 != swiftswiftAccountContainerCount { + errno = unix.ENOTEMPTY + return + } + } + delete(globals.swiftAccountMap, swiftAccountName) + } else { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func locateSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, errno syscall.Errno) { + var ( + err error + ok bool + swiftContainerAsValue sortedmap.Value + ) + + swiftContainerAsValue, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) + if nil != err { + panic(err) + } + if ok { + swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) + } else { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func createSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, errno syscall.Errno) { + var ( + err error + ok bool + ) + + _, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) + if nil != err { + panic(err) + } + if ok { + errno = unix.EEXIST + return + } else { + swiftContainer = &swiftContainerStruct{ + name: swiftContainerName, + swiftAccount: swiftAccount, + headers: make(http.Header), + swiftObjectTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), + } + _, err = swiftAccount.swiftContainerTree.Put(swiftContainerName, swiftContainer) + if nil != err { + panic(err) + } + } + errno = 0 + return +} + +func createOrLocateSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, wasCreated bool) { + var ( + err error + ok bool + swiftContainerAsValue sortedmap.Value + ) + + swiftContainerAsValue, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) + if nil != err { + panic(err) + } + if ok { + swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) + wasCreated = false + } else { + swiftContainer = &swiftContainerStruct{ + name: swiftContainerName, + swiftAccount: swiftAccount, + headers: make(http.Header), + swiftObjectTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), + } + _, err = swiftAccount.swiftContainerTree.Put(swiftContainerName, swiftContainer) + if nil != err { + panic(err) + } + wasCreated = true + } + return +} + +func deleteSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (errno syscall.Errno) { + var ( + err error + ok bool + swiftContainer *swiftContainerStruct + swiftContainerAsValue sortedmap.Value + swiftContainerObjectCount int + ) + + swiftContainerAsValue, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) + if nil != err { + panic(err) + } + if ok { + swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) + swiftContainerObjectCount, err = swiftContainer.swiftObjectTree.Len() + if nil != err { + panic(err) + } + if 0 != swiftContainerObjectCount { + errno = unix.ENOTEMPTY + return + } + _, err = swiftAccount.swiftContainerTree.DeleteByKey(swiftContainerName) + if nil != err { + panic(err) + } + } else { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func locateSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, errno syscall.Errno) { + var ( + err error + ok bool + swiftObjectAsValue sortedmap.Value + ) + + swiftObjectAsValue, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) + if nil != err { + panic(err) + } + if ok { + swiftObject = swiftObjectAsValue.(*swiftObjectStruct) + } else { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func createSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, errno syscall.Errno) { + var ( + err error + ok bool + ) + + _, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) + if nil != err { + panic(err) + } + if ok { + errno = unix.EEXIST + return + } else { + swiftObject = &swiftObjectStruct{name: swiftObjectName, swiftContainer: swiftContainer, contents: []byte{}} + _, err = swiftContainer.swiftObjectTree.Put(swiftObjectName, swiftObject) + if nil != err { + panic(err) + } + } + errno = 0 + return +} + +func createOrLocateSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, wasCreated bool) { + var ( + err error + ok bool + swiftObjectAsValue sortedmap.Value + ) + + swiftObjectAsValue, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) + if nil != err { + panic(err) + } + if ok { + swiftObject = swiftObjectAsValue.(*swiftObjectStruct) + wasCreated = false + } else { + swiftObject = &swiftObjectStruct{name: swiftObjectName, swiftContainer: swiftContainer, contents: []byte{}} + _, err = swiftContainer.swiftObjectTree.Put(swiftObjectName, swiftObject) + if nil != err { + panic(err) + } + wasCreated = true + } + return +} + +func deleteSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (errno syscall.Errno) { + var ( + err error + ok bool + ) + + _, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) + if nil != err { + panic(err) + } + if ok { + _, err = swiftContainer.swiftObjectTree.DeleteByKey(swiftObjectName) + if nil != err { + panic(err) + } + } else { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func doNoAuthDELETE(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + errno syscall.Errno + infoOnly bool + swiftAccount *swiftAccountStruct + swiftAccountName string + swiftContainer *swiftContainerStruct + swiftContainerName string + swiftObjectName string + ) + + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) + if infoOnly || ("" == swiftAccountName) { + responseWriter.WriteHeader(http.StatusForbidden) + } else { + if "" == swiftContainerName { + // DELETE SwiftAccount + errno = deleteSwiftAccount(swiftAccountName, false) + switch errno { + case 0: + responseWriter.WriteHeader(http.StatusNoContent) + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + case unix.ENOTEMPTY: + responseWriter.WriteHeader(http.StatusConflict) + default: + err = fmt.Errorf("deleteSwiftAccount(\"%v\", false) returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } else { + // DELETE SwiftContainer or SwiftObject + swiftAccount, errno = locateSwiftAccount(swiftAccountName) + switch errno { + case 0: + if "" == swiftObjectName { + // DELETE SwiftContainer + errno = deleteSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + responseWriter.WriteHeader(http.StatusNoContent) + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + case unix.ENOTEMPTY: + responseWriter.WriteHeader(http.StatusConflict) + default: + err = fmt.Errorf("deleteSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } else { + // DELETE SwiftObject + swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + errno = deleteSwiftObject(swiftContainer, swiftObjectName) + switch errno { + case 0: + responseWriter.WriteHeader(http.StatusNoContent) + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("deleteSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) + panic(err) + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } + } +} + +func doNoAuthGET(responseWriter http.ResponseWriter, request *http.Request) { + var ( + boundaryString string + containerIndex int + containerIndexLimit int + err error + errno syscall.Errno + found bool + headerName string + headerValue string + headerValueSlice []string + infoOnly bool + marker string + markerSlice []string + objectIndex int + objectIndexLimit int + ok bool + numContainers int + numObjects int + ranges []rangeStruct + rS rangeStruct + swiftAccount *swiftAccountStruct + swiftAccountName string + swiftContainer *swiftContainerStruct + swiftContainerName string + swiftContainerNameAsKey sortedmap.Key + swiftObject *swiftObjectStruct + swiftObjectName string + swiftObjectNameAsKey sortedmap.Key + ) + + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) + if infoOnly { + _, _ = responseWriter.Write(utils.StringToByteSlice("{")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"swift\": {")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_account_name_length\": " + strconv.Itoa(int(globals.config.MaxAccountNameLength)) + ",")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_container_name_length\": " + strconv.Itoa(int(globals.config.MaxContainerNameLength)) + ",")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_object_name_length\": " + strconv.Itoa(int(globals.config.MaxObjectNameLength)) + ",")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"account_listing_limit\": " + strconv.Itoa(int(globals.config.AccountListingLimit)) + ",")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"container_listing_limit\": " + strconv.Itoa(int(globals.config.ContainerListingLimit)))) + _, _ = responseWriter.Write(utils.StringToByteSlice("}")) + _, _ = responseWriter.Write(utils.StringToByteSlice("}")) + } else { + if "" == swiftAccountName { + responseWriter.WriteHeader(http.StatusForbidden) + } else { + swiftAccount, errno = locateSwiftAccount(swiftAccountName) + switch errno { + case 0: + if "" == swiftContainerName { + // GET SwiftAccount + for headerName, headerValueSlice = range swiftAccount.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + numContainers, err = swiftAccount.swiftContainerTree.Len() + if nil != err { + panic(err) + } + if 0 == numContainers { + responseWriter.WriteHeader(http.StatusNoContent) + } else { + marker = "" + markerSlice, ok = request.URL.Query()["marker"] + if ok && (0 < len(markerSlice)) { + marker = markerSlice[0] + } + containerIndex, found, err = swiftAccount.swiftContainerTree.BisectRight(marker) + if nil != err { + panic(err) + } + if found { + containerIndex++ + } + if containerIndex < numContainers { + containerIndexLimit = numContainers + if (containerIndexLimit - containerIndex) > int(globals.config.AccountListingLimit) { + containerIndexLimit = containerIndex + int(globals.config.AccountListingLimit) + } + for containerIndex < containerIndexLimit { + swiftContainerNameAsKey, _, _, err = swiftAccount.swiftContainerTree.GetByIndex(containerIndex) + if nil != err { + panic(err) + } + swiftContainerName = swiftContainerNameAsKey.(string) + _, _ = responseWriter.Write(utils.StringToByteSlice(swiftContainerName)) + _, _ = responseWriter.Write([]byte{'\n'}) + containerIndex++ + } + } else { + responseWriter.WriteHeader(http.StatusNoContent) + } + } + } else { + // GET SwiftContainer or SwiftObject + swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + if "" == swiftObjectName { + // GET SwiftContainer + for headerName, headerValueSlice = range swiftContainer.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + numObjects, err = swiftContainer.swiftObjectTree.Len() + if nil != err { + panic(err) + } + if 0 == numObjects { + responseWriter.WriteHeader(http.StatusNoContent) + } else { + marker = "" + markerSlice, ok = request.URL.Query()["marker"] + if ok && (0 < len(markerSlice)) { + marker = markerSlice[0] + } + objectIndex, found, err = swiftContainer.swiftObjectTree.BisectRight(marker) + if nil != err { + panic(err) + } + if found { + objectIndex++ + } + if objectIndex < numObjects { + objectIndexLimit = numObjects + if (objectIndexLimit - objectIndex) > int(globals.config.ContainerListingLimit) { + objectIndexLimit = objectIndex + int(globals.config.ContainerListingLimit) + } + for objectIndex < objectIndexLimit { + swiftObjectNameAsKey, _, _, err = swiftContainer.swiftObjectTree.GetByIndex(objectIndex) + if nil != err { + panic(err) + } + swiftObjectName = swiftObjectNameAsKey.(string) + _, _ = responseWriter.Write(utils.StringToByteSlice(swiftObjectName)) + _, _ = responseWriter.Write([]byte{'\n'}) + objectIndex++ + } + } else { + responseWriter.WriteHeader(http.StatusNoContent) + } + } + } else { + // GET SwiftObject + swiftObject, errno = locateSwiftObject(swiftContainer, swiftObjectName) + switch errno { + case 0: + for headerName, headerValueSlice = range swiftObject.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + ranges, err = parseRangeHeader(request, len(swiftObject.contents)) + if nil == err { + switch len(ranges) { + case 0: + responseWriter.Header().Add("Content-Type", "application/octet-stream") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(swiftObject.contents) + case 1: + responseWriter.Header().Add("Content-Type", "application/octet-stream") + responseWriter.Header().Add("Content-Range", fmt.Sprintf("bytes %d-%d/%d", ranges[0].startOffset, ranges[0].stopOffset, len(swiftObject.contents))) + responseWriter.WriteHeader(http.StatusPartialContent) + _, _ = responseWriter.Write(swiftObject.contents[ranges[0].startOffset:(ranges[0].stopOffset + 1)]) + default: + boundaryString = fmt.Sprintf("%016x%016x", rand.Uint64(), rand.Uint64()) + responseWriter.Header().Add("Content-Type", fmt.Sprintf("multipart/byteranges; boundary=%v", boundaryString)) + responseWriter.WriteHeader(http.StatusPartialContent) + for _, rS = range ranges { + _, _ = responseWriter.Write([]byte("--" + boundaryString + "\r\n")) + _, _ = responseWriter.Write([]byte("Content-Type: application/octet-stream\r\n")) + _, _ = responseWriter.Write([]byte(fmt.Sprintf("Content-Range: bytes %d-%d/%d\r\n", rS.startOffset, rS.stopOffset, len(swiftObject.contents)))) + _, _ = responseWriter.Write([]byte("\r\n")) + _, _ = responseWriter.Write(swiftObject.contents[rS.startOffset:(rS.stopOffset + 1)]) + _, _ = responseWriter.Write([]byte("\r\n")) + } + _, _ = responseWriter.Write([]byte("--" + boundaryString + "--")) + } + } else { + responseWriter.WriteHeader(http.StatusBadRequest) + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } + } +} + +func doNoAuthHEAD(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + errno syscall.Errno + headerName string + headerValue string + headerValueSlice []string + infoOnly bool + swiftAccount *swiftAccountStruct + swiftAccountName string + swiftContainer *swiftContainerStruct + swiftContainerName string + swiftObject *swiftObjectStruct + swiftObjectName string + ) + + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) + if infoOnly || ("" == swiftAccountName) { + responseWriter.WriteHeader(http.StatusForbidden) + } else { + swiftAccount, errno = locateSwiftAccount(swiftAccountName) + switch errno { + case 0: + if "" == swiftContainerName { + // HEAD SwiftAccount + for headerName, headerValueSlice = range swiftAccount.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + responseWriter.WriteHeader(http.StatusNoContent) + } else { + // HEAD SwiftContainer or SwiftObject + swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + if "" == swiftObjectName { + // HEAD SwiftContainer + for headerName, headerValueSlice = range swiftContainer.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + responseWriter.WriteHeader(http.StatusNoContent) + } else { + // HEAD SwiftObject + swiftObject, errno = locateSwiftObject(swiftContainer, swiftObjectName) + switch errno { + case 0: + for headerName, headerValueSlice = range swiftObject.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + responseWriter.Header().Set("Content-Length", strconv.Itoa(len(swiftObject.contents))) + responseWriter.WriteHeader(http.StatusOK) + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } +} + +func doNoAuthPOST(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + errno syscall.Errno + headerName string + headerValue string + headerValueSlice []string + headerValueSliceLen int + ignoreHeader bool + infoOnly bool + swiftAccount *swiftAccountStruct + swiftAccountName string + swiftContainer *swiftContainerStruct + swiftContainerName string + swiftObject *swiftObjectStruct + swiftObjectName string + ) + + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) + if infoOnly || ("" == swiftAccountName) { + responseWriter.WriteHeader(http.StatusForbidden) + } else { + swiftAccount, errno = locateSwiftAccount(swiftAccountName) + switch errno { + case 0: + if "" == swiftContainerName { + // POST SwiftAccount + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftAccount.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftAccount.headers[headerName] = append(swiftAccount.headers[headerName], headerValue) + } + } + if 0 == len(swiftAccount.headers[headerName]) { + delete(swiftAccount.headers, headerName) + } + } + } + } + responseWriter.WriteHeader(http.StatusNoContent) + } else { + // POST SwiftContainer or SwiftObject + swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + if "" == swiftObjectName { + // POST SwiftContainer + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftContainer.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftContainer.headers[headerName] = append(swiftContainer.headers[headerName], headerValue) + } + } + if 0 == len(swiftContainer.headers[headerName]) { + delete(swiftContainer.headers, headerName) + } + } + } + } + responseWriter.WriteHeader(http.StatusNoContent) + } else { + // POST SwiftObject + swiftObject, errno = locateSwiftObject(swiftContainer, swiftObjectName) + switch errno { + case 0: + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftObject.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftObject.headers[headerName] = append(swiftObject.headers[headerName], headerValue) + } + } + if 0 == len(swiftObject.headers[headerName]) { + delete(swiftObject.headers, headerName) + } + } + } + } + responseWriter.WriteHeader(http.StatusNoContent) + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } +} + +func doNoAuthPUT(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + errno syscall.Errno + headerName string + headerValue string + headerValueSlice []string + headerValueSliceLen int + ignoreHeader bool + infoOnly bool + swiftAccount *swiftAccountStruct + swiftAccountName string + swiftContainer *swiftContainerStruct + swiftContainerName string + swiftObject *swiftObjectStruct + swiftObjectName string + wasCreated bool + ) + + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) + if infoOnly || ("" == swiftAccountName) { + responseWriter.WriteHeader(http.StatusForbidden) + } else { + if "" == swiftContainerName { + // PUT SwiftAccount + swiftAccount, wasCreated = createOrLocateSwiftAccount(swiftAccountName) + if wasCreated { + swiftAccount.headers = make(http.Header) + } + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftAccount.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftAccount.headers[headerName] = append(swiftAccount.headers[headerName], headerValue) + } + } + if 0 == len(swiftAccount.headers[headerName]) { + delete(swiftAccount.headers, headerName) + } + } + } + } + if wasCreated { + responseWriter.WriteHeader(http.StatusCreated) + } else { + responseWriter.WriteHeader(http.StatusAccepted) + } + } else { + // PUT SwiftContainer or SwiftObject + swiftAccount, errno = locateSwiftAccount(swiftAccountName) + switch errno { + case 0: + if "" == swiftObjectName { + // PUT SwiftContainer + swiftContainer, wasCreated = createOrLocateSwiftContainer(swiftAccount, swiftContainerName) + if wasCreated { + swiftContainer.headers = make(http.Header) + } + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftContainer.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftContainer.headers[headerName] = append(swiftContainer.headers[headerName], headerValue) + } + } + if 0 == len(swiftContainer.headers[headerName]) { + delete(swiftContainer.headers, headerName) + } + } + } + } + if wasCreated { + responseWriter.WriteHeader(http.StatusCreated) + } else { + responseWriter.WriteHeader(http.StatusAccepted) + } + } else { + // PUT SwiftObject + swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + swiftObject, wasCreated = createOrLocateSwiftObject(swiftContainer, swiftObjectName) + if wasCreated { + swiftObject.headers = make(http.Header) + } + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftObject.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftObject.headers[headerName] = append(swiftObject.headers[headerName], headerValue) + } + } + if 0 == len(swiftObject.headers[headerName]) { + delete(swiftObject.headers, headerName) + } + } + } + } + swiftObject.contents, _ = ioutil.ReadAll(request.Body) + if wasCreated { + responseWriter.WriteHeader(http.StatusCreated) + } else { + responseWriter.WriteHeader(http.StatusCreated) + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusForbidden) + default: + err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusForbidden) + default: + err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } + } +} + +/* +func (h httpRequestHandler) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { + switch request.Method { + case http.MethodDelete: + doNoAuthDELETE(responseWriter, request) + case http.MethodGet: + doNoAuthGET(responseWriter, request) + case http.MethodHead: + doNoAuthHEAD(responseWriter, request) + case http.MethodPost: + doNoAuthPOST(responseWriter, request) + case http.MethodPut: + doNoAuthPUT(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func setupNoAuthSwift(confMap conf.ConfMap) { + var ( + err error + errno syscall.Errno + primaryPeerList []string + swiftAccountName string + volumeGroupNameList []string + volumeGroupName string + volumeGroupSectionName string + volumeName string + volumeNameList []string + volumeSectionName string + ) + + // Fetch and configure volumes for which "we" are the PrimaryPeer + + volumeGroupNameList, err = confMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") + if nil != err { + log.Fatalf("failed fetch of FSGlobals.VolumeGroupList: %v", err) + } + + for _, volumeGroupName = range volumeGroupNameList { + volumeGroupSectionName = "VolumeGroup:" + volumeGroupName + + primaryPeerList, err = confMap.FetchOptionValueStringSlice(volumeGroupSectionName, "PrimaryPeer") + if nil != err { + log.Fatalf("failed fetch of %v.PrimaryPeer: %v", volumeGroupSectionName, err) + } + if 0 == len(primaryPeerList) { + continue + } else if 1 == len(primaryPeerList) { + if globals.whoAmI != primaryPeerList[0] { + continue + } + } else { + log.Fatalf("fetch of %v.PrimaryPeer returned multiple values", volumeGroupSectionName) + } + + volumeNameList, err = confMap.FetchOptionValueStringSlice(volumeGroupSectionName, "VolumeList") + if nil != err { + log.Fatalf("failed fetch of %v.VolumeList: %v", volumeGroupSectionName, err) + } + + for _, volumeName = range volumeNameList { + volumeSectionName = "Volume:" + volumeName + + swiftAccountName, err = confMap.FetchOptionValueString(volumeSectionName, "AccountName") + if nil != err { + log.Fatalf("failed fetch of %v.AccountName: %v", volumeSectionName, err) + } + + _, errno = createSwiftAccount(swiftAccountName) + if 0 != errno { + log.Fatalf("failed create of %v: %v", swiftAccountName, err) + } + } + } + + // Create HTTP Server on the requested noAuthTCPPort + + globals.noAuthHTTPServer = &http.Server{ + Addr: globals.noAuthAddr, + Handler: httpRequestHandler{}, + } +} + +func serveNoAuthSwift() { + _ = globals.noAuthHTTPServer.ListenAndServe() + + globals.noAuthHTTPServerWG.Done() +} + +func Daemon(confFile string, confStrings []string, signalHandlerIsArmedWG *sync.WaitGroup, doneChan chan bool, signals ...os.Signal) { + var ( + confMap conf.ConfMap + err error + resp *http.Response + signalChan chan os.Signal + signalReceived os.Signal + ) + + // Initialization + + globals = &globalsStruct{} + + globals.swiftAccountMap = make(map[string]*swiftAccountStruct) + + // Compute confMap + + confMap, err = conf.MakeConfMapFromFile(confFile) + if nil != err { + log.Fatalf("failed to load config: %v", err) + } + + err = confMap.UpdateFromStrings(confStrings) + if nil != err { + log.Fatalf("failed to apply config overrides: %v", err) + } + + err = transitions.UpgradeConfMapIfNeeded(confMap) + if nil != err { + log.Fatalf("failed to upgrade confMap: %v", err) + } + + // Find out who "we" are + + globals.whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") + if nil != err { + log.Fatalf("failed fetch of Cluster.WhoAmI: %v", err) + } + + globals.noAuthTCPPort, err = confMap.FetchOptionValueUint16("SwiftClient", "NoAuthTCPPort") + if nil != err { + log.Fatalf("failed fetch of Swift.NoAuthTCPPort: %v", err) + } + + globals.noAuthAddr = "127.0.0.1:" + strconv.Itoa(int(globals.noAuthTCPPort)) + + // Kick off NoAuth Swift Proxy Emulator + + setupNoAuthSwift(confMap) + globals.noAuthHTTPServerWG.Add(1) + go serveNoAuthSwift() + + // Wait for serveNoAuthSwift() to begin serving + + for { + resp, err = http.Get("http://" + globals.noAuthAddr + "/info") + if nil != err { + log.Printf("failed GET of \"/info\": %v", err) + continue + } + if http.StatusOK == resp.StatusCode { + break + } + log.Printf("GET of \"/info\" returned Status %v", resp.Status) + time.Sleep(100 * time.Millisecond) + } + + // Arm signal handler used to indicate termination and wait on it + // + // Note: signalled chan must be buffered to avoid race with window between + // arming handler and blocking on the chan read + + signalChan = make(chan os.Signal, 1) + + signal.Notify(signalChan, signals...) + + if nil != signalHandlerIsArmedWG { + signalHandlerIsArmedWG.Done() + } + + // Await a signal - reloading confFile each SIGHUP - exiting otherwise + + for { + signalReceived = <-signalChan + + if unix.SIGHUP == signalReceived { + // recompute confMap and re-apply + + confMap, err = conf.MakeConfMapFromFile(confFile) + if nil != err { + log.Fatalf("failed to load updated config: %v", err) + } + + err = confMap.UpdateFromStrings(confStrings) + if nil != err { + log.Fatalf("failed to reapply config overrides: %v", err) + } + + err = transitions.UpgradeConfMapIfNeeded(confMap) + if nil != err { + log.Fatalf("failed to upgrade confMap: %v", err) + } + + updateConf(confMap) + } else { + // signalReceived either SIGINT or SIGTERM... so just exit + + err = globals.noAuthHTTPServer.Close() + + globals.noAuthHTTPServerWG.Wait() + + globals = nil + + doneChan <- true + + return + } + } +} +*/ diff --git a/emswift/emswiftpkg/setup_teardown_test.go_HIDE b/emswift/emswiftpkg/setup_teardown_test.go_HIDE new file mode 100644 index 000000000..5aa1e5b84 --- /dev/null +++ b/emswift/emswiftpkg/setup_teardown_test.go_HIDE @@ -0,0 +1,369 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package emswiftpkg + +import ( + "io/ioutil" + "net/http" + "os" + "strconv" + "sync" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/swiftstack/ProxyFS/conf" + "github.com/swiftstack/ProxyFS/proxyfsd" + "github.com/swiftstack/ProxyFS/ramswift" +) + +const ( + testAccountName = "AUTH_test" + testAuthKey = "testing" + testAuthUser = "test:tester" + testDaemonStartPollInterval = 1 * time.Second + testProxyFSDaemonHTTPPort = "15347" + testProxyFSDaemonIPAddr = "127.0.0.1" + testSwiftNoAuthIPAddr = "127.0.0.1" + testSwiftNoAuthPort = "38090" + testSwiftProxyAddr = "localhost:38080" +) + +type testDaemonGlobalsStruct struct { + proxyfsdErrChan chan error + proxyfsdWG sync.WaitGroup + ramswiftDoneChan chan bool +} + +var testDaemonGlobals testDaemonGlobalsStruct + +func testSetup(t *testing.T) { + var ( + err error + infoResponse *http.Response + ramswiftSignalHandlerIsArmedWG sync.WaitGroup + testConfMap conf.ConfMap + testConfStrings []string + testDir string + testPlugInEnvValue string + versionResponse *http.Response + ) + + testDir, err = ioutil.TempDir(os.TempDir(), "pfsagentd_test_") + if nil != err { + t.Fatalf("ioutil.TempDir() failed: %v", err) + } + + err = os.Chdir(testDir) + if nil != err { + t.Fatalf("os.Chdir() failed: %v", err) + } + + err = os.Mkdir("ProxyFSMountPointPath", 0777) // Agent.FUSEMountPointPath + if nil != err { + t.Fatalf("os.Mkdir() failed: %v", err) + } + + err = os.Mkdir("PfsAgentMountPointPath", 0777) // Volume:CommonVolume.FUSEMountPointName + if nil != err { + t.Fatalf("os.Mkdir() failed: %v", err) + } + + testPlugInEnvValue = "{\"AuthURL\":\"http://" + testPlugInEnvValue += testSwiftProxyAddr + testPlugInEnvValue += "/auth/v1.0\"\\u002C\"AuthUser\":\"" + testPlugInEnvValue += testAuthUser + testPlugInEnvValue += "\"\\u002C\"AuthKey\":\"" + testPlugInEnvValue += testAuthKey + testPlugInEnvValue += "\"\\u002C\"Account\":\"" + testPlugInEnvValue += testAccountName + testPlugInEnvValue += "\"}" + + testConfStrings = []string{ + "Agent.FUSEVolumeName=CommonVolume", + "Agent.FUSEMountPointPath=PfsAgentMountPointPath", + "Agent.FUSEUnMountRetryDelay=100ms", + "Agent.FUSEUnMountRetryCap=100", + "Agent.PlugInPath=/dev/null", // Using hard-coded AUTH + "Agent.PlugInEnvName=SwiftAuthBlob", + "Agent.PlugInEnvValue=" + testPlugInEnvValue, + "Agent.SwiftTimeout=20s", + "Agent.SwiftRetryLimit=10", + "Agent.SwiftRetryDelay=10ms", + "Agent.SwiftRetryDelayVariance=25", + "Agent.SwiftRetryExpBackoff=1.4", + "Agent.SwiftConnectionPoolSize=200", + "Agent.FetchExtentsFromFileOffset=32", + "Agent.FetchExtentsBeforeFileOffset=0", + "Agent.ReadCacheLineSize=1048576", + "Agent.ReadCacheLineCount=1000", + "Agent.LeaseRetryLimit=10", + "Agent.LeaseRetryDelay=10ms", + "Agent.LeaseRetryDelayVariance=25", + "Agent.LeaseRetryExpBackoff=1.4", + "Agent.SharedLeaseLimit=1000", + "Agent.ExclusiveLeaseLimit=100", + "Agent.ExtentMapEntryLimit=1048576", + "Agent.DirtyLogSegmentLimit=50", + "Agent.DirtyFileLimit=50", // TODO - obsolete this + "Agent.MaxFlushSize=10485760", + "Agent.MaxFlushTime=10s", + "Agent.LogFilePath=", + "Agent.LogToConsole=false", + "Agent.TraceEnabled=false", + "Agent.HTTPServerIPAddr=127.0.0.1", + "Agent.HTTPServerTCPPort=54323", + "Agent.ReadDirPlusEnabled=false", + "Agent.XAttrEnabled=false", + "Agent.EntryDuration=10s", + "Agent.AttrDuration=10s", + "Agent.AttrBlockSize=65536", + "Agent.ReaddirMaxEntries=1024", + "Agent.FUSEMaxBackground=100", + "Agent.FUSECongestionThreshhold=0", + "Agent.FUSEMaxWrite=131072", // Linux max... 128KiB is good enough for testing + "Agent.RetryRPCDeadlineIO=60s", + "Agent.RetryRPCKeepAlivePeriod=60s", + + "Stats.IPAddr=localhost", + "Stats.UDPPort=54324", + "Stats.BufferLength=100", + "Stats.MaxLatency=1s", + + "StatsLogger.Period=0m", + "StatsLogger.Verbose=false", + + "Logging.LogFilePath=/dev/null", + "Logging.LogToConsole=false", + + "Peer:Peer0.PublicIPAddr=" + testProxyFSDaemonIPAddr, + "Peer:Peer0.PrivateIPAddr=" + testProxyFSDaemonIPAddr, + "Peer:Peer0.ReadCacheQuotaFraction=0.20", + + "Cluster.WhoAmI=Peer0", + "Cluster.Peers=Peer0", + "Cluster.ServerGuid=a66488e9-a051-4ff7-865d-87bfb84cc2ae", + "Cluster.PrivateClusterUDPPort=54325", + "Cluster.UDPPacketSendSize=1400", + "Cluster.UDPPacketRecvSize=1500", + "Cluster.UDPPacketCapPerMessage=5", + "Cluster.HeartBeatDuration=1s", + "Cluster.HeartBeatMissLimit=3", + "Cluster.MessageQueueDepthPerPeer=4", + "Cluster.MaxRequestDuration=1s", + "Cluster.LivenessCheckRedundancy=2", + + "HTTPServer.TCPPort=" + testProxyFSDaemonHTTPPort, + + "SwiftClient.NoAuthIPAddr=" + testSwiftNoAuthIPAddr, + "SwiftClient.NoAuthTCPPort=" + testSwiftNoAuthPort, + "SwiftClient.Timeout=10s", + "SwiftClient.RetryLimit=1", + "SwiftClient.RetryLimitObject=1", + "SwiftClient.RetryDelay=10ms", + "SwiftClient.RetryDelayObject=10ms", + "SwiftClient.RetryExpBackoff=1.2", + "SwiftClient.RetryExpBackoffObject=2.0", + "SwiftClient.ChunkedConnectionPoolSize=64", + "SwiftClient.NonChunkedConnectionPoolSize=32", + "SwiftClient.SwiftReconChecksPerConfCheck=0", + + "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerStoragePolicy=silver", + "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerNamePrefix=Replicated3Way_", + "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainersPerPeer=10", + "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.MaxObjectsPerContainer=1000000", + + "Volume:CommonVolume.FSID=1", + "Volume:CommonVolume.FUSEMountPointName=ProxyFSMountPointPath", + "Volume:CommonVolume.NFSExportClientMapList=CommonVolumeNFSClient0", + "Volume:CommonVolume.SMBShareName=CommonShare", + "Volume:CommonVolume.PrimaryPeer=Peer0", + "Volume:CommonVolume.AccountName=AUTH_test", + "Volume:CommonVolume.CheckpointContainerName=.__checkpoint__", + "Volume:CommonVolume.CheckpointContainerStoragePolicy=gold", + "Volume:CommonVolume.CheckpointInterval=10s", + "Volume:CommonVolume.DefaultPhysicalContainerLayout=PhysicalContainerLayoutReplicated3Way", + "Volume:CommonVolume.MaxFlushSize=10485760", + "Volume:CommonVolume.MaxFlushTime=10s", + "Volume:CommonVolume.FileDefragmentChunkSize=10485760", + "Volume:CommonVolume.FileDefragmentChunkDelay=10ms", + "Volume:CommonVolume.NonceValuesToReserve=100", + "Volume:CommonVolume.MaxEntriesPerDirNode=32", + "Volume:CommonVolume.MaxExtentsPerFileNode=32", + "Volume:CommonVolume.MaxInodesPerMetadataNode=32", + "Volume:CommonVolume.MaxLogSegmentsPerMetadataNode=64", + "Volume:CommonVolume.MaxDirFileNodesPerMetadataNode=16", + "Volume:CommonVolume.MaxBytesInodeCache=100000", + "Volume:CommonVolume.InodeCacheEvictInterval=1s", + "Volume:CommonVolume.AutoFormat=true", + "Volume:CommonVolume.ActiveLeaseEvictLowLimit=5000", + "Volume:CommonVolume.ActiveLeaseEvictHighLimit=5010", + + "NFSClientMap:CommonVolumeNFSClient0.ClientPattern=*", + "NFSClientMap:CommonVolumeNFSClient0.AccessMode=rw", + "NFSClientMap:CommonVolumeNFSClient0.RootSquash=no_root_squash", + "NFSClientMap:CommonVolumeNFSClient0.Secure=insecure", + + "VolumeGroup:CommonVolumeGroup.VolumeList=CommonVolume", + "VolumeGroup:CommonVolumeGroup.VirtualIPAddr=", + "VolumeGroup:CommonVolumeGroup.PrimaryPeer=Peer0", + "VolumeGroup:CommonVolumeGroup.ReadCacheLineSize=1000000", + "VolumeGroup:CommonVolumeGroup.ReadCacheWeight=100", + + "FSGlobals.VolumeGroupList=CommonVolumeGroup", + "FSGlobals.CheckpointHeaderConsensusAttempts=5", + "FSGlobals.MountRetryLimit=6", + "FSGlobals.MountRetryDelay=1s", + "FSGlobals.MountRetryExpBackoff=2", + "FSGlobals.LogCheckpointHeaderPosts=true", + "FSGlobals.TryLockBackoffMin=10ms", + "FSGlobals.TryLockBackoffMax=50ms", + "FSGlobals.TryLockSerializationThreshhold=5", + "FSGlobals.SymlinkMax=32", + "FSGlobals.CoalesceElementChunkSize=16", + "FSGlobals.InodeRecCacheEvictLowLimit=10000", + "FSGlobals.InodeRecCacheEvictHighLimit=10010", + "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", + "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", + "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", + "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", + "FSGlobals.DirEntryCacheEvictLowLimit=10000", + "FSGlobals.DirEntryCacheEvictHighLimit=10010", + "FSGlobals.FileExtentMapEvictLowLimit=10000", + "FSGlobals.FileExtentMapEvictHighLimit=10010", + "FSGlobals.EtcdEnabled=false", + + "JSONRPCServer.TCPPort=54326", + "JSONRPCServer.FastTCPPort=54327", + "JSONRPCServer.RetryRPCPort=54328", + "JSONRPCServer.RetryRPCTTLCompleted=10s", + "JSONRPCServer.RetryRPCAckTrim=10ms", + "JSONRPCServer.DataPathLogging=false", + "JSONRPCServer.MinLeaseDuration=250ms", + "JSONRPCServer.LeaseInterruptInterval=250ms", + "JSONRPCServer.LeaseInterruptLimit=20", + } + + testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxAccountNameLength="+strconv.FormatUint(testMaxAccountNameLength, 10)) + testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxContainerNameLength="+strconv.FormatUint(testMaxContainerNameLength, 10)) + testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxObjectNameLength="+strconv.FormatUint(testMaxObjectNameLength, 10)) + testConfStrings = append(testConfStrings, "RamSwiftInfo.AccountListingLimit="+strconv.FormatUint(testAccountListingLimit, 10)) + testConfStrings = append(testConfStrings, "RamSwiftInfo.ContainerListingLimit="+strconv.FormatUint(testContainerListingLimit, 10)) + + ramswiftSignalHandlerIsArmedWG.Add(1) + testDaemonGlobals.ramswiftDoneChan = make(chan bool, 1) + + go ramswift.Daemon("/dev/null", testConfStrings, &ramswiftSignalHandlerIsArmedWG, testDaemonGlobals.ramswiftDoneChan, unix.SIGUSR1) + + ramswiftSignalHandlerIsArmedWG.Wait() + + for { + infoResponse, err = http.Get("http://" + testSwiftNoAuthIPAddr + ":" + testSwiftNoAuthPort + "/info") + if (nil == err) && (http.StatusOK == infoResponse.StatusCode) { + break + } + + time.Sleep(testDaemonStartPollInterval) + } + + testDaemonGlobals.proxyfsdErrChan = make(chan error, 1) // Must be buffered to avoid race + + go proxyfsd.Daemon("/dev/null", testConfStrings, testDaemonGlobals.proxyfsdErrChan, &testDaemonGlobals.proxyfsdWG, []string{}, unix.SIGUSR2) + + err = <-testDaemonGlobals.proxyfsdErrChan + if nil != err { + t.Fatalf("proxyfsd.Daemon() startup failed: %v", err) + } + + for { + versionResponse, err = http.Get("http://" + testProxyFSDaemonIPAddr + ":" + testProxyFSDaemonHTTPPort + "/version") + if (nil == err) && (http.StatusOK == versionResponse.StatusCode) { + break + } + + time.Sleep(testDaemonStartPollInterval) + } + + testConfMap, err = conf.MakeConfMapFromStrings(testConfStrings) + if nil != err { + t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) + } + + globals.logFile = nil + globals.config.LogFilePath = "" + globals.config.LogToConsole = true + + startSwiftProxyEmulator(t, testConfMap) + + initializeGlobals(testConfMap) + + // Fake out that plug-in auth has already obtained AuthToken & StorageURL + + globals.swiftAuthToken = testAuthToken + globals.swiftStorageURL = "http://" + testSwiftProxyAddr + "/proxyfs/" + testAccountName + + go testSwallowFissionErrChan(t, globals.fissionErrChan) + + doMountProxyFS() + + performMountFUSE() +} + +func testSwallowFissionErrChan(t *testing.T, fissionErrChan chan error) { + var ( + err error + ) + + err = <-fissionErrChan + if nil != err { + t.Fatalf("fissionErrChan received err: %v", err) + } +} + +func testTeardown(t *testing.T) { + var ( + err error + testDir string + ) + + performUnmountFUSE() + + emptyFileInodeDirtyListAndLogSegmentChan() + + doUnmountProxyFS() + + uninitializeGlobals() + + _ = unix.Kill(unix.Getpid(), unix.SIGUSR2) + + err = <-testDaemonGlobals.proxyfsdErrChan + + testDaemonGlobals.proxyfsdWG.Wait() + + if nil != err { + t.Fatalf("proxyfsd.Daemon() exited with error: %v", err) + } + + unix.Kill(unix.Getpid(), unix.SIGUSR1) + + _ = <-testDaemonGlobals.ramswiftDoneChan + + testDir, err = os.Getwd() + if nil != err { + t.Fatalf("os.Getwd() failed: %v", err) + } + + err = os.Chdir("..") + if nil != err { + t.Fatalf("os.Chdir() failed: %v", err) + } + + err = os.RemoveAll(testDir) + if nil != err { + t.Fatalf("os.RemoveAll() failed: %v", err) + } + + stopSwiftProxyEmulator() +} diff --git a/emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE b/emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE new file mode 100644 index 000000000..8f44734c6 --- /dev/null +++ b/emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE @@ -0,0 +1,411 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package emswiftpkg + +import ( + "context" + "io/ioutil" + "net" + "net/http" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/swiftstack/ProxyFS/conf" +) + +const ( + testAuthToken = "AUTH_tkTestToken" + testJrpcResponseBufSize = 1024 * 1024 +) + +type testSwiftProxyEmulatorGlobalsStruct struct { + sync.WaitGroup + t *testing.T + ramswiftNoAuthURL string + proxyfsdJrpcTCPAddr *net.TCPAddr + jrpcResponsePool *sync.Pool + httpClient *http.Client + httpServer *http.Server +} + +var testSwiftProxyEmulatorGlobals testSwiftProxyEmulatorGlobalsStruct + +func startSwiftProxyEmulator(t *testing.T, confMap conf.ConfMap) { + var ( + err error + infoResponse *http.Response + jrpcServerIPAddr string + jrpcServerTCPPort uint16 + swiftClientNoAuthIPAddr string + swiftClientNoAuthTCPPort uint16 + whoAmI string + ) + + testSwiftProxyEmulatorGlobals.t = t + + swiftClientNoAuthIPAddr, err = confMap.FetchOptionValueString("SwiftClient", "NoAuthIPAddr") + if nil != err { + t.Fatal(err) + } + + swiftClientNoAuthTCPPort, err = confMap.FetchOptionValueUint16("SwiftClient", "NoAuthTCPPort") + if nil != err { + t.Fatal(err) + } + + testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL = "http://" + net.JoinHostPort(swiftClientNoAuthIPAddr, strconv.FormatUint(uint64(swiftClientNoAuthTCPPort), 10)) + + whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") + if nil != err { + t.Fatal(err) + } + + jrpcServerIPAddr, err = confMap.FetchOptionValueString("Peer:"+whoAmI, "PrivateIPAddr") + if nil != err { + t.Fatal(err) + } + + jrpcServerTCPPort, err = confMap.FetchOptionValueUint16("JSONRPCServer", "TCPPort") + if nil != err { + t.Fatal(err) + } + + testSwiftProxyEmulatorGlobals.proxyfsdJrpcTCPAddr, err = net.ResolveTCPAddr("tcp", net.JoinHostPort(jrpcServerIPAddr, strconv.FormatUint(uint64(jrpcServerTCPPort), 10))) + if nil != err { + t.Fatal(err) + } + + testSwiftProxyEmulatorGlobals.httpClient = &http.Client{} + + testSwiftProxyEmulatorGlobals.httpServer = &http.Server{ + Addr: testSwiftProxyAddr, + Handler: &testSwiftProxyEmulatorGlobals, + } + + testSwiftProxyEmulatorGlobals.jrpcResponsePool = &sync.Pool{ + New: func() (bufAsInterface interface{}) { + var ( + bufAsByteSlice []byte + ) + + bufAsByteSlice = make([]byte, testJrpcResponseBufSize) + + bufAsInterface = bufAsByteSlice + + return + }, + } + + testSwiftProxyEmulatorGlobals.Add(1) + + go func() { + _ = testSwiftProxyEmulatorGlobals.httpServer.ListenAndServe() + + testSwiftProxyEmulatorGlobals.Done() + }() + + for { + infoResponse, err = http.Get("http://" + testSwiftProxyAddr + "/info") + if nil == err { + break + } + + time.Sleep(testDaemonStartPollInterval) + } + + if http.StatusOK != infoResponse.StatusCode { + t.Fatalf("GET /info from ServeHTTP() got unexpected status %s", infoResponse.Status) + } +} + +func stopSwiftProxyEmulator() { + var ( + err error + ) + + err = testSwiftProxyEmulatorGlobals.httpServer.Shutdown(context.Background()) + if nil != err { + testSwiftProxyEmulatorGlobals.t.Fatalf("testSwiftProxyEmulatorGlobals.httpServer.Shutdown() failed: %v", err) + } + + testSwiftProxyEmulatorGlobals.Wait() + + testSwiftProxyEmulatorGlobals.jrpcResponsePool = nil +} + +func (dummy *testSwiftProxyEmulatorGlobalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { + // Handle the GET of/on info & AuthURL cases + + if http.MethodGet == request.Method { + switch request.URL.Path { + case "/info": + doInfo(responseWriter) + return + case "/auth/v1.0": + doAuth(responseWriter, request) + return + default: + // Fall through to normal processing + } + } + + // Reject unauthorized requests + + if request.Header.Get("X-Auth-Token") != testAuthToken { + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + // Branch off to individual request method handlers + + switch request.Method { + case http.MethodGet: + doGET(responseWriter, request) + case http.MethodPut: + doPUT(responseWriter, request) + case "PROXYFS": + doRPC(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func doInfo(authResponseWriter http.ResponseWriter) { + var ( + err error + getBuf []byte + noAuthResponse *http.Response + ) + + noAuthResponse, err = http.Get(testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL + "/info") + if nil != err { + testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info from ramswift failed: %v", err) + } + + if http.StatusOK != noAuthResponse.StatusCode { + testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info from ramswift returned bad status: %v", noAuthResponse.Status) + } + + getBuf, err = ioutil.ReadAll(noAuthResponse.Body) + if nil != err { + testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info returned unreadable Body: %v", err) + } + + err = noAuthResponse.Body.Close() + if nil != err { + testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info returned uncloseable Body: %v", err) + } + + authResponseWriter.WriteHeader(http.StatusOK) + _, _ = authResponseWriter.Write(getBuf) +} + +func doAuth(authResponseWriter http.ResponseWriter, authRequest *http.Request) { + if authRequest.Header.Get("X-Auth-User") != testAuthUser { + authResponseWriter.WriteHeader(http.StatusUnauthorized) + return + } + if authRequest.Header.Get("X-Auth-Key") != testAuthKey { + authResponseWriter.WriteHeader(http.StatusUnauthorized) + return + } + authResponseWriter.Header().Add("X-Auth-Token", testAuthToken) + authResponseWriter.Header().Add("X-Storage-Url", "http://"+testSwiftProxyAddr+"/v1/"+testAccountName) + authResponseWriter.WriteHeader(http.StatusOK) +} + +// doGET proxies the GET over to ramswift +// +func doGET(authResponseWriter http.ResponseWriter, authRequest *http.Request) { + var ( + contentRangeHeader string + contentTypeHeader string + err error + getBuf []byte + hostHeader string + noAuthPath string + noAuthRequest *http.Request + noAuthResponse *http.Response + noAuthStatusCode int + rangeHeader string + ) + + if !strings.HasPrefix(authRequest.URL.Path, "/proxyfs/"+testAccountName) { + authResponseWriter.WriteHeader(http.StatusNotFound) + return + } + + noAuthPath = strings.Replace(authRequest.URL.Path, "proxyfs", "v1", 1) + + noAuthRequest, err = http.NewRequest("GET", testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL+noAuthPath, nil) + if nil != err { + authResponseWriter.WriteHeader(http.StatusBadRequest) + return + } + + hostHeader = authRequest.Header.Get("Host") + if "" != hostHeader { + noAuthRequest.Header.Add("Host", hostHeader) + } + + rangeHeader = authRequest.Header.Get("Range") + if "" != rangeHeader { + noAuthRequest.Header.Add("Range", rangeHeader) + } + + noAuthResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(noAuthRequest) + if nil != err { + authResponseWriter.WriteHeader(http.StatusBadRequest) + return + } + + noAuthStatusCode = noAuthResponse.StatusCode + + if (http.StatusOK != noAuthStatusCode) && (http.StatusPartialContent != noAuthStatusCode) { + _ = noAuthResponse.Body.Close() + authResponseWriter.WriteHeader(noAuthStatusCode) + return + } + + getBuf, err = ioutil.ReadAll(noAuthResponse.Body) + if nil != err { + _ = noAuthResponse.Body.Close() + authResponseWriter.WriteHeader(http.StatusBadRequest) + return + } + + err = noAuthResponse.Body.Close() + if nil != err { + authResponseWriter.WriteHeader(http.StatusBadRequest) + return + } + + contentTypeHeader = noAuthResponse.Header.Get("Content-Type") + if "" != contentTypeHeader { + authResponseWriter.Header().Add("Content-Type", contentTypeHeader) + } + + contentRangeHeader = noAuthResponse.Header.Get("Content-Range") + if "" != contentRangeHeader { + authResponseWriter.Header().Add("Content-Range", contentRangeHeader) + } + + authResponseWriter.WriteHeader(noAuthStatusCode) + + _, _ = authResponseWriter.Write(getBuf) +} + +// doPUT proxies the GET over to ramswift +// +func doPUT(authResponseWriter http.ResponseWriter, authRequest *http.Request) { + var ( + err error + hostHeader string + noAuthPath string + noAuthRequest *http.Request + noAuthResponse *http.Response + ) + + if !strings.HasPrefix(authRequest.URL.Path, "/proxyfs/"+testAccountName) { + _ = authRequest.Body.Close() + authResponseWriter.WriteHeader(http.StatusNotFound) + return + } + + noAuthPath = strings.Replace(authRequest.URL.Path, "proxyfs", "v1", 1) + + noAuthRequest, err = http.NewRequest("PUT", testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL+noAuthPath, authRequest.Body) + if nil != err { + _ = authRequest.Body.Close() + authResponseWriter.WriteHeader(http.StatusBadRequest) + return + } + + hostHeader = authRequest.Header.Get("Host") + if "" != hostHeader { + noAuthRequest.Header.Add("Host", hostHeader) + } + + noAuthResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(noAuthRequest) + if nil != err { + _ = authRequest.Body.Close() + authResponseWriter.WriteHeader(http.StatusBadRequest) + return + } + + err = authRequest.Body.Close() + if nil != err { + authResponseWriter.WriteHeader(http.StatusBadRequest) + return + } + + authResponseWriter.WriteHeader(noAuthResponse.StatusCode) +} + +// doRPC proxies the payload as a JSON RPC request over to proxyfsd +// +func doRPC(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + jrpcResponseBuf []byte + jrpcResponseLen int + jrpcRequestBuf []byte + tcpConn *net.TCPConn + ) + + if !strings.HasPrefix(request.URL.Path, "/proxyfs/"+testAccountName) { + _ = request.Body.Close() + responseWriter.WriteHeader(http.StatusNotFound) + return + } + + jrpcRequestBuf, err = ioutil.ReadAll(request.Body) + _ = request.Body.Close() + if nil != err { + responseWriter.WriteHeader(http.StatusBadRequest) + return + } + + if request.Header.Get("Content-Type") != "application/json" { + responseWriter.WriteHeader(http.StatusBadRequest) + return + } + + tcpConn, err = net.DialTCP("tcp", nil, testSwiftProxyEmulatorGlobals.proxyfsdJrpcTCPAddr) + if nil != err { + responseWriter.WriteHeader(http.StatusServiceUnavailable) + return + } + + _, err = tcpConn.Write(jrpcRequestBuf) + if nil != err { + _ = tcpConn.Close() + responseWriter.WriteHeader(http.StatusServiceUnavailable) + return + } + + jrpcResponseBuf = testSwiftProxyEmulatorGlobals.jrpcResponsePool.Get().([]byte) + + jrpcResponseLen, err = tcpConn.Read(jrpcResponseBuf) + if nil != err { + _ = tcpConn.Close() + responseWriter.WriteHeader(http.StatusServiceUnavailable) + return + } + + err = tcpConn.Close() + if nil != err { + responseWriter.WriteHeader(http.StatusServiceUnavailable) + return + } + + responseWriter.Header().Add("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(jrpcResponseBuf[:jrpcResponseLen]) + + testSwiftProxyEmulatorGlobals.jrpcResponsePool.Put(jrpcResponseBuf) +} diff --git a/emswift/main.go b/emswift/main.go new file mode 100644 index 000000000..ed6b83fc9 --- /dev/null +++ b/emswift/main.go @@ -0,0 +1,69 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "os/signal" + + "golang.org/x/sys/unix" + + "github.com/swiftstack/ProxyFS/conf" + "github.com/swiftstack/ProxyFS/emswift/emswiftpkg" +) + +func main() { + var ( + confMap conf.ConfMap + err error + signalChan chan os.Signal + ) + + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "no .conf file specified\n") + os.Exit(1) + } + + confMap, err = conf.MakeConfMapFromFile(os.Args[1]) + if nil != err { + fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err) + os.Exit(1) + } + + err = confMap.UpdateFromStrings(os.Args[2:]) + if nil != err { + fmt.Fprintf(os.Stderr, "failed to apply config overrides: %v\n", err) + os.Exit(1) + } + + // Arm signal handler used to indicate termination and wait on it + // + // Note: signalled chan must be buffered to avoid race with window between + // arming handler and blocking on the chan read + + signalChan = make(chan os.Signal, 1) + + signal.Notify(signalChan, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) + + // Start Swift Emulation + + err = emswiftpkg.Start(confMap) + if nil != err { + fmt.Fprintf(os.Stderr, "emswiftpkg.Start(confMap) failed: %v\n", err) + os.Exit(1) + } + + // Await any of the above specified signals + + _ = <-signalChan + + // Stop Swift Emulation + + emswiftpkg.Stop() + if nil != err { + fmt.Fprintf(os.Stderr, "emswiftpkg.Stop() failed: %v\n", err) + os.Exit(1) + } +} diff --git a/imgr/imgrpkg/Makefile b/imgr/imgrpkg/Makefile new file mode 100644 index 000000000..9fe6f3f1e --- /dev/null +++ b/imgr/imgrpkg/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/swiftstack/ProxyFS/imgr/imgrpkg + +include ../../GoMakefile diff --git a/imgr/imgrpkg/dummy.go b/imgr/imgrpkg/dummy.go new file mode 100644 index 000000000..e1af09375 --- /dev/null +++ b/imgr/imgrpkg/dummy.go @@ -0,0 +1,4 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg diff --git a/imgr/imgrpkg/dummy_test.go b/imgr/imgrpkg/dummy_test.go new file mode 100644 index 000000000..0b0d310b5 --- /dev/null +++ b/imgr/imgrpkg/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} From c053f24a4e71d88d6f3b1db051a178ec62c6d7d0 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 26 Jan 2021 10:51:33 -0800 Subject: [PATCH 012/160] Ported over package ramswift's daemon_test.go to emswiftpkg - Auth stub'd / NoAuth fails (for now) --- emswift/emswiftpkg/dummy_test.go | 11 - emswift/emswiftpkg/pkg_test.go | 1283 ++++++++++++++++++++++++++++++ emswift/main.go | 20 +- 3 files changed, 1292 insertions(+), 22 deletions(-) delete mode 100644 emswift/emswiftpkg/dummy_test.go create mode 100644 emswift/emswiftpkg/pkg_test.go diff --git a/emswift/emswiftpkg/dummy_test.go b/emswift/emswiftpkg/dummy_test.go deleted file mode 100644 index 7c7334abd..000000000 --- a/emswift/emswiftpkg/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package emswiftpkg - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/emswift/emswiftpkg/pkg_test.go b/emswift/emswiftpkg/pkg_test.go new file mode 100644 index 000000000..a19ad55d8 --- /dev/null +++ b/emswift/emswiftpkg/pkg_test.go @@ -0,0 +1,1283 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package emswiftpkg + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/swiftstack/ProxyFS/conf" + "github.com/swiftstack/ProxyFS/utils" +) + +func TestAuthEmulation(t *testing.T) { + var ( + confMap conf.ConfMap + confStrings = []string{ + "EMSWIFT.AuthIPAddr=127.0.0.1", + "EMSWIFT.AuthTCPPort=9998", + "EMSWIFT.NoAuthIPAddr=127.0.0.1", + "EMSWIFT.NoAuthTCPPort=9999", + "EMSWIFT.MaxAccountNameLength=256", + "EMSWIFT.MaxContainerNameLength=256", + "EMSWIFT.MaxObjectNameLength=1024", + "EMSWIFT.AccountListingLimit=10000", + "EMSWIFT.ContainerListingLimit=10000", + } + err error + ) + + confMap, err = conf.MakeConfMapFromStrings(confStrings) + if nil != err { + t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned unexpected error: %v", err) + } + + err = Start(confMap) + if nil != err { + t.Fatalf("Start(confMap) returned unexpected error: %v", err) + } + + t.Logf("TODO: Implement TestAuthEmulation() test steps") + + err = Stop() + if nil != err { + t.Fatalf("Stop() returned unexpected error: %v", err) + } +} + +func TestNoAuthEmulation(t *testing.T) { + var ( + confMap conf.ConfMap + confStrings = []string{ + "EMSWIFT.NoAuthIPAddr=127.0.0.1", + "EMSWIFT.NoAuthTCPPort=9999", + "EMSWIFT.MaxAccountNameLength=256", + "EMSWIFT.MaxContainerNameLength=256", + "EMSWIFT.MaxObjectNameLength=1024", + "EMSWIFT.AccountListingLimit=10000", + "EMSWIFT.ContainerListingLimit=10000", + } + contentType string + contentTypeMultiPartBoundary string + err error + errChan chan error + expectedBuf []byte + expectedInfo string + httpClient *http.Client + httpRequest *http.Request + httpResponse *http.Response + mouseHeaderPresent bool + pipeReader *io.PipeReader + pipeWriter *io.PipeWriter + readBuf []byte + urlForInfo string + urlPrefix string + ) + + confMap, err = conf.MakeConfMapFromStrings(confStrings) + if nil != err { + t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned unexpected error: %v", err) + } + + err = Start(confMap) + if nil != err { + t.Fatalf("Start(confMap) returned unexpected error: %v", err) + } + + // Setup http.Client that we will use for all HTTP requests + + httpClient = &http.Client{} + + // Send a GET for "/info" expecting [RamSwiftInfo] data in compact JSON form + + httpRequest, err = http.NewRequest("GET", urlForInfo, nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + expectedInfo = "{\"swift\": {\"max_account_name_length\": 256,\"max_container_name_length\": 256,\"max_object_name_length\": 1024,\"account_listing_limit\": 10000,\"container_listing_limit\": 10000}}" + if int64(len(expectedInfo)) != httpResponse.ContentLength { + t.Fatalf("GET of /info httpResponse.ContentLength unexpected") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if expectedInfo != utils.ByteSliceToString(readBuf) { + t.Fatalf("GET of /info httpResponse.Body contents unexpected") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for account "TestAccount" and header Cat: Dog + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Cat", "Dog") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for account "TestAccount" expecting header Cat: Dog + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for account "TestAccount" and header Mouse: Bird + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "Bird") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if httpResponse.Header.Get("Mouse") != "Bird" { + t.Fatalf("TestAccount should have header Mouse: Bird") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for account "TestAccount" deleting header Mouse + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + _, mouseHeaderPresent = httpResponse.Header["Mouse"] + if mouseHeaderPresent { + t.Fatalf("TestAccount should not have header Mouse") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for account "TestAccount" and header Mouse: Bird + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "Bird") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusAccepted != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if httpResponse.Header.Get("Mouse") != "Bird" { + t.Fatalf("TestAccount should have header Mouse: Bird") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for account "TestAccount" deleting header Mouse + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusAccepted != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + _, mouseHeaderPresent = httpResponse.Header["Mouse"] + if mouseHeaderPresent { + t.Fatalf("TestAccount should not have header Mouse") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for container "TestContainer" and header Cat: Dog + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Cat", "Dog") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" expecting "TestContainer\n" and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if int64(len("TestContainer\n")) != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if "TestContainer\n" != utils.ByteSliceToString(readBuf) { + t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" with marker "AAA" expecting "TestContainer\n" and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount?marker=AAA", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if int64(len("TestContainer\n")) != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if "TestContainer\n" != utils.ByteSliceToString(readBuf) { + t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" with marker "ZZZ" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount?marker=ZZZ", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for container "TestContainer" expecting header Cat: Dog + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for container "TestContainer" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for container "TestContainer" and header Mouse: Bird + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "Bird") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + if httpResponse.Header.Get("Mouse") != "Bird" { + t.Fatalf("TestContainer should have header Mouse: Bird") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for container "TestContainer" deleting header Mouse + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + _, mouseHeaderPresent = httpResponse.Header["Mouse"] + if mouseHeaderPresent { + t.Fatalf("TestContainer should not have header Mouse") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for container "TestContainer" and header Mouse: Bird + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "Bird") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusAccepted != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + if httpResponse.Header.Get("Mouse") != "Bird" { + t.Fatalf("TestContainer should have header Mouse: Bird") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for container "TestContainer" deleting header Mouse + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusAccepted != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + _, mouseHeaderPresent = httpResponse.Header["Mouse"] + if mouseHeaderPresent { + t.Fatalf("TestContainer should not have header Mouse") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a non-chunked PUT for object "Foo" to contain []byte{0x00, 0x01, 0x02} + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/Foo", bytes.NewReader([]byte{0x00, 0x01, 0x02})) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a chunked PUT for object "Bar"" with 1st chunk being []byte{0xAA, 0xBB} & 2nd chunk being []byte{0xCC, 0xDD, 0xEE} + + pipeReader, pipeWriter = io.Pipe() + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/Bar", pipeReader) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.ContentLength = -1 + httpRequest.Header.Del("Content-Length") + errChan = make(chan error, 1) + go func() { + var ( + nonShadowingErr error + nonShadowingHTTPResponse *http.Response + ) + nonShadowingHTTPResponse, nonShadowingErr = httpClient.Do(httpRequest) + if nil == nonShadowingErr { + httpResponse = nonShadowingHTTPResponse + } + errChan <- nonShadowingErr + }() + _, err = pipeWriter.Write([]byte{0xAA, 0xBB}) + if nil != err { + t.Fatalf("pipeWriter.Write() returned unexpected error: %v", err) + } + _, err = pipeWriter.Write([]byte{0xCC, 0xDD, 0xEE}) + if nil != err { + t.Fatalf("pipeWriter.Write() returned unexpected error: %v", err) + } + err = pipeWriter.Close() + if nil != err { + t.Fatalf("pipeWriter.Close() returned unexpected error: %v", err) + } + err = <-errChan + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for container "TestContainer" expecting "Bar\nFoo\n" and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if int64(len("Bar\nFoo\n")) != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if "Bar\nFoo\n" != utils.ByteSliceToString(readBuf) { + t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for container "TestContainer" with marker "AAA" expecting "Bar\nFoo\n" and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer?marker=AAA", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if int64(len("Bar\nFoo\n")) != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if "Bar\nFoo\n" != utils.ByteSliceToString(readBuf) { + t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for container "TestContainer" with marker "ZZZ" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer?marker=ZZZ", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for object "Foo" expecting Content-Length: 3 + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/Foo", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if 3 != httpResponse.ContentLength { + t.Fatalf("httpResponse.ContentLength contained unexpected value: %v", httpResponse.ContentLength) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a full object GET for object "Foo" expecting []byte{0x00, 0x01, 0x02} + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Foo", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if int64(len([]byte{0x00, 0x01, 0x02})) != httpResponse.ContentLength { + t.Fatalf("Foo should contain precisely []byte{0x00, 0x01, 0x02}") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if 0 != bytes.Compare([]byte{0x00, 0x01, 0x02}, readBuf) { + t.Fatalf("Foo should contain precisely []byte{0x00, 0x01, 0x02}") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a range GET of bytes at offset 1 for length 3 for object "Bar" expecting []byte{0xBB, 0xCC, 0xDD} + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Range", "bytes=1-3") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusPartialContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if int64(len([]byte{0xBB, 0xCC, 0xDD})) != httpResponse.ContentLength { + t.Fatalf("Bar's bytes 1-3 should contain precisely []byte{0xBB, 0xCC, 0xDD}") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if 0 != bytes.Compare([]byte{0xBB, 0xCC, 0xDD}, readBuf) { + t.Fatalf("Bar's bytes 1-3 should contain precisely []byte{0xBB, 0xCC, 0xDD}") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a range GET of bytes at offset 0 for length 2 + // and offset 3 for length of 1 for object "Bar" + // expecting two MIME parts: []byte{0xAA, 0xBB} and []byte{0xDD} + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Range", "bytes=0-1,3-3") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusPartialContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + contentType = httpResponse.Header.Get("Content-Type") + contentTypeMultiPartBoundary = strings.TrimPrefix(contentType, "multipart/byteranges; boundary=") + if (len(contentType) == len(contentTypeMultiPartBoundary)) || (0 == len(contentTypeMultiPartBoundary)) { + t.Fatalf("httpReponse.Header[\"Content-Type\"] contained unexpected value: \"%v\"", contentType) + } + expectedBuf = make([]byte, 0, httpResponse.ContentLength) + expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"\r\n")...) + expectedBuf = append(expectedBuf, []byte("Content-Type: application/octet-stream\r\n")...) + expectedBuf = append(expectedBuf, []byte("Content-Range: bytes 0-1/5\r\n")...) + expectedBuf = append(expectedBuf, []byte("\r\n")...) + expectedBuf = append(expectedBuf, []byte{0xAA, 0xBB}...) + expectedBuf = append(expectedBuf, []byte("\r\n")...) + expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"\r\n")...) + expectedBuf = append(expectedBuf, []byte("Content-Type: application/octet-stream\r\n")...) + expectedBuf = append(expectedBuf, []byte("Content-Range: bytes 3-3/5\r\n")...) + expectedBuf = append(expectedBuf, []byte("\r\n")...) + expectedBuf = append(expectedBuf, []byte{0xDD}...) + expectedBuf = append(expectedBuf, []byte("\r\n")...) + expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"--")...) + if int64(len(expectedBuf)) != httpResponse.ContentLength { + t.Fatalf("Unexpected multi-part GET response Content-Length") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if 0 != bytes.Compare(expectedBuf, readBuf) { + t.Fatalf("Unexpected payload of multi-part GET response") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a tail GET of the last 2 bytes for object "Bar" expecting []byte{0xDD, 0xEE} + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Range", "bytes=-2") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusPartialContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if int64(len([]byte{0xDD, 0xEE})) != httpResponse.ContentLength { + t.Fatalf("Bar's last 2 bytes should contain precisely []byte{0xDD, 0xEE}") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if 0 != bytes.Compare([]byte{0xDD, 0xEE}, readBuf) { + t.Fatalf("Bar's last 2 bytes should contain precisely []byte{0xDD, 0xEE}") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for object "ZigZag" and header Cat: Dog + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Cat", "Dog") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for object "ZigZag" expecting header Cat: Dog + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for object "ZigZag" expecting header Cat: Dog + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for object "ZigZag" and header Mouse: Bird + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "Bird") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for object "ZigZag" expecting header Cat: Dog & Mouse: Bird + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if httpResponse.Header.Get("Mouse") != "Bird" { + t.Fatalf("TestAccount should have header Mouse: Bird") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for object "ZigZag" deleting header Mouse + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for object "ZigZag" expecting header Cat: Dog & no Mouse header + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + _, mouseHeaderPresent = httpResponse.Header["Mouse"] + if mouseHeaderPresent { + t.Fatalf("TestAccount should not have header Mouse") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a DELETE for object "Foo" + + httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/Foo", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a DELETE for object "Bar" + + httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/Bar", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a DELETE for object "ZigZag" + + httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for container "TestContainer" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a DELETE for container "TestContainer" + + httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a DELETE for account "TestAccount" + + httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + err = Stop() + if nil != err { + t.Fatalf("Stop() returned unexpected error: %v", err) + } +} diff --git a/emswift/main.go b/emswift/main.go index ed6b83fc9..94a14a148 100644 --- a/emswift/main.go +++ b/emswift/main.go @@ -38,15 +38,6 @@ func main() { os.Exit(1) } - // Arm signal handler used to indicate termination and wait on it - // - // Note: signalled chan must be buffered to avoid race with window between - // arming handler and blocking on the chan read - - signalChan = make(chan os.Signal, 1) - - signal.Notify(signalChan, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) - // Start Swift Emulation err = emswiftpkg.Start(confMap) @@ -55,13 +46,20 @@ func main() { os.Exit(1) } - // Await any of the above specified signals + // Arm signal handler used to indicate termination & wait on it + // + // Note: signal'd chan must be buffered to avoid race with window between + // arming handler and blocking on the chan read + + signalChan = make(chan os.Signal, 1) + + signal.Notify(signalChan, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) _ = <-signalChan // Stop Swift Emulation - emswiftpkg.Stop() + err = emswiftpkg.Stop() if nil != err { fmt.Fprintf(os.Stderr, "emswiftpkg.Stop() failed: %v\n", err) os.Exit(1) From 57384f2f786302b5c50fad68d17cccd510c06911 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 26 Jan 2021 14:18:33 -0800 Subject: [PATCH 013/160] Fixed startup of NoAuthEmulator and got NoAuthEmulation to pass pkg_test.go ported from package ramswift --- emswift/emswiftpkg/impl.go | 59 ++++++++++++++++++++++++++++++++-- emswift/emswiftpkg/pkg_test.go | 5 +++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/emswift/emswiftpkg/impl.go b/emswift/emswiftpkg/impl.go index 3ab3ea7d8..6aa756a41 100644 --- a/emswift/emswiftpkg/impl.go +++ b/emswift/emswiftpkg/impl.go @@ -13,6 +13,7 @@ import ( "strings" "sync" "syscall" + "time" "golang.org/x/sys/unix" @@ -76,6 +77,11 @@ type globalsStruct struct { var globals globalsStruct +const ( + startGETInfoMaxRetries = 10 + startGETInfoRetryDelay = 100 * time.Millisecond +) + type httpRequestHandler struct{} type rangeStruct struct { @@ -195,7 +201,8 @@ func uninitializeGlobals() { func startAuthIfRequested() (err error) { var ( - authEmulator *authEmulatorStruct + authEmulator *authEmulatorStruct + startGETInfoNumRetries int ) if "" == globals.config.AuthIPAddr { @@ -220,6 +227,22 @@ func startAuthIfRequested() (err error) { globals.authEmulator.wg.Done() }() + startGETInfoNumRetries = 0 + + for { + _, err = http.Get("http://" + globals.authEmulator.httpServer.Addr + "/info") + if nil == err { + break + } + startGETInfoNumRetries++ + if startGETInfoNumRetries > startGETInfoMaxRetries { + _ = stopAuthIfRequested() + err = fmt.Errorf("startAuthIfRequested() failed to establish that authEmulator is up") + return + } + time.Sleep(startGETInfoRetryDelay) + } + err = nil return } @@ -244,7 +267,8 @@ func stopAuthIfRequested() (err error) { func startNoAuth() (err error) { var ( - noAuthEmulator *noAuthEmulatorStruct + noAuthEmulator *noAuthEmulatorStruct + startGETInfoNumRetries int ) noAuthEmulator = &noAuthEmulatorStruct{ @@ -263,6 +287,22 @@ func startNoAuth() (err error) { globals.noAuthEmulator.wg.Done() }() + startGETInfoNumRetries = 0 + + for { + _, err = http.Get("http://" + globals.noAuthEmulator.httpServer.Addr + "/info") + if nil == err { + break + } + startGETInfoNumRetries++ + if startGETInfoNumRetries > startGETInfoMaxRetries { + _ = stopNoAuth() + err = fmt.Errorf("startNoAuth() failed to establish that noAuthEmulator is up") + return + } + time.Sleep(startGETInfoRetryDelay) + } + err = nil return } @@ -285,7 +325,20 @@ func (dummy *authEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, r } func (dummy *noAuthEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - // TODO + switch request.Method { + case http.MethodDelete: + doNoAuthDELETE(responseWriter, request) + case http.MethodGet: + doNoAuthGET(responseWriter, request) + case http.MethodHead: + doNoAuthHEAD(responseWriter, request) + case http.MethodPost: + doNoAuthPOST(responseWriter, request) + case http.MethodPut: + doNoAuthPUT(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusMethodNotAllowed) + } } func (dummy *globalsStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { diff --git a/emswift/emswiftpkg/pkg_test.go b/emswift/emswiftpkg/pkg_test.go index a19ad55d8..d7faae864 100644 --- a/emswift/emswiftpkg/pkg_test.go +++ b/emswift/emswiftpkg/pkg_test.go @@ -89,6 +89,11 @@ func TestNoAuthEmulation(t *testing.T) { t.Fatalf("Start(confMap) returned unexpected error: %v", err) } + // Format URLs + + urlForInfo = "http://" + globals.noAuthEmulator.httpServer.Addr + "/info" + urlPrefix = "http://" + globals.noAuthEmulator.httpServer.Addr + "/v1/" + // Setup http.Client that we will use for all HTTP requests httpClient = &http.Client{} From d990df58f4320374bbeb586f88ec18e0ddc4d6d9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 26 Jan 2021 15:41:33 -0800 Subject: [PATCH 014/160] Implemented non-RPC portion of emswift AuthEmulation --- emswift/emswift.conf | 4 +- emswift/emswiftpkg/impl.go | 286 ++++++++++--------------------------- 2 files changed, 76 insertions(+), 214 deletions(-) diff --git a/emswift/emswift.conf b/emswift/emswift.conf index 7bbdf9f28..dabf7f0a5 100644 --- a/emswift/emswift.conf +++ b/emswift/emswift.conf @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 [EMSWIFT] -AuthIPAddr: # 0.0.0.0 -AuthTCPPort: # 8080 +AuthIPAddr: 127.0.0.1 # If AuthIPAddr left blank or missing, +AuthTCPPort: 8080 # AuthTCPPort is ignored and no AuthEmulator is launched NoAuthIPAddr: 127.0.0.1 NoAuthTCPPort: 8090 diff --git a/emswift/emswiftpkg/impl.go b/emswift/emswiftpkg/impl.go index 6aa756a41..5d53638f3 100644 --- a/emswift/emswiftpkg/impl.go +++ b/emswift/emswiftpkg/impl.go @@ -82,6 +82,11 @@ const ( startGETInfoRetryDelay = 100 * time.Millisecond ) +const ( + fixedAuthToken = "AUTH_tk0123456789abcde0123456789abcdef0" + fixedUserToAccountPrefix = "AUTH_" // Prefixed to User truncated before colon (":") if necessary +) + type httpRequestHandler struct{} type rangeStruct struct { @@ -321,7 +326,75 @@ func stopNoAuth() (err error) { } func (dummy *authEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - // TODO + var ( + noAuthPath string + xAuthKey string + xAuthUser string + xAuthUserSplit2OnColon []string + xStorageURL string + ) + + // Handle the GET of/on info & AuthURL cases + + if http.MethodGet == request.Method { + switch request.URL.Path { + case "/info": + _ = request.Body.Close() + doNoAuthGET(responseWriter, request) + return + case "/auth/v1.0": + _ = request.Body.Close() + xAuthUser = request.Header.Get("X-Auth-User") + xAuthKey = request.Header.Get("X-Auth-Key") + if ("" == xAuthUser) || ("" == xAuthKey) { + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + xAuthUserSplit2OnColon = strings.SplitN(xAuthUser, ":", 2) + xStorageURL = "http://" + globals.authEmulator.httpServer.Addr + "/v1/" + fixedUserToAccountPrefix + xAuthUserSplit2OnColon[0] + responseWriter.Header().Add("X-Auth-Token", fixedAuthToken) + responseWriter.Header().Add("X-Storage-Url", xStorageURL) + return + default: + // Fall through to normal processing + } + } + + // Require X-Auth-Token to match fixedAuthToken + + if fixedAuthToken != request.Header.Get("X-Auth-Token") { + _ = request.Body.Close() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + // Require "version" portion of request.URL.Path to be "proxyfs" + + if !strings.HasPrefix(request.URL.Path, "/proxyfs/") { + _ = request.Body.Close() + responseWriter.WriteHeader(http.StatusNotFound) + return + } + + // Branch off to individual request method handlers + + switch request.Method { + case http.MethodGet: + noAuthPath = strings.Replace(request.URL.Path, "proxyfs", "v1", 1) + request.URL.Path = noAuthPath + doNoAuthGET(responseWriter, request) + case http.MethodPut: + noAuthPath = strings.Replace(request.URL.Path, "proxyfs", "v1", 1) + request.URL.Path = noAuthPath + doNoAuthPUT(responseWriter, request) + case "PROXYFS": + // TODO: doRPC(responseWriter, request) + _ = request.Body.Close() + responseWriter.WriteHeader(http.StatusNotImplemented) + return + default: + responseWriter.WriteHeader(http.StatusMethodNotAllowed) + } } func (dummy *noAuthEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { @@ -1382,214 +1455,3 @@ func doNoAuthPUT(responseWriter http.ResponseWriter, request *http.Request) { } } } - -/* -func (h httpRequestHandler) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - switch request.Method { - case http.MethodDelete: - doNoAuthDELETE(responseWriter, request) - case http.MethodGet: - doNoAuthGET(responseWriter, request) - case http.MethodHead: - doNoAuthHEAD(responseWriter, request) - case http.MethodPost: - doNoAuthPOST(responseWriter, request) - case http.MethodPut: - doNoAuthPUT(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusMethodNotAllowed) - } -} - -func setupNoAuthSwift(confMap conf.ConfMap) { - var ( - err error - errno syscall.Errno - primaryPeerList []string - swiftAccountName string - volumeGroupNameList []string - volumeGroupName string - volumeGroupSectionName string - volumeName string - volumeNameList []string - volumeSectionName string - ) - - // Fetch and configure volumes for which "we" are the PrimaryPeer - - volumeGroupNameList, err = confMap.FetchOptionValueStringSlice("FSGlobals", "VolumeGroupList") - if nil != err { - log.Fatalf("failed fetch of FSGlobals.VolumeGroupList: %v", err) - } - - for _, volumeGroupName = range volumeGroupNameList { - volumeGroupSectionName = "VolumeGroup:" + volumeGroupName - - primaryPeerList, err = confMap.FetchOptionValueStringSlice(volumeGroupSectionName, "PrimaryPeer") - if nil != err { - log.Fatalf("failed fetch of %v.PrimaryPeer: %v", volumeGroupSectionName, err) - } - if 0 == len(primaryPeerList) { - continue - } else if 1 == len(primaryPeerList) { - if globals.whoAmI != primaryPeerList[0] { - continue - } - } else { - log.Fatalf("fetch of %v.PrimaryPeer returned multiple values", volumeGroupSectionName) - } - - volumeNameList, err = confMap.FetchOptionValueStringSlice(volumeGroupSectionName, "VolumeList") - if nil != err { - log.Fatalf("failed fetch of %v.VolumeList: %v", volumeGroupSectionName, err) - } - - for _, volumeName = range volumeNameList { - volumeSectionName = "Volume:" + volumeName - - swiftAccountName, err = confMap.FetchOptionValueString(volumeSectionName, "AccountName") - if nil != err { - log.Fatalf("failed fetch of %v.AccountName: %v", volumeSectionName, err) - } - - _, errno = createSwiftAccount(swiftAccountName) - if 0 != errno { - log.Fatalf("failed create of %v: %v", swiftAccountName, err) - } - } - } - - // Create HTTP Server on the requested noAuthTCPPort - - globals.noAuthHTTPServer = &http.Server{ - Addr: globals.noAuthAddr, - Handler: httpRequestHandler{}, - } -} - -func serveNoAuthSwift() { - _ = globals.noAuthHTTPServer.ListenAndServe() - - globals.noAuthHTTPServerWG.Done() -} - -func Daemon(confFile string, confStrings []string, signalHandlerIsArmedWG *sync.WaitGroup, doneChan chan bool, signals ...os.Signal) { - var ( - confMap conf.ConfMap - err error - resp *http.Response - signalChan chan os.Signal - signalReceived os.Signal - ) - - // Initialization - - globals = &globalsStruct{} - - globals.swiftAccountMap = make(map[string]*swiftAccountStruct) - - // Compute confMap - - confMap, err = conf.MakeConfMapFromFile(confFile) - if nil != err { - log.Fatalf("failed to load config: %v", err) - } - - err = confMap.UpdateFromStrings(confStrings) - if nil != err { - log.Fatalf("failed to apply config overrides: %v", err) - } - - err = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != err { - log.Fatalf("failed to upgrade confMap: %v", err) - } - - // Find out who "we" are - - globals.whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - log.Fatalf("failed fetch of Cluster.WhoAmI: %v", err) - } - - globals.noAuthTCPPort, err = confMap.FetchOptionValueUint16("SwiftClient", "NoAuthTCPPort") - if nil != err { - log.Fatalf("failed fetch of Swift.NoAuthTCPPort: %v", err) - } - - globals.noAuthAddr = "127.0.0.1:" + strconv.Itoa(int(globals.noAuthTCPPort)) - - // Kick off NoAuth Swift Proxy Emulator - - setupNoAuthSwift(confMap) - globals.noAuthHTTPServerWG.Add(1) - go serveNoAuthSwift() - - // Wait for serveNoAuthSwift() to begin serving - - for { - resp, err = http.Get("http://" + globals.noAuthAddr + "/info") - if nil != err { - log.Printf("failed GET of \"/info\": %v", err) - continue - } - if http.StatusOK == resp.StatusCode { - break - } - log.Printf("GET of \"/info\" returned Status %v", resp.Status) - time.Sleep(100 * time.Millisecond) - } - - // Arm signal handler used to indicate termination and wait on it - // - // Note: signalled chan must be buffered to avoid race with window between - // arming handler and blocking on the chan read - - signalChan = make(chan os.Signal, 1) - - signal.Notify(signalChan, signals...) - - if nil != signalHandlerIsArmedWG { - signalHandlerIsArmedWG.Done() - } - - // Await a signal - reloading confFile each SIGHUP - exiting otherwise - - for { - signalReceived = <-signalChan - - if unix.SIGHUP == signalReceived { - // recompute confMap and re-apply - - confMap, err = conf.MakeConfMapFromFile(confFile) - if nil != err { - log.Fatalf("failed to load updated config: %v", err) - } - - err = confMap.UpdateFromStrings(confStrings) - if nil != err { - log.Fatalf("failed to reapply config overrides: %v", err) - } - - err = transitions.UpgradeConfMapIfNeeded(confMap) - if nil != err { - log.Fatalf("failed to upgrade confMap: %v", err) - } - - updateConf(confMap) - } else { - // signalReceived either SIGINT or SIGTERM... so just exit - - err = globals.noAuthHTTPServer.Close() - - globals.noAuthHTTPServerWG.Wait() - - globals = nil - - doneChan <- true - - return - } - } -} -*/ From 3c95fea25ec517d73c62f7e4d3e93616b6356130 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 27 Jan 2021 08:40:40 -0800 Subject: [PATCH 015/160] Added non-functional PROXYFS->JSON-RPC hooks to emswift --- emswift/emswift.conf | 3 + emswift/emswiftpkg/impl.go | 42 +++++++++- emswift/emswiftpkg/pkg_test.go | 147 ++++++++++++++++++++++++++++++++- 3 files changed, 184 insertions(+), 8 deletions(-) diff --git a/emswift/emswift.conf b/emswift/emswift.conf index dabf7f0a5..d36206bfc 100644 --- a/emswift/emswift.conf +++ b/emswift/emswift.conf @@ -5,6 +5,9 @@ AuthIPAddr: 127.0.0.1 # If AuthIPAddr left blank or missing, AuthTCPPort: 8080 # AuthTCPPort is ignored and no AuthEmulator is launched +JRPCIPAddr: 127.0.0.1 # If AuthIPAddr left blank or missing, +JRPCTCPPort: 12345 # JRPC{IPAddr|TCPPort} are ignored + NoAuthIPAddr: 127.0.0.1 NoAuthTCPPort: 8090 diff --git a/emswift/emswiftpkg/impl.go b/emswift/emswiftpkg/impl.go index 5d53638f3..03eaa542a 100644 --- a/emswift/emswiftpkg/impl.go +++ b/emswift/emswiftpkg/impl.go @@ -47,6 +47,9 @@ type configStruct struct { AuthIPAddr string // Only required if Auth Swift Proxy enabled AuthTCPPort uint16 // Only required if Auth Swift Proxy enabled + JRPCIPAddr string // Only required if Auth Swift Proxy enabled + JRPCTCPPort uint16 // Only required if Auth Swift Proxy enabled + NoAuthIPAddr string NoAuthTCPPort uint16 @@ -87,6 +90,25 @@ const ( fixedUserToAccountPrefix = "AUTH_" // Prefixed to User truncated before colon (":") if necessary ) +type jrpcRequestStruct struct { + JSONrpc string `json:"jsonrpc"` + Method string `json:"method"` + ID uint64 `json:"id"` + Params [1]interface{} `json:"params"` +} + +type jrpcRequestEmptyParamStruct struct{} + +type jrpcResponseIDAndErrorStruct struct { + ID uint64 `json:"id"` + Error string `json:"error"` +} + +type jrpcResponseNoErrorStruct struct { + ID uint64 `json:"id"` + Result interface{} `json:"result"` +} + type httpRequestHandler struct{} type rangeStruct struct { @@ -141,6 +163,15 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { if nil != err { return } + + globals.config.JRPCIPAddr, err = confMap.FetchOptionValueString("EMSWIFT", "JRPCIPAddr") + if nil != err { + return + } + globals.config.JRPCTCPPort, err = confMap.FetchOptionValueUint16("EMSWIFT", "JRPCTCPPort") + if nil != err { + return + } } else { err = nil globals.config.AuthIPAddr = "" @@ -388,15 +419,18 @@ func (dummy *authEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, r request.URL.Path = noAuthPath doNoAuthPUT(responseWriter, request) case "PROXYFS": - // TODO: doRPC(responseWriter, request) - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusNotImplemented) - return + doAuthPROXYFS(responseWriter, request) default: responseWriter.WriteHeader(http.StatusMethodNotAllowed) } } +func doAuthPROXYFS(responseWriter http.ResponseWriter, request *http.Request) { + fmt.Println("TODO: doAuthPROXYFS(responseWriter, request)") + _ = request.Body.Close() + responseWriter.WriteHeader(http.StatusNotImplemented) +} + func (dummy *noAuthEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { switch request.Method { case http.MethodDelete: diff --git a/emswift/emswiftpkg/pkg_test.go b/emswift/emswiftpkg/pkg_test.go index d7faae864..56172d98d 100644 --- a/emswift/emswiftpkg/pkg_test.go +++ b/emswift/emswiftpkg/pkg_test.go @@ -5,22 +5,32 @@ package emswiftpkg import ( "bytes" + "fmt" "io" "io/ioutil" + "net" "net/http" "strings" + "sync" "testing" "github.com/swiftstack/ProxyFS/conf" "github.com/swiftstack/ProxyFS/utils" ) +type testJRPCServerHandlerStruct struct { + httpServer *http.Server + wg sync.WaitGroup +} + func TestAuthEmulation(t *testing.T) { var ( confMap conf.ConfMap confStrings = []string{ "EMSWIFT.AuthIPAddr=127.0.0.1", - "EMSWIFT.AuthTCPPort=9998", + "EMSWIFT.AuthTCPPort=9997", + "EMSWIFT.JRPCIPAddr=127.0.0.1", + "EMSWIFT.JRPCTCPPort=9998", "EMSWIFT.NoAuthIPAddr=127.0.0.1", "EMSWIFT.NoAuthTCPPort=9999", "EMSWIFT.MaxAccountNameLength=256", @@ -29,7 +39,17 @@ func TestAuthEmulation(t *testing.T) { "EMSWIFT.AccountListingLimit=10000", "EMSWIFT.ContainerListingLimit=10000", } - err error + err error + expectedInfo string + expectedStorageURL string + httpClient *http.Client + httpRequest *http.Request + httpResponse *http.Response + readBuf []byte + testJRPCServerHandler *testJRPCServerHandlerStruct + urlForAuth string + urlForInfo string + urlPrefix string ) confMap, err = conf.MakeConfMapFromStrings(confStrings) @@ -42,7 +62,122 @@ func TestAuthEmulation(t *testing.T) { t.Fatalf("Start(confMap) returned unexpected error: %v", err) } - t.Logf("TODO: Implement TestAuthEmulation() test steps") + testJRPCServerHandler = &testJRPCServerHandlerStruct{ + httpServer: &http.Server{ + Addr: net.JoinHostPort(globals.config.JRPCIPAddr, fmt.Sprintf("%d", globals.config.JRPCTCPPort)), + }, + } + testJRPCServerHandler.httpServer.Handler = testJRPCServerHandler + + // Format URLs + + urlForInfo = "http://" + globals.authEmulator.httpServer.Addr + "/info" + urlForAuth = "http://" + globals.authEmulator.httpServer.Addr + "/auth/v1.0" + urlPrefix = "http://" + globals.authEmulator.httpServer.Addr + "/proxyfs/" + + expectedStorageURL = "http://" + globals.authEmulator.httpServer.Addr + "/v1/" + "AUTH_test" + + // Setup http.Client that we will use for all HTTP requests + + httpClient = &http.Client{} + + // Send a GET for "/info" expecting [EMSWIFT] data in compact JSON form + + httpRequest, err = http.NewRequest("GET", urlForInfo, nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + expectedInfo = "{\"swift\": {\"max_account_name_length\": 256,\"max_container_name_length\": 256,\"max_object_name_length\": 1024,\"account_listing_limit\": 10000,\"container_listing_limit\": 10000}}" + if int64(len(expectedInfo)) != httpResponse.ContentLength { + t.Fatalf("GET of /info httpResponse.ContentLength unexpected") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if expectedInfo != utils.ByteSliceToString(readBuf) { + t.Fatalf("GET of /info httpResponse.Body contents unexpected") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for "/auth/v1.0" expecting X-Auth-Token & X-Storage-Url + + httpRequest, err = http.NewRequest("GET", urlForAuth, nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-User", "test:tester") + httpRequest.Header.Add("X-Auth-Key", "testing") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + if httpResponse.Header.Get("X-Auth-Token") != fixedAuthToken { + t.Fatalf("Auth response should have header X-Auth-Token: %s", fixedAuthToken) + } + if httpResponse.Header.Get("X-Storage-Url") != expectedStorageURL { + t.Fatalf("Auth response should have header X-Storage-Url: %s", expectedStorageURL) + } + + // Send a PUT for account "TestAccount" + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", fixedAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" expecting Content-Length: 0 + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", fixedAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + t.Logf("TODO: Implement remaining TestAuthEmulation() PROXYFS RPC Method") err = Stop() if nil != err { @@ -50,6 +185,10 @@ func TestAuthEmulation(t *testing.T) { } } +func testAuthEmulationJRPCServer() { + // TODO +} + func TestNoAuthEmulation(t *testing.T) { var ( confMap conf.ConfMap @@ -98,7 +237,7 @@ func TestNoAuthEmulation(t *testing.T) { httpClient = &http.Client{} - // Send a GET for "/info" expecting [RamSwiftInfo] data in compact JSON form + // Send a GET for "/info" expecting [EMSWIFT] data in compact JSON form httpRequest, err = http.NewRequest("GET", urlForInfo, nil) if nil != err { From f454a2da217f5a1f43fee408c96987a53317e62d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 27 Jan 2021 10:29:18 -0800 Subject: [PATCH 016/160] emswiftpkg now supports PROXYFS JSON-RPC traffic... just missing a simple "echo" test for it --- emswift/emswiftpkg/impl.go | 105 +++++++++++++++++++++++++++-- emswift/emswiftpkg/pkg_test.go | 119 ++++++++++++++++++++++++++------- 2 files changed, 195 insertions(+), 29 deletions(-) diff --git a/emswift/emswiftpkg/impl.go b/emswift/emswiftpkg/impl.go index 03eaa542a..4e8fde1de 100644 --- a/emswift/emswiftpkg/impl.go +++ b/emswift/emswiftpkg/impl.go @@ -61,17 +61,19 @@ type configStruct struct { } type authEmulatorStruct struct { - httpServer *http.Server - wg sync.WaitGroup + sync.Mutex + httpServer *http.Server + resolvedJRPCTCPAddr *net.TCPAddr + wg sync.WaitGroup } type noAuthEmulatorStruct struct { + sync.Mutex httpServer *http.Server wg sync.WaitGroup } type globalsStruct struct { - sync.Mutex config configStruct authEmulator *authEmulatorStruct noAuthEmulator *noAuthEmulatorStruct @@ -254,6 +256,11 @@ func startAuthIfRequested() (err error) { } authEmulator.httpServer.Handler = authEmulator + authEmulator.resolvedJRPCTCPAddr, err = net.ResolveTCPAddr("tcp", net.JoinHostPort(globals.config.JRPCIPAddr, fmt.Sprintf("%d", globals.config.JRPCTCPPort))) + if nil != err { + return + } + authEmulator.wg.Add(1) globals.authEmulator = authEmulator @@ -426,9 +433,82 @@ func (dummy *authEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, r } func doAuthPROXYFS(responseWriter http.ResponseWriter, request *http.Request) { - fmt.Println("TODO: doAuthPROXYFS(responseWriter, request)") + var ( + bytesWritten int + bytesWrittenInTotal int + err error + jrpcRequest []byte + jrpcResponse []byte + jrpcResponseNestedLeftBraces uint32 + jrpcResponseSingleByte []byte + jrpcTCPConn *net.TCPConn + ) + + globals.authEmulator.Lock() + defer globals.authEmulator.Unlock() + + jrpcRequest, err = ioutil.ReadAll(request.Body) + if nil != err { + panic(err) + } _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusNotImplemented) + + jrpcTCPConn, err = net.DialTCP("tcp", nil, globals.authEmulator.resolvedJRPCTCPAddr) + if nil != err { + panic(err) + } + + bytesWrittenInTotal = 0 + + for bytesWrittenInTotal < len(jrpcRequest) { + bytesWritten, err = jrpcTCPConn.Write(jrpcRequest[len(jrpcRequest)-bytesWrittenInTotal:]) + if nil != err { + panic(err) + } + bytesWrittenInTotal += bytesWritten + } + + jrpcResponse = make([]byte, 0) + jrpcResponseSingleByte = make([]byte, 1) + + _, err = jrpcTCPConn.Read(jrpcResponseSingleByte) + if nil != err { + panic(err) + } + jrpcResponse = append(jrpcResponse, jrpcResponseSingleByte[0]) + + if '{' != jrpcResponseSingleByte[0] { + err = fmt.Errorf("Opening character of jrpcResponse must be '{'") + panic(err) + } + + jrpcResponseNestedLeftBraces = 1 + + for 0 < jrpcResponseNestedLeftBraces { + _, err = jrpcTCPConn.Read(jrpcResponseSingleByte) + if nil != err { + panic(err) + } + jrpcResponse = append(jrpcResponse, jrpcResponseSingleByte[0]) + + switch jrpcResponseSingleByte[0] { + case '{': + jrpcResponseNestedLeftBraces++ + case '}': + jrpcResponseNestedLeftBraces-- + default: + // Nothing special to do here + } + } + + err = jrpcTCPConn.Close() + if nil != err { + panic(err) + } + + responseWriter.Header().Add("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(jrpcResponse) } func (dummy *noAuthEmulatorStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { @@ -886,6 +966,9 @@ func doNoAuthDELETE(responseWriter http.ResponseWriter, request *http.Request) { swiftObjectName string ) + globals.noAuthEmulator.Lock() + defer globals.noAuthEmulator.Unlock() + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) if infoOnly || ("" == swiftAccountName) { responseWriter.WriteHeader(http.StatusForbidden) @@ -986,6 +1069,9 @@ func doNoAuthGET(responseWriter http.ResponseWriter, request *http.Request) { swiftObjectNameAsKey sortedmap.Key ) + globals.noAuthEmulator.Lock() + defer globals.noAuthEmulator.Unlock() + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) if infoOnly { _, _ = responseWriter.Write(utils.StringToByteSlice("{")) @@ -1178,6 +1264,9 @@ func doNoAuthHEAD(responseWriter http.ResponseWriter, request *http.Request) { swiftObjectName string ) + globals.noAuthEmulator.Lock() + defer globals.noAuthEmulator.Unlock() + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) if infoOnly || ("" == swiftAccountName) { responseWriter.WriteHeader(http.StatusForbidden) @@ -1259,6 +1348,9 @@ func doNoAuthPOST(responseWriter http.ResponseWriter, request *http.Request) { swiftObjectName string ) + globals.noAuthEmulator.Lock() + defer globals.noAuthEmulator.Unlock() + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) if infoOnly || ("" == swiftAccountName) { responseWriter.WriteHeader(http.StatusForbidden) @@ -1376,6 +1468,9 @@ func doNoAuthPUT(responseWriter http.ResponseWriter, request *http.Request) { wasCreated bool ) + globals.noAuthEmulator.Lock() + defer globals.noAuthEmulator.Unlock() + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) if infoOnly || ("" == swiftAccountName) { responseWriter.WriteHeader(http.StatusForbidden) diff --git a/emswift/emswiftpkg/pkg_test.go b/emswift/emswiftpkg/pkg_test.go index 56172d98d..e89f6cda3 100644 --- a/emswift/emswiftpkg/pkg_test.go +++ b/emswift/emswiftpkg/pkg_test.go @@ -5,7 +5,6 @@ package emswiftpkg import ( "bytes" - "fmt" "io" "io/ioutil" "net" @@ -18,11 +17,6 @@ import ( "github.com/swiftstack/ProxyFS/utils" ) -type testJRPCServerHandlerStruct struct { - httpServer *http.Server - wg sync.WaitGroup -} - func TestAuthEmulation(t *testing.T) { var ( confMap conf.ConfMap @@ -39,17 +33,18 @@ func TestAuthEmulation(t *testing.T) { "EMSWIFT.AccountListingLimit=10000", "EMSWIFT.ContainerListingLimit=10000", } - err error - expectedInfo string - expectedStorageURL string - httpClient *http.Client - httpRequest *http.Request - httpResponse *http.Response - readBuf []byte - testJRPCServerHandler *testJRPCServerHandlerStruct - urlForAuth string - urlForInfo string - urlPrefix string + err error + expectedInfo string + expectedStorageURL string + httpClient *http.Client + httpRequest *http.Request + httpResponse *http.Response + readBuf []byte + testJRPCServerHandlerTCPListener *net.TCPListener + testJRPCServerHandlerWG *sync.WaitGroup + urlForAuth string + urlForInfo string + urlPrefix string ) confMap, err = conf.MakeConfMapFromStrings(confStrings) @@ -62,12 +57,14 @@ func TestAuthEmulation(t *testing.T) { t.Fatalf("Start(confMap) returned unexpected error: %v", err) } - testJRPCServerHandler = &testJRPCServerHandlerStruct{ - httpServer: &http.Server{ - Addr: net.JoinHostPort(globals.config.JRPCIPAddr, fmt.Sprintf("%d", globals.config.JRPCTCPPort)), - }, + testJRPCServerHandlerTCPListener, err = net.ListenTCP("tcp", globals.authEmulator.resolvedJRPCTCPAddr) + if nil != err { + t.Fatalf("net.ListenTCP() returned unexpected error: %v", err) } - testJRPCServerHandler.httpServer.Handler = testJRPCServerHandler + + testJRPCServerHandlerWG = new(sync.WaitGroup) + testJRPCServerHandlerWG.Add(1) + go testJRPCServerHandler(testJRPCServerHandlerTCPListener, testJRPCServerHandlerWG) // Format URLs @@ -179,14 +176,88 @@ func TestAuthEmulation(t *testing.T) { t.Logf("TODO: Implement remaining TestAuthEmulation() PROXYFS RPC Method") + err = testJRPCServerHandlerTCPListener.Close() + if nil != err { + t.Fatalf("testJRPCServerHandlerTCPListener.Close() returned unexpected error: %v", err) + } + + testJRPCServerHandlerWG.Wait() + err = Stop() if nil != err { t.Fatalf("Stop() returned unexpected error: %v", err) } } -func testAuthEmulationJRPCServer() { - // TODO +func testJRPCServerHandler(testJRPCServerHandlerTCPListener *net.TCPListener, testJRPCServerHandlerWG *sync.WaitGroup) { + var ( + bytesWritten int + bytesWrittenInTotal int + err error + jrpcRequest []byte + jrpcRequestNestedLeftBraces uint32 + jrpcRequestSingleByte []byte + jrpcResponse []byte + jrpcTCPConn *net.TCPConn + ) + +DoAcceptTCP: + jrpcTCPConn, err = testJRPCServerHandlerTCPListener.AcceptTCP() + if nil != err { + testJRPCServerHandlerWG.Done() + return + } + + jrpcRequest = make([]byte, 0) + jrpcRequestSingleByte = make([]byte, 1) + + _, err = jrpcTCPConn.Read(jrpcRequestSingleByte) + if nil != err { + _ = jrpcTCPConn.Close() + goto DoAcceptTCP + } + jrpcRequest = append(jrpcRequest, jrpcRequestSingleByte[0]) + + if '{' != jrpcRequestSingleByte[0] { + _ = jrpcTCPConn.Close() + goto DoAcceptTCP + } + + jrpcRequestNestedLeftBraces = 1 + + for 0 < jrpcRequestNestedLeftBraces { + _, err = jrpcTCPConn.Read(jrpcRequestSingleByte) + if nil != err { + _ = jrpcTCPConn.Close() + goto DoAcceptTCP + } + jrpcRequest = append(jrpcRequest, jrpcRequestSingleByte[0]) + + switch jrpcRequestSingleByte[0] { + case '{': + jrpcRequestNestedLeftBraces++ + case '}': + jrpcRequestNestedLeftBraces-- + default: + // Nothing special to do here + } + } + + jrpcResponse = jrpcRequest + + bytesWrittenInTotal = 0 + + for bytesWrittenInTotal < len(jrpcResponse) { + bytesWritten, err = jrpcTCPConn.Write(jrpcResponse[len(jrpcResponse)-bytesWrittenInTotal:]) + if nil != err { + _ = jrpcTCPConn.Close() + goto DoAcceptTCP + } + bytesWrittenInTotal += bytesWritten + } + + _ = jrpcTCPConn.Close() + goto DoAcceptTCP } func TestNoAuthEmulation(t *testing.T) { From ae06288ba2b3c8943fca84141493f4d4d7e27fd5 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 27 Jan 2021 10:53:32 -0800 Subject: [PATCH 017/160] Completed unit test of emswift --- emswift/emswiftpkg/impl.go | 2 +- emswift/emswiftpkg/pkg_test.go | 35 ++++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/emswift/emswiftpkg/impl.go b/emswift/emswiftpkg/impl.go index 4e8fde1de..1473fb024 100644 --- a/emswift/emswiftpkg/impl.go +++ b/emswift/emswiftpkg/impl.go @@ -461,7 +461,7 @@ func doAuthPROXYFS(responseWriter http.ResponseWriter, request *http.Request) { bytesWrittenInTotal = 0 for bytesWrittenInTotal < len(jrpcRequest) { - bytesWritten, err = jrpcTCPConn.Write(jrpcRequest[len(jrpcRequest)-bytesWrittenInTotal:]) + bytesWritten, err = jrpcTCPConn.Write(jrpcRequest[bytesWrittenInTotal:]) if nil != err { panic(err) } diff --git a/emswift/emswiftpkg/pkg_test.go b/emswift/emswiftpkg/pkg_test.go index e89f6cda3..949b0fb56 100644 --- a/emswift/emswiftpkg/pkg_test.go +++ b/emswift/emswiftpkg/pkg_test.go @@ -174,7 +174,38 @@ func TestAuthEmulation(t *testing.T) { t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) } - t.Logf("TODO: Implement remaining TestAuthEmulation() PROXYFS RPC Method") + // Send a PROXYFS JSON-RPC "request" that simply gets echo'd + + httpRequest, err = http.NewRequest("PROXYFS", urlPrefix+"TestAccount", bytes.NewReader([]byte{'{', 'p', 'i', 'n', 'g', '}'})) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", fixedAuthToken) + httpRequest.Header.Add("Content-Type", "application/json") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Content-Type") != "application/json" { + t.Fatalf("TestAccount should have header Content-Type: application/json") + } + if int64(len("{ping}")) != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain only \"{ping}\"") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if "{ping}" != utils.ByteSliceToString(readBuf) { + t.Fatalf("TestContainer should contain only \"{ping}\"") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } err = testJRPCServerHandlerTCPListener.Close() if nil != err { @@ -248,7 +279,7 @@ DoAcceptTCP: bytesWrittenInTotal = 0 for bytesWrittenInTotal < len(jrpcResponse) { - bytesWritten, err = jrpcTCPConn.Write(jrpcResponse[len(jrpcResponse)-bytesWrittenInTotal:]) + bytesWritten, err = jrpcTCPConn.Write(jrpcResponse[bytesWrittenInTotal:]) if nil != err { _ = jrpcTCPConn.Close() goto DoAcceptTCP From a43be90cbaf08959990f06c25631eb89a655decd Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 28 Jan 2021 13:59:24 -0800 Subject: [PATCH 018/160] Temporary work-around for SwiftStack Swift Fork in the process of moving... --- pfs_middleware/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pfs_middleware/tox.ini b/pfs_middleware/tox.ini index 0e5c5a19e..d162fedc4 100644 --- a/pfs_middleware/tox.ini +++ b/pfs_middleware/tox.ini @@ -7,7 +7,7 @@ usedevelop = True deps = lint: flake8 !lint: -r{toxinidir}/test-requirements.txt - release: git+git://github.com/swiftstack/swift.git@ss-release-2.26.0.10 + release: git+git://github.com/openstack/swift.git@2.25.1 minver: http://tarballs.openstack.org/swift/swift-2.9.0.tar.gz master: http://tarballs.openstack.org/swift/swift-master.tar.gz commands = python -m unittest discover From 91b3fc26492ff05f0dc6dfd531237ec1317a9f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Thu, 28 Jan 2021 15:12:34 -0800 Subject: [PATCH 019/160] Remove reference to SMB & NFS in helper scripts (#575) --- .../proxyfs/files/default/usr/bin/disable_s3 | 2 +- .../proxyfs/files/default/usr/bin/enable_s3 | 2 +- .../files/default/usr/bin/start_and_mount_pfs | 111 +++++++ .../default/usr/bin/unmount_and_stop_pfs | 43 --- cookbooks/proxyfs/recipes/default.rb | 21 +- .../default/usr/bin/start_and_mount_pfs.erb | 298 ------------------ 6 files changed, 116 insertions(+), 361 deletions(-) create mode 100755 cookbooks/proxyfs/files/default/usr/bin/start_and_mount_pfs delete mode 100755 cookbooks/proxyfs/templates/default/usr/bin/start_and_mount_pfs.erb diff --git a/cookbooks/proxyfs/files/default/usr/bin/disable_s3 b/cookbooks/proxyfs/files/default/usr/bin/disable_s3 index 8f27621d7..1cf385cc7 100755 --- a/cookbooks/proxyfs/files/default/usr/bin/disable_s3 +++ b/cookbooks/proxyfs/files/default/usr/bin/disable_s3 @@ -32,4 +32,4 @@ PROXYSERVERCONF=/etc/swift/proxy-server.conf sed -i "/^pipeline/ s/${LIBRARY} tempauth/tempauth/" $PROXYSERVERCONF echo "${LIBRARY} was successfully disabled. Remember to restart Swift or just run:" -echo " start_and_mount_pfs keepmounts" +echo " start_and_mount_pfs" diff --git a/cookbooks/proxyfs/files/default/usr/bin/enable_s3 b/cookbooks/proxyfs/files/default/usr/bin/enable_s3 index e546499b3..a24c3bd03 100755 --- a/cookbooks/proxyfs/files/default/usr/bin/enable_s3 +++ b/cookbooks/proxyfs/files/default/usr/bin/enable_s3 @@ -36,4 +36,4 @@ sed -i '/^pipeline/ s/s3api tempauth/tempauth/' $PROXYSERVERCONF sed -i "/^pipeline/ s/tempauth/${LIBRARY} tempauth/" $PROXYSERVERCONF echo "${LIBRARY} was successfully enabled. Remember to restart Swift or just run:" -echo " start_and_mount_pfs keepmounts" +echo " start_and_mount_pfs" diff --git a/cookbooks/proxyfs/files/default/usr/bin/start_and_mount_pfs b/cookbooks/proxyfs/files/default/usr/bin/start_and_mount_pfs new file mode 100755 index 000000000..10a13b279 --- /dev/null +++ b/cookbooks/proxyfs/files/default/usr/bin/start_and_mount_pfs @@ -0,0 +1,111 @@ +#!/bin/bash + +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +# A simple script to start the services and mount the sample mount point +# The PATH, etc should already be setup by systemctl environment + +function await_proxyfsd_startup { + /usr/bin/systemctl -q is-active proxyfsd + if [ $? -ne 0 ]; then + echo "ProxyFS failed to start. Exiting..." + exit 1 + fi + while true; do + curl http://127.0.0.1:15346/ 2>/dev/null >/dev/null + if [ $? -eq 0 ]; then + break + fi + echo "Waiting for ProxyFS to be started..." + sleep 1 + done +} + +function await_pfsagentd_startup { + /usr/bin/systemctl -q is-active pfsagentd + if [ $? -ne 0 ]; then + echo "PFSAgent failed to start. Exiting..." + exit 1 + fi + while true; do + curl http://127.0.0.1:9090/version 2>/dev/null >/dev/null + if [ $? -eq 0 ]; then + # Service is active and curl to the HTTP server succeeded. + # We're go to go. + break + fi + echo "Waiting for PFSAgent to be started..." + sleep 1 + done +} + +function await_swift_startup { + while true + do + curl http://127.0.0.1:8090/info 2>/dev/null >/dev/null + if [ $? -eq 0 ] + then + break + fi + echo "Waiting for Swift to be started..." + sleep 1 + done +} + +function format_volume_if_necessary { + sudo $GOPATH/bin/mkproxyfs -I $1 /etc/proxyfsd/saioproxyfsd0.conf SwiftClient.RetryLimit=1 + if [ $? -ne 0 ] + then + echo "Could not pre-format $1" + exit 1 + fi +} + +function containsElement () { + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 0; done + return 1 +} + +# Making sure $GOPATH contains something: +source ~/.bashrc +if [ -z "$GOPATH" ]; then + GOPATH=/home/swift/code/ProxyFS +fi + +sudo mount -a + +echo "Shutting down services and mount points..." +/usr/bin/unmount_and_stop_pfs +echo + +echo "Bringing up services..." +if [ -f /usr/bin/systemctl ]; then + # Centos + MOUNT=/usr/bin/mount + sudo /usr/bin/systemctl start memcached + sudo /usr/bin/swift-init main start + await_swift_startup + format_volume_if_necessary CommonVolume + sudo /usr/bin/systemctl start proxyfsd + await_proxyfsd_startup + echo "ProxyFS successfully started" + echo "Starting PFSAgent..." + sudo /usr/bin/systemctl start pfsagentd + await_pfsagentd_startup +else + # Ubuntu (not tested!) + MOUNT=/bin/mount + sudo /usr/sbin/service memcached start + sudo /usr/bin/swift-init main start + await_swift_startup + format_volume_if_necessary CommonVolume + sudo /usr/sbin/service proxyfsd start + await_proxyfsd_startup + echo "ProxyFS successfully started" + # Here we should start pfsagentd (if 'all' or 'pfsa' are present in + # $MOUNT_OPTIONS), but we don't support Ubuntu +fi +echo diff --git a/cookbooks/proxyfs/files/default/usr/bin/unmount_and_stop_pfs b/cookbooks/proxyfs/files/default/usr/bin/unmount_and_stop_pfs index ba8491139..38599bfd0 100755 --- a/cookbooks/proxyfs/files/default/usr/bin/unmount_and_stop_pfs +++ b/cookbooks/proxyfs/files/default/usr/bin/unmount_and_stop_pfs @@ -32,46 +32,9 @@ function await_pfsagentd_shutdown { done } -MOUNT_POINT_NFS=127.0.0.1:/CommonMountPoint -SHARE_NFS=/mnt/nfs_proxyfs_mount/ - -MOUNT_POINT_SMB=//127.0.0.1/proxyfs -SHARE_SMB=/mnt/smb_proxyfs_mount/ - -if [ $# = 0 ]; then - SHOULD_UNMOUNT=1 -else - if [ $1 = "keepmounts" ]; then - SHOULD_UNMOUNT=0 - else - echo "Invalid argument: $1" - exit 1 - fi -fi - -if [ $SHOULD_UNMOUNT = 1 ]; then - mountpoint -q $SHARE_NFS - if [ $? -eq 0 ]; then - echo "Unmounting NFS mount point at '${SHARE_NFS}'..." - sudo umount $SHARE_NFS - fi - - mountpoint -q $SHARE_SMB - if [ $? -eq 0 ]; then - echo "Unmounting SMB mount point at '${SHARE_SMB}'..." - sudo umount $SHARE_SMB - fi -fi - if [ -f /usr/bin/systemctl ]; then # Centos sudo /usr/bin/systemctl stop pfsagentd - sudo /usr/bin/systemctl stop nfs-idmap - sudo /usr/bin/systemctl stop nfs-lock - sudo /usr/bin/systemctl stop nfs-server - sudo /usr/bin/systemctl stop rpcbind - sudo /usr/bin/systemctl stop smb - sudo /usr/bin/systemctl stop nmb # We need to make sure PFSAgent is stopped before we stop ProxyFS, but we # don't care if other services are stopped in the meantime. await_pfsagentd_shutdown @@ -82,12 +45,6 @@ if [ -f /usr/bin/systemctl ]; then else # Ubuntu (not tested!) # Here we should stop pfsagentd, but we don't support Ubuntu - sudo /usr/sbin/service nfs-idmap stop - sudo /usr/sbin/service nfs-lock stop - sudo /usr/sbin/service nfs-server stop - sudo /usr/sbin/service rpcbind stop - sudo /usr/sbin/service smbd stop - sudo /usr/sbin/service nmbd stop sudo /usr/sbin/service proxyfsd stop await_proxyfsd_shutdown sudo /usr/bin/swift-init main stop diff --git a/cookbooks/proxyfs/recipes/default.rb b/cookbooks/proxyfs/recipes/default.rb index 2cb653925..20173fa8c 100644 --- a/cookbooks/proxyfs/recipes/default.rb +++ b/cookbooks/proxyfs/recipes/default.rb @@ -162,6 +162,7 @@ command "install -m 0755 #{PROXYFS_SRC_DIR}/pfs-swift-load/pfs-swift-load-plot #{PROXYFS_BIN_DIR}/" end +# TODO: install aws cli v2 execute "Install awscli and awscli-plugin-endpoint" do command "pip install awscli awscli-plugin-endpoint" end @@ -214,14 +215,8 @@ group proxyfs_group end -template "/usr/bin/start_and_mount_pfs" do - mode '0755' - source "usr/bin/start_and_mount_pfs.erb" - variables({ - :swift_user => node['swift_user'], - :swift_uid => node['swift_uid'], - :swift_gid => node['swift_gid'] - }) +execute "Provision start_and_mount_pfs" do + command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/start_and_mount_pfs /usr/bin" end execute "Provision start_swift_only" do @@ -428,16 +423,6 @@ # # Create mount point and fstab entry # -execute "Create SMB mount point" do - command "mkdir /mnt/smb_proxyfs_mount" - not_if { ::Dir.exists?("/mnt/smb_proxyfs_mount") } -end - -execute "Create NFS mount point" do - command "mkdir /mnt/nfs_proxyfs_mount" - not_if { ::Dir.exists?("/mnt/nfs_proxyfs_mount") } -end - execute "Create PFSAgent mount point" do command "mkdir /mnt/pfsa_proxyfs_mount" not_if { ::Dir.exists?("/mnt/pfsa_proxyfs_mount") } diff --git a/cookbooks/proxyfs/templates/default/usr/bin/start_and_mount_pfs.erb b/cookbooks/proxyfs/templates/default/usr/bin/start_and_mount_pfs.erb deleted file mode 100755 index b62e86338..000000000 --- a/cookbooks/proxyfs/templates/default/usr/bin/start_and_mount_pfs.erb +++ /dev/null @@ -1,298 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -# A simple script to start the services and mount the sample mount point -# The PATH, etc should already be setup by systemctl environment - -function await_proxyfsd_startup { - /usr/bin/systemctl -q is-active proxyfsd - if [ $? -ne 0 ]; then - echo "ProxyFS failed to start. Exiting..." - exit 1 - fi - while true; do - curl http://127.0.0.1:15346/ 2>/dev/null >/dev/null - if [ $? -eq 0 ]; then - break - fi - echo "Waiting for ProxyFS to be started..." - sleep 1 - done -} - -function await_pfsagentd_startup { - /usr/bin/systemctl -q is-active pfsagentd - if [ $? -ne 0 ]; then - echo "PFSAgent failed to start. Exiting..." - exit 1 - fi - while true; do - curl http://127.0.0.1:9090/version 2>/dev/null >/dev/null - if [ $? -eq 0 ]; then - # Service is active and curl to the HTTP server succeeded. - # We're go to go. - break - fi - echo "Waiting for PFSAgent to be started..." - sleep 1 - done -} - -function await_swift_startup { - while true - do - curl http://127.0.0.1:8090/info 2>/dev/null >/dev/null - if [ $? -eq 0 ] - then - break - fi - echo "Waiting for Swift to be started..." - sleep 1 - done -} - -function format_volume_if_necessary { - sudo $GOPATH/bin/mkproxyfs -I $1 /etc/proxyfsd/saioproxyfsd0.conf SwiftClient.RetryLimit=1 - if [ $? -ne 0 ] - then - echo "Could not pre-format $1" - exit 1 - fi -} - -function containsElement () { - local e match="$1" - shift - for e; do [[ "$e" == "$match" ]] && return 0; done - return 1 -} - -help() { - echo "Usage: $0 [mount type]" - echo "Mount type options:" - echo " all: NFS v3 and SMB v3.0 (default option)" - echo " keepmounts: don't umount/mount any mountpoints" - echo " nfs: NFS v3" - echo " smb: SMB v3.0" - echo " smb1: SMB v1.0" - echo " smb2: SMB v2.1" - echo " smb3: SMB v3.0" - echo " pfsa: PFSAgent" -} - -MOUNT_OPTIONS=() -if [ $# -gt 0 ]; then - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -h|--help) - help - exit 0 - ;; - all|keepmounts|smb|smb1|smb2|smb3|nfs|pfsa) - if containsElement "$key" "${MOUNT_OPTIONS[@]}"; then - echo "Error: duplicated option '${key}'." - echo - help - exit 1 - fi - MOUNT_OPTIONS+=("$1") - shift - ;; - *) - echo "Invalid argument '$key'." - echo - help - exit 1 - esac - done - - # Now that we allow more than one mount option to be passed, we have to check - # for incompatible options. - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "all" "${MOUNT_OPTIONS[@]}"; then - echo "Error: no other options are allowed while using 'all'." - echo - help - exit 1 - fi - - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "keepmounts" "${MOUNT_OPTIONS[@]}"; then - echo "Error: no other options are allowed while using 'keepmounts'." - echo - help - exit 1 - fi - - # I know the next 4 'if' clauses are gross, but I don't have the patience to - # find out the proper way to do it in bash. - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "smb" "${MOUNT_OPTIONS[@]}"; then - if containsElement "smb1" "${MOUNT_OPTIONS[@]}" || containsElement "smb2" "${MOUNT_OPTIONS[@]}" || containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - echo "Error: only one SMB option can be specified at a time." - echo - help - exit 1 - fi - fi - - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "smb1" "${MOUNT_OPTIONS[@]}"; then - if containsElement "smb" "${MOUNT_OPTIONS[@]}" || containsElement "smb2" "${MOUNT_OPTIONS[@]}" || containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - echo "Error: only one SMB option can be specified at a time." - echo - help - exit 1 - fi - fi - - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "smb2" "${MOUNT_OPTIONS[@]}"; then - if containsElement "smb1" "${MOUNT_OPTIONS[@]}" || containsElement "smb" "${MOUNT_OPTIONS[@]}" || containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - echo "Error: only one SMB option can be specified at a time." - echo - help - exit 1 - fi - fi - - if [ ${#MOUNT_OPTIONS[@]} -gt 1 ] && containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - if containsElement "smb1" "${MOUNT_OPTIONS[@]}" || containsElement "smb2" "${MOUNT_OPTIONS[@]}" || containsElement "smb" "${MOUNT_OPTIONS[@]}"; then - echo "Error: only one SMB option can be specified at a time." - echo - help - exit 1 - fi - fi -else - MOUNT_OPTIONS+=("all") -fi - -if containsElement "all" "${MOUNT_OPTIONS[@]}"; then - NFS_VERS=3 - SMB_VERS=3.0 -else - if containsElement "nfs" "${MOUNT_OPTIONS[@]}"; then - NFS_VERS=3 - fi - - if containsElement "smb1" "${MOUNT_OPTIONS[@]}"; then - SMB_VERS=1.0 - elif containsElement "smb2" "${MOUNT_OPTIONS[@]}"; then - SMB_VERS=2.1 - elif containsElement "smb" "${MOUNT_OPTIONS[@]}" || containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - SMB_VERS=3.0 - fi -fi - -MOUNT_POINT_NFS=127.0.0.1:/CommonMountPoint -SHARE_NFS=/mnt/nfs_proxyfs_mount/ - -MOUNT_POINT_SMB=//127.0.0.1/proxyfs -SHARE_SMB=/mnt/smb_proxyfs_mount/ - -# Making sure $GOPATH contains something: -source ~/.bashrc -if [ -z "$GOPATH" ]; then - GOPATH=/home/swift/code/ProxyFS -fi - -UID_SMB=`id -u` -GID_SMB=`id -g` - -sudo mount -a - -# "keepmounts" means "keep everything as it was", but PFSAgent works differently, -# so just preserving the mount points is not enough. What we'll do is check if -# it's running before trying to stop it, and if it is, save it as a "MOUNT_OPTION" -if [ -f /usr/bin/systemctl ] && containsElement "keepmounts" "${MOUNT_OPTIONS[@]}"; then - /usr/bin/systemctl is-active --quiet pfsagentd && MOUNT_OPTIONS+=("pfsa") -fi - -if containsElement "keepmounts" "${MOUNT_OPTIONS[@]}"; then - echo "Shutting down services..." - /usr/bin/unmount_and_stop_pfs keepmounts -else - echo "Shutting down services and mount points..." - /usr/bin/unmount_and_stop_pfs -fi -echo - -echo "Bringing up services..." -if [ -f /usr/bin/systemctl ]; then - # Centos - MOUNT=/usr/bin/mount - sudo /usr/bin/systemctl start memcached - sudo /usr/bin/swift-init main start - await_swift_startup - format_volume_if_necessary CommonVolume - sudo /usr/bin/systemctl start proxyfsd - await_proxyfsd_startup - echo "ProxyFS successfully started" - sudo /usr/bin/systemctl start nmb - sudo /usr/bin/systemctl start smb - sudo /usr/bin/systemctl start rpcbind - sudo /usr/bin/systemctl start nfs-server - sudo /usr/bin/systemctl start nfs-lock - sudo /usr/bin/systemctl start nfs-idmap - if containsElement "all" "${MOUNT_OPTIONS[@]}" || containsElement "pfsa" "${MOUNT_OPTIONS[@]}"; then - echo "Starting PFSAgent..." - sudo /usr/bin/systemctl start pfsagentd - await_pfsagentd_startup - fi -else - # Ubuntu (not tested!) - MOUNT=/bin/mount - sudo /usr/sbin/service memcached start - sudo /usr/bin/swift-init main start - await_swift_startup - format_volume_if_necessary CommonVolume - sudo /usr/sbin/service proxyfsd start - await_proxyfsd_startup - echo "ProxyFS successfully started" - sudo /usr/sbin/service nmbd start - sudo /usr/sbin/service smbd start - sudo /usr/sbin/service rpcbind start - sudo /usr/sbin/service nfs-server start - sudo /usr/sbin/service nfs-lock start - sudo /usr/sbin/service nfs-idmap start - # Here we should start pfsagentd (if 'all' or 'pfsa' are present in - # $MOUNT_OPTIONS), but we don't support Ubuntu -fi -echo - -if containsElement "all" "${MOUNT_OPTIONS[@]}" || containsElement "nfs" "${MOUNT_OPTIONS[@]}"; then - mountpoint -q $SHARE_NFS - if [ $? -ne 0 ]; then - for i in $(seq 5); do - sleep 5 - sudo $MOUNT -t nfs -o vers=$NFS_VERS $MOUNT_POINT_NFS $SHARE_NFS && break - echo "Mount of $SHARE_NFS failed. Retrying..." - done - fi - - mountpoint -q $SHARE_NFS - if [ $? -ne 0 ]; then - echo "ERROR: Could not mount $SHARE_NFS." - exit 1 - else - echo "$SHARE_NFS successfully mounted" - fi -fi - -if containsElement "all" "${MOUNT_OPTIONS[@]}" || containsElement "smb" "${MOUNT_OPTIONS[@]}" || containsElement "smb1" "${MOUNT_OPTIONS[@]}" || containsElement "smb2" "${MOUNT_OPTIONS[@]}" || containsElement "smb3" "${MOUNT_OPTIONS[@]}"; then - mountpoint -q $SHARE_SMB - if [ $? -ne 0 ]; then - for i in $(seq 5); do - sleep 5 - sudo $MOUNT -t cifs -o user=<%= @swift_user %>,password=<%= @swift_user %>,domain=LOCALHOST,uid=$UID_SMB,gid=$GID_SMB,vers=$SMB_VERS,iocharset=utf8,actimeo=0 $MOUNT_POINT_SMB $SHARE_SMB && break - echo "Mount of $SHARE_SMB failed. Retrying..." - done - fi - - mountpoint -q $SHARE_SMB - if [ $? -ne 0 ]; then - echo "ERROR: Could not mount $SHARE_SMB." - exit 1 - else - echo "$SHARE_SMB successfully mounted" - fi -fi From 12c1724186e62e4b060535c150f9c4ae0ec3cd90 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 29 Jan 2021 12:31:51 -0800 Subject: [PATCH 020/160] Forced cachedStat loading & missing unlock in DoRead()/DoWrite() --- pfsagentd/fission.go | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/pfsagentd/fission.go b/pfsagentd/fission.go index 466da600d..c98118c96 100644 --- a/pfsagentd/fission.go +++ b/pfsagentd/fission.go @@ -108,6 +108,34 @@ func unixTimeToNs(sec uint64, nsec uint32) (ns uint64) { return } +func (fileInode *fileInodeStruct) ensureCachedStatPopulatedWhileLocked() (err error) { + var ( + getStatReply *jrpcfs.StatStruct + getStatRequest *jrpcfs.GetStatRequest + ) + + if nil == fileInode.cachedStat { + getStatRequest = &jrpcfs.GetStatRequest{ + InodeHandle: jrpcfs.InodeHandle{ + MountID: globals.mountID, + InodeNumber: int64(fileInode.InodeNumber), + }, + } + + getStatReply = &jrpcfs.StatStruct{} + + err = globals.retryRPCClient.Send("RpcGetStat", getStatRequest, getStatReply) + if nil != err { + return + } + + fileInode.cachedStat = getStatReply + } + + err = nil + return +} + func (dummy *globalsStruct) DoLookup(inHeader *fission.InHeader, lookupIn *fission.LookupIn) (lookupOut *fission.LookupOut, errno syscall.Errno) { var ( aTimeNSec uint32 @@ -1238,11 +1266,19 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R } // defer fileInode.dereference() + err = fileInode.ensureCachedStatPopulatedWhileLocked() + if nil != err { + fileInode.unlock(true) + errno = convertErrToErrno(err, syscall.EIO) + return + } + // grantedLock = fileInode.getSharedLock() // defer grantedLock.release() err = fileInode.populateExtentMap(uint64(readIn.Offset), uint64(readIn.Size)) if nil != err { + fileInode.unlock(true) errno = convertErrToErrno(err, syscall.EIO) return } @@ -1277,6 +1313,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R logSegmentCacheElement = fetchLogSegmentCacheLine(readPlanStepAsMultiObjectExtentStruct.containerName, readPlanStepAsMultiObjectExtentStruct.objectName, curObjectOffset) if logSegmentCacheElementStateGetFailed == logSegmentCacheElement.state { + fileInode.unlock(true) errno = syscall.EIO return } @@ -1318,6 +1355,8 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R } } + fileInode.unlock(true) + _ = atomic.AddUint64(&globals.metrics.FUSE_DoRead_bytes, uint64(len(readOut.Data))) errno = 0 @@ -1328,6 +1367,7 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission var ( chunkedPutContext *chunkedPutContextStruct chunkedPutContextElement *list.Element + err error fhInodeNumber uint64 fileInode *fileInodeStruct ok bool @@ -1355,6 +1395,12 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission logFatalf("DoWrite(NodeID=%v,FH=%v) called for non-FileInode", inHeader.NodeID, writeIn.FH) } + err = fileInode.ensureCachedStatPopulatedWhileLocked() + if nil != err { + errno = convertErrToErrno(err, syscall.EIO) + return + } + // Grab quota to start a fresh chunkedPutContext before calling fileInode.getExclusiveLock() // since, in the pathologic case where only fileInode has any outstanding chunkedPutContext's, // they can only be complete()'d while holding fileInode.getExclusiveLock() and we would @@ -1578,6 +1624,7 @@ func (dummy *globalsStruct) DoFSync(inHeader *fission.InHeader, fSyncIn *fission globals.Unlock() + fileInode = lockInodeWithExclusiveLease(inode.InodeNumber(inHeader.NodeID)) // fileInode = referenceFileInode(inode.InodeNumber(inHeader.NodeID)) if nil == fileInode { logFatalf("DoFSync(NodeID=%v,FH=%v) called for non-FileInode", inHeader.NodeID, fSyncIn.FH) @@ -1586,6 +1633,7 @@ func (dummy *globalsStruct) DoFSync(inHeader *fission.InHeader, fSyncIn *fission fileInode.doFlushIfNecessary() // fileInode.dereference() + fileInode.unlock(false) errno = 0 return From b8a45311937ca28cf797b549910499f355811416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Fri, 29 Jan 2021 15:41:07 -0800 Subject: [PATCH 021/160] Stop installing smb nfs runway (#577) This means we can stop using "SwiftStack packages", so removing all the references to that from code as well. --- ci/ansible/chef_files/ci.attribs.json | 3 +- ci/ansible/chef_files/packages/debian.json | 18 -- ci/ansible/chef_files/packages/rhel.json | 17 -- ci/ansible/chef_files/packages/rhel_ss.json | 37 ---- ci/ansible/chef_files/runway.attribs.json | 1 - ci/ansible/chef_files/runway_ss.attribs.json | 35 ---- ci/ansible/chef_files/runway_ss.cfg | 3 - ci/ansible/chef_files/vagrant.attribs.json | 3 +- ci/ansible/install_proxyfs.sh | 9 +- .../usr/bin/reprovision_proxyfs.template | 2 +- ci/ansible/vars/runway_ss.yml | 6 - cookbooks/proxyfs/recipes/default.rb | 182 +----------------- 12 files changed, 9 insertions(+), 307 deletions(-) delete mode 100644 ci/ansible/chef_files/packages/rhel_ss.json delete mode 100644 ci/ansible/chef_files/runway_ss.attribs.json delete mode 100644 ci/ansible/chef_files/runway_ss.cfg delete mode 100644 ci/ansible/vars/runway_ss.yml diff --git a/ci/ansible/chef_files/ci.attribs.json b/ci/ansible/chef_files/ci.attribs.json index 9260af429..a86f6a0ec 100644 --- a/ci/ansible/chef_files/ci.attribs.json +++ b/ci/ansible/chef_files/ci.attribs.json @@ -29,6 +29,5 @@ "swift_bench_repo": "git://github.com/openstack/swift-bench.git", "swift_bench_repo_branch": "master", "extra_key": "", - "source_root": "/home/swiftstack/provisioning", - "use_swiftstack_packages": false + "source_root": "/home/swiftstack/provisioning" } diff --git a/ci/ansible/chef_files/packages/debian.json b/ci/ansible/chef_files/packages/debian.json index 8b0c59378..e6e50dae8 100644 --- a/ci/ansible/chef_files/packages/debian.json +++ b/ci/ansible/chef_files/packages/debian.json @@ -1,18 +1,4 @@ { - "samba_packages": [ - ["samba"], - ["smbclient"] - ], - "samba_deps": [ - ["gcc"], - ["python-dev"], - ["libgnutls-dev"], - ["libacl1-dev"], - ["libldap2-dev"], - ["pkg-config"], - ["cifs-utils"], - ["libpam0g-dev"] - ], "proxyfs_packages": [ ["libjson-c-dev"], ["fuse"] @@ -24,10 +10,6 @@ "ssh_packages": [ ["sshpass"] ], - "nfs_packages": [ - ["nfs-kernel-server"], - ["nfs-common"] - ], "gdb_packages": [ ["gdb"] ], diff --git a/ci/ansible/chef_files/packages/rhel.json b/ci/ansible/chef_files/packages/rhel.json index 988718339..5d6b8f8b1 100644 --- a/ci/ansible/chef_files/packages/rhel.json +++ b/ci/ansible/chef_files/packages/rhel.json @@ -1,18 +1,4 @@ { - "samba_packages": [ - ["samba"], - ["samba-client"] - ], - "samba_deps": [ - ["gcc"], - ["gcc-c++"], - ["python-devel"], - ["gnutls-devel"], - ["libacl-devel"], - ["openldap-devel"], - ["cifs-utils"], - ["pam-devel"] - ], "proxyfs_packages": [ ["json-c-devel"], ["fuse"] @@ -24,9 +10,6 @@ "ssh_packages": [ ["sshpass"] ], - "nfs_packages": [ - ["nfs-utils"] - ], "gdb_packages": [ ["gdb"], ["yum-utils"] diff --git a/ci/ansible/chef_files/packages/rhel_ss.json b/ci/ansible/chef_files/packages/rhel_ss.json deleted file mode 100644 index f0577b8de..000000000 --- a/ci/ansible/chef_files/packages/rhel_ss.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "samba_packages": [ - ["ss-samba", "4.6.2-3.el7"] - ], - "samba_deps": [ - ["gcc", "4.8.5-44.el7"], - ["gcc-c++", "4.8.5-44.el7"], - ["python-devel", "2.7.5-90.el7"], - ["gnutls-devel", "3.3.29-9.el7_6"], - ["libacl-devel", "2.2.51-15.el7"], - ["openldap-devel", "2.4.44-22.el7"], - ["cifs-utils", "6.2-10.el7"], - ["pam-devel", "1.1.8-23.el7"] - ], - "proxyfs_packages": [ - ["json-c-devel", "0.11-4.el7_0"], - ["fuse", "2.9.3-5.el7"] - ], - "wireshark_packages": [ - ["wireshark"], - ["libcap"] - ], - "ssh_packages": [ - ["sshpass"] - ], - "nfs_packages": [ - ["nfs-utils", "1.3.0-0.68.el7"] - ], - "gdb_packages": [ - ["gdb", "7.6.1-120.el7"], - ["yum-utils"] - ], - "utils_packages": [ - ["atop"], - ["vim-common"] - ] -} diff --git a/ci/ansible/chef_files/runway.attribs.json b/ci/ansible/chef_files/runway.attribs.json index d18d0490b..a8b584d01 100644 --- a/ci/ansible/chef_files/runway.attribs.json +++ b/ci/ansible/chef_files/runway.attribs.json @@ -30,6 +30,5 @@ "swift_bench_repo_branch": "master", "extra_key": "", "source_root": "/home/swift/code/ProxyFS", - "use_swiftstack_packages": false, "package_spec_path": "/home/swift/code/ProxyFS/src/github.com/swiftstack/ProxyFS/ci/ansible/chef_files/packages" } diff --git a/ci/ansible/chef_files/runway_ss.attribs.json b/ci/ansible/chef_files/runway_ss.attribs.json deleted file mode 100644 index cdf8c6b65..000000000 --- a/ci/ansible/chef_files/runway_ss.attribs.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "run_list": [ "recipe[pfs_middleware]", "recipe[meta_middleware]", "recipe[proxyfs_swift]", "recipe[proxyfs]" ], - "full_reprovision": "false", - "loopback_gb": 4, - "extra_packages": [], - "storage_policies": ["default"], - "ec_policy": "", - "servers_per_port": 0, - "object_sync_method": "rsync", - "post_as_copy": "true", - "part_power": 10, - "replicas": 3, - "ec_replicas": 6, - "regions": 1, - "zones": 4, - "nodes": 4, - "disks": 4, - "ec_disks": 8, - "swift_user": "swift", - "swift_group": "swift", - "swift_uid": 1000, - "swift_gid": 1000, - "proxyfs_user": "swift", - "proxyfs_group": "swift", - "swift_repo": "git://github.com/openstack/swift.git", - "swift_repo_branch": "master", - "swiftclient_repo": "git://github.com/openstack/python-swiftclient.git", - "swiftclient_repo_branch": "master", - "swift_bench_repo": "git://github.com/openstack/swift-bench.git", - "swift_bench_repo_branch": "master", - "extra_key": "", - "source_root": "/home/swift/code/ProxyFS", - "use_swiftstack_packages": true, - "package_spec_path": "/home/swift/code/ProxyFS/src/github.com/swiftstack/ProxyFS/ci/ansible/chef_files/packages" -} diff --git a/ci/ansible/chef_files/runway_ss.cfg b/ci/ansible/chef_files/runway_ss.cfg deleted file mode 100644 index f9aee8f6d..000000000 --- a/ci/ansible/chef_files/runway_ss.cfg +++ /dev/null @@ -1,3 +0,0 @@ -file_cache_path "/home/swift/chef" -cookbook_path "/home/swift/code/ProxyFS/src/github.com/swiftstack/ProxyFS/cookbooks" -json_attribs "/home/swift/code/ProxyFS/src/github.com/swiftstack/ProxyFS/ci/ansible/chef_files/runway_ss.attribs.json" diff --git a/ci/ansible/chef_files/vagrant.attribs.json b/ci/ansible/chef_files/vagrant.attribs.json index 4b17de57d..b8257f798 100644 --- a/ci/ansible/chef_files/vagrant.attribs.json +++ b/ci/ansible/chef_files/vagrant.attribs.json @@ -29,6 +29,5 @@ "swift_bench_repo": "git://github.com/openstack/swift-bench.git", "swift_bench_repo_branch": "master", "extra_key": "", - "source_root": "/vagrant", - "use_swiftstack_packages": false + "source_root": "/vagrant" } diff --git a/ci/ansible/install_proxyfs.sh b/ci/ansible/install_proxyfs.sh index 625d78d87..2cb2954d2 100755 --- a/ci/ansible/install_proxyfs.sh +++ b/ci/ansible/install_proxyfs.sh @@ -6,23 +6,18 @@ set -e ENV_NAME=$1 -SS_PACKAGES=$2 -GOLANG_VERSION=$3 +GOLANG_VERSION=$2 if [ -z "$ENV_NAME" ]; then echo "usage: $0 " exit 1 fi -if [ -n "$SS_PACKAGES" ] && [ "$ENV_NAME" == "runway" ] && [ "$SS_PACKAGES" == "swiftstack" ]; then - ENV_NAME="runway_ss" -fi - if [ -z "$GOLANG_VERSION" ]; then GOLANG_VERSION="current" fi SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -ansible-playbook -i "localhost," -c local -e env=$ENV_NAME -e env_arg="$1" -e ss_packages_arg="$2" -e golang_version="$GOLANG_VERSION" "$SCRIPT_DIR"/tasks/main.yml +ansible-playbook -i "localhost," -c local -e env=$ENV_NAME -e env_arg="$1" -e golang_version="$GOLANG_VERSION" "$SCRIPT_DIR"/tasks/main.yml chef-solo -c "$SCRIPT_DIR"/chef_files/$ENV_NAME.cfg diff --git a/ci/ansible/tasks/templates/usr/bin/reprovision_proxyfs.template b/ci/ansible/tasks/templates/usr/bin/reprovision_proxyfs.template index 0cf6d1ecf..07f6a375d 100644 --- a/ci/ansible/tasks/templates/usr/bin/reprovision_proxyfs.template +++ b/ci/ansible/tasks/templates/usr/bin/reprovision_proxyfs.template @@ -3,5 +3,5 @@ set -e /usr/bin/unmount_and_stop_pfs cd $GOPATH/src/github.com/swiftstack/ProxyFS -ci/ansible/install_proxyfs.sh {{ env_arg }} {{ ss_packages_arg }} {{ active_golang_version }} +ci/ansible/install_proxyfs.sh {{ env_arg }} {{ active_golang_version }} /usr/bin/start_and_mount_pfs "$@" diff --git a/ci/ansible/vars/runway_ss.yml b/ci/ansible/vars/runway_ss.yml deleted file mode 100644 index d094123b9..000000000 --- a/ci/ansible/vars/runway_ss.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -proxyfs_user: "swift" -proxyfs_group: "swift" -source_root: "/home/{{ proxyfs_user }}/code/ProxyFS" -REPO_CLONE_PARENT_DIR: "{{ source_root }}/src/github.com/swiftstack" -PROXYFS_SRC_DIR: "{{ REPO_CLONE_PARENT_DIR }}/ProxyFS" diff --git a/cookbooks/proxyfs/recipes/default.rb b/cookbooks/proxyfs/recipes/default.rb index 20173fa8c..e09a75be5 100644 --- a/cookbooks/proxyfs/recipes/default.rb +++ b/cookbooks/proxyfs/recipes/default.rb @@ -4,7 +4,6 @@ proxyfs_user = node['proxyfs_user'] proxyfs_group = node['proxyfs_group'] is_dev = node['is_dev_environment'] -ss_packages = node['use_swiftstack_packages'] package_spec_path = node['package_spec_path'] HOME_DIR = "/home/#{proxyfs_user}" @@ -16,30 +15,6 @@ REPO_CLONE_PARENT_DIR = "#{source_root}/src/github.com/swiftstack" PROXYFS_BIN_DIR = "#{source_root}/bin" PROXYFS_SRC_DIR = "#{REPO_CLONE_PARENT_DIR}/ProxyFS" -VFS_SRC_DIR = "#{PROXYFS_SRC_DIR}/vfs" -JRPCCLIENT_SRC_DIR = "#{PROXYFS_SRC_DIR}/jrpcclient" -# We're doing this to only need to change SAMBA_PARENT_DIR in case we decide to -# change the location of samba again in the future. -SAMBA_PARENT_DIR = "#{VFS_SRC_DIR}" -SAMBA_SRC_DIR = "#{SAMBA_PARENT_DIR}/samba" - -if node[:platform_family].include?("rhel") and ss_packages - cookbook_file "/etc/yum.repos.d/swiftstack-controller.repo" do - source "etc/yum.repos.d/swiftstack-controller.repo" - owner "root" - group "root" - end - - cookbook_file "/etc/pki/rpm-gpg/RPM-GPG-KEY-swiftstack-controller" do - source "etc/pki/rpm-gpg/RPM-GPG-KEY-swiftstack-controller" - owner "root" - group "root" - end - - execute "yum makecache" do - command "yum makecache" - end -end ruby_block "update_profile_and_bashrc" do block do @@ -73,14 +48,9 @@ file = Chef::Util::FileEdit.new(DOT_BASHRC) file.insert_line_if_no_match(/export GOPATH/, "export GOPATH=#{source_root}") - if ss_packages - file.insert_line_if_no_match(%r{usr/local/go/bin}, "export PATH=$GOPATH/bin:$PATH:/usr/local/go/bin:/opt/ss/bin:/opt/ss/sbin") - else - file.insert_line_if_no_match(%r{usr/local/go/bin}, "export PATH=$GOPATH/bin:$PATH:/usr/local/go/bin") - end + file.insert_line_if_no_match(%r{usr/local/go/bin}, "export PATH=$GOPATH/bin:$PATH:/usr/local/go/bin") file.insert_line_if_no_match(/cdpfs/, "alias cdpfs='cd $GOPATH/src/github.com/swiftstack/ProxyFS'") file.insert_line_if_no_match(/cdfun/, "alias cdfun='cd /home/swift/code/functional-tests'") - file.insert_line_if_no_match(/cdsamba/, "alias cdsamba='cd #{SAMBA_SRC_DIR}'") file.insert_line_if_no_match(/ls -lha/, "alias la='ls -lha'") file.insert_line_if_no_match(/ls -liha/, "alias li='ls -liha'") file.insert_line_if_no_match(/statmnt/, "alias statmnt='stat /mnt/*'") @@ -108,14 +78,9 @@ file = Chef::Util::FileEdit.new(ROOT_DOT_BASHRC) file.insert_line_if_no_match(/export GOPATH/, "export GOPATH=#{source_root}") - if ss_packages - file.insert_line_if_no_match(%r{usr/local/go/bin}, "export PATH=$GOPATH/bin:$PATH:/usr/local/go/bin:/opt/ss/bin:/opt/ss/sbin") - else - file.insert_line_if_no_match(%r{usr/local/go/bin}, "export PATH=$GOPATH/bin:$PATH:/usr/local/go/bin") - end + file.insert_line_if_no_match(%r{usr/local/go/bin}, "export PATH=$GOPATH/bin:$PATH:/usr/local/go/bin") file.insert_line_if_no_match(/cdpfs/, "alias cdpfs='cd $GOPATH/src/github.com/swiftstack/ProxyFS'") file.insert_line_if_no_match(/cdfun/, "alias cdfun='cd /home/swift/code/functional-tests'") - file.insert_line_if_no_match(/cdsamba/, "alias cdsamba='cd #{SAMBA_SRC_DIR}'") file.insert_line_if_no_match(/ls -lha/, "alias la='ls -lha'") file.insert_line_if_no_match(/ls -liha/, "alias li='ls -liha'") file.insert_line_if_no_match(/statmnt/, "alias statmnt='stat /mnt/*'") @@ -270,20 +235,6 @@ only_if { ::File.directory?("/usr/lib/systemd/system/") } end -if node[:platform_family].include?("rhel") and ss_packages - cookbook_file "/usr/lib/systemd/system/smb.service" do - source "usr/lib/systemd/system/smb.service" - # notifies :restart, 'service[smb]' - only_if { ::File.directory?("/usr/lib/systemd/system/") } - end - - cookbook_file "/usr/lib/systemd/system/nmb.service" do - source "usr/lib/systemd/system/nmb.service" - # notifies :restart, 'service[nmb]' - only_if { ::File.directory?("/usr/lib/systemd/system/") } - end -end - cookbook_file "/etc/init/proxyfsd.conf" do source "etc/init/proxyfsd.upstart" # notifies :restart, 'service[proxyfsd]' @@ -295,17 +246,13 @@ # Dependency lists by OS # if node[:platform_family].include?("rhel") - if ss_packages - package_spec_file_path = File.read(package_spec_path + '/rhel_ss.json') - else - package_spec_file_path = File.read(package_spec_path + '/rhel.json') - end + package_spec_file_path = File.read(package_spec_path + '/rhel.json') else # assume debian package_spec_file_path = File.read(package_spec_path + '/debian.json') end package_spec = JSON.parse(package_spec_file_path) -packages = package_spec['samba_packages'] + package_spec['samba_deps'] + package_spec['proxyfs_packages'] + package_spec['nfs_packages'] + package_spec['gdb_packages'] + package_spec['utils_packages'] +packages = package_spec['proxyfs_packages'] + package_spec['gdb_packages'] + package_spec['utils_packages'] packages += package_spec['wireshark_packages'] if is_dev packages += package_spec['ssh_packages'] if is_dev @@ -339,87 +286,6 @@ end end -# -# Always remake the samba symbolic link since the user may be switching between -# Centos and Ubuntu -# -execute "Remove samba symbolic link" do - command "rm -f samba" - cwd SAMBA_PARENT_DIR -end - - -# -# Check out and build samba -# -# For now we're hard-coding the OS. It should be parametrized. -OS_DISTRO="centos" -OS_DISTRO_VERSION="7.5" - -SAMBA_VERSION = ss_packages ? "4.6.2" : "" - -bash 'Check out samba + build headers if needed' do - code <<-EOH - if [ "#{SAMBA_VERSION}" = "" ]; then - SAMBA_VERSION="`smbstatus -V | cut -d' ' -f2 | tr -d '\\n'`" - else - SAMBA_VERSION="#{SAMBA_VERSION}" - fi - SAMBA_DIR="build-samba-`echo ${SAMBA_VERSION} | sed -e 's:\\.:-:g'`-#{OS_DISTRO}-#{OS_DISTRO_VERSION.gsub(".", "-")}" - - if [ -d "#{SAMBA_PARENT_DIR}/${SAMBA_DIR}" ]; then - ln -s ${SAMBA_DIR} #{SAMBA_SRC_DIR} - else - # Check out samba - git clone -b samba-${SAMBA_VERSION} --single-branch --depth 1 https://github.com/samba-team/samba.git ${SAMBA_DIR} - - ln -s ${SAMBA_DIR} #{SAMBA_SRC_DIR} - - cd #{SAMBA_SRC_DIR} - - # Configure samba src - # lockfile dropped by `waf configure` - if [ ! -f "#{SAMBA_SRC_DIR}/.lock-wscript" ]; then - ./configure - fi - - # Build samba headers - if [ ! -f "#{SAMBA_SRC_DIR}/bin/default/librpc/gen_ndr/server_id.h" ]; then - make GEN_NDR_TABLES - fi - fi - EOH - cwd SAMBA_PARENT_DIR -end - -# -# Configure Samba -# - -if ss_packages - smb_conf = "/opt/ss/etc/samba/smb.conf" -else - smb_conf = "/etc/samba/smb.conf" -end - -execute "Setup #{smb_conf}" do - command "cat sample_entry_smb_conf.txt > #{smb_conf}" - cwd "#{VFS_SRC_DIR}" -end - -ruby_block "update_smb_conf" do - block do - file = Chef::Util::FileEdit.new(smb_conf) - file.search_file_replace(/valid users = CHANGEME/, "valid users = #{node['swift_user']}") - file.write_file - end -end - -smbpasswd_path = ss_packages ? "/opt/ss/bin/smbpasswd" : "/bin/smbpasswd" -execute "Setup Samba password" do - command "printf \"#{node['swift_user']}\n#{node['swift_user']}\n\" | #{smbpasswd_path} -a -s #{node['swift_user']}" -end - # # Create mount point and fstab entry # @@ -491,46 +357,6 @@ group proxyfs_group end -if ss_packages - # Creating link to jrpcclient's libs into the new /opt/ss path - link '/opt/ss/lib64/libproxyfs.so.1.0.0' do - to "#{JRPCCLIENT_SRC_DIR}/libproxyfs.so.1.0.0" - link_type :symbolic - owner "root" - group "root" - end - - link '/opt/ss/lib64/libproxyfs.so.1' do - to "#{JRPCCLIENT_SRC_DIR}/libproxyfs.so.1.0.0" - link_type :symbolic - owner "root" - group "root" - end - - link '/opt/ss/lib64/libproxyfs.so' do - to "#{JRPCCLIENT_SRC_DIR}/libproxyfs.so.1.0.0" - link_type :symbolic - owner "root" - group "root" - end - - # Creating link to vfs' libs into the new /opt/ss path - directory '/opt/ss/lib64/samba/vfs' do - owner 'root' - group 'root' - mode '0755' - action :create - end - - link '/opt/ss/lib64/samba/vfs/proxyfs.so' do - to "#{VFS_SRC_DIR}/proxyfs.so" - link_type :symbolic - owner "root" - group "root" - mode '0755' - end -end - cookbook_file "#{HOME_DIR}/.gdbinit" do source "home/unprivileged_user/.gdbinit" owner "#{proxyfs_user}" From 1c3f56fee3ee430f6bf13bd9420ff3567d56d9d4 Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Mon, 1 Feb 2021 11:20:18 -0800 Subject: [PATCH 022/160] Add EtcdCertPath to conf file (#578) * Add EtcdCertDir to conf file --- CONFIGURING.md | 1 + .../sample-proxyfs-configuration/proxyfs.conf | 1 + etcdclient/api.go | 22 +++++++++---------- headhunter/config.go | 11 +++++++--- mkproxyfs/api.go | 11 +++++++--- pfs-fsck/main.go | 11 +++++++--- proxyfsd/default.conf | 1 + proxyfsd/file_server.conf | 1 + proxyfsd/file_server_mac_3_peers.conf | 1 + saio/container/proxyfs.conf | 1 + 10 files changed, 41 insertions(+), 20 deletions(-) diff --git a/CONFIGURING.md b/CONFIGURING.md index 740af9f06..e6c29853c 100644 --- a/CONFIGURING.md +++ b/CONFIGURING.md @@ -126,6 +126,7 @@ For each of the keys supported, the following table will list whether or not its | | EtcdEnabled | No | false | Yes but don't re-enable | No | | | EtcdEndpoints | If enabled | | Yes | No | | | EtcdAutoSyncInterval | If enabled | | Yes | No | +| | EtcdCertDir | If enabled | /etc/ssl/etcd/ssl/ | Yes | No | | | EtcdDialTimeout | If enabled | | Yes | No | | | EtcdOpTimeout | If enabled | | Yes | No | | | MetadataRecycleBin | No | false | Yes | No | diff --git a/confgen/sample-proxyfs-configuration/proxyfs.conf b/confgen/sample-proxyfs-configuration/proxyfs.conf index 3d7651b20..faa4219a3 100644 --- a/confgen/sample-proxyfs-configuration/proxyfs.conf +++ b/confgen/sample-proxyfs-configuration/proxyfs.conf @@ -126,6 +126,7 @@ HAMode: on EtcdEnabled: true EtcdAutoSyncInterval: 60000ms +EtcdCertDir: /etc/ssl/etcd/ssl/ EtcdDialTimeout: 10000ms EtcdOpTimeout: 20000ms diff --git a/etcdclient/api.go b/etcdclient/api.go index 528c8efb2..21b599a7c 100644 --- a/etcdclient/api.go +++ b/etcdclient/api.go @@ -13,8 +13,7 @@ import ( ) const ( - certPath = "/etc/ssl/etcd/ssl/" - trustedCAFile = certPath + "ca.pem" + trustedCAFile = "/ca.pem" ) // New initializes etcd config structures and returns an etcd client @@ -33,27 +32,28 @@ func New(tlsInfo *transport.TLSInfo, endPoints []string, autoSyncInterval time.D return } -// GetCertFile returns the name of the cert file for the local node -func GetCertFile() string { +// GetCertFilePath returns the name of the cert file for the local node +func GetCertFilePath(certDir string) string { h, _ := os.Hostname() - return certPath + "node-" + h + ".pem" + return certDir + "/node-" + h + ".pem" } -// GetKeyFile returns the name of the key file for the local node -func GetKeyFile() string { +// GetKeyFilePath returns the name of the key file for the local node +func GetKeyFilePath(certDir string) string { h, _ := os.Hostname() - return certPath + "node-" + h + "-key.pem" + return certDir + "/node-" + h + "-key.pem" } // GetCA returns the name of the certificate authority for the local node -func GetCA() string { +func GetCA(certDir string) string { var ( caFile string ) - _, statErr := os.Stat(trustedCAFile) + trustedCAFilePath := certDir + trustedCAFile + _, statErr := os.Stat(trustedCAFilePath) if os.IsExist(statErr) { - caFile = trustedCAFile + caFile = trustedCAFilePath } return caFile diff --git a/headhunter/config.go b/headhunter/config.go index 63c86c0be..e8c2c1794 100644 --- a/headhunter/config.go +++ b/headhunter/config.go @@ -200,6 +200,7 @@ type globalsStruct struct { etcdEnabled bool etcdEndpoints []string etcdAutoSyncInterval time.Duration + etcdCertDir string etcdDialTimeout time.Duration etcdOpTimeout time.Duration @@ -460,6 +461,10 @@ func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { if nil != err { return } + globals.etcdCertDir, err = confMap.FetchOptionValueString("FSGlobals", "EtcdCertDir") + if nil != err { + return + } globals.etcdDialTimeout, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdDialTimeout") if nil != err { return @@ -472,9 +477,9 @@ func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { // Initialize etcd Client & KV objects tlsInfo := transport.TLSInfo{ - CertFile: etcdclient.GetCertFile(), - KeyFile: etcdclient.GetKeyFile(), - TrustedCAFile: etcdclient.GetCA(), + CertFile: etcdclient.GetCertFilePath(globals.etcdCertDir), + KeyFile: etcdclient.GetKeyFilePath(globals.etcdCertDir), + TrustedCAFile: etcdclient.GetCA(globals.etcdCertDir), } globals.etcdClient, err = etcdclient.New(&tlsInfo, globals.etcdEndpoints, diff --git a/mkproxyfs/api.go b/mkproxyfs/api.go index 68842bacd..0c6901d1d 100644 --- a/mkproxyfs/api.go +++ b/mkproxyfs/api.go @@ -45,6 +45,7 @@ func Format(mode Mode, volumeNameToFormat string, confFile string, confStrings [ etcdClient *etcd.Client etcdDialTimeout time.Duration etcdEnabled bool + etcdCertDir string etcdEndpoints []string etcdKV etcd.KV etcdOpTimeout time.Duration @@ -111,6 +112,10 @@ func Format(mode Mode, volumeNameToFormat string, confFile string, confStrings [ } if etcdEnabled { + etcdCertDir, err = confMap.FetchOptionValueString("FSGlobals", "EtcdCertDir") + if nil != err { + return + } etcdEndpoints, err = confMap.FetchOptionValueStringSlice("FSGlobals", "EtcdEndpoints") if nil != err { return @@ -134,9 +139,9 @@ func Format(mode Mode, volumeNameToFormat string, confFile string, confStrings [ } tlsInfo := transport.TLSInfo{ - CertFile: etcdclient.GetCertFile(), - KeyFile: etcdclient.GetKeyFile(), - TrustedCAFile: etcdclient.GetCA(), + CertFile: etcdclient.GetCertFilePath(etcdCertDir), + KeyFile: etcdclient.GetKeyFilePath(etcdCertDir), + TrustedCAFile: etcdclient.GetCA(etcdCertDir), } // Initialize etcd Client & KV objects diff --git a/pfs-fsck/main.go b/pfs-fsck/main.go index d2b6be1b3..8945383da 100644 --- a/pfs-fsck/main.go +++ b/pfs-fsck/main.go @@ -51,6 +51,7 @@ type globalsStruct struct { etcdClient *etcd.Client etcdDialTimeout time.Duration etcdEnabled bool + etcdCertDir string etcdEndpoints []string etcdKV etcd.KV etcdOpTimeout time.Duration @@ -365,6 +366,10 @@ func setup() { if nil != err { log.Fatal(err) } + globals.etcdCertDir, err = confMap.FetchOptionValueString("FSGlobals", "EtcdCertDir") + if nil != err { + return + } globals.etcdDialTimeout, err = confMap.FetchOptionValueDuration("FSGlobals", "EtcdDialTimeout") if nil != err { log.Fatal(err) @@ -379,9 +384,9 @@ func setup() { } tlsInfo := transport.TLSInfo{ - CertFile: etcdclient.GetCertFile(), - KeyFile: etcdclient.GetKeyFile(), - TrustedCAFile: etcdclient.GetCA(), + CertFile: etcdclient.GetCertFilePath(globals.etcdCertDir), + KeyFile: etcdclient.GetKeyFilePath(globals.etcdCertDir), + TrustedCAFile: etcdclient.GetCA(globals.etcdCertDir), } globals.etcdClient, err = etcdclient.New(&tlsInfo, globals.etcdEndpoints, diff --git a/proxyfsd/default.conf b/proxyfsd/default.conf index d5437dc50..5e750e60e 100644 --- a/proxyfsd/default.conf +++ b/proxyfsd/default.conf @@ -200,6 +200,7 @@ FileExtentMapEvictHighLimit: 10010 EtcdEnabled: false EtcdEndpoints: 127.0.0.1:2379 EtcdAutoSyncInterval: 1m +EtcdCertDir: /etc/ssl/etcd/ssl/ EtcdDialTimeout: 10s EtcdOpTimeout: 20s MetadataRecycleBin: false diff --git a/proxyfsd/file_server.conf b/proxyfsd/file_server.conf index 79b7b38b2..3db18e868 100644 --- a/proxyfsd/file_server.conf +++ b/proxyfsd/file_server.conf @@ -135,6 +135,7 @@ FileExtentMapEvictHighLimit: 10010 EtcdEnabled: false EtcdEndpoints: 127.0.0.1:2379 EtcdAutoSyncInterval: 1m +EtcdCertDir: /etc/ssl/etcd/ssl/ EtcdDialTimeout: 10s EtcdOpTimeout: 20s MetadataRecycleBin: false diff --git a/proxyfsd/file_server_mac_3_peers.conf b/proxyfsd/file_server_mac_3_peers.conf index 652f59651..c64ca229b 100644 --- a/proxyfsd/file_server_mac_3_peers.conf +++ b/proxyfsd/file_server_mac_3_peers.conf @@ -143,6 +143,7 @@ FileExtentMapEvictHighLimit: 10010 EtcdEnabled: false EtcdEndpoints: 127.0.0.1:2379 EtcdAutoSyncInterval: 1m +EtcdCertDir: /etc/ssl/etcd/ssl/ EtcdDialTimeout: 10s EtcdOpTimeout: 20s MetadataRecycleBin: false diff --git a/saio/container/proxyfs.conf b/saio/container/proxyfs.conf index 6ce935bf5..d87ea9e5c 100644 --- a/saio/container/proxyfs.conf +++ b/saio/container/proxyfs.conf @@ -172,6 +172,7 @@ FileExtentMapEvictHighLimit: 10010 EtcdEnabled: false EtcdEndpoints: 127.0.0.1:2379 EtcdAutoSyncInterval: 1m +EtcdCertDir: /etc/ssl/etcd/ssl/ EtcdDialTimeout: 10s EtcdOpTimeout: 20s MetadataRecycleBin: false From b5bdd730d68810d50131610bad2ae49f33f52cac Mon Sep 17 00:00:00 2001 From: Darrell Bishop Date: Wed, 3 Feb 2021 09:41:28 -0800 Subject: [PATCH 023/160] Adding Contributer License Agreement v1.0.1 --- CLA | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 CLA diff --git a/CLA b/CLA new file mode 100644 index 000000000..102ee1de7 --- /dev/null +++ b/CLA @@ -0,0 +1,151 @@ +Contribution License Agreement + +This Contribution License Agreement (“Agreement”) is agreed to by the party +signing below (“You”), and conveys certain license rights to NVIDIA Corporation +and its affiliates (“NVIDIA”) for Your contributions to NVIDIA open source +projects. This Agreement is effective as of the latest signature date below. + +1. Definitions. + +“Code” means the computer software code, whether in human-readable or +machine-executable form, that is delivered by You to NVIDIA under this +Agreement. + +“Project” means any of the projects owned or managed by NVIDIA in which +software is offered under a license approved by the Open Source Initiative +(OSI) (www.opensource.org) and documentation offered under an OSI or a +Creative Commons license (https://creativecommons.org/licenses). + +“Submit” is the act of uploading, submitting, transmitting, or distributing +code or other content to any Project, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Project for the purpose of discussing and improving that Project, but +excluding communication that is conspicuously marked or otherwise +designated in writing by You as “Not a Submission.” + +“Submission” means the Code and any other copyrightable material Submitted +by You, including any associated comments and documentation. + +2. Your Submission. You must agree to the terms of this Agreement before +making a Submission to any Project. This Agreement covers any and all +Submissions that You, now or in the future (except as described in Section +4 below), Submit to any Project. + +3. Originality of Work. You represent that each of Your Submissions is +entirely Your original work. Should You wish to Submit materials that are +not Your original work, You may Submit them separately to the Project if +You (a) retain all copyright and license information that was in the +materials as You received them, (b) in the description accompanying Your +Submission, include the phrase “Submission containing materials of a +third party:” followed by the names of the third party and any licenses +or other restrictions of which You are aware, and (c) follow any other +instructions in the Project’s written guidelines concerning Submissions. + +4. Your Employer. References to “employer” in this Agreement include Your +employer or anyone else for whom You are acting in making Your Submission, +e.g. as a contractor, vendor, or agent. If Your Submission is made in the +course of Your work for an employer or Your employer has intellectual +property rights in Your Submission by contract or applicable law, You must +secure permission from Your employer to make the Submission before signing +this Agreement. In that case, the term “You” in this Agreement will refer +to You and the employer collectively. If You change employers in the +future and desire to Submit additional Submissions for the new employer, +then You agree to sign a new Agreement and secure permission from the +new employer before Submitting those Submissions. + + +5. Licenses. + +a. Copyright License. You grant NVIDIA, and those who receive the Submission +directly or indirectly from NVIDIA, a perpetual, worldwide, non-exclusive, +royalty-free, irrevocable license in the Submission to reproduce, prepare +derivative works of, publicly display, publicly perform, and distribute the +Submission and such derivative works, and to sublicense any or all of the +foregoing rights to third parties. + +b. Patent License. You grant NVIDIA, and those who receive the Submission +directly or indirectly from NVIDIA, a perpetual, worldwide, non-exclusive, +royalty-free, irrevocable license under Your patent claims that are +necessarily infringed by the Submission or the combination of the Submission +with the Project to which it was Submitted to make, have made, use, offer to +sell, sell and import or otherwise dispose of the Submission alone or with +the Project. + +c. Other Rights Reserved. Each party reserves all rights not expressly +granted in this Agreement. No additional licenses or rights whatsoever +(including, without limitation, any implied licenses) are granted by +implication, exhaustion, estoppel or otherwise. + +6. Representations and Warranties. You represent that You are legally +entitled to grant the above licenses. You represent that each of Your +Submissions is entirely Your original work (except as You may have +disclosed under Section 3). You represent that You have secured permission +from Your employer to make the Submission in cases where Your Submission +is made in the course of Your work for Your employer or Your employer has +intellectual property rights in Your Submission by contract or applicable +law. If You are signing this Agreement on behalf of Your employer, You +represent and warrant that You have the necessary authority to bind the +listed employer to the obligations contained in this Agreement. You are +not expected to provide support for Your Submission, unless You choose to +do so. UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING, AND +EXCEPT FOR THE WARRANTIES EXPRESSLY STATED IN SECTIONS 3, 4, AND 6, THE +SUBMISSION PROVIDED UNDER THIS AGREEMENT IS PROVIDED WITHOUT WARRANTY OF +ANY KIND, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTY OF NONINFRINGEMENT, +MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + +7. Notice to NVIDIA. You agree to notify NVIDIA in writing of any facts +or circumstances of which You later become aware that would make Your +representations in this Agreement inaccurate in any respect. + +8. Information about Submissions. You agree that contributions to Projects +and information about contributions may be maintained indefinitely and +disclosed publicly, including Your name and other information that You +submit with Your Submission. + +9. Governing Law/Jurisdiction. Claims arising under this Agreement shall +be governed by the laws of Delaware, excluding its principles of conflict +of laws and the United Nations Convention on Contracts for the Sale of +Goods. The state and/or federal courts residing in Santa Clara County, +California shall have exclusive jurisdiction over any dispute or claim +arising out of this Agreement. You may not export the Software in +violation of applicable export laws and regulations. + +10. Entire Agreement/Assignment. This Agreement is the entire agreement +between the parties, and supersedes any and all prior agreements, +understandings or communications, written or oral, between the parties +relating to the subject matter hereof. This Agreement may be assigned by +NVIDIA. + + + + +Please select one of the options below and sign as indicated. By signing, +You accept and agree to the terms of this Contribution License Agreement +for Your present and future Submissions to NVIDIA. + +___ I have sole ownership of intellectual property rights to my Submissions +and I am not making Submissions in the course of work for my employer. + +Name (“You”): _________________________________________ +Signature: _________________________________________ +Date: _________________________________________ +GitHub Login: _________________________________________ +Email: _________________________________________ +Address: _________________________________________ + +___ I am making Submissions in the course of work for my employer (or my +employer has intellectual property rights in my Submissions by contract or +applicable law). I have permission from my employer to make Submissions and +enter into this Agreement on behalf of my employer. By signing below, the +defined term “You” includes me and my employer. + +Company Name: _________________________________________ +Signature: _________________________________________ +By: _________________________________________ +Title: _________________________________________ +Date: _________________________________________ +GitHub Login: _________________________________________ +Email: _________________________________________ +Address: _________________________________________ + From f2777aee166f644f362a46118d384cbb5ba85994 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 1 Feb 2021 15:41:03 -0600 Subject: [PATCH 024/160] bump swift to ss-release-2.26.0.11 --- saio/container/Dockerfile | 2 +- saio/vagrant_provision.sh | 2 +- sait/vagrant_provision.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 41d140f31..81adbaf1e 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -3,7 +3,7 @@ FROM centos:7.4.1708 -ARG SwiftVersion=ss-release-2.26.0.10 +ARG SwiftVersion=ss-release-2.26.0.11 ARG GolangVersion=1.15.5 ARG ProxyFS_Version=stable diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index 76b672ccc..7b1ef11be 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -228,7 +228,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/swiftstack/swift.git cd swift -git checkout ss-release-2.26.0.10 +git checkout ss-release-2.26.0.11 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index e369f8293..cfae6e7ba 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -230,7 +230,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/swiftstack/swift.git cd swift -git checkout ss-release-2.26.0.10 +git checkout ss-release-2.26.0.11 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet From 3b3cdb3530a369b724642677ef64d4426023fee9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 3 Feb 2021 14:04:58 -0800 Subject: [PATCH 025/160] Redirect (& unhack) movement of reference Swift fork from swiftstack to NVIDIA Also converted pfs_middleware/tox.ini to use https: instead of git:... it's a read-only repo clone anyway --- pfs_middleware/tox.ini | 2 +- saio/container/Dockerfile | 2 +- saio/vagrant_provision.sh | 2 +- sait/vagrant_provision.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pfs_middleware/tox.ini b/pfs_middleware/tox.ini index d162fedc4..f89a1de95 100644 --- a/pfs_middleware/tox.ini +++ b/pfs_middleware/tox.ini @@ -7,7 +7,7 @@ usedevelop = True deps = lint: flake8 !lint: -r{toxinidir}/test-requirements.txt - release: git+git://github.com/openstack/swift.git@2.25.1 + release: git+https://github.com/NVIDIA/swift.git@ss-release-2.26.0.11 minver: http://tarballs.openstack.org/swift/swift-2.9.0.tar.gz master: http://tarballs.openstack.org/swift/swift-master.tar.gz commands = python -m unittest discover diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 81adbaf1e..eae3677cf 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -169,7 +169,7 @@ RUN yum install -y make fuse RUN mkdir -p $GOPATH/src/github.com/swiftstack WORKDIR $GOPATH/src/github.com/swiftstack -RUN git clone https://github.com/swiftstack/ProxyFS.git +RUN git clone https://github.com/NVIDIA/ProxyFS.git WORKDIR $GOPATH/src/github.com/swiftstack/ProxyFS RUN git checkout $ProxyFS_Version diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index 7b1ef11be..35e11daf4 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -226,7 +226,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile # Now we can actually install Swift from source cd ~swift -git clone https://github.com/swiftstack/swift.git +git clone https://github.com/NVIDIA/swift.git cd swift git checkout ss-release-2.26.0.11 pip install wheel diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index cfae6e7ba..efb425149 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -228,7 +228,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile # Now we can actually install Swift from source cd ~swift -git clone https://github.com/swiftstack/swift.git +git clone https://github.com/NVIDIA/swift.git cd swift git checkout ss-release-2.26.0.11 pip install wheel From c20fdf27df31cfe107f6202d6a452b4726feeba6 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 3 Feb 2021 14:37:42 -0800 Subject: [PATCH 026/160] Avoid upgrade to pip 21.0 or later since Py2 support was deprecated there --- saio/container/Dockerfile | 2 +- saio/vagrant_provision.sh | 2 +- sait/vagrant_provision.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index eae3677cf..4473b181a 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -88,7 +88,7 @@ RUN ln -s /opt/rh/rh-python36/root/usr/include /opt/rh/rh-python36/root/include RUN yum install -y epel-release RUN yum install -y python-pip -RUN pip install --upgrade pip +RUN pip install --upgrade 'pip<21.0' RUN pip install --upgrade setuptools WORKDIR /home/swift diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index 35e11daf4..d8100bd10 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -103,7 +103,7 @@ ln -s /opt/rh/rh-python36/root/usr/include /opt/rh/rh-python36/root/include yum -y install epel-release yum -y install python-pip -pip install --upgrade pip +pip install --upgrade 'pip<21.0' # Setup ProxyFS build environment diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index efb425149..e51e55649 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -105,7 +105,7 @@ ln -s /opt/rh/rh-python36/root/usr/include /opt/rh/rh-python36/root/include yum -y install epel-release yum -y install python-pip -pip install --upgrade pip +pip install --upgrade 'pip<21.0' # Setup ProxyFS build environment From 6af9838150b110cd27a0d51f0c21680c386aeb0e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 3 Feb 2021 14:41:28 -0800 Subject: [PATCH 027/160] Missed saio/container/Dockerfile's reference to the old swift repo location --- saio/container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 4473b181a..bc4c0e37d 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -109,7 +109,7 @@ RUN pip install -e . RUN pip install -r test-requirements.txt WORKDIR /home/swift -RUN git clone https://github.com/swiftstack/swift.git +RUN git clone https://github.com/NVIDIA/swift.git WORKDIR /home/swift/swift RUN git checkout $SwiftVersion RUN pip install wheel From 4017c267d62fc709d88c67553f9da34d1da50ada Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 3 Feb 2021 15:08:42 -0800 Subject: [PATCH 028/160] And the reason for the "miss" was changing the wrong repo (ProxyFS is still in swiftstack) --- saio/container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index bc4c0e37d..77e5f3dea 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -169,7 +169,7 @@ RUN yum install -y make fuse RUN mkdir -p $GOPATH/src/github.com/swiftstack WORKDIR $GOPATH/src/github.com/swiftstack -RUN git clone https://github.com/NVIDIA/ProxyFS.git +RUN git clone https://github.com/swiftstack/ProxyFS.git WORKDIR $GOPATH/src/github.com/swiftstack/ProxyFS RUN git checkout $ProxyFS_Version From 8995d8b4416ad64ac7a41c86e621ab36790fdb5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Fri, 5 Feb 2021 16:30:36 -0800 Subject: [PATCH 029/160] First batch of changes related to CI --- .travis.yml | 2 +- README.md | 25 +++++------- ci/ansible/chef_files/ci.cfg | 4 +- ci/ansible/chef_files/runway.attribs.json | 4 +- ci/ansible/chef_files/runway.cfg | 4 +- ci/ansible/chef_files/vagrant.cfg | 4 +- ci/ansible/tasks/installer_without_chef.yml | 14 +++---- .../tasks/templates/usr/bin/goswitch.template | 4 +- .../usr/bin/reprovision_proxyfs.template | 2 +- ci/ansible/vars/ci.yml | 4 +- ci/ansible/vars/runway.yml | 6 +-- ci/ansible/vars/vagrant.yml | 4 +- ci/chef.cfg | 4 +- cookbooks/meta_middleware/recipes/default.rb | 2 +- cookbooks/pfs_middleware/recipes/default.rb | 2 +- cookbooks/proxyfs/recipes/default.rb | 38 +++++++++---------- run_docker_tests.sh | 4 +- test/container/Dockerfile | 2 +- test/container/launch.sh | 2 +- 19 files changed, 62 insertions(+), 69 deletions(-) diff --git a/.travis.yml b/.travis.yml index e64ba31f8..0ccebea4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,4 +14,4 @@ notifications: on_pull_requests: true script: -- docker run -e "COVERALLS_TOKEN=$COVERALLS_TOKEN" -e "TRAVIS_BRANCH=$TRAVIS_BRANCH" --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/swiftstack/ProxyFS swiftstack/proxyfs_unit_tests +- docker run -e "COVERALLS_TOKEN=$COVERALLS_TOKEN" -e "TRAVIS_BRANCH=$TRAVIS_BRANCH" --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs swiftstack/proxyfs_unit_tests diff --git a/README.md b/README.md index 3ee4da5dc..ad662f409 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![Build Status](https://travis-ci.com/swiftstack/ProxyFS.svg?branch=development)](https://travis-ci.com/swiftstack/ProxyFS) -[![Coverage Status](https://coveralls.io/repos/github/swiftstack/ProxyFS/badge.svg?branch=development)](https://coveralls.io/github/swiftstack/ProxyFS?branch=development) +[![Build Status](https://travis-ci.com/NVIDIA/proxyfs.svg?branch=development)](https://travis-ci.com/NVIDIA/proxyfs) +[![Coverage Status](https://coveralls.io/repos/github/NVIDIA/proxyfs/badge.svg?branch=development)](https://coveralls.io/github/NVIDIA/proxyfs?branch=development) # ProxyFS Integrated File and Object Access for Swift Object Storage @@ -18,7 +18,7 @@ to object API clients. All code contributions for ProxyFS go through GitHub. -https://github.com/swiftstack/ProxyFS +https://github.com/NVIDIA/proxyfs Please feel free to contribute by opening a pull request to the `development` branch. If you see an open pull request, feel free to @@ -32,28 +32,21 @@ in the [ProxyFS dev mailing list](https://lists.proxyfs.org/mailman/listinfo) or the [ProxyFS Slack group](https://proxyfs.slack.com), which you can join through [this inivitation](https://join.slack.com/t/proxyfs/shared_invite/enQtMzA2NTQwMDU4NTkyLWM4ZjhkYmE0NWEzMTYzZGZkNThkNzcxMzg0NWIzMmQ4MTU5MGQyMDRlY2UzMDU0YjBlNGZkMzk4N2NkNTRjNjY). -## Development Environment - -The officially supported development environment for ProxyFS is -[Runway](https://github.com/swiftstack/runway). Please read Runway's -documentation and make sure to use the ProxyFS manifest instead of the default -one. - -## How to get the code (if not using Runway) +## How to get the code * Define your GOPATH as desired (where your bin/, /pkg, and /src/ directory trees will appear) * cd $GOPATH -* mkdir -p src/github.com/swiftstack -* cd src/github.com/swiftstack -* git clone git@github.com:swiftstack/ProxyFS.git +* mkdir -p src/github.com/NVIDIA +* cd src/github.com/NVIDIA +* git clone git@github.com:NVIDIA/proxyfs.git * cd ProxyFS ## How to run unit tests (in your Development Environment) -* Install/update to at least Go 1.8.3 (if not using Runway) +* Install/update to at least Go 1.8.3 * Ensure $GOPATH/bin is in your $PATH -* cd $GOPATH/src/github.com/swiftstack/ProxyFS +* cd $GOPATH/src/github.com/NVIDIA/proxyfs * make ## License diff --git a/ci/ansible/chef_files/ci.cfg b/ci/ansible/chef_files/ci.cfg index 018609296..9ad6f599a 100644 --- a/ci/ansible/chef_files/ci.cfg +++ b/ci/ansible/chef_files/ci.cfg @@ -1,3 +1,3 @@ file_cache_path "/home/swiftstack/chef" -cookbook_path "/home/swiftstack/provisioning/src/github.com/swiftstack/ProxyFS/cookbooks" -json_attribs "/home/swiftstack/provisioning/src/github.com/swiftstack/ProxyFS/ci/ansible/chef_files/ci.attribs.json" +cookbook_path "/home/swiftstack/provisioning/src/github.com/NVIDIA/proxyfs/cookbooks" +json_attribs "/home/swiftstack/provisioning/src/github.com/NVIDIA/proxyfs/ci/ansible/chef_files/ci.attribs.json" diff --git a/ci/ansible/chef_files/runway.attribs.json b/ci/ansible/chef_files/runway.attribs.json index a8b584d01..9157dd64a 100644 --- a/ci/ansible/chef_files/runway.attribs.json +++ b/ci/ansible/chef_files/runway.attribs.json @@ -29,6 +29,6 @@ "swift_bench_repo": "git://github.com/openstack/swift-bench.git", "swift_bench_repo_branch": "master", "extra_key": "", - "source_root": "/home/swift/code/ProxyFS", - "package_spec_path": "/home/swift/code/ProxyFS/src/github.com/swiftstack/ProxyFS/ci/ansible/chef_files/packages" + "source_root": "/home/swift/code/proxyfs", + "package_spec_path": "/home/swift/code/proxyfs/src/github.com/NVIDIA/proxyfs/ci/ansible/chef_files/packages" } diff --git a/ci/ansible/chef_files/runway.cfg b/ci/ansible/chef_files/runway.cfg index 45ec42638..db824545d 100644 --- a/ci/ansible/chef_files/runway.cfg +++ b/ci/ansible/chef_files/runway.cfg @@ -1,3 +1,3 @@ file_cache_path "/home/swift/chef" -cookbook_path "/home/swift/code/ProxyFS/src/github.com/swiftstack/ProxyFS/cookbooks" -json_attribs "/home/swift/code/ProxyFS/src/github.com/swiftstack/ProxyFS/ci/ansible/chef_files/runway.attribs.json" +cookbook_path "/home/swift/code/proxyfs/src/github.com/NVIDIA/proxyfs/cookbooks" +json_attribs "/home/swift/code/proxyfs/src/github.com/NVIDIA/proxyfs/ci/ansible/chef_files/runway.attribs.json" diff --git a/ci/ansible/chef_files/vagrant.cfg b/ci/ansible/chef_files/vagrant.cfg index 4d63a47ff..cbd9bfe53 100644 --- a/ci/ansible/chef_files/vagrant.cfg +++ b/ci/ansible/chef_files/vagrant.cfg @@ -1,3 +1,3 @@ file_cache_path "/home/vagrant/chef" -cookbook_path "/vagrant/src/github.com/swiftstack/ProxyFS/cookbooks" -json_attribs "/vagrant/src/github.com/swiftstack/ProxyFS/ci/ansible/chef_files/vagrant.attribs.json" +cookbook_path "/vagrant/src/github.com/NVIDIA/proxyfs/cookbooks" +json_attribs "/vagrant/src/github.com/NVIDIA/proxyfs/ci/ansible/chef_files/vagrant.attribs.json" diff --git a/ci/ansible/tasks/installer_without_chef.yml b/ci/ansible/tasks/installer_without_chef.yml index ac45b8d74..71f018fbc 100644 --- a/ci/ansible/tasks/installer_without_chef.yml +++ b/ci/ansible/tasks/installer_without_chef.yml @@ -13,9 +13,9 @@ GOROOT: "/usr/local/go" HOME_DIR: "/home/{{ proxyfs_user }}" BASH_DOT_PROFILE: "{{ ansible_env.HOME }}/.bash_profile" - REPO_CLONE_PARENT_DIR: "{{ source_root }}/src/github.com/swiftstack" + REPO_CLONE_PARENT_DIR: "{{ source_root }}/src/github.com/NVIDIA" SAMBA_SRC_DIR: "{{ REPO_CLONE_PARENT_DIR }}/samba" - PROXYFS_SRC_DIR: "{{ REPO_CLONE_PARENT_DIR }}/ProxyFS" + PROXYFS_SRC_DIR: "{{ REPO_CLONE_PARENT_DIR }}/proxyfs" dot_bashrc: "{{ HOME_DIR }}/.bashrc" is_dev: False rhel_packages: @@ -50,14 +50,14 @@ tasks: - name: Clone ProxyFS repo git: - repo: https://github.com/swiftstack/ProxyFS.git + repo: https://github.com/NVIDIA/proxyfs.git dest: "{{ PROXYFS_SRC_DIR }}/" version: "{{ ansible_env.proxyfs_branch }}" - name: Install pfs_middleware in development mode shell: "python setup.py develop" args: - chdir: "{{ ansible_env.HOME }}/provisioning/github.com/swiftstack/ProxyFS/pfs_middleware" + chdir: "{{ ansible_env.HOME }}/provisioning/github.com/NVIDIA/proxyfs/pfs_middleware" - stat: path={{GOROOT}} register: go_installed @@ -105,7 +105,7 @@ create: True owner: {{proxyfs_user}} with_items: - - {regexp: "/cdpfs/", line: "alias cdpfs='cd $GOPATH/src/github.com/swiftstack/ProxyFS'"} + - {regexp: "/cdpfs/", line: "alias cdpfs='cd $GOPATH/src/github.com/NVIDIA/proxyfs'"} - {regexp: "/cdsamba/", line: "alias cdsamba='cd {{SAMBA_SRC_DIR}}'"} - name: "Disable SELinux" @@ -154,7 +154,7 @@ - name: "Link to proxyfsd" file: - src: "{{source_root}}/src/github.com/swiftstack/ProxyFS/proxyfsd/" + src: "{{source_root}}/src/github.com/NVIDIA/proxyfs/proxyfsd/" dest: "/etc/proxyfsd" state: link owner: {{proxyfs_user}} @@ -168,7 +168,7 @@ mode: 0755 - name: "Provision pfs_stat" - shell: "install -m 0755 {{source_root}}/src/github.com/swiftstack/ProxyFS/bin/pfs_stat /usr/bin" + shell: "install -m 0755 {{source_root}}/src/github.com/NVIDIA/proxyfs/bin/pfs_stat /usr/bin" - stat: path=/usr/lib/systemd/system/ register: system_exists diff --git a/ci/ansible/tasks/templates/usr/bin/goswitch.template b/ci/ansible/tasks/templates/usr/bin/goswitch.template index 9aa5ed288..368ebd9fa 100644 --- a/ci/ansible/tasks/templates/usr/bin/goswitch.template +++ b/ci/ansible/tasks/templates/usr/bin/goswitch.template @@ -14,9 +14,9 @@ GO_DIR_NAME = "{{ go_dir_name }}" GO_DIR_PATH = os.path.join(GO_DIR_PARENT_PATH, GO_DIR_NAME) SOURCE_ROOT = "{{ source_root }}" REPO_CLONE_PARENT_DIR = os.path.join( - SOURCE_ROOT, "src", "github.com", "swiftstack" + SOURCE_ROOT, "src", "github.com", "NVIDIA" ) -PROXYFS_SRC_DIR = os.path.join(REPO_CLONE_PARENT_DIR, "ProxyFS") +PROXYFS_SRC_DIR = os.path.join(REPO_CLONE_PARENT_DIR, "proxyfs") class GoDirNotLinkError(Exception): diff --git a/ci/ansible/tasks/templates/usr/bin/reprovision_proxyfs.template b/ci/ansible/tasks/templates/usr/bin/reprovision_proxyfs.template index 07f6a375d..1f3fe4e43 100644 --- a/ci/ansible/tasks/templates/usr/bin/reprovision_proxyfs.template +++ b/ci/ansible/tasks/templates/usr/bin/reprovision_proxyfs.template @@ -2,6 +2,6 @@ set -e /usr/bin/unmount_and_stop_pfs -cd $GOPATH/src/github.com/swiftstack/ProxyFS +cd $GOPATH/src/github.com/NVIDIA/proxyfs ci/ansible/install_proxyfs.sh {{ env_arg }} {{ active_golang_version }} /usr/bin/start_and_mount_pfs "$@" diff --git a/ci/ansible/vars/ci.yml b/ci/ansible/vars/ci.yml index 787b43626..8ba350bb7 100644 --- a/ci/ansible/vars/ci.yml +++ b/ci/ansible/vars/ci.yml @@ -2,5 +2,5 @@ proxyfs_user: "swiftstack" proxyfs_group: "swiftsatck" source_root: "/home/{{ proxyfs_user }}/provisioning" -REPO_CLONE_PARENT_DIR: "{{ source_root }}/src/github.com/swiftstack" -PROXYFS_SRC_DIR: "{{ REPO_CLONE_PARENT_DIR }}/ProxyFS" +REPO_CLONE_PARENT_DIR: "{{ source_root }}/src/github.com/NVIDIA" +PROXYFS_SRC_DIR: "{{ REPO_CLONE_PARENT_DIR }}/proxyfs" diff --git a/ci/ansible/vars/runway.yml b/ci/ansible/vars/runway.yml index d094123b9..b54c163d7 100644 --- a/ci/ansible/vars/runway.yml +++ b/ci/ansible/vars/runway.yml @@ -1,6 +1,6 @@ --- proxyfs_user: "swift" proxyfs_group: "swift" -source_root: "/home/{{ proxyfs_user }}/code/ProxyFS" -REPO_CLONE_PARENT_DIR: "{{ source_root }}/src/github.com/swiftstack" -PROXYFS_SRC_DIR: "{{ REPO_CLONE_PARENT_DIR }}/ProxyFS" +source_root: "/home/{{ proxyfs_user }}/code/proxyfs" +REPO_CLONE_PARENT_DIR: "{{ source_root }}/src/github.com/NVIDIA" +PROXYFS_SRC_DIR: "{{ REPO_CLONE_PARENT_DIR }}/proxyfs" diff --git a/ci/ansible/vars/vagrant.yml b/ci/ansible/vars/vagrant.yml index 8e7c76a23..545723ff2 100644 --- a/ci/ansible/vars/vagrant.yml +++ b/ci/ansible/vars/vagrant.yml @@ -2,5 +2,5 @@ proxyfs_user: "vagrant" proxyfs_group: "vagrant" source_root: "/vagrant" -REPO_CLONE_PARENT_DIR: "{{ source_root }}/src/github.com/swiftstack" -PROXYFS_SRC_DIR: "{{ REPO_CLONE_PARENT_DIR }}/ProxyFS" +REPO_CLONE_PARENT_DIR: "{{ source_root }}/src/github.com/NVIDIA" +PROXYFS_SRC_DIR: "{{ REPO_CLONE_PARENT_DIR }}/proxyfs" diff --git a/ci/chef.cfg b/ci/chef.cfg index 749d279ed..501a8d758 100644 --- a/ci/chef.cfg +++ b/ci/chef.cfg @@ -1,3 +1,3 @@ file_cache_path "/home/swiftstack/chef" -cookbook_path "/home/swiftstack/provisioning/src/github.com/swiftstack/ProxyFS/cookbooks" -json_attribs "/home/swiftstack/provisioning/src/github.com/swiftstack/ProxyFS/ci/chef.attribs.json" +cookbook_path "/home/swiftstack/provisioning/src/github.com/NVIDIA/proxyfs/cookbooks" +json_attribs "/home/swiftstack/provisioning/src/github.com/NVIDIA/proxyfs/ci/chef.attribs.json" diff --git a/cookbooks/meta_middleware/recipes/default.rb b/cookbooks/meta_middleware/recipes/default.rb index 37ca6eae6..bea0435e2 100644 --- a/cookbooks/meta_middleware/recipes/default.rb +++ b/cookbooks/meta_middleware/recipes/default.rb @@ -8,5 +8,5 @@ # install middleware in development mode execute 'meta_middleware - setup.py develop' do command "python setup.py develop" - cwd "#{node['source_root']}/src/github.com/swiftstack/ProxyFS/meta_middleware" + cwd "#{node['source_root']}/src/github.com/NVIDIA/proxyfs/meta_middleware" end diff --git a/cookbooks/pfs_middleware/recipes/default.rb b/cookbooks/pfs_middleware/recipes/default.rb index fa32d4a73..9da1519b1 100644 --- a/cookbooks/pfs_middleware/recipes/default.rb +++ b/cookbooks/pfs_middleware/recipes/default.rb @@ -8,5 +8,5 @@ # install middleware in development mode execute 'pfs_middleware - setup.py develop' do command "python setup.py develop" - cwd "#{node['source_root']}/src/github.com/swiftstack/ProxyFS/pfs_middleware" + cwd "#{node['source_root']}/src/github.com/NVIDIA/proxyfs/pfs_middleware" end diff --git a/cookbooks/proxyfs/recipes/default.rb b/cookbooks/proxyfs/recipes/default.rb index e09a75be5..f85186439 100644 --- a/cookbooks/proxyfs/recipes/default.rb +++ b/cookbooks/proxyfs/recipes/default.rb @@ -12,9 +12,9 @@ ROOT_DOT_BASH_PROFILE = "/root/.bash_profile" ROOT_DOT_BASHRC = "/root/.bashrc" ETC_BASHRC = "/etc/bashrc" -REPO_CLONE_PARENT_DIR = "#{source_root}/src/github.com/swiftstack" +REPO_CLONE_PARENT_DIR = "#{source_root}/src/github.com/NVIDIA" PROXYFS_BIN_DIR = "#{source_root}/bin" -PROXYFS_SRC_DIR = "#{REPO_CLONE_PARENT_DIR}/ProxyFS" +PROXYFS_SRC_DIR = "#{REPO_CLONE_PARENT_DIR}/proxyfs" ruby_block "update_profile_and_bashrc" do block do @@ -49,7 +49,7 @@ file = Chef::Util::FileEdit.new(DOT_BASHRC) file.insert_line_if_no_match(/export GOPATH/, "export GOPATH=#{source_root}") file.insert_line_if_no_match(%r{usr/local/go/bin}, "export PATH=$GOPATH/bin:$PATH:/usr/local/go/bin") - file.insert_line_if_no_match(/cdpfs/, "alias cdpfs='cd $GOPATH/src/github.com/swiftstack/ProxyFS'") + file.insert_line_if_no_match(/cdpfs/, "alias cdpfs='cd $GOPATH/src/github.com/NVIDIA/proxyfs'") file.insert_line_if_no_match(/cdfun/, "alias cdfun='cd /home/swift/code/functional-tests'") file.insert_line_if_no_match(/ls -lha/, "alias la='ls -lha'") file.insert_line_if_no_match(/ls -liha/, "alias li='ls -liha'") @@ -79,7 +79,7 @@ file = Chef::Util::FileEdit.new(ROOT_DOT_BASHRC) file.insert_line_if_no_match(/export GOPATH/, "export GOPATH=#{source_root}") file.insert_line_if_no_match(%r{usr/local/go/bin}, "export PATH=$GOPATH/bin:$PATH:/usr/local/go/bin") - file.insert_line_if_no_match(/cdpfs/, "alias cdpfs='cd $GOPATH/src/github.com/swiftstack/ProxyFS'") + file.insert_line_if_no_match(/cdpfs/, "alias cdpfs='cd $GOPATH/src/github.com/NVIDIA/proxyfs'") file.insert_line_if_no_match(/cdfun/, "alias cdfun='cd /home/swift/code/functional-tests'") file.insert_line_if_no_match(/ls -lha/, "alias la='ls -lha'") file.insert_line_if_no_match(/ls -liha/, "alias li='ls -liha'") @@ -118,12 +118,12 @@ command "pip install -r #{PROXYFS_SRC_DIR}/pfs-swift-load/requirements.txt" end -execute "Create ProxyFS/bin dir" do +execute "Create proxyfs/bin dir" do command "mkdir #{PROXYFS_BIN_DIR}" not_if { ::Dir.exists?("#{PROXYFS_BIN_DIR}") } end -execute "Copy pfs-swift-load-plot at /home/swift/code/ProxyFS/bin/" do +execute "Copy pfs-swift-load-plot at /home/swift/code/proxyfs/bin/" do command "install -m 0755 #{PROXYFS_SRC_DIR}/pfs-swift-load/pfs-swift-load-plot #{PROXYFS_BIN_DIR}/" end @@ -167,61 +167,61 @@ end link '/etc/proxyfsd' do - to "#{source_root}/src/github.com/swiftstack/ProxyFS/proxyfsd/" + to "#{source_root}/src/github.com/NVIDIA/proxyfs/proxyfsd/" link_type :symbolic owner proxyfs_user group proxyfs_group end link '/etc/pfsagentd' do - to "#{source_root}/src/github.com/swiftstack/ProxyFS/pfsagentd/" + to "#{source_root}/src/github.com/NVIDIA/proxyfs/pfsagentd/" link_type :symbolic owner proxyfs_user group proxyfs_group end execute "Provision start_and_mount_pfs" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/start_and_mount_pfs /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/cookbooks/proxyfs/files/default/usr/bin/start_and_mount_pfs /usr/bin" end execute "Provision start_swift_only" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/start_swift_only /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/cookbooks/proxyfs/files/default/usr/bin/start_swift_only /usr/bin" end execute "Provision start_proxyfsd_only" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/start_proxyfsd_only /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/cookbooks/proxyfs/files/default/usr/bin/start_proxyfsd_only /usr/bin" end execute "Provision stop_proxyfsd_only" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/stop_proxyfsd_only /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/cookbooks/proxyfs/files/default/usr/bin/stop_proxyfsd_only /usr/bin" end execute "Provision unmount_and_stop_pfs" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/unmount_and_stop_pfs /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/cookbooks/proxyfs/files/default/usr/bin/unmount_and_stop_pfs /usr/bin" end execute "Provision set_up_s3api" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/set_up_s3api /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/cookbooks/proxyfs/files/default/usr/bin/set_up_s3api /usr/bin" end execute "Provision set_up_swift3" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/set_up_swift3 /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/cookbooks/proxyfs/files/default/usr/bin/set_up_swift3 /usr/bin" end execute "Provision enable_s3" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/enable_s3 /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/cookbooks/proxyfs/files/default/usr/bin/enable_s3 /usr/bin" end execute "Provision disable_s3" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/disable_s3 /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/cookbooks/proxyfs/files/default/usr/bin/disable_s3 /usr/bin" end execute "Provision detect_s3" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/cookbooks/proxyfs/files/default/usr/bin/detect_s3 /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/cookbooks/proxyfs/files/default/usr/bin/detect_s3 /usr/bin" end execute "Provision pfs_stat" do - command "install -m 0755 #{source_root}/src/github.com/swiftstack/ProxyFS/bin/pfs_stat /usr/bin" + command "install -m 0755 #{source_root}/src/github.com/NVIDIA/proxyfs/bin/pfs_stat /usr/bin" end cookbook_file "/usr/lib/systemd/system/proxyfsd.service" do diff --git a/run_docker_tests.sh b/run_docker_tests.sh index f41cadebb..da306674a 100755 --- a/run_docker_tests.sh +++ b/run_docker_tests.sh @@ -17,11 +17,11 @@ if [ "$IMAGE" == "build" ]; then set -x # That's how you run it from scratch docker build -t proxyfs_unit_tests test/container - docker run --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/swiftstack/ProxyFS proxyfs_unit_tests + docker run --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs proxyfs_unit_tests elif [ "$IMAGE" == "pull" ]; then set -x # That's how you run it using the image on Docker Hub - docker run --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/swiftstack/ProxyFS swiftstack/proxyfs_unit_tests + docker run --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs swiftstack/proxyfs_unit_tests else echo "Bad argument: $IMAGE" exit 1 diff --git a/test/container/Dockerfile b/test/container/Dockerfile index 589fcdf68..de5555529 100644 --- a/test/container/Dockerfile +++ b/test/container/Dockerfile @@ -127,4 +127,4 @@ RUN cd /pyeclib && pip install -e . && pip install -r test-requirements.txt RUN go get github.com/ory/go-acc github.com/mattn/goveralls -CMD ["/bin/bash", "/gopathroot/src/github.com/swiftstack/ProxyFS/test/container/launch.sh"] +CMD ["/bin/bash", "/gopathroot/src/github.com/NVIDIA/proxyfs/test/container/launch.sh"] diff --git a/test/container/launch.sh b/test/container/launch.sh index 4df25c0d8..c22b9a90e 100755 --- a/test/container/launch.sh +++ b/test/container/launch.sh @@ -7,7 +7,7 @@ set -e set -x # Build ProxyFS and run tests -cd $GOPATH/src/github.com/swiftstack/ProxyFS +cd $GOPATH/src/github.com/NVIDIA/proxyfs make ci # $COVERALLS_TOKEN must be configured in TravisCI if [ -n "$COVERALLS_TOKEN" ] && [ -n "$TRAVIS_BRANCH" ]; then From ee47ff879f62ec74e4fd9c006bee9fcc854dc794 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Fri, 5 Feb 2021 16:38:59 -0800 Subject: [PATCH 030/160] Remove TravisCI badges --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index ad662f409..ca2277677 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,3 @@ -[![Build Status](https://travis-ci.com/NVIDIA/proxyfs.svg?branch=development)](https://travis-ci.com/NVIDIA/proxyfs) -[![Coverage Status](https://coveralls.io/repos/github/NVIDIA/proxyfs/badge.svg?branch=development)](https://coveralls.io/github/NVIDIA/proxyfs?branch=development) - # ProxyFS Integrated File and Object Access for Swift Object Storage From d22201d9531eea818adb724633dfbdfcf4aeb62c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 8 Feb 2021 10:00:44 -0800 Subject: [PATCH 031/160] Migrated all Makefile's --- Makefile | 2 +- blunder/Makefile | 2 +- bucketstats/Makefile | 2 +- cleanproxyfs/Makefile | 2 +- conf/Makefile | 2 +- confgen/Makefile | 2 +- confgen/confgen/Makefile | 2 +- dlm/Makefile | 2 +- emswift/Makefile | 2 +- emswift/emswiftpkg/Makefile | 2 +- etcdclient/Makefile | 2 +- evtlog/Makefile | 2 +- evtlog/pfsevtlogd/Makefile | 2 +- fs/Makefile | 2 +- fsworkout/Makefile | 2 +- fuse/Makefile | 2 +- halter/Makefile | 2 +- headhunter/Makefile | 2 +- httpserver/Makefile | 2 +- imgr/Makefile | 2 +- imgr/imgrpkg/Makefile | 2 +- inode/Makefile | 2 +- inodeworkout/Makefile | 2 +- jrpcfs/Makefile | 2 +- jrpcfs/test/Makefile | 2 +- liveness/Makefile | 2 +- logger/Makefile | 2 +- make-static-content/Makefile | 2 +- mkproxyfs/Makefile | 2 +- mkproxyfs/mkproxyfs/Makefile | 2 +- pfs-crash/Makefile | 2 +- pfs-fsck/Makefile | 2 +- pfs-jrpc/Makefile | 2 +- pfs-restart-test/Makefile | 2 +- pfs-stress/Makefile | 2 +- pfs-swift-load/Makefile | 2 +- pfsagentConfig/Makefile | 2 +- pfsagentConfig/pfsagentConfig/Makefile | 2 +- pfsagentd/Makefile | 2 +- pfsagentd/pfsagentd-init/Makefile | 2 +- pfsagentd/pfsagentd-swift-auth-plugin/Makefile | 2 +- pfsconfjson/Makefile | 2 +- pfsconfjsonpacked/Makefile | 2 +- pfsworkout/Makefile | 2 +- platform/Makefile | 2 +- proxyfsd/Makefile | 2 +- proxyfsd/proxyfsd/Makefile | 2 +- ramswift/Makefile | 2 +- ramswift/ramswift/Makefile | 2 +- retryrpc/Makefile | 2 +- retryrpc/rpctest/Makefile | 2 +- stats/Makefile | 2 +- statslogger/Makefile | 2 +- swiftclient/Makefile | 2 +- trackedlock/Makefile | 2 +- transitions/Makefile | 2 +- utils/Makefile | 2 +- version/Makefile | 2 +- 58 files changed, 58 insertions(+), 58 deletions(-) diff --git a/Makefile b/Makefile index e78d746ad..f7462c9d2 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,7 @@ gobinsubdirsforci = \ proxyfsd/proxyfsd gosubdirsforci = $(gopkgsubdirs) $(gobinsubdirsforci); -gosubdirspathsforci = $(addprefix github.com/swiftstack/ProxyFS/,$(gosubdirsforci)) +gosubdirspathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(gosubdirsforci)) uname = $(shell uname) machine = $(shell uname -m) diff --git a/blunder/Makefile b/blunder/Makefile index 3710054cd..9466c3178 100644 --- a/blunder/Makefile +++ b/blunder/Makefile @@ -1,7 +1,7 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/blunder +gosubdir := github.com/NVIDIA/proxyfs/blunder generatedfiles := \ fserror_string.go diff --git a/bucketstats/Makefile b/bucketstats/Makefile index 27605ff0e..7d2f51140 100644 --- a/bucketstats/Makefile +++ b/bucketstats/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/bucketstats +gosubdir := github.com/NVIDIA/proxyfs/bucketstats include ../GoMakefile diff --git a/cleanproxyfs/Makefile b/cleanproxyfs/Makefile index 4fdd5b0b7..381b9e6e9 100644 --- a/cleanproxyfs/Makefile +++ b/cleanproxyfs/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/cleanproxyfs +gosubdir := github.com/NVIDIA/proxyfs/cleanproxyfs include ../GoMakefile diff --git a/conf/Makefile b/conf/Makefile index 201b26edc..aaae46c26 100644 --- a/conf/Makefile +++ b/conf/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/conf +gosubdir := github.com/NVIDIA/proxyfs/conf include ../GoMakefile diff --git a/confgen/Makefile b/confgen/Makefile index 0b00720c2..d63911273 100644 --- a/confgen/Makefile +++ b/confgen/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/confgen +gosubdir := github.com/NVIDIA/proxyfs/confgen include ../GoMakefile diff --git a/confgen/confgen/Makefile b/confgen/confgen/Makefile index ed0c48a2b..073b6b37d 100644 --- a/confgen/confgen/Makefile +++ b/confgen/confgen/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/confgen/confgen +gosubdir := github.com/NVIDIA/proxyfs/confgen/confgen include ../../GoMakefile diff --git a/dlm/Makefile b/dlm/Makefile index 4018099be..f54a4b861 100644 --- a/dlm/Makefile +++ b/dlm/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/dlm +gosubdir := github.com/NVIDIA/proxyfs/dlm include ../GoMakefile diff --git a/emswift/Makefile b/emswift/Makefile index 74b2d5323..1deda5018 100644 --- a/emswift/Makefile +++ b/emswift/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/emswift +gosubdir := github.com/NVIDIA/proxyfs/emswift include ../GoMakefile diff --git a/emswift/emswiftpkg/Makefile b/emswift/emswiftpkg/Makefile index e40813d02..ee8ca60de 100644 --- a/emswift/emswiftpkg/Makefile +++ b/emswift/emswiftpkg/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/emswift/emswiftpkg +gosubdir := github.com/NVIDIA/proxyfs/emswift/emswiftpkg include ../../GoMakefile diff --git a/etcdclient/Makefile b/etcdclient/Makefile index 41aba670b..302c61c44 100644 --- a/etcdclient/Makefile +++ b/etcdclient/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/etcdclient +gosubdir := github.com/NVIDIA/proxyfs/etcdclient include ../GoMakefile diff --git a/evtlog/Makefile b/evtlog/Makefile index 8520cb99c..54a10b7a7 100644 --- a/evtlog/Makefile +++ b/evtlog/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/evtlog +gosubdir := github.com/NVIDIA/proxyfs/evtlog include ../GoMakefile diff --git a/evtlog/pfsevtlogd/Makefile b/evtlog/pfsevtlogd/Makefile index d86ae15e0..f9804187f 100644 --- a/evtlog/pfsevtlogd/Makefile +++ b/evtlog/pfsevtlogd/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/evtlog/pfsevtlogd +gosubdir := github.com/NVIDIA/proxyfs/evtlog/pfsevtlogd include ../../GoMakefile diff --git a/fs/Makefile b/fs/Makefile index 9ea8bf26a..cb3621560 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/fs +gosubdir := github.com/NVIDIA/proxyfs/fs include ../GoMakefile diff --git a/fsworkout/Makefile b/fsworkout/Makefile index b952ccda5..29e827699 100644 --- a/fsworkout/Makefile +++ b/fsworkout/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/fsworkout +gosubdir := github.com/NVIDIA/proxyfs/fsworkout include ../GoMakefile diff --git a/fuse/Makefile b/fuse/Makefile index 08f887e2e..38e27e074 100644 --- a/fuse/Makefile +++ b/fuse/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/fuse +gosubdir := github.com/NVIDIA/proxyfs/fuse include ../GoMakefile diff --git a/halter/Makefile b/halter/Makefile index cdafa6665..0f9999ec2 100644 --- a/halter/Makefile +++ b/halter/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/halter +gosubdir := github.com/NVIDIA/proxyfs/halter include ../GoMakefile diff --git a/headhunter/Makefile b/headhunter/Makefile index faeee50b5..a5302e8b0 100644 --- a/headhunter/Makefile +++ b/headhunter/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/headhunter +gosubdir := github.com/NVIDIA/proxyfs/headhunter include ../GoMakefile diff --git a/httpserver/Makefile b/httpserver/Makefile index 3b3143213..1f29bddd5 100644 --- a/httpserver/Makefile +++ b/httpserver/Makefile @@ -1,7 +1,7 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/httpserver +gosubdir := github.com/NVIDIA/proxyfs/httpserver generatedfiles := \ bootstrap_dot_min_dot_css_.go \ diff --git a/imgr/Makefile b/imgr/Makefile index f699a7896..8eace0306 100644 --- a/imgr/Makefile +++ b/imgr/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/imgr +gosubdir := github.com/NVIDIA/proxyfs/imgr include ../GoMakefile diff --git a/imgr/imgrpkg/Makefile b/imgr/imgrpkg/Makefile index 9fe6f3f1e..a2b7d2743 100644 --- a/imgr/imgrpkg/Makefile +++ b/imgr/imgrpkg/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/imgr/imgrpkg +gosubdir := github.com/NVIDIA/proxyfs/imgr/imgrpkg include ../../GoMakefile diff --git a/inode/Makefile b/inode/Makefile index 1e5093a83..1122a376b 100644 --- a/inode/Makefile +++ b/inode/Makefile @@ -1,7 +1,7 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/inode +gosubdir := github.com/NVIDIA/proxyfs/inode generatedfiles := \ inodetype_string.go diff --git a/inodeworkout/Makefile b/inodeworkout/Makefile index f10219ffc..381033738 100644 --- a/inodeworkout/Makefile +++ b/inodeworkout/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/inodeworkout +gosubdir := github.com/NVIDIA/proxyfs/inodeworkout include ../GoMakefile diff --git a/jrpcfs/Makefile b/jrpcfs/Makefile index d7ef57f80..37cc18439 100644 --- a/jrpcfs/Makefile +++ b/jrpcfs/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/jrpcfs +gosubdir := github.com/NVIDIA/proxyfs/jrpcfs include ../GoMakefile diff --git a/jrpcfs/test/Makefile b/jrpcfs/test/Makefile index 46e13461f..8c3d6a4a6 100644 --- a/jrpcfs/test/Makefile +++ b/jrpcfs/test/Makefile @@ -3,7 +3,7 @@ CC=gcc CFLAGS=-I. -JSON_C_DIR=/Users/kmalone/code/gowork/src/github.com/swiftstack/samba/json-c +JSON_C_DIR=/Users/kmalone/code/gowork/src/github.com/NVIDIA/samba/json-c CFLAGS += -I $(JSON_C_DIR)/include/json-c LDFLAGS+= -L $(JSON_C_DIR)/lib -ljson-c DEPS = diff --git a/liveness/Makefile b/liveness/Makefile index 23e81fd77..b18e1302f 100644 --- a/liveness/Makefile +++ b/liveness/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/liveness +gosubdir := github.com/NVIDIA/proxyfs/liveness include ../GoMakefile diff --git a/logger/Makefile b/logger/Makefile index 7738b86e0..369af8406 100644 --- a/logger/Makefile +++ b/logger/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/logger +gosubdir := github.com/NVIDIA/proxyfs/logger include ../GoMakefile diff --git a/make-static-content/Makefile b/make-static-content/Makefile index 92a9b4209..e70a62277 100644 --- a/make-static-content/Makefile +++ b/make-static-content/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/make-static-content +gosubdir := github.com/NVIDIA/proxyfs/make-static-content include ../GoMakefile diff --git a/mkproxyfs/Makefile b/mkproxyfs/Makefile index 7c4ac9128..fd2018643 100644 --- a/mkproxyfs/Makefile +++ b/mkproxyfs/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/mkproxyfs +gosubdir := github.com/NVIDIA/proxyfs/mkproxyfs include ../GoMakefile diff --git a/mkproxyfs/mkproxyfs/Makefile b/mkproxyfs/mkproxyfs/Makefile index 4cb5486f4..66c1094a4 100644 --- a/mkproxyfs/mkproxyfs/Makefile +++ b/mkproxyfs/mkproxyfs/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/mkproxyfs/mkproxyfs +gosubdir := github.com/NVIDIA/proxyfs/mkproxyfs/mkproxyfs include ../../GoMakefile diff --git a/pfs-crash/Makefile b/pfs-crash/Makefile index 29b1e3fb1..20de57d27 100644 --- a/pfs-crash/Makefile +++ b/pfs-crash/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfs-crash +gosubdir := github.com/NVIDIA/proxyfs/pfs-crash include ../GoMakefile diff --git a/pfs-fsck/Makefile b/pfs-fsck/Makefile index c8238cc79..e8e6dce1c 100644 --- a/pfs-fsck/Makefile +++ b/pfs-fsck/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfs-fsck +gosubdir := github.com/NVIDIA/proxyfs/pfs-fsck include ../GoMakefile diff --git a/pfs-jrpc/Makefile b/pfs-jrpc/Makefile index c1533ec88..741445ad4 100644 --- a/pfs-jrpc/Makefile +++ b/pfs-jrpc/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfs-jrpc +gosubdir := github.com/NVIDIA/proxyfs/pfs-jrpc include ../GoMakefile diff --git a/pfs-restart-test/Makefile b/pfs-restart-test/Makefile index e493afa18..83b2e56c8 100644 --- a/pfs-restart-test/Makefile +++ b/pfs-restart-test/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfs-restart-test +gosubdir := github.com/NVIDIA/proxyfs/pfs-restart-test include ../GoMakefile diff --git a/pfs-stress/Makefile b/pfs-stress/Makefile index a3239593e..48c005561 100644 --- a/pfs-stress/Makefile +++ b/pfs-stress/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfs-stress +gosubdir := github.com/NVIDIA/proxyfs/pfs-stress include ../GoMakefile diff --git a/pfs-swift-load/Makefile b/pfs-swift-load/Makefile index d6a7ad42f..fdcb9dc14 100644 --- a/pfs-swift-load/Makefile +++ b/pfs-swift-load/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfs-swift-load +gosubdir := github.com/NVIDIA/proxyfs/pfs-swift-load include ../GoMakefile diff --git a/pfsagentConfig/Makefile b/pfsagentConfig/Makefile index 431d5a9cb..1ae0a6c6b 100644 --- a/pfsagentConfig/Makefile +++ b/pfsagentConfig/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfsagentConfig +gosubdir := github.com/NVIDIA/proxyfs/pfsagentConfig include ../GoMakefile diff --git a/pfsagentConfig/pfsagentConfig/Makefile b/pfsagentConfig/pfsagentConfig/Makefile index 65670235e..d43c7af26 100644 --- a/pfsagentConfig/pfsagentConfig/Makefile +++ b/pfsagentConfig/pfsagentConfig/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfsagentConfig/pfsagentConfig +gosubdir := github.com/NVIDIA/proxyfs/pfsagentConfig/pfsagentConfig include ../../GoMakefile diff --git a/pfsagentd/Makefile b/pfsagentd/Makefile index f0e2f3226..d49ed7722 100644 --- a/pfsagentd/Makefile +++ b/pfsagentd/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfsagentd +gosubdir := github.com/NVIDIA/proxyfs/pfsagentd include ../GoMakefile diff --git a/pfsagentd/pfsagentd-init/Makefile b/pfsagentd/pfsagentd-init/Makefile index e47578cc3..1251963f8 100644 --- a/pfsagentd/pfsagentd-init/Makefile +++ b/pfsagentd/pfsagentd-init/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfsagentd/pfsagentd-init +gosubdir := github.com/NVIDIA/proxyfs/pfsagentd/pfsagentd-init include ../../GoMakefile diff --git a/pfsagentd/pfsagentd-swift-auth-plugin/Makefile b/pfsagentd/pfsagentd-swift-auth-plugin/Makefile index c8ec40d10..2b2feefe7 100644 --- a/pfsagentd/pfsagentd-swift-auth-plugin/Makefile +++ b/pfsagentd/pfsagentd-swift-auth-plugin/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfsagentd/pfsagentd-swift-auth-plugin +gosubdir := github.com/NVIDIA/proxyfs/pfsagentd/pfsagentd-swift-auth-plugin include ../../GoMakefile diff --git a/pfsconfjson/Makefile b/pfsconfjson/Makefile index f81150fef..63e2cd571 100644 --- a/pfsconfjson/Makefile +++ b/pfsconfjson/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfsconfjson +gosubdir := github.com/NVIDIA/proxyfs/pfsconfjson include ../GoMakefile diff --git a/pfsconfjsonpacked/Makefile b/pfsconfjsonpacked/Makefile index a84882ca2..069317fbf 100644 --- a/pfsconfjsonpacked/Makefile +++ b/pfsconfjsonpacked/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfsconfjsonpacked +gosubdir := github.com/NVIDIA/proxyfs/pfsconfjsonpacked include ../GoMakefile diff --git a/pfsworkout/Makefile b/pfsworkout/Makefile index 9a9be24f1..eab2cae90 100644 --- a/pfsworkout/Makefile +++ b/pfsworkout/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/pfsworkout +gosubdir := github.com/NVIDIA/proxyfs/pfsworkout include ../GoMakefile diff --git a/platform/Makefile b/platform/Makefile index d053f9ae0..31637cc06 100644 --- a/platform/Makefile +++ b/platform/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/platform +gosubdir := github.com/NVIDIA/proxyfs/platform include ../GoMakefile diff --git a/proxyfsd/Makefile b/proxyfsd/Makefile index a5e91398c..850db1fa8 100644 --- a/proxyfsd/Makefile +++ b/proxyfsd/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/proxyfsd +gosubdir := github.com/NVIDIA/proxyfs/proxyfsd include ../GoMakefile diff --git a/proxyfsd/proxyfsd/Makefile b/proxyfsd/proxyfsd/Makefile index a1200f3a4..83d406207 100644 --- a/proxyfsd/proxyfsd/Makefile +++ b/proxyfsd/proxyfsd/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/proxyfsd/proxyfsd +gosubdir := github.com/NVIDIA/proxyfs/proxyfsd/proxyfsd include ../../GoMakefile diff --git a/ramswift/Makefile b/ramswift/Makefile index 4319b2cb7..fa8948bd5 100644 --- a/ramswift/Makefile +++ b/ramswift/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/ramswift +gosubdir := github.com/NVIDIA/proxyfs/ramswift include ../GoMakefile diff --git a/ramswift/ramswift/Makefile b/ramswift/ramswift/Makefile index e617a9e25..3c34d0687 100644 --- a/ramswift/ramswift/Makefile +++ b/ramswift/ramswift/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/ramswift/ramswift +gosubdir := github.com/NVIDIA/proxyfs/ramswift/ramswift include ../../GoMakefile diff --git a/retryrpc/Makefile b/retryrpc/Makefile index 69fd8bb6f..42dedddfe 100644 --- a/retryrpc/Makefile +++ b/retryrpc/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/retryrpc +gosubdir := github.com/NVIDIA/proxyfs/retryrpc include ../GoMakefile diff --git a/retryrpc/rpctest/Makefile b/retryrpc/rpctest/Makefile index 10472ef9a..85d4ff7ee 100644 --- a/retryrpc/rpctest/Makefile +++ b/retryrpc/rpctest/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/retryrpc/rpctest +gosubdir := github.com/NVIDIA/proxyfs/retryrpc/rpctest include ../../GoMakefile diff --git a/stats/Makefile b/stats/Makefile index 35caea243..5b4416209 100644 --- a/stats/Makefile +++ b/stats/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/stats +gosubdir := github.com/NVIDIA/proxyfs/stats include ../GoMakefile diff --git a/statslogger/Makefile b/statslogger/Makefile index cb2586dbd..dd3b90488 100644 --- a/statslogger/Makefile +++ b/statslogger/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/statslogger +gosubdir := github.com/NVIDIA/proxyfs/statslogger include ../GoMakefile diff --git a/swiftclient/Makefile b/swiftclient/Makefile index 725ecb28d..d92557039 100644 --- a/swiftclient/Makefile +++ b/swiftclient/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/swiftclient +gosubdir := github.com/NVIDIA/proxyfs/swiftclient include ../GoMakefile diff --git a/trackedlock/Makefile b/trackedlock/Makefile index d5c683823..d047b0e9e 100644 --- a/trackedlock/Makefile +++ b/trackedlock/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/trackedlock +gosubdir := github.com/NVIDIA/proxyfs/trackedlock include ../GoMakefile diff --git a/transitions/Makefile b/transitions/Makefile index a5e23279c..37a959a61 100644 --- a/transitions/Makefile +++ b/transitions/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/transitions +gosubdir := github.com/NVIDIA/proxyfs/transitions include ../GoMakefile diff --git a/utils/Makefile b/utils/Makefile index d7d7254b7..1af05292d 100644 --- a/utils/Makefile +++ b/utils/Makefile @@ -1,6 +1,6 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/utils +gosubdir := github.com/NVIDIA/proxyfs/utils include ../GoMakefile diff --git a/version/Makefile b/version/Makefile index 99075641d..20962736b 100644 --- a/version/Makefile +++ b/version/Makefile @@ -1,7 +1,7 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -gosubdir := github.com/swiftstack/ProxyFS/version +gosubdir := github.com/NVIDIA/proxyfs/version generatedfiles := \ proxyfs_version.go From e6b11b5f62f21984a3f5d31d3953089c5add0162 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 8 Feb 2021 10:18:41 -0800 Subject: [PATCH 032/160] Migrated all .go files --- blunder/api.go | 2 +- blunder/api_test.go | 6 ++--- cleanproxyfs/main.go | 6 ++--- confgen/api_internal.go | 2 +- confgen/api_test.go | 2 +- confgen/confgen/main.go | 2 +- dlm/api.go | 2 +- dlm/config.go | 6 ++--- dlm/llm.go | 4 +-- dlm/llm_test.go | 10 +++---- emswift/emswiftpkg/api.go | 2 +- emswift/emswiftpkg/impl.go | 6 ++--- emswift/emswiftpkg/pkg_test.go | 4 +-- emswift/main.go | 4 +-- evtlog/api_test.go | 4 +-- evtlog/benchmark_test.go | 4 +-- evtlog/config.go | 8 +++--- evtlog/pfsevtlogd/main.go | 6 ++--- fs/api.go | 4 +-- fs/api_internal.go | 10 +++---- fs/api_test.go | 4 +-- fs/config.go | 14 +++++----- fs/metadata_stress_test.go | 4 +-- fs/resolve_path.go | 10 +++---- fs/resolve_path_test.go | 2 +- fs/setup_teardown_test.go | 6 ++--- fs/treewalk.go | 16 ++++++------ fs/utils.go | 2 +- fsworkout/main.go | 10 +++---- fuse/config.go | 10 +++---- fuse/dir.go | 8 +++--- fuse/file.go | 6 ++--- fuse/fuse.go | 6 ++--- fuse/symlink.go | 6 ++--- halter/api.go | 2 +- halter/api_test.go | 4 +-- halter/config.go | 6 ++--- headhunter/api.go | 2 +- headhunter/api_internal.go | 8 +++--- headhunter/api_test.go | 6 ++--- headhunter/checkpoint.go | 18 ++++++------- headhunter/config.go | 20 +++++++------- headhunter/sortedmap_helper.go | 12 ++++----- headhunter/stress_test.go | 6 ++--- httpserver/config.go | 20 +++++++------- httpserver/request_handler.go | 26 +++++++++---------- httpserver/setup_teardown_test.go | 6 ++--- imgr/globals.go | 4 +-- imgr/main.go | 2 +- inode/api.go | 6 ++--- inode/api_test.go | 4 +-- inode/benchmark_test.go | 2 +- inode/cache.go | 6 ++--- inode/coalesce_test.go | 2 +- inode/config.go | 20 +++++++------- inode/config_test.go | 2 +- inode/cron.go | 6 ++--- inode/cron_test.go | 2 +- inode/dir.go | 12 ++++----- inode/dir_stress_test.go | 4 +-- inode/file.go | 12 ++++----- inode/file_extent_test.go | 2 +- inode/file_flusher.go | 10 +++---- inode/gc_test.go | 2 +- inode/inode.go | 26 +++++++++---------- inode/locker.go | 2 +- inode/locker_test.go | 4 +-- inode/payload.go | 10 +++---- inode/setup_teardown_test.go | 6 ++--- inode/symlink.go | 4 +-- inode/validate_test.go | 4 +-- inode/volume.go | 6 ++--- inodeworkout/main.go | 8 +++--- jrpcfs/api.go | 6 ++--- jrpcfs/config.go | 12 ++++----- jrpcfs/filesystem.go | 12 ++++----- jrpcfs/io.go | 12 ++++----- jrpcfs/lease.go | 10 +++---- jrpcfs/lease_test.go | 2 +- jrpcfs/middleware.go | 10 +++---- jrpcfs/middleware_test.go | 16 ++++++------ jrpcfs/ping.go | 2 +- jrpcfs/retryrpc.go | 4 +-- liveness/config.go | 10 +++---- liveness/messages.go | 4 +-- liveness/polling.go | 6 ++--- liveness/states.go | 4 +-- logger/api.go | 2 +- logger/api_test.go | 4 +-- logger/config.go | 2 +- mkproxyfs/api.go | 16 ++++++------ mkproxyfs/mkproxyfs/main.go | 2 +- pfs-crash/main.go | 6 ++--- pfs-fsck/main.go | 10 +++---- pfs-jrpc/main.go | 4 +-- pfs-restart-test/main.go | 2 +- pfs-stress/main.go | 2 +- pfs-swift-load/main.go | 2 +- pfsagentConfig/pfsagentConfig/main.go | 2 +- pfsagentConfig/utils.go | 2 +- pfsagentd/file_inode.go | 6 ++--- pfsagentd/fission.go | 10 +++---- pfsagentd/globals.go | 18 ++++++------- pfsagentd/http_server.go | 6 ++--- pfsagentd/lease.go | 4 +-- pfsagentd/main.go | 2 +- pfsagentd/pfsagentd-init/main.go | 2 +- pfsagentd/pfsagentd-swift-auth-plugin/main.go | 2 +- pfsagentd/request.go | 6 ++--- pfsagentd/setup_teardown_test.go | 6 ++--- pfsagentd/swift_proxy_emulator_test.go | 2 +- pfsagentd/unit_test.go | 2 +- pfsconfjson/main.go | 4 +-- pfsconfjsonpacked/main.go | 4 +-- pfsworkout/main.go | 16 ++++++------ proxyfsd/daemon.go | 10 +++---- proxyfsd/daemon_test.go | 6 ++--- proxyfsd/proxyfsd/main.go | 2 +- ramswift/daemon.go | 8 +++--- ramswift/daemon_test.go | 2 +- ramswift/ramswift/main.go | 2 +- retryrpc/api.go | 4 +-- retryrpc/api_internal.go | 4 +-- retryrpc/client.go | 4 +-- retryrpc/retryrpc_test.go | 4 +-- retryrpc/rpctest/ping.go | 2 +- retryrpc/server.go | 4 +-- retryrpc/stress_test.go | 2 +- retryrpc/upcall_test.go | 2 +- stats/api_test.go | 4 +-- stats/config.go | 6 ++--- stats/sender.go | 2 +- statslogger/config.go | 12 ++++----- statslogger/config_test.go | 8 +++--- swiftclient/account.go | 8 +++--- swiftclient/api_test.go | 12 ++++----- swiftclient/config.go | 10 +++---- swiftclient/container.go | 8 +++--- swiftclient/http_status.go | 2 +- swiftclient/object.go | 12 ++++----- swiftclient/retry.go | 6 ++--- swiftclient/utils.go | 6 ++--- trackedlock/api.go | 2 +- trackedlock/api_internal.go | 4 +-- trackedlock/api_test.go | 4 +-- trackedlock/config.go | 6 ++--- transitions/api.go | 6 ++--- transitions/api_internal.go | 4 +-- transitions/api_test.go | 2 +- utils/api.go | 10 +++---- 150 files changed, 474 insertions(+), 474 deletions(-) diff --git a/blunder/api.go b/blunder/api.go index 5639861e0..cd78e441d 100644 --- a/blunder/api.go +++ b/blunder/api.go @@ -36,7 +36,7 @@ import ( "github.com/ansel1/merry" "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/logger" ) // Error constants to be used in the ProxyFS namespace. diff --git a/blunder/api_test.go b/blunder/api_test.go index 31874c6bd..9a1d05bf5 100644 --- a/blunder/api_test.go +++ b/blunder/api_test.go @@ -9,9 +9,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/transitions" ) var testConfMap conf.ConfMap diff --git a/cleanproxyfs/main.go b/cleanproxyfs/main.go index 1324c7377..fbcac8555 100644 --- a/cleanproxyfs/main.go +++ b/cleanproxyfs/main.go @@ -13,9 +13,9 @@ import ( "os" "strings" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/transitions" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/transitions" + "github.com/NVIDIA/proxyfs/utils" ) const ( diff --git a/confgen/api_internal.go b/confgen/api_internal.go index db629da28..2a635fa52 100644 --- a/confgen/api_internal.go +++ b/confgen/api_internal.go @@ -11,7 +11,7 @@ import ( "strconv" "strings" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) type envSettingsStruct struct { diff --git a/confgen/api_test.go b/confgen/api_test.go index b5b9f2353..dc5cd9fa5 100644 --- a/confgen/api_test.go +++ b/confgen/api_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) func getConfMap(t *testing.T, filename string) (confMap conf.ConfMap, err error) { diff --git a/confgen/confgen/main.go b/confgen/confgen/main.go index f971dffeb..a8f05ddf5 100644 --- a/confgen/confgen/main.go +++ b/confgen/confgen/main.go @@ -16,7 +16,7 @@ import ( "os" "strings" - "github.com/swiftstack/ProxyFS/confgen" + "github.com/NVIDIA/proxyfs/confgen" ) func usage() { diff --git a/dlm/api.go b/dlm/api.go index 61167c9b7..843a0a9c1 100644 --- a/dlm/api.go +++ b/dlm/api.go @@ -54,7 +54,7 @@ package dlm import ( "fmt" - "github.com/swiftstack/ProxyFS/trackedlock" + "github.com/NVIDIA/proxyfs/trackedlock" ) type NotifyReason uint32 diff --git a/dlm/config.go b/dlm/config.go index fbf0583cd..a89056300 100644 --- a/dlm/config.go +++ b/dlm/config.go @@ -6,9 +6,9 @@ package dlm // Configuration variables for DLM import ( - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) type globalsStruct struct { diff --git a/dlm/llm.go b/dlm/llm.go index aa0b7e1b1..4a3090ad0 100644 --- a/dlm/llm.go +++ b/dlm/llm.go @@ -10,8 +10,8 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/trackedlock" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/trackedlock" ) // This struct is used by LLM to track a lock. diff --git a/dlm/llm_test.go b/dlm/llm_test.go index 7ab6ed16f..7682f1700 100644 --- a/dlm/llm_test.go +++ b/dlm/llm_test.go @@ -13,11 +13,11 @@ import ( "github.com/stretchr/testify/assert" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) // Test string for passing inode 1 diff --git a/emswift/emswiftpkg/api.go b/emswift/emswiftpkg/api.go index 080d66b3b..308cbd9da 100644 --- a/emswift/emswiftpkg/api.go +++ b/emswift/emswiftpkg/api.go @@ -4,7 +4,7 @@ package emswiftpkg import ( - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) // Start is called to start serving the NoAuth Swift Proxy Port and, diff --git a/emswift/emswiftpkg/impl.go b/emswift/emswiftpkg/impl.go index 1473fb024..34296a622 100644 --- a/emswift/emswiftpkg/impl.go +++ b/emswift/emswiftpkg/impl.go @@ -17,10 +17,10 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/utils" ) type swiftAccountStruct struct { diff --git a/emswift/emswiftpkg/pkg_test.go b/emswift/emswiftpkg/pkg_test.go index 949b0fb56..31a3c896e 100644 --- a/emswift/emswiftpkg/pkg_test.go +++ b/emswift/emswiftpkg/pkg_test.go @@ -13,8 +13,8 @@ import ( "sync" "testing" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/utils" ) func TestAuthEmulation(t *testing.T) { diff --git a/emswift/main.go b/emswift/main.go index 94a14a148..4e9073845 100644 --- a/emswift/main.go +++ b/emswift/main.go @@ -10,8 +10,8 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/emswift/emswiftpkg" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/emswift/emswiftpkg" ) func main() { diff --git a/evtlog/api_test.go b/evtlog/api_test.go index fc60df978..95cb15111 100644 --- a/evtlog/api_test.go +++ b/evtlog/api_test.go @@ -6,8 +6,8 @@ package evtlog import ( "testing" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/transitions" ) func TestAPI(t *testing.T) { diff --git a/evtlog/benchmark_test.go b/evtlog/benchmark_test.go index 8c60d4c09..cd4476050 100644 --- a/evtlog/benchmark_test.go +++ b/evtlog/benchmark_test.go @@ -6,8 +6,8 @@ package evtlog import ( "testing" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/transitions" ) var ( diff --git a/evtlog/config.go b/evtlog/config.go index 52a394996..786a743d0 100644 --- a/evtlog/config.go +++ b/evtlog/config.go @@ -8,10 +8,10 @@ import ( "syscall" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) // #include diff --git a/evtlog/pfsevtlogd/main.go b/evtlog/pfsevtlogd/main.go index a35c9ed0a..a585da13e 100644 --- a/evtlog/pfsevtlogd/main.go +++ b/evtlog/pfsevtlogd/main.go @@ -13,9 +13,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/evtlog" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/evtlog" + "github.com/NVIDIA/proxyfs/transitions" ) func usage() { diff --git a/fs/api.go b/fs/api.go index 85f8aa125..350bf07cc 100644 --- a/fs/api.go +++ b/fs/api.go @@ -10,8 +10,8 @@ import "C" import ( "time" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/utils" ) // ReadRangeIn is the ReadPlan range requested diff --git a/fs/api_internal.go b/fs/api_internal.go index 6e67c3511..17a7c5fc7 100644 --- a/fs/api_internal.go +++ b/fs/api_internal.go @@ -14,11 +14,11 @@ import ( "syscall" "time" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/dlm" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/dlm" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/utils" ) // Shorthand for our internal API debug log id; global to the package diff --git a/fs/api_test.go b/fs/api_test.go index a4d030a88..62c944539 100644 --- a/fs/api_test.go +++ b/fs/api_test.go @@ -13,8 +13,8 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/inode" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/inode" ) // TODO: Enhance this to do a stat() as well and check number of files diff --git a/fs/config.go b/fs/config.go index 30d9f50ce..32af7807c 100644 --- a/fs/config.go +++ b/fs/config.go @@ -9,13 +9,13 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) type inFlightFileInodeDataStruct struct { diff --git a/fs/metadata_stress_test.go b/fs/metadata_stress_test.go index ad22f8660..a4e9ee263 100644 --- a/fs/metadata_stress_test.go +++ b/fs/metadata_stress_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/utils" ) // diff --git a/fs/resolve_path.go b/fs/resolve_path.go index 8d038f879..0de667c08 100644 --- a/fs/resolve_path.go +++ b/fs/resolve_path.go @@ -7,11 +7,11 @@ import ( "fmt" "strings" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/dlm" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/dlm" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/utils" ) type resolvePathOption uint32 diff --git a/fs/resolve_path_test.go b/fs/resolve_path_test.go index c968337d6..5a6a521d2 100644 --- a/fs/resolve_path_test.go +++ b/fs/resolve_path_test.go @@ -6,7 +6,7 @@ package fs import ( "testing" - "github.com/swiftstack/ProxyFS/inode" + "github.com/NVIDIA/proxyfs/inode" ) func TestResolvePath(t *testing.T) { diff --git a/fs/setup_teardown_test.go b/fs/setup_teardown_test.go index 19976e24b..4ef4d7516 100644 --- a/fs/setup_teardown_test.go +++ b/fs/setup_teardown_test.go @@ -13,9 +13,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/ramswift" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/ramswift" + "github.com/NVIDIA/proxyfs/transitions" ) var ( diff --git a/fs/treewalk.go b/fs/treewalk.go index 9148fe20f..1faf29e49 100644 --- a/fs/treewalk.go +++ b/fs/treewalk.go @@ -14,14 +14,14 @@ import ( "sync" "time" - "github.com/swiftstack/sortedmap" - - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/dlm" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/swiftclient" + "github.com/NVIDIA/sortedmap" + + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/dlm" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/swiftclient" ) const ( diff --git a/fs/utils.go b/fs/utils.go index aadeb3e11..5df8460db 100644 --- a/fs/utils.go +++ b/fs/utils.go @@ -10,7 +10,7 @@ import ( "math/big" "time" - "github.com/swiftstack/ProxyFS/stats" + "github.com/NVIDIA/proxyfs/stats" ) func (tryLockBackoffContext *tryLockBackoffContextStruct) backoff() { diff --git a/fsworkout/main.go b/fsworkout/main.go index 1295ff7c9..7cf58d684 100644 --- a/fsworkout/main.go +++ b/fsworkout/main.go @@ -10,11 +10,11 @@ import ( "strconv" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) const ( diff --git a/fuse/config.go b/fuse/config.go index e5103db3d..5d9d84aff 100644 --- a/fuse/config.go +++ b/fuse/config.go @@ -15,11 +15,11 @@ import ( fuselib "bazil.org/fuse" fusefslib "bazil.org/fuse/fs" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) const ( diff --git a/fuse/dir.go b/fuse/dir.go index 2af93486a..687e9ba3d 100644 --- a/fuse/dir.go +++ b/fuse/dir.go @@ -12,10 +12,10 @@ import ( fusefslib "bazil.org/fuse/fs" "golang.org/x/net/context" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" ) type Dir struct { diff --git a/fuse/file.go b/fuse/file.go index a4fc8a509..39ea8e793 100644 --- a/fuse/file.go +++ b/fuse/file.go @@ -12,9 +12,9 @@ import ( fuselib "bazil.org/fuse" "golang.org/x/net/context" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" ) type File struct { diff --git a/fuse/fuse.go b/fuse/fuse.go index 80f54a39e..a2293eb5b 100644 --- a/fuse/fuse.go +++ b/fuse/fuse.go @@ -11,9 +11,9 @@ import ( fusefslib "bazil.org/fuse/fs" "golang.org/x/net/context" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" ) type ProxyFUSE struct { diff --git a/fuse/symlink.go b/fuse/symlink.go index 6890e5eb8..cec57f442 100644 --- a/fuse/symlink.go +++ b/fuse/symlink.go @@ -11,9 +11,9 @@ import ( fuselib "bazil.org/fuse" "golang.org/x/net/context" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" ) type Symlink struct { diff --git a/halter/api.go b/halter/api.go index be70ab80d..8d64fcdca 100644 --- a/halter/api.go +++ b/halter/api.go @@ -8,7 +8,7 @@ import ( "os" "syscall" - "github.com/swiftstack/ProxyFS/evtlog" + "github.com/NVIDIA/proxyfs/evtlog" ) // Note 1: Following const block and HaltLabelStrings should be kept in sync diff --git a/halter/api_test.go b/halter/api_test.go index 0da377efb..1be723f3b 100644 --- a/halter/api_test.go +++ b/halter/api_test.go @@ -7,8 +7,8 @@ import ( "fmt" "testing" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/transitions" ) var ( diff --git a/halter/config.go b/halter/config.go index 462856e57..bf112b532 100644 --- a/halter/config.go +++ b/halter/config.go @@ -4,9 +4,9 @@ package halter import ( - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) type globalsStruct struct { diff --git a/headhunter/api.go b/headhunter/api.go index 097795710..8fecdebc1 100644 --- a/headhunter/api.go +++ b/headhunter/api.go @@ -8,7 +8,7 @@ import ( "fmt" "time" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" ) type BPlusTreeType uint32 diff --git a/headhunter/api_internal.go b/headhunter/api_internal.go index 94a15f3e6..eb3d3804f 100644 --- a/headhunter/api_internal.go +++ b/headhunter/api_internal.go @@ -9,11 +9,11 @@ import ( "math/big" "time" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/evtlog" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/swiftclient" + "github.com/NVIDIA/proxyfs/evtlog" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/swiftclient" ) func (volume *volumeStruct) RegisterForEvents(listener VolumeEventListener) { diff --git a/headhunter/api_test.go b/headhunter/api_test.go index acc051859..2557ef9a4 100644 --- a/headhunter/api_test.go +++ b/headhunter/api_test.go @@ -10,9 +10,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/ramswift" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/ramswift" + "github.com/NVIDIA/proxyfs/transitions" ) func inodeRecPutGet(t *testing.T, volume VolumeHandle, key uint64, value []byte) { diff --git a/headhunter/checkpoint.go b/headhunter/checkpoint.go index 5bd58451c..de33bfb5d 100644 --- a/headhunter/checkpoint.go +++ b/headhunter/checkpoint.go @@ -19,15 +19,15 @@ import ( etcd "go.etcd.io/etcd/clientv3" - "github.com/swiftstack/cstruct" - "github.com/swiftstack/sortedmap" - - "github.com/swiftstack/ProxyFS/evtlog" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/platform" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/cstruct" + "github.com/NVIDIA/sortedmap" + + "github.com/NVIDIA/proxyfs/evtlog" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/platform" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/utils" ) var ( diff --git a/headhunter/config.go b/headhunter/config.go index e8c2c1794..cd684bbca 100644 --- a/headhunter/config.go +++ b/headhunter/config.go @@ -14,16 +14,16 @@ import ( etcd "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/pkg/transport" - "github.com/swiftstack/cstruct" - "github.com/swiftstack/sortedmap" - - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/etcdclient" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/cstruct" + "github.com/NVIDIA/sortedmap" + + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/etcdclient" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) const ( diff --git a/headhunter/sortedmap_helper.go b/headhunter/sortedmap_helper.go index 73c3aa53b..7b447afc3 100644 --- a/headhunter/sortedmap_helper.go +++ b/headhunter/sortedmap_helper.go @@ -6,13 +6,13 @@ package headhunter import ( "fmt" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/evtlog" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/evtlog" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/utils" ) func (bPlusTreeWrapper *bPlusTreeWrapperStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { diff --git a/headhunter/stress_test.go b/headhunter/stress_test.go index 2731541bc..76ebe3f67 100644 --- a/headhunter/stress_test.go +++ b/headhunter/stress_test.go @@ -13,9 +13,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/ramswift" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/ramswift" + "github.com/NVIDIA/proxyfs/transitions" ) const ( diff --git a/httpserver/config.go b/httpserver/config.go index 69caaff4a..0c026f5f1 100644 --- a/httpserver/config.go +++ b/httpserver/config.go @@ -10,19 +10,19 @@ import ( "sync" "time" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/transitions" // Force importing of the following next "top-most" packages - _ "github.com/swiftstack/ProxyFS/fuse" - _ "github.com/swiftstack/ProxyFS/jrpcfs" - _ "github.com/swiftstack/ProxyFS/statslogger" - "github.com/swiftstack/ProxyFS/trackedlock" + _ "github.com/NVIDIA/proxyfs/fuse" + _ "github.com/NVIDIA/proxyfs/jrpcfs" + _ "github.com/NVIDIA/proxyfs/statslogger" + "github.com/NVIDIA/proxyfs/trackedlock" ) type ExtentMapElementStruct struct { diff --git a/httpserver/request_handler.go b/httpserver/request_handler.go index 813a27845..b8eed5c94 100644 --- a/httpserver/request_handler.go +++ b/httpserver/request_handler.go @@ -19,19 +19,19 @@ import ( "strings" "time" - "github.com/swiftstack/sortedmap" - - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/halter" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/jrpcfs" - "github.com/swiftstack/ProxyFS/liveness" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/utils" - "github.com/swiftstack/ProxyFS/version" + "github.com/NVIDIA/sortedmap" + + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/halter" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/jrpcfs" + "github.com/NVIDIA/proxyfs/liveness" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/utils" + "github.com/NVIDIA/proxyfs/version" ) type httpRequestHandler struct{} diff --git a/httpserver/setup_teardown_test.go b/httpserver/setup_teardown_test.go index 26f9be836..5c138c419 100644 --- a/httpserver/setup_teardown_test.go +++ b/httpserver/setup_teardown_test.go @@ -10,9 +10,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/ramswift" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/ramswift" + "github.com/NVIDIA/proxyfs/transitions" ) var ( diff --git a/imgr/globals.go b/imgr/globals.go index 2f7955d82..9e7623f9f 100644 --- a/imgr/globals.go +++ b/imgr/globals.go @@ -9,8 +9,8 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/conf" ) type configStruct struct { diff --git a/imgr/main.go b/imgr/main.go index 9a86ffee9..25ebc728a 100644 --- a/imgr/main.go +++ b/imgr/main.go @@ -10,7 +10,7 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) func main() { diff --git a/inode/api.go b/inode/api.go index 832b763bb..0bd698669 100644 --- a/inode/api.go +++ b/inode/api.go @@ -10,10 +10,10 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/dlm" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/dlm" + "github.com/NVIDIA/proxyfs/utils" ) type InodeNumber uint64 diff --git a/inode/api_test.go b/inode/api_test.go index d47a46559..00c17a435 100644 --- a/inode/api_test.go +++ b/inode/api_test.go @@ -11,8 +11,8 @@ import ( "time" "github.com/stretchr/testify/assert" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/utils" ) const durationToDelayOrSkew = "100ms" diff --git a/inode/benchmark_test.go b/inode/benchmark_test.go index 312599a97..262f602ee 100644 --- a/inode/benchmark_test.go +++ b/inode/benchmark_test.go @@ -7,7 +7,7 @@ import ( "strconv" "testing" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/utils" ) func writeBenchmarkHelper(b *testing.B, byteSize uint64) { diff --git a/inode/cache.go b/inode/cache.go index 3a44f51d8..53a0a216f 100644 --- a/inode/cache.go +++ b/inode/cache.go @@ -7,9 +7,9 @@ import ( "fmt" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/platform" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/platform" ) func adoptVolumeGroupReadCacheParameters(confMap conf.ConfMap) (err error) { diff --git a/inode/coalesce_test.go b/inode/coalesce_test.go index 99dad38d1..57b5eabf9 100644 --- a/inode/coalesce_test.go +++ b/inode/coalesce_test.go @@ -8,7 +8,7 @@ import ( "time" "github.com/stretchr/testify/assert" - "github.com/swiftstack/ProxyFS/blunder" + "github.com/NVIDIA/proxyfs/blunder" ) // NB: test setup and such is in api_test.go (look for TestMain function) diff --git a/inode/config.go b/inode/config.go index 79c1837c1..1536b5e3d 100644 --- a/inode/config.go +++ b/inode/config.go @@ -9,16 +9,16 @@ import ( "time" "unsafe" - "github.com/swiftstack/cstruct" - "github.com/swiftstack/sortedmap" - - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/cstruct" + "github.com/NVIDIA/sortedmap" + + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) type readCacheKeyStruct struct { diff --git a/inode/config_test.go b/inode/config_test.go index b1e64d484..c10a6d9c8 100644 --- a/inode/config_test.go +++ b/inode/config_test.go @@ -6,7 +6,7 @@ package inode import ( "testing" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/transitions" "github.com/stretchr/testify/assert" ) diff --git a/inode/cron.go b/inode/cron.go index aa32a816d..0ce548466 100644 --- a/inode/cron.go +++ b/inode/cron.go @@ -10,9 +10,9 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/logger" ) type snapShotScheduleStruct struct { diff --git a/inode/cron_test.go b/inode/cron_test.go index d5c142fa0..f4921e83d 100644 --- a/inode/cron_test.go +++ b/inode/cron_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) func TestLoadSnapShotPolicy(t *testing.T) { diff --git a/inode/dir.go b/inode/dir.go index a4ccf7d7f..ab6668553 100644 --- a/inode/dir.go +++ b/inode/dir.go @@ -7,13 +7,13 @@ import ( "fmt" "time" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/utils" ) func (vS *volumeStruct) createRootOrSubDir(filePerm InodeMode, userID InodeUserID, groupID InodeGroupID, isRootDir bool) (dirInodeNumber InodeNumber, err error) { diff --git a/inode/dir_stress_test.go b/inode/dir_stress_test.go index a81e0d90a..a92af896c 100644 --- a/inode/dir_stress_test.go +++ b/inode/dir_stress_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/trackedlock" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/trackedlock" ) const ( diff --git a/inode/file.go b/inode/file.go index d7ea024fe..ac77bbef3 100644 --- a/inode/file.go +++ b/inode/file.go @@ -8,13 +8,13 @@ import ( "strconv" "time" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/utils" ) type fileExtentStruct struct { diff --git a/inode/file_extent_test.go b/inode/file_extent_test.go index 95421ca7f..1882422a9 100644 --- a/inode/file_extent_test.go +++ b/inode/file_extent_test.go @@ -11,7 +11,7 @@ import ( mathRand "math/rand" "testing" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" ) const ( diff --git a/inode/file_flusher.go b/inode/file_flusher.go index f68bfa9af..779cfd804 100644 --- a/inode/file_flusher.go +++ b/inode/file_flusher.go @@ -6,11 +6,11 @@ package inode import ( "fmt" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/utils" ) func openLogSegmentLRUInsertWhileLocked(inFlightLogSegment *inFlightLogSegmentStruct) { diff --git a/inode/gc_test.go b/inode/gc_test.go index 28f1c9653..8cc68afc8 100644 --- a/inode/gc_test.go +++ b/inode/gc_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/swiftstack/ProxyFS/swiftclient" + "github.com/NVIDIA/proxyfs/swiftclient" ) type testObjectLocationStruct struct { diff --git a/inode/inode.go b/inode/inode.go index ede7fcd66..25b64c2d5 100644 --- a/inode/inode.go +++ b/inode/inode.go @@ -13,19 +13,19 @@ import ( "time" "github.com/ansel1/merry" - "github.com/swiftstack/cstruct" - "github.com/swiftstack/sortedmap" - - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/dlm" - "github.com/swiftstack/ProxyFS/evtlog" - "github.com/swiftstack/ProxyFS/halter" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/cstruct" + "github.com/NVIDIA/sortedmap" + + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/dlm" + "github.com/NVIDIA/proxyfs/evtlog" + "github.com/NVIDIA/proxyfs/halter" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/utils" ) // Shorthand for inode internal API debug log id; global to the package diff --git a/inode/locker.go b/inode/locker.go index 5e665e0c1..6f32d1577 100644 --- a/inode/locker.go +++ b/inode/locker.go @@ -13,7 +13,7 @@ package inode import ( "fmt" - "github.com/swiftstack/ProxyFS/dlm" + "github.com/NVIDIA/proxyfs/dlm" ) // MakeLockID creates the ID of an inode used in volume diff --git a/inode/locker_test.go b/inode/locker_test.go index 7776f2834..17adc4687 100644 --- a/inode/locker_test.go +++ b/inode/locker_test.go @@ -6,8 +6,8 @@ package inode import ( "testing" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/dlm" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/dlm" ) func TestLockerStuff(t *testing.T) { diff --git a/inode/payload.go b/inode/payload.go index dedb9603c..48331c73d 100644 --- a/inode/payload.go +++ b/inode/payload.go @@ -7,12 +7,12 @@ import ( "bytes" "fmt" - "github.com/swiftstack/ProxyFS/evtlog" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" + "github.com/NVIDIA/proxyfs/evtlog" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" - "github.com/swiftstack/cstruct" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/cstruct" + "github.com/NVIDIA/sortedmap" ) // For DirType inodes, our payload tree-map is from basenames to inode numbers. diff --git a/inode/setup_teardown_test.go b/inode/setup_teardown_test.go index 6348f78ef..d2c307f42 100644 --- a/inode/setup_teardown_test.go +++ b/inode/setup_teardown_test.go @@ -13,9 +13,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/ramswift" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/ramswift" + "github.com/NVIDIA/proxyfs/transitions" ) var ( diff --git a/inode/symlink.go b/inode/symlink.go index 649fbd8a2..d5abb19f0 100644 --- a/inode/symlink.go +++ b/inode/symlink.go @@ -6,8 +6,8 @@ package inode import ( "fmt" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" ) func (vS *volumeStruct) CreateSymlink(target string, filePerm InodeMode, userID InodeUserID, groupID InodeGroupID) (symlinkInodeNumber InodeNumber, err error) { diff --git a/inode/validate_test.go b/inode/validate_test.go index 922eb24b1..1aba9f325 100644 --- a/inode/validate_test.go +++ b/inode/validate_test.go @@ -7,8 +7,8 @@ import ( "strings" "testing" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/swiftclient" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/swiftclient" ) // Helper to fetch a volume handle and create a file for a test. Return the diff --git a/inode/volume.go b/inode/volume.go index 17e916bb7..3e8b3563b 100644 --- a/inode/volume.go +++ b/inode/volume.go @@ -6,9 +6,9 @@ package inode import ( "fmt" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/sortedmap" ) func (vS *volumeStruct) GetFSID() (fsid uint64) { diff --git a/inodeworkout/main.go b/inodeworkout/main.go index 040f08584..1a1e9d22c 100644 --- a/inodeworkout/main.go +++ b/inodeworkout/main.go @@ -10,10 +10,10 @@ import ( "strconv" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) const ( diff --git a/jrpcfs/api.go b/jrpcfs/api.go index 8696a0150..5b31debfa 100644 --- a/jrpcfs/api.go +++ b/jrpcfs/api.go @@ -150,9 +150,9 @@ package jrpcfs import ( - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/inode" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/inode" ) // The first section of this file defines the structs used by protocols like Samba. diff --git a/jrpcfs/config.go b/jrpcfs/config.go index 51cb46ee7..31e96e341 100644 --- a/jrpcfs/config.go +++ b/jrpcfs/config.go @@ -10,12 +10,12 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/retryrpc" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/retryrpc" + "github.com/NVIDIA/proxyfs/transitions" ) type leaseRequestOperationStruct struct { diff --git a/jrpcfs/filesystem.go b/jrpcfs/filesystem.go index 90147a737..ea6ab978c 100644 --- a/jrpcfs/filesystem.go +++ b/jrpcfs/filesystem.go @@ -16,12 +16,12 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/utils" ) // RPC server handle diff --git a/jrpcfs/io.go b/jrpcfs/io.go index a4a0cefbf..55a17bce5 100644 --- a/jrpcfs/io.go +++ b/jrpcfs/io.go @@ -12,12 +12,12 @@ import ( "time" "unsafe" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/utils" ) // Server handle, used to track io-related ops diff --git a/jrpcfs/lease.go b/jrpcfs/lease.go index 33fa956fc..11acf3ac5 100644 --- a/jrpcfs/lease.go +++ b/jrpcfs/lease.go @@ -11,11 +11,11 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/evtlog" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/evtlog" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/sortedmap" ) // RpcLease is called to either request a Shared|Exclusive Lease or to diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index 6d8464cd2..f0517c1de 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/swiftstack/ProxyFS/retryrpc" + "github.com/NVIDIA/proxyfs/retryrpc" ) const ( diff --git a/jrpcfs/middleware.go b/jrpcfs/middleware.go index fc56f3c4a..1fb422f68 100644 --- a/jrpcfs/middleware.go +++ b/jrpcfs/middleware.go @@ -6,11 +6,11 @@ package jrpcfs import ( "fmt" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/utils" ) // JSON RPC Server on top of the FS package. diff --git a/jrpcfs/middleware_test.go b/jrpcfs/middleware_test.go index 04f995ec9..b699c287f 100644 --- a/jrpcfs/middleware_test.go +++ b/jrpcfs/middleware_test.go @@ -15,14 +15,14 @@ import ( "golang.org/x/sys/unix" "github.com/stretchr/testify/assert" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/ramswift" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/ramswift" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/transitions" ) // Shorthand for our testing debug log id; global to the package diff --git a/jrpcfs/ping.go b/jrpcfs/ping.go index 583dffcaf..9a6ce2688 100644 --- a/jrpcfs/ping.go +++ b/jrpcfs/ping.go @@ -7,7 +7,7 @@ package jrpcfs import ( "fmt" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/logger" ) func (s *Server) RpcPing(in *PingReq, reply *PingReply) (err error) { diff --git a/jrpcfs/retryrpc.go b/jrpcfs/retryrpc.go index e6593ad76..850b43b33 100644 --- a/jrpcfs/retryrpc.go +++ b/jrpcfs/retryrpc.go @@ -6,8 +6,8 @@ package jrpcfs import ( "time" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/retryrpc" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/retryrpc" ) func retryRPCServerUp(jserver *Server, publicIPAddr string, retryRPCPort uint16, diff --git a/liveness/config.go b/liveness/config.go index 58b77aaa4..f85d060bb 100644 --- a/liveness/config.go +++ b/liveness/config.go @@ -13,11 +13,11 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) const ( diff --git a/liveness/messages.go b/liveness/messages.go index 023c3c123..3337b2c04 100644 --- a/liveness/messages.go +++ b/liveness/messages.go @@ -12,8 +12,8 @@ import ( "reflect" "time" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" ) // Each UDP Packet Header is made up of (also in LittleEndian byte order): diff --git a/liveness/polling.go b/liveness/polling.go index 603f0b508..4ce6e85af 100644 --- a/liveness/polling.go +++ b/liveness/polling.go @@ -19,9 +19,9 @@ import ( "regexp" "time" - "github.com/swiftstack/ProxyFS/jrpcfs" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/jrpcfs" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/utils" ) type pingReqStruct struct { diff --git a/liveness/states.go b/liveness/states.go index c17281a4b..3997388cb 100644 --- a/liveness/states.go +++ b/liveness/states.go @@ -10,8 +10,8 @@ import ( "runtime" "time" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" ) func stateMachine() { diff --git a/logger/api.go b/logger/api.go index 179122e90..86753031d 100644 --- a/logger/api.go +++ b/logger/api.go @@ -26,7 +26,7 @@ import ( log "github.com/sirupsen/logrus" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/utils" ) type Level int diff --git a/logger/api_test.go b/logger/api_test.go index 814aabfbe..64ff7d6de 100644 --- a/logger/api_test.go +++ b/logger/api_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/utils" ) func testNestedFunc() { diff --git a/logger/config.go b/logger/config.go index a6bf0836d..e5c26d959 100644 --- a/logger/config.go +++ b/logger/config.go @@ -12,7 +12,7 @@ import ( log "github.com/sirupsen/logrus" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) // RFC3339 format with microsecond precision diff --git a/mkproxyfs/api.go b/mkproxyfs/api.go index 0c6901d1d..c2b92c88c 100644 --- a/mkproxyfs/api.go +++ b/mkproxyfs/api.go @@ -14,14 +14,14 @@ import ( etcd "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/pkg/transport" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/etcdclient" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/transitions" - "github.com/swiftstack/ProxyFS/version" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/etcdclient" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/transitions" + "github.com/NVIDIA/proxyfs/version" ) type Mode int diff --git a/mkproxyfs/mkproxyfs/main.go b/mkproxyfs/mkproxyfs/main.go index c9078f6ad..5d4610ac9 100644 --- a/mkproxyfs/mkproxyfs/main.go +++ b/mkproxyfs/mkproxyfs/main.go @@ -9,7 +9,7 @@ import ( "fmt" "os" - "github.com/swiftstack/ProxyFS/mkproxyfs" + "github.com/NVIDIA/proxyfs/mkproxyfs" ) func usage() { diff --git a/pfs-crash/main.go b/pfs-crash/main.go index 7968af1e9..e39541129 100644 --- a/pfs-crash/main.go +++ b/pfs-crash/main.go @@ -26,9 +26,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/httpserver" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/httpserver" + "github.com/NVIDIA/proxyfs/transitions" ) const ( diff --git a/pfs-fsck/main.go b/pfs-fsck/main.go index 8945383da..e903cf5f1 100644 --- a/pfs-fsck/main.go +++ b/pfs-fsck/main.go @@ -19,12 +19,12 @@ import ( etcd "go.etcd.io/etcd/clientv3" "go.etcd.io/etcd/pkg/transport" - "github.com/swiftstack/cstruct" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/cstruct" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/etcdclient" - "github.com/swiftstack/ProxyFS/headhunter" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/etcdclient" + "github.com/NVIDIA/proxyfs/headhunter" ) const ( diff --git a/pfs-jrpc/main.go b/pfs-jrpc/main.go index 53b820407..2bf7d548b 100644 --- a/pfs-jrpc/main.go +++ b/pfs-jrpc/main.go @@ -13,8 +13,8 @@ import ( "os" "strings" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/jrpcfs" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/jrpcfs" ) type jrpcRequestStruct struct { diff --git a/pfs-restart-test/main.go b/pfs-restart-test/main.go index 53c9ee633..befd6c083 100644 --- a/pfs-restart-test/main.go +++ b/pfs-restart-test/main.go @@ -16,7 +16,7 @@ import ( "strconv" "time" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" "golang.org/x/sys/unix" ) diff --git a/pfs-stress/main.go b/pfs-stress/main.go index 376783447..08b7d5669 100644 --- a/pfs-stress/main.go +++ b/pfs-stress/main.go @@ -16,7 +16,7 @@ import ( "sync/atomic" "time" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) var ( diff --git a/pfs-swift-load/main.go b/pfs-swift-load/main.go index 8bf9d09ef..a18801997 100644 --- a/pfs-swift-load/main.go +++ b/pfs-swift-load/main.go @@ -15,7 +15,7 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) const ( diff --git a/pfsagentConfig/pfsagentConfig/main.go b/pfsagentConfig/pfsagentConfig/main.go index ef0fe7ce8..a20373814 100644 --- a/pfsagentConfig/pfsagentConfig/main.go +++ b/pfsagentConfig/pfsagentConfig/main.go @@ -7,7 +7,7 @@ import ( "fmt" "os" - pfsagentConfig "github.com/swiftstack/ProxyFS/pfsagentConfig" + pfsagentConfig "github.com/NVIDIA/proxyfs/pfsagentConfig" ) const ( diff --git a/pfsagentConfig/utils.go b/pfsagentConfig/utils.go index 376c33e02..3fe8ffa89 100644 --- a/pfsagentConfig/utils.go +++ b/pfsagentConfig/utils.go @@ -11,7 +11,7 @@ import ( "os" "os/user" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) const ( diff --git a/pfsagentd/file_inode.go b/pfsagentd/file_inode.go index 296ab0863..004ae4bcb 100644 --- a/pfsagentd/file_inode.go +++ b/pfsagentd/file_inode.go @@ -14,10 +14,10 @@ import ( "sync/atomic" "time" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/jrpcfs" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/jrpcfs" ) func (fileInode *fileInodeStruct) doFlushIfNecessary() { diff --git a/pfsagentd/fission.go b/pfsagentd/fission.go index c98118c96..14302789f 100644 --- a/pfsagentd/fission.go +++ b/pfsagentd/fission.go @@ -12,12 +12,12 @@ import ( "syscall" "time" - "github.com/swiftstack/fission" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/fission" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/jrpcfs" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/jrpcfs" ) const ( diff --git a/pfsagentd/globals.go b/pfsagentd/globals.go index 4cd45af1d..38995ceef 100644 --- a/pfsagentd/globals.go +++ b/pfsagentd/globals.go @@ -17,15 +17,15 @@ import ( "bazil.org/fuse" - "github.com/swiftstack/fission" - "github.com/swiftstack/sortedmap" - - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/jrpcfs" - "github.com/swiftstack/ProxyFS/retryrpc" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/fission" + "github.com/NVIDIA/sortedmap" + + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/jrpcfs" + "github.com/NVIDIA/proxyfs/retryrpc" + "github.com/NVIDIA/proxyfs/utils" ) type configStruct struct { diff --git a/pfsagentd/http_server.go b/pfsagentd/http_server.go index 486d54b13..faa8833dd 100644 --- a/pfsagentd/http_server.go +++ b/pfsagentd/http_server.go @@ -19,10 +19,10 @@ import ( "strings" "sync/atomic" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/version" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/version" ) type leaseReportStruct struct { diff --git a/pfsagentd/lease.go b/pfsagentd/lease.go index 2d0f3d3f7..359dc77b3 100644 --- a/pfsagentd/lease.go +++ b/pfsagentd/lease.go @@ -8,8 +8,8 @@ import ( "encoding/json" "time" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/jrpcfs" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/jrpcfs" ) func (dummy *globalsStruct) Interrupt(rpcInterruptBuf []byte) { diff --git a/pfsagentd/main.go b/pfsagentd/main.go index 0378809dd..e0f9eb753 100644 --- a/pfsagentd/main.go +++ b/pfsagentd/main.go @@ -10,7 +10,7 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) func main() { diff --git a/pfsagentd/pfsagentd-init/main.go b/pfsagentd/pfsagentd-init/main.go index db9ae62b9..c251317a5 100644 --- a/pfsagentd/pfsagentd-init/main.go +++ b/pfsagentd/pfsagentd-init/main.go @@ -18,7 +18,7 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" "golang.org/x/sys/unix" ) diff --git a/pfsagentd/pfsagentd-swift-auth-plugin/main.go b/pfsagentd/pfsagentd-swift-auth-plugin/main.go index 1d04c9cb9..7e9037b5e 100644 --- a/pfsagentd/pfsagentd-swift-auth-plugin/main.go +++ b/pfsagentd/pfsagentd-swift-auth-plugin/main.go @@ -11,7 +11,7 @@ import ( "os" "strings" - "github.com/swiftstack/ProxyFS/version" + "github.com/NVIDIA/proxyfs/version" ) type authInStruct struct { diff --git a/pfsagentd/request.go b/pfsagentd/request.go index 5981a88d4..8631dddc6 100644 --- a/pfsagentd/request.go +++ b/pfsagentd/request.go @@ -17,9 +17,9 @@ import ( "sync/atomic" "time" - "github.com/swiftstack/ProxyFS/jrpcfs" - "github.com/swiftstack/ProxyFS/retryrpc" - "github.com/swiftstack/ProxyFS/version" + "github.com/NVIDIA/proxyfs/jrpcfs" + "github.com/NVIDIA/proxyfs/retryrpc" + "github.com/NVIDIA/proxyfs/version" ) const ( diff --git a/pfsagentd/setup_teardown_test.go b/pfsagentd/setup_teardown_test.go index 31696ac90..552391ca9 100644 --- a/pfsagentd/setup_teardown_test.go +++ b/pfsagentd/setup_teardown_test.go @@ -14,9 +14,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/proxyfsd" - "github.com/swiftstack/ProxyFS/ramswift" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/proxyfsd" + "github.com/NVIDIA/proxyfs/ramswift" ) const ( diff --git a/pfsagentd/swift_proxy_emulator_test.go b/pfsagentd/swift_proxy_emulator_test.go index d006671c3..48511fa45 100644 --- a/pfsagentd/swift_proxy_emulator_test.go +++ b/pfsagentd/swift_proxy_emulator_test.go @@ -14,7 +14,7 @@ import ( "testing" "time" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) const ( diff --git a/pfsagentd/unit_test.go b/pfsagentd/unit_test.go index c579d6022..2a168bf63 100644 --- a/pfsagentd/unit_test.go +++ b/pfsagentd/unit_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/swiftstack/ProxyFS/jrpcfs" + "github.com/NVIDIA/proxyfs/jrpcfs" ) const ( diff --git a/pfsconfjson/main.go b/pfsconfjson/main.go index d828533be..180db5177 100644 --- a/pfsconfjson/main.go +++ b/pfsconfjson/main.go @@ -10,8 +10,8 @@ import ( "log" "os" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/transitions" ) func main() { diff --git a/pfsconfjsonpacked/main.go b/pfsconfjsonpacked/main.go index cd3182c24..fcd9576f2 100644 --- a/pfsconfjsonpacked/main.go +++ b/pfsconfjsonpacked/main.go @@ -9,8 +9,8 @@ import ( "log" "os" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/transitions" ) func main() { diff --git a/pfsworkout/main.go b/pfsworkout/main.go index 8ec7a8923..b575ccc5c 100644 --- a/pfsworkout/main.go +++ b/pfsworkout/main.go @@ -10,14 +10,14 @@ import ( "strconv" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/headhunter" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/headhunter" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" + "github.com/NVIDIA/proxyfs/utils" ) const ( diff --git a/proxyfsd/daemon.go b/proxyfsd/daemon.go index 6b4b47ef8..0b9002b9f 100644 --- a/proxyfsd/daemon.go +++ b/proxyfsd/daemon.go @@ -14,13 +14,13 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/transitions" - "github.com/swiftstack/ProxyFS/version" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/transitions" + "github.com/NVIDIA/proxyfs/version" // Force importing of the following "top-most" package - _ "github.com/swiftstack/ProxyFS/httpserver" + _ "github.com/NVIDIA/proxyfs/httpserver" ) // Daemon is launched as a GoRoutine that launches ProxyFS. During startup, the parent should read errChan diff --git a/proxyfsd/daemon_test.go b/proxyfsd/daemon_test.go index 9a3a834e6..84980b224 100644 --- a/proxyfsd/daemon_test.go +++ b/proxyfsd/daemon_test.go @@ -16,9 +16,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/fs" - "github.com/swiftstack/ProxyFS/inode" - "github.com/swiftstack/ProxyFS/ramswift" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/ramswift" ) func TestMain(m *testing.M) { diff --git a/proxyfsd/proxyfsd/main.go b/proxyfsd/proxyfsd/main.go index 9abb351e9..82933f7f9 100644 --- a/proxyfsd/proxyfsd/main.go +++ b/proxyfsd/proxyfsd/main.go @@ -13,7 +13,7 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/proxyfsd" + "github.com/NVIDIA/proxyfs/proxyfsd" ) func main() { diff --git a/ramswift/daemon.go b/ramswift/daemon.go index ab7f4d864..48108c6d6 100644 --- a/ramswift/daemon.go +++ b/ramswift/daemon.go @@ -22,11 +22,11 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/transitions" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/transitions" + "github.com/NVIDIA/proxyfs/utils" ) type swiftAccountStruct struct { diff --git a/ramswift/daemon_test.go b/ramswift/daemon_test.go index c87393b77..dcfff25c4 100644 --- a/ramswift/daemon_test.go +++ b/ramswift/daemon_test.go @@ -14,7 +14,7 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/utils" ) func TestViaNoAuthClient(t *testing.T) { diff --git a/ramswift/ramswift/main.go b/ramswift/ramswift/main.go index 7d11cd9df..a94373f1e 100644 --- a/ramswift/ramswift/main.go +++ b/ramswift/ramswift/main.go @@ -9,7 +9,7 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/ramswift" + "github.com/NVIDIA/proxyfs/ramswift" ) func main() { diff --git a/retryrpc/api.go b/retryrpc/api.go index 589e5bad2..0463a1266 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -22,8 +22,8 @@ import ( "time" "github.com/google/btree" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/logger" ) // ServerCreds tracks the root CA and the diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index a3e070652..2e25c1f4c 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -23,8 +23,8 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/logger" ) // PayloadProtocols defines the supported protocols for the payload diff --git a/retryrpc/client.go b/retryrpc/client.go index bbc79c065..4c2b8728d 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -13,8 +13,8 @@ import ( "time" "github.com/google/btree" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/logger" ) const ( diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index d99fdc3ed..ac0f6b22f 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -8,8 +8,8 @@ import ( "time" "github.com/stretchr/testify/assert" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/retryrpc/rpctest" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/retryrpc/rpctest" ) // Test basic retryrpc primitives diff --git a/retryrpc/rpctest/ping.go b/retryrpc/rpctest/ping.go index 24f70a91f..f008a1156 100644 --- a/retryrpc/rpctest/ping.go +++ b/retryrpc/rpctest/ping.go @@ -8,7 +8,7 @@ import ( "bytes" "fmt" - "github.com/swiftstack/ProxyFS/blunder" + "github.com/NVIDIA/proxyfs/blunder" ) func encodeErrno(e *error) { diff --git a/retryrpc/server.go b/retryrpc/server.go index ede0708b6..c44bd624e 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -14,8 +14,8 @@ import ( "sync" "time" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/logger" "golang.org/x/sys/unix" ) diff --git a/retryrpc/stress_test.go b/retryrpc/stress_test.go index 97c0e6501..aa083774e 100644 --- a/retryrpc/stress_test.go +++ b/retryrpc/stress_test.go @@ -15,7 +15,7 @@ import ( */ "github.com/stretchr/testify/assert" - "github.com/swiftstack/ProxyFS/retryrpc/rpctest" + "github.com/NVIDIA/proxyfs/retryrpc/rpctest" ) func TestStress(t *testing.T) { diff --git a/retryrpc/upcall_test.go b/retryrpc/upcall_test.go index 626126e56..35d7124e7 100644 --- a/retryrpc/upcall_test.go +++ b/retryrpc/upcall_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/stretchr/testify/assert" - "github.com/swiftstack/ProxyFS/retryrpc/rpctest" + "github.com/NVIDIA/proxyfs/retryrpc/rpctest" ) // Test Upcall() functionality diff --git a/stats/api_test.go b/stats/api_test.go index 080377f03..54c15ed90 100644 --- a/stats/api_test.go +++ b/stats/api_test.go @@ -12,8 +12,8 @@ import ( "testing" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/transitions" ) type testGlobalsStruct struct { diff --git a/stats/config.go b/stats/config.go index 3df90b1db..916d65cbe 100644 --- a/stats/config.go +++ b/stats/config.go @@ -9,9 +9,9 @@ import ( "strconv" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) const ( diff --git a/stats/sender.go b/stats/sender.go index 124e50b68..0955b5862 100644 --- a/stats/sender.go +++ b/stats/sender.go @@ -8,7 +8,7 @@ import ( "net" "strconv" // XXX TODO: Can't call logger from here since logger calls stats. - //"github.com/swiftstack/ProxyFS/logger" + //"github.com/NVIDIA/proxyfs/logger" ) func sender() { diff --git a/statslogger/config.go b/statslogger/config.go index 77b2f037b..0c753eb58 100644 --- a/statslogger/config.go +++ b/statslogger/config.go @@ -8,12 +8,12 @@ import ( "strings" "time" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/transitions" ) type globalsStruct struct { diff --git a/statslogger/config_test.go b/statslogger/config_test.go index f52d8f714..904af1148 100644 --- a/statslogger/config_test.go +++ b/statslogger/config_test.go @@ -22,10 +22,10 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/ramswift" - "github.com/swiftstack/ProxyFS/swiftclient" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/ramswift" + "github.com/NVIDIA/proxyfs/swiftclient" + "github.com/NVIDIA/proxyfs/transitions" ) func (tOCCS *testObjectCopyCallbackStruct) BytesRemaining(bytesRemaining uint64) (chunkSize uint64) { diff --git a/swiftclient/account.go b/swiftclient/account.go index 9e4dd7949..33bec5d96 100644 --- a/swiftclient/account.go +++ b/swiftclient/account.go @@ -9,10 +9,10 @@ import ( "fmt" "net/url" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/evtlog" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/evtlog" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" ) func accountDeleteWithRetry(accountName string) (err error) { diff --git a/swiftclient/api_test.go b/swiftclient/api_test.go index c6546cfa8..7ed025c0d 100644 --- a/swiftclient/api_test.go +++ b/swiftclient/api_test.go @@ -18,12 +18,12 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/ramswift" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/ramswift" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/transitions" ) type testObjectCopyCallbackStruct struct { diff --git a/swiftclient/config.go b/swiftclient/config.go index 9dbba5263..a6b3176d5 100644 --- a/swiftclient/config.go +++ b/swiftclient/config.go @@ -10,11 +10,11 @@ import ( "strconv" "time" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/trackedlock" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/trackedlock" + "github.com/NVIDIA/proxyfs/transitions" ) type connectionStruct struct { diff --git a/swiftclient/container.go b/swiftclient/container.go index a17a046e9..0936dc516 100644 --- a/swiftclient/container.go +++ b/swiftclient/container.go @@ -9,10 +9,10 @@ import ( "fmt" "net/url" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/evtlog" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/evtlog" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" ) func containerDeleteWithRetry(accountName string, containerName string) (err error) { diff --git a/swiftclient/http_status.go b/swiftclient/http_status.go index bb8200daa..8a685498f 100644 --- a/swiftclient/http_status.go +++ b/swiftclient/http_status.go @@ -6,7 +6,7 @@ package swiftclient import ( - "github.com/swiftstack/ProxyFS/blunder" + "github.com/NVIDIA/proxyfs/blunder" ) func httpStatusIsSuccess(httpStatus int) (isSuccess bool) { diff --git a/swiftclient/object.go b/swiftclient/object.go index b57ab18a5..936fdcd88 100644 --- a/swiftclient/object.go +++ b/swiftclient/object.go @@ -13,13 +13,13 @@ import ( "time" "github.com/creachadair/cityhash" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/blunder" - "github.com/swiftstack/ProxyFS/evtlog" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/trackedlock" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/evtlog" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/trackedlock" ) func objectContentLengthWithRetry(accountName string, containerName string, objectName string) (uint64, error) { diff --git a/swiftclient/retry.go b/swiftclient/retry.go index 9cf426c89..f6defc4f1 100644 --- a/swiftclient/retry.go +++ b/swiftclient/retry.go @@ -9,9 +9,9 @@ import ( "fmt" "time" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" ) // Swift requests should be retried if they fail with a retriable error. diff --git a/swiftclient/utils.go b/swiftclient/utils.go index acc452aca..ec344fb46 100644 --- a/swiftclient/utils.go +++ b/swiftclient/utils.go @@ -14,9 +14,9 @@ import ( "sync" "sync/atomic" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/stats" - "github.com/swiftstack/ProxyFS/version" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/stats" + "github.com/NVIDIA/proxyfs/version" ) const swiftVersion = "v1" diff --git a/trackedlock/api.go b/trackedlock/api.go index 8a9234ee5..6e81d6e3d 100644 --- a/trackedlock/api.go +++ b/trackedlock/api.go @@ -8,7 +8,7 @@ import ( "sync" "sync/atomic" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/logger" ) /* diff --git a/trackedlock/api_internal.go b/trackedlock/api_internal.go index 3c69bea32..9ccb45639 100644 --- a/trackedlock/api_internal.go +++ b/trackedlock/api_internal.go @@ -9,8 +9,8 @@ import ( "sync/atomic" "time" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/utils" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/utils" ) type globalsStruct struct { diff --git a/trackedlock/api_test.go b/trackedlock/api_test.go index e4a2273d9..bfc9f8829 100644 --- a/trackedlock/api_test.go +++ b/trackedlock/api_test.go @@ -19,8 +19,8 @@ import ( "time" "unsafe" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" ) // Common configuration for all tests (unless overridden) diff --git a/trackedlock/config.go b/trackedlock/config.go index 4648e11b9..5a61f3cc7 100644 --- a/trackedlock/config.go +++ b/trackedlock/config.go @@ -7,9 +7,9 @@ import ( "sync/atomic" "time" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" - "github.com/swiftstack/ProxyFS/transitions" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/transitions" ) func parseConfMap(confMap conf.ConfMap) (err error) { diff --git a/transitions/api.go b/transitions/api.go index fa9ede358..217d3ebf0 100644 --- a/transitions/api.go +++ b/transitions/api.go @@ -4,7 +4,7 @@ package transitions import ( - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) // Callbacks is the interface implemented by each package desiring notification of @@ -57,8 +57,8 @@ type Callbacks interface { // // package foo // -// import "github.com/swiftstack/ProxyFS/conf" -// import "github.com/swiftstack/ProxyFS/transitions" +// import "github.com/NVIDIA/proxyfs/conf" +// import "github.com/NVIDIA/proxyfs/transitions" // // type transitionsCallbackInterfaceStruct struct { // } diff --git a/transitions/api_internal.go b/transitions/api_internal.go index 8793d2c9f..a4ad494db 100644 --- a/transitions/api_internal.go +++ b/transitions/api_internal.go @@ -8,8 +8,8 @@ import ( "fmt" "sync" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/logger" ) type loggerCallbacksInterfaceStruct struct { diff --git a/transitions/api_test.go b/transitions/api_test.go index d05a82774..960b1649f 100644 --- a/transitions/api_test.go +++ b/transitions/api_test.go @@ -8,7 +8,7 @@ import ( "sort" "testing" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) type testCallbacksInterfaceStruct struct { diff --git a/utils/api.go b/utils/api.go index fb9a37ca1..d5874391d 100644 --- a/utils/api.go +++ b/utils/api.go @@ -196,19 +196,19 @@ func StackTraceToGoId(buf []byte) uint64 { // // goroutine 1 [running]: // main.main() -// /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/swiftstack/ProxyFS/stacktrace.go:27 +0x21e +// /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/NVIDIA/proxyfs/stacktrace.go:27 +0x21e // // goroutine 5 [runnable]: // main.killTime() -// /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/swiftstack/ProxyFS/stacktrace.go:9 +// /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/NVIDIA/proxyfs/stacktrace.go:9 // created by main.main -// /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/swiftstack/ProxyFS/stacktrace.go:14 +0x47 +// /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/NVIDIA/proxyfs/stacktrace.go:14 +0x47 // // goroutine 7 [runnable]: // main.killTime() -// /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/swiftstack/ProxyFS/stacktrace.go:9 +// /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/NVIDIA/proxyfs/stacktrace.go:9 // created by main.main -// /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/swiftstack/ProxyFS/stacktrace.go:16 +0x77 +// /vagrant/guest_workspaces/swift-runway-001/ProxyFS/src/github.com/NVIDIA/proxyfs/stacktrace.go:16 +0x77 // func StackTracesToMap(buf []byte) (traceMap map[uint64]string, stateMap map[uint64]string) { From fe4adcb8379c8f45ed00d9b862c5ccdce2ac30dc Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 8 Feb 2021 10:25:28 -0800 Subject: [PATCH 033/160] Migrated .service files --- saio/usr/lib/systemd/system/pfsagentd.service | 2 +- saio/usr/lib/systemd/system/proxyfsd.service | 2 +- sait/sait1/usr/lib/systemd/system/proxyfsd.service | 2 +- sait/sait2/usr/lib/systemd/system/proxyfsd.service | 2 +- sait/sait3/usr/lib/systemd/system/proxyfsd.service | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/saio/usr/lib/systemd/system/pfsagentd.service b/saio/usr/lib/systemd/system/pfsagentd.service index 2b674e103..c0db0533f 100644 --- a/saio/usr/lib/systemd/system/pfsagentd.service +++ b/saio/usr/lib/systemd/system/pfsagentd.service @@ -6,7 +6,7 @@ After=proxyfsd.service Environment=PATH=/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/sbin:/bin:/vagrant/bin Environment=GOTRACEBACK=crash LimitCORE=infinity -ExecStart=/vagrant/bin/pfsagentd /vagrant/src/github.com/swiftstack/ProxyFS/pfsagentd/pfsagent.conf +ExecStart=/vagrant/bin/pfsagentd /vagrant/src/github.com/NVIDIA/proxyfs/pfsagentd/pfsagent.conf ExecReload=/usr/bin/kill -HUP $MAINPID Restart=always diff --git a/saio/usr/lib/systemd/system/proxyfsd.service b/saio/usr/lib/systemd/system/proxyfsd.service index 41960917b..0ae0258f8 100644 --- a/saio/usr/lib/systemd/system/proxyfsd.service +++ b/saio/usr/lib/systemd/system/proxyfsd.service @@ -4,7 +4,7 @@ Description=Swift ProxyFS service [Service] Environment=GOTRACEBACK=crash LimitCORE=infinity -ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/swiftstack/ProxyFS/saio/proxyfs.conf +ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/NVIDIA/proxyfs/saio/proxyfs.conf ExecReload=/usr/bin/kill -HUP $MAINPID Restart=no KillMode=process diff --git a/sait/sait1/usr/lib/systemd/system/proxyfsd.service b/sait/sait1/usr/lib/systemd/system/proxyfsd.service index 38a806146..abfc477b6 100644 --- a/sait/sait1/usr/lib/systemd/system/proxyfsd.service +++ b/sait/sait1/usr/lib/systemd/system/proxyfsd.service @@ -4,7 +4,7 @@ Description=Swift ProxyFS service [Service] Environment=GOTRACEBACK=crash LimitCORE=infinity -ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/swiftstack/ProxyFS/sait/sait1/proxyfs.conf +ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/NVIDIA/proxyfs/sait/sait1/proxyfs.conf ExecReload=/usr/bin/kill -HUP $MAINPID Restart=no KillMode=process diff --git a/sait/sait2/usr/lib/systemd/system/proxyfsd.service b/sait/sait2/usr/lib/systemd/system/proxyfsd.service index 90f779cf9..eb1cae9fc 100644 --- a/sait/sait2/usr/lib/systemd/system/proxyfsd.service +++ b/sait/sait2/usr/lib/systemd/system/proxyfsd.service @@ -4,7 +4,7 @@ Description=Swift ProxyFS service [Service] Environment=GOTRACEBACK=crash LimitCORE=infinity -ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/swiftstack/ProxyFS/sait/sait2/proxyfs.conf +ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/NVIDIA/proxyfs/sait/sait2/proxyfs.conf ExecReload=/usr/bin/kill -HUP $MAINPID Restart=no KillMode=process diff --git a/sait/sait3/usr/lib/systemd/system/proxyfsd.service b/sait/sait3/usr/lib/systemd/system/proxyfsd.service index ba2d6dacf..2ca6a19d3 100644 --- a/sait/sait3/usr/lib/systemd/system/proxyfsd.service +++ b/sait/sait3/usr/lib/systemd/system/proxyfsd.service @@ -4,7 +4,7 @@ Description=Swift ProxyFS service [Service] Environment=GOTRACEBACK=crash LimitCORE=infinity -ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/swiftstack/ProxyFS/sait/sait3/proxyfs.conf +ExecStart=/vagrant/bin/proxyfsd /vagrant/src/github.com/NVIDIA/proxyfs/sait/sait3/proxyfs.conf ExecReload=/usr/bin/kill -HUP $MAINPID Restart=no KillMode=process From 00bbfc68d065cc19725a27f4802bad224066dcf1 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 8 Feb 2021 10:46:27 -0800 Subject: [PATCH 034/160] Migrated saio/sait files ...also migraded ".go_HIDE" files --- emswift/emswiftpkg/setup_teardown_test.go_HIDE | 6 +++--- .../swift_proxy_emulator_test.go_HIDE | 2 +- imgr/http_server.go_HIDE | 6 +++--- pfsagentd/container/build/Dockerfile | 8 ++++---- .../container/insert/MakePFSAgentDockerfile | 4 ++-- saio/Vagrantfile | 6 +++--- saio/bin/provision_middleware | 4 ++-- saio/bin/start_proxyfs_and_swift | 4 ++-- saio/bin/start_swift_only | 2 +- saio/container/Dockerfile | 8 ++++---- saio/vagrant_provision.sh | 18 +++++++++--------- sait/Vagrantfile | 6 +++--- sait/bin/provision_middleware | 4 ++-- sait/bin/start_proxyfs_and_swift | 4 ++-- sait/bin/start_swift_only | 2 +- sait/vagrant_provision.sh | 18 +++++++++--------- 16 files changed, 51 insertions(+), 51 deletions(-) diff --git a/emswift/emswiftpkg/setup_teardown_test.go_HIDE b/emswift/emswiftpkg/setup_teardown_test.go_HIDE index 5aa1e5b84..45d709cf6 100644 --- a/emswift/emswiftpkg/setup_teardown_test.go_HIDE +++ b/emswift/emswiftpkg/setup_teardown_test.go_HIDE @@ -14,9 +14,9 @@ import ( "golang.org/x/sys/unix" - "github.com/swiftstack/ProxyFS/conf" - "github.com/swiftstack/ProxyFS/proxyfsd" - "github.com/swiftstack/ProxyFS/ramswift" + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/proxyfsd" + "github.com/NVIDIA/proxyfs/ramswift" ) const ( diff --git a/emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE b/emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE index 8f44734c6..10ff83386 100644 --- a/emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE +++ b/emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE @@ -14,7 +14,7 @@ import ( "testing" "time" - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) const ( diff --git a/imgr/http_server.go_HIDE b/imgr/http_server.go_HIDE index 486d54b13..faa8833dd 100644 --- a/imgr/http_server.go_HIDE +++ b/imgr/http_server.go_HIDE @@ -19,10 +19,10 @@ import ( "strings" "sync/atomic" - "github.com/swiftstack/sortedmap" + "github.com/NVIDIA/sortedmap" - "github.com/swiftstack/ProxyFS/bucketstats" - "github.com/swiftstack/ProxyFS/version" + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/version" ) type leaseReportStruct struct { diff --git a/pfsagentd/container/build/Dockerfile b/pfsagentd/container/build/Dockerfile index 8e3dc06ac..5319f90d2 100644 --- a/pfsagentd/container/build/Dockerfile +++ b/pfsagentd/container/build/Dockerfile @@ -20,10 +20,10 @@ RUN tar -C /usr/local -xzf $GolangBasename ENV GOPATH /opt/PFSAgent/GOPATH ENV PATH $PATH:/usr/local/go/bin:$GOPATH/bin -RUN mkdir -p $GOPATH/src/github.com/swiftstack -WORKDIR $GOPATH/src/github.com/swiftstack -RUN git clone https://github.com/swiftstack/ProxyFS.git -WORKDIR $GOPATH/src/github.com/swiftstack/ProxyFS +RUN mkdir -p $GOPATH/src/github.com/NVIDIA +WORKDIR $GOPATH/src/github.com/NVIDIA +RUN git clone https://github.com/NVIDIA/proxyfs.git +WORKDIR $GOPATH/src/github.com/NVIDIA/proxyfs RUN git checkout $ProxyFS_Version RUN make version pfsagent diff --git a/pfsagentd/container/insert/MakePFSAgentDockerfile b/pfsagentd/container/insert/MakePFSAgentDockerfile index 97daf616c..d9de1037d 100755 --- a/pfsagentd/container/insert/MakePFSAgentDockerfile +++ b/pfsagentd/container/insert/MakePFSAgentDockerfile @@ -120,7 +120,7 @@ do docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd . docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd-init . docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd-swift-auth-plugin . - docker cp $CONTAINER:/opt/PFSAgent/GOPATH/src/github.com/swiftstack/ProxyFS/pfsagentd/pfsagent.conf pfsagent.conf_TEMPLATE + docker cp $CONTAINER:/opt/PFSAgent/GOPATH/src/github.com/NVIDIA/proxyfs/pfsagentd/pfsagent.conf pfsagent.conf_TEMPLATE docker container rm $CONTAINER > /dev/null else if [[ $NEW_SECTION_NAME = "TEMPLATE" ]] @@ -180,7 +180,7 @@ then docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd . docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd-init . docker cp $CONTAINER:/opt/PFSAgent/GOPATH/bin/pfsagentd-swift-auth-plugin . - docker cp $CONTAINER:/opt/PFSAgent/GOPATH/src/github.com/swiftstack/ProxyFS/pfsagentd/pfsagent.conf pfsagent.conf_TEMPLATE + docker cp $CONTAINER:/opt/PFSAgent/GOPATH/src/github.com/NVIDIA/proxyfs/pfsagentd/pfsagent.conf pfsagent.conf_TEMPLATE docker container rm $CONTAINER > /dev/null fi diff --git a/saio/Vagrantfile b/saio/Vagrantfile index b3062cd46..621fdd2af 100644 --- a/saio/Vagrantfile +++ b/saio/Vagrantfile @@ -6,9 +6,9 @@ # 1) vboxnet0 is assumed to be a host-only network @ address 172.28.128.1 (DHCP disabled) # 2) Though not required, GOPATH is assumed to be the ../../../../../ directory # 3) The directory on the VM Host will be /vagrant on the VM and be the path in GOPATH -# 4) ProxyFS repo git clone'd to $GOPATH/src/github.com/swiftstack/ -# 5) samba repo automatically staged in $GOPATH/src/github.com/swiftstack/ProxyFS/saio/ -# 6) Swift repos et. al. git clone'd to $GOPATH/src/github.com/swiftstack/ProxyFS/saio/ +# 4) ProxyFS repo git clone'd to $GOPATH/src/github.com/NVIDIA/ +# 5) samba repo automatically staged in $GOPATH/src/github.com/NVIDIA/proxyfs/saio/ +# 6) Swift repos et. al. git clone'd to $GOPATH/src/github.com/NVIDIA/proxyfs/saio/ # 7) ../Makefile will be ready to be executed after `cdpfs` inside the VM # 8) As GOPATH is effectively shared between Host and VM, builds in the two environments # will collide in the contents of the $GOPATH/bin (only executables, not libraries) diff --git a/saio/bin/provision_middleware b/saio/bin/provision_middleware index 7cb669aca..1c1559e98 100755 --- a/saio/bin/provision_middleware +++ b/saio/bin/provision_middleware @@ -3,7 +3,7 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -cd $GOPATH/src/github.com/swiftstack/ProxyFS/pfs_middleware +cd $GOPATH/src/github.com/NVIDIA/proxyfs/pfs_middleware sudo python setup.py develop -cd $GOPATH/src/github.com/swiftstack/ProxyFS/meta_middleware +cd $GOPATH/src/github.com/NVIDIA/proxyfs/meta_middleware sudo python setup.py develop diff --git a/saio/bin/start_proxyfs_and_swift b/saio/bin/start_proxyfs_and_swift index dd8128dd3..6bf87bb3f 100755 --- a/saio/bin/start_proxyfs_and_swift +++ b/saio/bin/start_proxyfs_and_swift @@ -38,7 +38,7 @@ function await_swift_startup { } function format_volume_if_necessary { - sudo /vagrant/bin/mkproxyfs -I $1 /vagrant/src/github.com/swiftstack/ProxyFS/saio/proxyfs.conf SwiftClient.RetryLimit=1 + sudo /vagrant/bin/mkproxyfs -I $1 /vagrant/src/github.com/NVIDIA/proxyfs/saio/proxyfs.conf SwiftClient.RetryLimit=1 if [ $? -ne 0 ] then echo "Could not pre-format $1" @@ -49,7 +49,7 @@ function format_volume_if_necessary { sudo mount -a echo "Shutting down services and mount points..." -/vagrant/src/github.com/swiftstack/ProxyFS/saio/bin/unmount_and_stop_pfs +/vagrant/src/github.com/NVIDIA/proxyfs/saio/bin/unmount_and_stop_pfs echo echo "Bringing up services..." if [ -f /usr/bin/systemctl ]; then diff --git a/saio/bin/start_swift_only b/saio/bin/start_swift_only index b2e58f984..45edb5195 100755 --- a/saio/bin/start_swift_only +++ b/saio/bin/start_swift_only @@ -9,7 +9,7 @@ sudo mount -a echo "Shutting down services and mount points..." -/vagrant/src/github.com/swiftstack/ProxyFS/saio/bin/unmount_and_stop_pfs +/vagrant/src/github.com/NVIDIA/proxyfs/saio/bin/unmount_and_stop_pfs echo echo "Bringing up services..." if [ -f /usr/bin/systemctl ]; then diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 77e5f3dea..634a94f82 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -167,10 +167,10 @@ ENV PATH $PATH:/usr/local/go/bin:$GOPATH/bin RUN yum install -y make fuse -RUN mkdir -p $GOPATH/src/github.com/swiftstack -WORKDIR $GOPATH/src/github.com/swiftstack -RUN git clone https://github.com/swiftstack/ProxyFS.git -WORKDIR $GOPATH/src/github.com/swiftstack/ProxyFS +RUN mkdir -p $GOPATH/src/github.com/NVIDIA +WORKDIR $GOPATH/src/github.com/NVIDIA +RUN git clone https://github.com/NVIDIA/proxyfs.git +WORKDIR $GOPATH/src/github.com/NVIDIA/proxyfs RUN git checkout $ProxyFS_Version RUN cd pfs_middleware && python setup.py develop diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index d8100bd10..895a39770 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -77,7 +77,7 @@ echo "export PATH=\$PATH:/usr/local/go/bin" >> ~vagrant/.bash_profile # Patch Golang's GDB runtime plug-in mv /usr/local/go/src/runtime/runtime-gdb.py /usr/local/go/src/runtime/runtime-gdb.py_ORIGINAL -cp /vagrant/src/github.com/swiftstack/ProxyFS/saio/usr/local/go/src/runtime/runtime-gdb.py /usr/local/go/src/runtime/. +cp /vagrant/src/github.com/NVIDIA/proxyfs/saio/usr/local/go/src/runtime/runtime-gdb.py /usr/local/go/src/runtime/. # Install GDB and enable above Golang GDB runtime plug-in as well as other niceties @@ -112,7 +112,7 @@ yum -y install json-c-devel yum -y install fuse echo "export GOPATH=/vagrant" >> ~vagrant/.bash_profile echo "export PATH=\$PATH:\$GOPATH/bin" >> ~vagrant/.bash_profile -echo "alias cdpfs=\"cd \$GOPATH/src/github.com/swiftstack/ProxyFS\"" >> ~vagrant/.bash_profile +echo "alias cdpfs=\"cd \$GOPATH/src/github.com/NVIDIA/proxyfs\"" >> ~vagrant/.bash_profile echo "alias goclean=\"go clean;go clean --cache;go clean --testcache\"" >> ~vagrant/.bash_profile echo "alias gogetdlv=\"go get -u github.com/go-delve/delve/cmd/dlv\"" >> ~vagrant/.bash_profile echo "user_allow_other" >> /etc/fuse.conf @@ -263,7 +263,7 @@ systemctl start memcached.service # [Setup Swift] Configuring each node rm -rf /etc/swift -cp -R /vagrant/src/github.com/swiftstack/ProxyFS/saio/etc/swift /etc/swift +cp -R /vagrant/src/github.com/NVIDIA/proxyfs/saio/etc/swift /etc/swift chown -R swift:swift /etc/swift # [Setup Swift] Setting up scripts for running Swift @@ -271,19 +271,19 @@ chown -R swift:swift /etc/swift mkdir -p ~swift/bin cd ~swift/bin -cp /vagrant/src/github.com/swiftstack/ProxyFS/saio/home/swift/bin/* . +cp /vagrant/src/github.com/NVIDIA/proxyfs/saio/home/swift/bin/* . echo "export PATH=\$PATH:~swift/bin" >> ~vagrant/.bash_profile ~swift/bin/remakerings # Install ProxyFS's pfs_middleware into the "normal" Swift Proxy pipeline -cd /vagrant/src/github.com/swiftstack/ProxyFS/pfs_middleware +cd /vagrant/src/github.com/NVIDIA/proxyfs/pfs_middleware python setup.py develop # Install ProxyFS's meta_middleware into the "NoAuth" Swift Proxy pipeline -cd /vagrant/src/github.com/swiftstack/ProxyFS/meta_middleware +cd /vagrant/src/github.com/NVIDIA/proxyfs/meta_middleware python setup.py develop # Setup AWS access for local vagrant user @@ -329,8 +329,8 @@ chmod 777 /CommonMountPoint # Install systemd .service files for ProxyFS -cp /vagrant/src/github.com/swiftstack/ProxyFS/saio/usr/lib/systemd/system/proxyfsd.service /usr/lib/systemd/system/. -cp /vagrant/src/github.com/swiftstack/ProxyFS/saio/usr/lib/systemd/system/pfsagentd.service /usr/lib/systemd/system/. +cp /vagrant/src/github.com/NVIDIA/proxyfs/saio/usr/lib/systemd/system/proxyfsd.service /usr/lib/systemd/system/. +cp /vagrant/src/github.com/NVIDIA/proxyfs/saio/usr/lib/systemd/system/pfsagentd.service /usr/lib/systemd/system/. # Place symlink in root's $PATH to locate pfsagentd-swift-auth-plugin referenced without a path @@ -344,7 +344,7 @@ semanage port -a -t smbd_port_t -p tcp 32345 # Enable start/stop tools -echo "export PATH=\$PATH:/vagrant/src/github.com/swiftstack/ProxyFS/saio/bin" >> ~vagrant/.bash_profile +echo "export PATH=\$PATH:/vagrant/src/github.com/NVIDIA/proxyfs/saio/bin" >> ~vagrant/.bash_profile # Install wireshark diff --git a/sait/Vagrantfile b/sait/Vagrantfile index e0f993930..1b8b3eb91 100644 --- a/sait/Vagrantfile +++ b/sait/Vagrantfile @@ -6,9 +6,9 @@ # 1) vboxnet0 is assumed to be a host-only network @ address 172.28.128.1 (DHCP disabled) # 2) Though not required, GOPATH is assumed to be the ../../../../../ directory # 3) The directory on the VM Host will be /vagrant on the VM and be the path in GOPATH -# 4) ProxyFS repo git clone'd to $GOPATH/src/github.com/swiftstack/ -# 5) samba repo automatically staged in $GOPATH/src/github.com/swiftstack/ProxyFS/saio/ -# 6) Swift repos et. al. git clone'd to $GOPATH/src/github.com/swiftstack/ProxyFS/saio/ +# 4) ProxyFS repo git clone'd to $GOPATH/src/github.com/NVIDIA/ +# 5) samba repo automatically staged in $GOPATH/src/github.com/NVIDIA/proxyfs/saio/ +# 6) Swift repos et. al. git clone'd to $GOPATH/src/github.com/NVIDIA/proxyfs/saio/ # 7) ../Makefile will be ready to be executed after `cdpfs` inside the VM # 8) As GOPATH is effectively shared between Host and VM, builds in the two environments # will collide in the contents of the $GOPATH/bin (only executables, not libraries) diff --git a/sait/bin/provision_middleware b/sait/bin/provision_middleware index 7cb669aca..1c1559e98 100755 --- a/sait/bin/provision_middleware +++ b/sait/bin/provision_middleware @@ -3,7 +3,7 @@ # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -cd $GOPATH/src/github.com/swiftstack/ProxyFS/pfs_middleware +cd $GOPATH/src/github.com/NVIDIA/proxyfs/pfs_middleware sudo python setup.py develop -cd $GOPATH/src/github.com/swiftstack/ProxyFS/meta_middleware +cd $GOPATH/src/github.com/NVIDIA/proxyfs/meta_middleware sudo python setup.py develop diff --git a/sait/bin/start_proxyfs_and_swift b/sait/bin/start_proxyfs_and_swift index 49e7e1f64..28e2ba686 100755 --- a/sait/bin/start_proxyfs_and_swift +++ b/sait/bin/start_proxyfs_and_swift @@ -56,7 +56,7 @@ function await_swift_startup { function format_volume_if_necessary { if [ "" != "$1" ] then - sudo /vagrant/bin/mkproxyfs -I $1 /vagrant/src/github.com/swiftstack/ProxyFS/sait/$SAIT_DIR/proxyfs.conf SwiftClient.RetryLimit=1 + sudo /vagrant/bin/mkproxyfs -I $1 /vagrant/src/github.com/NVIDIA/proxyfs/sait/$SAIT_DIR/proxyfs.conf SwiftClient.RetryLimit=1 if [ $? -ne 0 ] then echo "Could not pre-format $1" @@ -68,7 +68,7 @@ function format_volume_if_necessary { sudo mount -a echo "Shutting down services and mount points..." -/vagrant/src/github.com/swiftstack/ProxyFS/sait/bin/unmount_and_stop_pfs +/vagrant/src/github.com/NVIDIA/proxyfs/sait/bin/unmount_and_stop_pfs echo echo "Bringing up services..." if [ -f /usr/bin/systemctl ]; then diff --git a/sait/bin/start_swift_only b/sait/bin/start_swift_only index 52f4cc618..3269977bf 100755 --- a/sait/bin/start_swift_only +++ b/sait/bin/start_swift_only @@ -6,7 +6,7 @@ sudo mount -a echo "Shutting down services and mount points..." -/vagrant/src/github.com/swiftstack/ProxyFS/sait/bin/unmount_and_stop_pfs +/vagrant/src/github.com/NVIDIA/proxyfs/sait/bin/unmount_and_stop_pfs echo echo "Bringing up services..." if [ -f /usr/bin/systemctl ]; then diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index e51e55649..6508454d1 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -79,7 +79,7 @@ echo "export PATH=\$PATH:/usr/local/go/bin" >> ~vagrant/.bash_profile # Patch Golang's GDB runtime plug-in mv /usr/local/go/src/runtime/runtime-gdb.py /usr/local/go/src/runtime/runtime-gdb.py_ORIGINAL -cp /vagrant/src/github.com/swiftstack/ProxyFS/sait/usr/local/go/src/runtime/runtime-gdb.py /usr/local/go/src/runtime/. +cp /vagrant/src/github.com/NVIDIA/proxyfs/sait/usr/local/go/src/runtime/runtime-gdb.py /usr/local/go/src/runtime/. # Install GDB and enable above Golang GDB runtime plug-in as well as other niceties @@ -114,7 +114,7 @@ yum -y install json-c-devel yum -y install fuse echo "export GOPATH=/vagrant" >> ~vagrant/.bash_profile echo "export PATH=\$PATH:\$GOPATH/bin" >> ~vagrant/.bash_profile -echo "alias cdpfs=\"cd \$GOPATH/src/github.com/swiftstack/ProxyFS\"" >> ~vagrant/.bash_profile +echo "alias cdpfs=\"cd \$GOPATH/src/github.com/NVIDIA/proxyfs\"" >> ~vagrant/.bash_profile echo "alias goclean=\"go clean;go clean --cache;go clean --testcache\"" >> ~vagrant/.bash_profile echo "alias gogetdlv=\"go get -u github.com/go-delve/delve/cmd/dlv\"" >> ~vagrant/.bash_profile echo "user_allow_other" >> /etc/fuse.conf @@ -265,8 +265,8 @@ systemctl start memcached.service # [Setup Swift] Configuring each node rm -rf /etc/swift -cp -R /vagrant/src/github.com/swiftstack/ProxyFS/sait/etc/swift /etc/. -cp -R /vagrant/src/github.com/swiftstack/ProxyFS/sait/sait$SAIT_INSTANCE/etc/swift /etc/. +cp -R /vagrant/src/github.com/NVIDIA/proxyfs/sait/etc/swift /etc/. +cp -R /vagrant/src/github.com/NVIDIA/proxyfs/sait/sait$SAIT_INSTANCE/etc/swift /etc/. chown -R swift:swift /etc/swift # [Setup Swift] Setting up scripts for running Swift @@ -274,19 +274,19 @@ chown -R swift:swift /etc/swift mkdir -p ~swift/bin cd ~swift/bin -cp /vagrant/src/github.com/swiftstack/ProxyFS/sait/home/swift/bin/* . +cp /vagrant/src/github.com/NVIDIA/proxyfs/sait/home/swift/bin/* . echo "export PATH=\$PATH:~swift/bin" >> ~vagrant/.bash_profile ~swift/bin/remakerings # Install ProxyFS's pfs_middleware into the "normal" Swift Proxy pipeline -cd /vagrant/src/github.com/swiftstack/ProxyFS/pfs_middleware +cd /vagrant/src/github.com/NVIDIA/proxyfs/pfs_middleware python setup.py develop # Install ProxyFS's meta_middleware into the "NoAuth" Swift Proxy pipeline -cd /vagrant/src/github.com/swiftstack/ProxyFS/meta_middleware +cd /vagrant/src/github.com/NVIDIA/proxyfs/meta_middleware python setup.py develop # Setup AWS access for local vagrant user @@ -335,11 +335,11 @@ fi # Install systemd .service files for ProxyFS -cp /vagrant/src/github.com/swiftstack/ProxyFS/sait/sait$SAIT_INSTANCE/usr/lib/systemd/system/proxyfsd.service /usr/lib/systemd/system/. +cp /vagrant/src/github.com/NVIDIA/proxyfs/sait/sait$SAIT_INSTANCE/usr/lib/systemd/system/proxyfsd.service /usr/lib/systemd/system/. # Enable start/stop tools -echo "export PATH=\$PATH:/vagrant/src/github.com/swiftstack/ProxyFS/sait/bin" >> ~vagrant/.bash_profile +echo "export PATH=\$PATH:/vagrant/src/github.com/NVIDIA/proxyfs/sait/bin" >> ~vagrant/.bash_profile # Install wireshark From 8843c5cbbdeb601f063b6ff55e9532fb761859a9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 8 Feb 2021 10:56:34 -0800 Subject: [PATCH 035/160] Migrated all but Go Modules files go.{mod|sum} --- docs/source/api/index.rst | 2 +- docs/source/architecture/access.rst | 4 ++-- docs/source/theme/swiftopensource/footer.html | 4 ++-- docs/source/theme/swiftopensource/header.html | 4 ++-- enmesh_in_gopath.sh | 8 ++++---- logger/README.md | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/source/api/index.rst b/docs/source/api/index.rst index a44b16044..7147449ec 100644 --- a/docs/source/api/index.rst +++ b/docs/source/api/index.rst @@ -12,7 +12,7 @@ Bimodal access A primary of the motivation for ProxyFS was to have the ability to read and write the same data in the same account whether accessing it via the object API or file system API. Such access is possible via the ``pfs`` `proxy middleware -`__, +`__, but there are numerous caveats. Writing via object API diff --git a/docs/source/architecture/access.rst b/docs/source/architecture/access.rst index 74ec88855..ceb9a2016 100644 --- a/docs/source/architecture/access.rst +++ b/docs/source/architecture/access.rst @@ -35,9 +35,9 @@ SMB Access – ProxyFS Samba VFS ProxyFS provides a Samba VFS module written in C that integrates directly with the Samba server processes. This module is provided in a separate repository called -`proxyfs-vfs `__. A ProxyFS +`proxyfs-vfs `__. A ProxyFS JSON RPC client is provided in a separate repository called -`proxyfs-jrpc-client `__. +`proxyfs-jrpc-client `__. The ProxyFS Samba VFS module utilizes the ProxyFS JSON RPC interface for direct access into ProxyFS for a given volume. diff --git a/docs/source/theme/swiftopensource/footer.html b/docs/source/theme/swiftopensource/footer.html index e919c8689..0a1f13aa6 100644 --- a/docs/source/theme/swiftopensource/footer.html +++ b/docs/source/theme/swiftopensource/footer.html @@ -4,7 +4,7 @@

Source

@@ -12,7 +12,7 @@

OpenStack Swift

diff --git a/docs/source/theme/swiftopensource/header.html b/docs/source/theme/swiftopensource/header.html index 6b0a64f9a..5874bce30 100644 --- a/docs/source/theme/swiftopensource/header.html +++ b/docs/source/theme/swiftopensource/header.html @@ -14,14 +14,14 @@
  • - Source + Source
  • OpenStack Swift
  • diff --git a/enmesh_in_gopath.sh b/enmesh_in_gopath.sh index 401b8bf6d..cead999a3 100755 --- a/enmesh_in_gopath.sh +++ b/enmesh_in_gopath.sh @@ -4,16 +4,16 @@ # SPDX-License-Identifier: Apache-2.0 # This script takes a clean checkout of the ProxyFS repo, turns it into a -# working directory (e.g., enmeshes it into src/github.com/swiftstack/Proxyfs) +# working directory (e.g., enmeshes it into src/github.com/NVIDIA/proxyfs) # It is intended for use by CI/CD systems. set -e shopt -s extglob ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -SWIFTSTACK_DIR=src/github.com/swiftstack -REPO_CLONE_PARENT_DIR=$ROOT_DIR/$SWIFTSTACK_DIR -PROXYFS_DIR=$REPO_CLONE_PARENT_DIR/ProxyFS +NVIDIA_DIR=src/github.com/NVIDIA +REPO_CLONE_PARENT_DIR=$ROOT_DIR/$NVIDIA_DIR +PROXYFS_DIR=$REPO_CLONE_PARENT_DIR/proxyfs if [ ! -d $PROXYFS_DIR ]; then cd $ROOT_DIR diff --git a/logger/README.md b/logger/README.md index 0c43faf8c..03d3bbb36 100644 --- a/logger/README.md +++ b/logger/README.md @@ -74,7 +74,7 @@ The simplest way to use Logger is as a replacement for the stdlib logger: package main import ( - "github.com/swiftstack/ProxyFS/logger" + "github.com/NVIDIA/proxyfs/logger" ) func main() { From d09218593cdd14dfdcda4a851a552bffc8d2eb53 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 8 Feb 2021 10:58:52 -0800 Subject: [PATCH 036/160] Migrated go modules --- go.mod | 8 ++++---- go.sum | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 2bb32f4bf..347655b46 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,12 @@ -module github.com/swiftstack/ProxyFS +module github.com/NVIDIA/proxyfs go 1.15 require ( bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898 + github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a + github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0 + github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec github.com/ansel1/merry v1.0.1 github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect @@ -15,9 +18,6 @@ require ( github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f // indirect github.com/sirupsen/logrus v1.2.0 github.com/stretchr/testify v1.3.0 - github.com/swiftstack/cstruct v0.0.0-20210119215647-26f9e390f1a4 - github.com/swiftstack/fission v0.0.0-20210119215758-8345a19492f6 - github.com/swiftstack/sortedmap v0.0.0-20210119215928-78f1ae2b7386 go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 go.uber.org/multierr v1.5.0 // indirect go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f // indirect diff --git a/go.sum b/go.sum index f32c59d5c..ee33bdbfb 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,12 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxo cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= +github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= +github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0 h1:GQZP8OZqvkcMt7l7QAdtfa17PDUqiamz64df2DWNCDM= +github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= +github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= +github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/ansel1/merry v1.0.1 h1:NEqlG+pyb6XKY55oRdGStyMkQ35YucmgFVG2tpybaP8= @@ -128,12 +134,6 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/swiftstack/cstruct v0.0.0-20210119215647-26f9e390f1a4 h1:a8h1nMi7X3Pt3nki0EGrYwcGVxSVdTH0mhmlnM9MdGI= -github.com/swiftstack/cstruct v0.0.0-20210119215647-26f9e390f1a4/go.mod h1:M9lpLN5EdXU71XfQYsfp/Om0LBYmbhAH0egd/JoC7n0= -github.com/swiftstack/fission v0.0.0-20210119215758-8345a19492f6 h1:NTNyX6NGekgMb/jFgVkCUCRhrO57TOYlNvlMShSf3Ik= -github.com/swiftstack/fission v0.0.0-20210119215758-8345a19492f6/go.mod h1:Cfh/Fpms2DJoorCv+82iLQCaJNymAA6SBm2PjfKvQu0= -github.com/swiftstack/sortedmap v0.0.0-20210119215928-78f1ae2b7386 h1:6g01lFENjg9Lcoyp6LL3Z2QJeDmaKxLd9WEP6/cPh8I= -github.com/swiftstack/sortedmap v0.0.0-20210119215928-78f1ae2b7386/go.mod h1:PIKb6sduhN4itGEKgEN+zxDUs45bpx0XuJPo6UjF/vQ= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= From a94696ae15d60aab682c9238b7f9394363057e86 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 8 Feb 2021 11:25:06 -0800 Subject: [PATCH 037/160] Post-migration go format import reorderings --- confgen/api_test.go | 2 +- inode/api_test.go | 2 +- inode/coalesce_test.go | 2 +- inode/inode.go | 2 +- jrpcfs/middleware_test.go | 2 +- retryrpc/api.go | 2 +- retryrpc/client.go | 2 +- retryrpc/retryrpc_test.go | 2 +- retryrpc/stress_test.go | 2 +- retryrpc/upcall_test.go | 2 +- swiftclient/object.go | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/confgen/api_test.go b/confgen/api_test.go index dc5cd9fa5..5d7345461 100644 --- a/confgen/api_test.go +++ b/confgen/api_test.go @@ -8,8 +8,8 @@ import ( "os" "testing" - "github.com/stretchr/testify/assert" "github.com/NVIDIA/proxyfs/conf" + "github.com/stretchr/testify/assert" ) func getConfMap(t *testing.T, filename string) (confMap conf.ConfMap, err error) { diff --git a/inode/api_test.go b/inode/api_test.go index 00c17a435..2538824e0 100644 --- a/inode/api_test.go +++ b/inode/api_test.go @@ -10,9 +10,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" "github.com/NVIDIA/proxyfs/swiftclient" "github.com/NVIDIA/proxyfs/utils" + "github.com/stretchr/testify/assert" ) const durationToDelayOrSkew = "100ms" diff --git a/inode/coalesce_test.go b/inode/coalesce_test.go index 57b5eabf9..5dad6c352 100644 --- a/inode/coalesce_test.go +++ b/inode/coalesce_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" "github.com/NVIDIA/proxyfs/blunder" + "github.com/stretchr/testify/assert" ) // NB: test setup and such is in api_test.go (look for TestMain function) diff --git a/inode/inode.go b/inode/inode.go index 25b64c2d5..abfd5fff2 100644 --- a/inode/inode.go +++ b/inode/inode.go @@ -12,9 +12,9 @@ import ( "sync" "time" - "github.com/ansel1/merry" "github.com/NVIDIA/cstruct" "github.com/NVIDIA/sortedmap" + "github.com/ansel1/merry" "github.com/NVIDIA/proxyfs/blunder" "github.com/NVIDIA/proxyfs/dlm" diff --git a/jrpcfs/middleware_test.go b/jrpcfs/middleware_test.go index b699c287f..24f3b827f 100644 --- a/jrpcfs/middleware_test.go +++ b/jrpcfs/middleware_test.go @@ -14,7 +14,6 @@ import ( "golang.org/x/sys/unix" - "github.com/stretchr/testify/assert" "github.com/NVIDIA/proxyfs/blunder" "github.com/NVIDIA/proxyfs/conf" "github.com/NVIDIA/proxyfs/fs" @@ -23,6 +22,7 @@ import ( "github.com/NVIDIA/proxyfs/ramswift" "github.com/NVIDIA/proxyfs/swiftclient" "github.com/NVIDIA/proxyfs/transitions" + "github.com/stretchr/testify/assert" ) // Shorthand for our testing debug log id; global to the package diff --git a/retryrpc/api.go b/retryrpc/api.go index 0463a1266..7bec8b118 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -21,9 +21,9 @@ import ( "sync" "time" - "github.com/google/btree" "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/logger" + "github.com/google/btree" ) // ServerCreds tracks the root CA and the diff --git a/retryrpc/client.go b/retryrpc/client.go index 4c2b8728d..068564efd 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -12,9 +12,9 @@ import ( "os" "time" - "github.com/google/btree" "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/logger" + "github.com/google/btree" ) const ( diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index ac0f6b22f..fcfd81f78 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -7,9 +7,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/retryrpc/rpctest" + "github.com/stretchr/testify/assert" ) // Test basic retryrpc primitives diff --git a/retryrpc/stress_test.go b/retryrpc/stress_test.go index aa083774e..953c3b2ca 100644 --- a/retryrpc/stress_test.go +++ b/retryrpc/stress_test.go @@ -14,8 +14,8 @@ import ( _ "net/http/pprof" */ - "github.com/stretchr/testify/assert" "github.com/NVIDIA/proxyfs/retryrpc/rpctest" + "github.com/stretchr/testify/assert" ) func TestStress(t *testing.T) { diff --git a/retryrpc/upcall_test.go b/retryrpc/upcall_test.go index 35d7124e7..349ab86ef 100644 --- a/retryrpc/upcall_test.go +++ b/retryrpc/upcall_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" "github.com/NVIDIA/proxyfs/retryrpc/rpctest" + "github.com/stretchr/testify/assert" ) // Test Upcall() functionality diff --git a/swiftclient/object.go b/swiftclient/object.go index 936fdcd88..12fb5081c 100644 --- a/swiftclient/object.go +++ b/swiftclient/object.go @@ -12,8 +12,8 @@ import ( "sync/atomic" "time" - "github.com/creachadair/cityhash" "github.com/NVIDIA/sortedmap" + "github.com/creachadair/cityhash" "github.com/NVIDIA/proxyfs/blunder" "github.com/NVIDIA/proxyfs/evtlog" From 24298b36d768a88ad28bc0958772f8d65faac1b2 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 8 Feb 2021 09:35:11 -0600 Subject: [PATCH 038/160] bump swift to ss-release-2.26.0.12 --- pfs_middleware/tox.ini | 2 +- saio/container/Dockerfile | 2 +- saio/vagrant_provision.sh | 2 +- sait/vagrant_provision.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pfs_middleware/tox.ini b/pfs_middleware/tox.ini index f89a1de95..a4c1363b3 100644 --- a/pfs_middleware/tox.ini +++ b/pfs_middleware/tox.ini @@ -7,7 +7,7 @@ usedevelop = True deps = lint: flake8 !lint: -r{toxinidir}/test-requirements.txt - release: git+https://github.com/NVIDIA/swift.git@ss-release-2.26.0.11 + release: git+https://github.com/NVIDIA/swift.git@ss-release-2.26.0.12 minver: http://tarballs.openstack.org/swift/swift-2.9.0.tar.gz master: http://tarballs.openstack.org/swift/swift-master.tar.gz commands = python -m unittest discover diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 634a94f82..03cafab48 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -3,7 +3,7 @@ FROM centos:7.4.1708 -ARG SwiftVersion=ss-release-2.26.0.11 +ARG SwiftVersion=ss-release-2.26.0.12 ARG GolangVersion=1.15.5 ARG ProxyFS_Version=stable diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index 895a39770..083ee6485 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -228,7 +228,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.26.0.11 +git checkout ss-release-2.26.0.12 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index 6508454d1..8baa7e9e5 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -230,7 +230,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.26.0.11 +git checkout ss-release-2.26.0.12 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet From f0406e401a4fc3a041338a4d18378306960001b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Tue, 9 Feb 2021 14:41:47 -0800 Subject: [PATCH 039/160] Install pip < 21.0 in TravisCI --- test/container/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/container/Dockerfile b/test/container/Dockerfile index de5555529..6296c652a 100644 --- a/test/container/Dockerfile +++ b/test/container/Dockerfile @@ -104,7 +104,7 @@ RUN yum -y install rh-python36 && \ # - requests ## Install Python tox # - tox -RUN pip install --upgrade pip +RUN pip install --upgrade 'pip<21.0' RUN pip install requests tox==3.5.3 # Install Golang From 216d496a525de37671d2545c8cb6ca65ff2367ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Tue, 9 Feb 2021 14:55:41 -0800 Subject: [PATCH 040/160] Restore TravisCI and Coveralls badges --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ca2277677..bb31e5004 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +[![Build Status](https://travis-ci.com/NVIDIA/proxyfs.svg?branch=development)](https://travis-ci.com/NVIDIA/proxyfs) +[![Coverage Status](https://coveralls.io/repos/github/swiftstack/ProxyFS/badge.svg?branch=development)](https://coveralls.io/github/swiftstack/ProxyFS?branch=development) + # ProxyFS Integrated File and Object Access for Swift Object Storage From 51e3a6d66b1e5eee65fcc641c46e5b3c088a9e18 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 16 Feb 2021 10:18:14 -0800 Subject: [PATCH 041/160] Fix missing inode flush in inode.AddDirEntry() --- inode/dir.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/inode/dir.go b/inode/dir.go index ab6668553..f0d59a4b0 100644 --- a/inode/dir.go +++ b/inode/dir.go @@ -1317,7 +1317,11 @@ func (vS *volumeStruct) AddDirEntry(dirInodeNumber InodeNumber, dirEntryName str dirEntryInode.onDiskInodeV1Struct.LinkCount++ } - err = nil + dirInode.dirty = true + dirEntryInode.dirty = true + + err = vS.flushInodes([]*inMemoryInodeStruct{dirInode, dirEntryInode}) + return } From d7bc1123f4ab72f53a12c398e7202b74d227b051 Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Fri, 19 Feb 2021 16:29:31 -0800 Subject: [PATCH 042/160] Add dummy test file to avoid annoying error message (#586) --- etcdclient/dummy_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 etcdclient/dummy_test.go diff --git a/etcdclient/dummy_test.go b/etcdclient/dummy_test.go new file mode 100644 index 000000000..baf4898dc --- /dev/null +++ b/etcdclient/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package etcdclient + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} From 48747ba76980d613742531a4821a828b644ab20a Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 22 Feb 2021 09:51:56 -0600 Subject: [PATCH 043/160] bump swift to ss-release-2.26.0.13 --- pfs_middleware/tox.ini | 2 +- saio/container/Dockerfile | 2 +- saio/vagrant_provision.sh | 2 +- sait/vagrant_provision.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pfs_middleware/tox.ini b/pfs_middleware/tox.ini index a4c1363b3..4bd691bd1 100644 --- a/pfs_middleware/tox.ini +++ b/pfs_middleware/tox.ini @@ -7,7 +7,7 @@ usedevelop = True deps = lint: flake8 !lint: -r{toxinidir}/test-requirements.txt - release: git+https://github.com/NVIDIA/swift.git@ss-release-2.26.0.12 + release: git+https://github.com/NVIDIA/swift.git@ss-release-2.26.0.13 minver: http://tarballs.openstack.org/swift/swift-2.9.0.tar.gz master: http://tarballs.openstack.org/swift/swift-master.tar.gz commands = python -m unittest discover diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 03cafab48..5c5992937 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -3,7 +3,7 @@ FROM centos:7.4.1708 -ARG SwiftVersion=ss-release-2.26.0.12 +ARG SwiftVersion=ss-release-2.26.0.13 ARG GolangVersion=1.15.5 ARG ProxyFS_Version=stable diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index 083ee6485..d991cb3c7 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -228,7 +228,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.26.0.12 +git checkout ss-release-2.26.0.13 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index 8baa7e9e5..f05fbebfd 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -230,7 +230,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.26.0.12 +git checkout ss-release-2.26.0.13 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet From ef964c8b109241001e2fa24791eb58613cbf1a77 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 2 Feb 2021 12:06:05 -0800 Subject: [PATCH 044/160] Beginnings of work on imgrpkg --- .../emswiftpkg/setup_teardown_test.go_HIDE | 369 ---------------- .../swift_proxy_emulator_test.go_HIDE | 411 ------------------ imgr/imgrpkg/api.go | 29 ++ imgr/imgrpkg/dummy.go | 16 + 4 files changed, 45 insertions(+), 780 deletions(-) delete mode 100644 emswift/emswiftpkg/setup_teardown_test.go_HIDE delete mode 100644 emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE create mode 100644 imgr/imgrpkg/api.go diff --git a/emswift/emswiftpkg/setup_teardown_test.go_HIDE b/emswift/emswiftpkg/setup_teardown_test.go_HIDE deleted file mode 100644 index 45d709cf6..000000000 --- a/emswift/emswiftpkg/setup_teardown_test.go_HIDE +++ /dev/null @@ -1,369 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package emswiftpkg - -import ( - "io/ioutil" - "net/http" - "os" - "strconv" - "sync" - "testing" - "time" - - "golang.org/x/sys/unix" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/proxyfsd" - "github.com/NVIDIA/proxyfs/ramswift" -) - -const ( - testAccountName = "AUTH_test" - testAuthKey = "testing" - testAuthUser = "test:tester" - testDaemonStartPollInterval = 1 * time.Second - testProxyFSDaemonHTTPPort = "15347" - testProxyFSDaemonIPAddr = "127.0.0.1" - testSwiftNoAuthIPAddr = "127.0.0.1" - testSwiftNoAuthPort = "38090" - testSwiftProxyAddr = "localhost:38080" -) - -type testDaemonGlobalsStruct struct { - proxyfsdErrChan chan error - proxyfsdWG sync.WaitGroup - ramswiftDoneChan chan bool -} - -var testDaemonGlobals testDaemonGlobalsStruct - -func testSetup(t *testing.T) { - var ( - err error - infoResponse *http.Response - ramswiftSignalHandlerIsArmedWG sync.WaitGroup - testConfMap conf.ConfMap - testConfStrings []string - testDir string - testPlugInEnvValue string - versionResponse *http.Response - ) - - testDir, err = ioutil.TempDir(os.TempDir(), "pfsagentd_test_") - if nil != err { - t.Fatalf("ioutil.TempDir() failed: %v", err) - } - - err = os.Chdir(testDir) - if nil != err { - t.Fatalf("os.Chdir() failed: %v", err) - } - - err = os.Mkdir("ProxyFSMountPointPath", 0777) // Agent.FUSEMountPointPath - if nil != err { - t.Fatalf("os.Mkdir() failed: %v", err) - } - - err = os.Mkdir("PfsAgentMountPointPath", 0777) // Volume:CommonVolume.FUSEMountPointName - if nil != err { - t.Fatalf("os.Mkdir() failed: %v", err) - } - - testPlugInEnvValue = "{\"AuthURL\":\"http://" - testPlugInEnvValue += testSwiftProxyAddr - testPlugInEnvValue += "/auth/v1.0\"\\u002C\"AuthUser\":\"" - testPlugInEnvValue += testAuthUser - testPlugInEnvValue += "\"\\u002C\"AuthKey\":\"" - testPlugInEnvValue += testAuthKey - testPlugInEnvValue += "\"\\u002C\"Account\":\"" - testPlugInEnvValue += testAccountName - testPlugInEnvValue += "\"}" - - testConfStrings = []string{ - "Agent.FUSEVolumeName=CommonVolume", - "Agent.FUSEMountPointPath=PfsAgentMountPointPath", - "Agent.FUSEUnMountRetryDelay=100ms", - "Agent.FUSEUnMountRetryCap=100", - "Agent.PlugInPath=/dev/null", // Using hard-coded AUTH - "Agent.PlugInEnvName=SwiftAuthBlob", - "Agent.PlugInEnvValue=" + testPlugInEnvValue, - "Agent.SwiftTimeout=20s", - "Agent.SwiftRetryLimit=10", - "Agent.SwiftRetryDelay=10ms", - "Agent.SwiftRetryDelayVariance=25", - "Agent.SwiftRetryExpBackoff=1.4", - "Agent.SwiftConnectionPoolSize=200", - "Agent.FetchExtentsFromFileOffset=32", - "Agent.FetchExtentsBeforeFileOffset=0", - "Agent.ReadCacheLineSize=1048576", - "Agent.ReadCacheLineCount=1000", - "Agent.LeaseRetryLimit=10", - "Agent.LeaseRetryDelay=10ms", - "Agent.LeaseRetryDelayVariance=25", - "Agent.LeaseRetryExpBackoff=1.4", - "Agent.SharedLeaseLimit=1000", - "Agent.ExclusiveLeaseLimit=100", - "Agent.ExtentMapEntryLimit=1048576", - "Agent.DirtyLogSegmentLimit=50", - "Agent.DirtyFileLimit=50", // TODO - obsolete this - "Agent.MaxFlushSize=10485760", - "Agent.MaxFlushTime=10s", - "Agent.LogFilePath=", - "Agent.LogToConsole=false", - "Agent.TraceEnabled=false", - "Agent.HTTPServerIPAddr=127.0.0.1", - "Agent.HTTPServerTCPPort=54323", - "Agent.ReadDirPlusEnabled=false", - "Agent.XAttrEnabled=false", - "Agent.EntryDuration=10s", - "Agent.AttrDuration=10s", - "Agent.AttrBlockSize=65536", - "Agent.ReaddirMaxEntries=1024", - "Agent.FUSEMaxBackground=100", - "Agent.FUSECongestionThreshhold=0", - "Agent.FUSEMaxWrite=131072", // Linux max... 128KiB is good enough for testing - "Agent.RetryRPCDeadlineIO=60s", - "Agent.RetryRPCKeepAlivePeriod=60s", - - "Stats.IPAddr=localhost", - "Stats.UDPPort=54324", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - - "StatsLogger.Period=0m", - "StatsLogger.Verbose=false", - - "Logging.LogFilePath=/dev/null", - "Logging.LogToConsole=false", - - "Peer:Peer0.PublicIPAddr=" + testProxyFSDaemonIPAddr, - "Peer:Peer0.PrivateIPAddr=" + testProxyFSDaemonIPAddr, - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - - "Cluster.WhoAmI=Peer0", - "Cluster.Peers=Peer0", - "Cluster.ServerGuid=a66488e9-a051-4ff7-865d-87bfb84cc2ae", - "Cluster.PrivateClusterUDPPort=54325", - "Cluster.UDPPacketSendSize=1400", - "Cluster.UDPPacketRecvSize=1500", - "Cluster.UDPPacketCapPerMessage=5", - "Cluster.HeartBeatDuration=1s", - "Cluster.HeartBeatMissLimit=3", - "Cluster.MessageQueueDepthPerPeer=4", - "Cluster.MaxRequestDuration=1s", - "Cluster.LivenessCheckRedundancy=2", - - "HTTPServer.TCPPort=" + testProxyFSDaemonHTTPPort, - - "SwiftClient.NoAuthIPAddr=" + testSwiftNoAuthIPAddr, - "SwiftClient.NoAuthTCPPort=" + testSwiftNoAuthPort, - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=1", - "SwiftClient.RetryLimitObject=1", - "SwiftClient.RetryDelay=10ms", - "SwiftClient.RetryDelayObject=10ms", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - "SwiftClient.ChunkedConnectionPoolSize=64", - "SwiftClient.NonChunkedConnectionPoolSize=32", - "SwiftClient.SwiftReconChecksPerConfCheck=0", - - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerStoragePolicy=silver", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainerNamePrefix=Replicated3Way_", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.ContainersPerPeer=10", - "PhysicalContainerLayout:PhysicalContainerLayoutReplicated3Way.MaxObjectsPerContainer=1000000", - - "Volume:CommonVolume.FSID=1", - "Volume:CommonVolume.FUSEMountPointName=ProxyFSMountPointPath", - "Volume:CommonVolume.NFSExportClientMapList=CommonVolumeNFSClient0", - "Volume:CommonVolume.SMBShareName=CommonShare", - "Volume:CommonVolume.PrimaryPeer=Peer0", - "Volume:CommonVolume.AccountName=AUTH_test", - "Volume:CommonVolume.CheckpointContainerName=.__checkpoint__", - "Volume:CommonVolume.CheckpointContainerStoragePolicy=gold", - "Volume:CommonVolume.CheckpointInterval=10s", - "Volume:CommonVolume.DefaultPhysicalContainerLayout=PhysicalContainerLayoutReplicated3Way", - "Volume:CommonVolume.MaxFlushSize=10485760", - "Volume:CommonVolume.MaxFlushTime=10s", - "Volume:CommonVolume.FileDefragmentChunkSize=10485760", - "Volume:CommonVolume.FileDefragmentChunkDelay=10ms", - "Volume:CommonVolume.NonceValuesToReserve=100", - "Volume:CommonVolume.MaxEntriesPerDirNode=32", - "Volume:CommonVolume.MaxExtentsPerFileNode=32", - "Volume:CommonVolume.MaxInodesPerMetadataNode=32", - "Volume:CommonVolume.MaxLogSegmentsPerMetadataNode=64", - "Volume:CommonVolume.MaxDirFileNodesPerMetadataNode=16", - "Volume:CommonVolume.MaxBytesInodeCache=100000", - "Volume:CommonVolume.InodeCacheEvictInterval=1s", - "Volume:CommonVolume.AutoFormat=true", - "Volume:CommonVolume.ActiveLeaseEvictLowLimit=5000", - "Volume:CommonVolume.ActiveLeaseEvictHighLimit=5010", - - "NFSClientMap:CommonVolumeNFSClient0.ClientPattern=*", - "NFSClientMap:CommonVolumeNFSClient0.AccessMode=rw", - "NFSClientMap:CommonVolumeNFSClient0.RootSquash=no_root_squash", - "NFSClientMap:CommonVolumeNFSClient0.Secure=insecure", - - "VolumeGroup:CommonVolumeGroup.VolumeList=CommonVolume", - "VolumeGroup:CommonVolumeGroup.VirtualIPAddr=", - "VolumeGroup:CommonVolumeGroup.PrimaryPeer=Peer0", - "VolumeGroup:CommonVolumeGroup.ReadCacheLineSize=1000000", - "VolumeGroup:CommonVolumeGroup.ReadCacheWeight=100", - - "FSGlobals.VolumeGroupList=CommonVolumeGroup", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=32", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.InodeRecCacheEvictLowLimit=10000", - "FSGlobals.InodeRecCacheEvictHighLimit=10010", - "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", - "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", - "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", - "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", - "FSGlobals.DirEntryCacheEvictLowLimit=10000", - "FSGlobals.DirEntryCacheEvictHighLimit=10010", - "FSGlobals.FileExtentMapEvictLowLimit=10000", - "FSGlobals.FileExtentMapEvictHighLimit=10010", - "FSGlobals.EtcdEnabled=false", - - "JSONRPCServer.TCPPort=54326", - "JSONRPCServer.FastTCPPort=54327", - "JSONRPCServer.RetryRPCPort=54328", - "JSONRPCServer.RetryRPCTTLCompleted=10s", - "JSONRPCServer.RetryRPCAckTrim=10ms", - "JSONRPCServer.DataPathLogging=false", - "JSONRPCServer.MinLeaseDuration=250ms", - "JSONRPCServer.LeaseInterruptInterval=250ms", - "JSONRPCServer.LeaseInterruptLimit=20", - } - - testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxAccountNameLength="+strconv.FormatUint(testMaxAccountNameLength, 10)) - testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxContainerNameLength="+strconv.FormatUint(testMaxContainerNameLength, 10)) - testConfStrings = append(testConfStrings, "RamSwiftInfo.MaxObjectNameLength="+strconv.FormatUint(testMaxObjectNameLength, 10)) - testConfStrings = append(testConfStrings, "RamSwiftInfo.AccountListingLimit="+strconv.FormatUint(testAccountListingLimit, 10)) - testConfStrings = append(testConfStrings, "RamSwiftInfo.ContainerListingLimit="+strconv.FormatUint(testContainerListingLimit, 10)) - - ramswiftSignalHandlerIsArmedWG.Add(1) - testDaemonGlobals.ramswiftDoneChan = make(chan bool, 1) - - go ramswift.Daemon("/dev/null", testConfStrings, &ramswiftSignalHandlerIsArmedWG, testDaemonGlobals.ramswiftDoneChan, unix.SIGUSR1) - - ramswiftSignalHandlerIsArmedWG.Wait() - - for { - infoResponse, err = http.Get("http://" + testSwiftNoAuthIPAddr + ":" + testSwiftNoAuthPort + "/info") - if (nil == err) && (http.StatusOK == infoResponse.StatusCode) { - break - } - - time.Sleep(testDaemonStartPollInterval) - } - - testDaemonGlobals.proxyfsdErrChan = make(chan error, 1) // Must be buffered to avoid race - - go proxyfsd.Daemon("/dev/null", testConfStrings, testDaemonGlobals.proxyfsdErrChan, &testDaemonGlobals.proxyfsdWG, []string{}, unix.SIGUSR2) - - err = <-testDaemonGlobals.proxyfsdErrChan - if nil != err { - t.Fatalf("proxyfsd.Daemon() startup failed: %v", err) - } - - for { - versionResponse, err = http.Get("http://" + testProxyFSDaemonIPAddr + ":" + testProxyFSDaemonHTTPPort + "/version") - if (nil == err) && (http.StatusOK == versionResponse.StatusCode) { - break - } - - time.Sleep(testDaemonStartPollInterval) - } - - testConfMap, err = conf.MakeConfMapFromStrings(testConfStrings) - if nil != err { - t.Fatalf("conf.MakeConfMapFromStrings() failed: %v", err) - } - - globals.logFile = nil - globals.config.LogFilePath = "" - globals.config.LogToConsole = true - - startSwiftProxyEmulator(t, testConfMap) - - initializeGlobals(testConfMap) - - // Fake out that plug-in auth has already obtained AuthToken & StorageURL - - globals.swiftAuthToken = testAuthToken - globals.swiftStorageURL = "http://" + testSwiftProxyAddr + "/proxyfs/" + testAccountName - - go testSwallowFissionErrChan(t, globals.fissionErrChan) - - doMountProxyFS() - - performMountFUSE() -} - -func testSwallowFissionErrChan(t *testing.T, fissionErrChan chan error) { - var ( - err error - ) - - err = <-fissionErrChan - if nil != err { - t.Fatalf("fissionErrChan received err: %v", err) - } -} - -func testTeardown(t *testing.T) { - var ( - err error - testDir string - ) - - performUnmountFUSE() - - emptyFileInodeDirtyListAndLogSegmentChan() - - doUnmountProxyFS() - - uninitializeGlobals() - - _ = unix.Kill(unix.Getpid(), unix.SIGUSR2) - - err = <-testDaemonGlobals.proxyfsdErrChan - - testDaemonGlobals.proxyfsdWG.Wait() - - if nil != err { - t.Fatalf("proxyfsd.Daemon() exited with error: %v", err) - } - - unix.Kill(unix.Getpid(), unix.SIGUSR1) - - _ = <-testDaemonGlobals.ramswiftDoneChan - - testDir, err = os.Getwd() - if nil != err { - t.Fatalf("os.Getwd() failed: %v", err) - } - - err = os.Chdir("..") - if nil != err { - t.Fatalf("os.Chdir() failed: %v", err) - } - - err = os.RemoveAll(testDir) - if nil != err { - t.Fatalf("os.RemoveAll() failed: %v", err) - } - - stopSwiftProxyEmulator() -} diff --git a/emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE b/emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE deleted file mode 100644 index 10ff83386..000000000 --- a/emswift/emswiftpkg/swift_proxy_emulator_test.go_HIDE +++ /dev/null @@ -1,411 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package emswiftpkg - -import ( - "context" - "io/ioutil" - "net" - "net/http" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/NVIDIA/proxyfs/conf" -) - -const ( - testAuthToken = "AUTH_tkTestToken" - testJrpcResponseBufSize = 1024 * 1024 -) - -type testSwiftProxyEmulatorGlobalsStruct struct { - sync.WaitGroup - t *testing.T - ramswiftNoAuthURL string - proxyfsdJrpcTCPAddr *net.TCPAddr - jrpcResponsePool *sync.Pool - httpClient *http.Client - httpServer *http.Server -} - -var testSwiftProxyEmulatorGlobals testSwiftProxyEmulatorGlobalsStruct - -func startSwiftProxyEmulator(t *testing.T, confMap conf.ConfMap) { - var ( - err error - infoResponse *http.Response - jrpcServerIPAddr string - jrpcServerTCPPort uint16 - swiftClientNoAuthIPAddr string - swiftClientNoAuthTCPPort uint16 - whoAmI string - ) - - testSwiftProxyEmulatorGlobals.t = t - - swiftClientNoAuthIPAddr, err = confMap.FetchOptionValueString("SwiftClient", "NoAuthIPAddr") - if nil != err { - t.Fatal(err) - } - - swiftClientNoAuthTCPPort, err = confMap.FetchOptionValueUint16("SwiftClient", "NoAuthTCPPort") - if nil != err { - t.Fatal(err) - } - - testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL = "http://" + net.JoinHostPort(swiftClientNoAuthIPAddr, strconv.FormatUint(uint64(swiftClientNoAuthTCPPort), 10)) - - whoAmI, err = confMap.FetchOptionValueString("Cluster", "WhoAmI") - if nil != err { - t.Fatal(err) - } - - jrpcServerIPAddr, err = confMap.FetchOptionValueString("Peer:"+whoAmI, "PrivateIPAddr") - if nil != err { - t.Fatal(err) - } - - jrpcServerTCPPort, err = confMap.FetchOptionValueUint16("JSONRPCServer", "TCPPort") - if nil != err { - t.Fatal(err) - } - - testSwiftProxyEmulatorGlobals.proxyfsdJrpcTCPAddr, err = net.ResolveTCPAddr("tcp", net.JoinHostPort(jrpcServerIPAddr, strconv.FormatUint(uint64(jrpcServerTCPPort), 10))) - if nil != err { - t.Fatal(err) - } - - testSwiftProxyEmulatorGlobals.httpClient = &http.Client{} - - testSwiftProxyEmulatorGlobals.httpServer = &http.Server{ - Addr: testSwiftProxyAddr, - Handler: &testSwiftProxyEmulatorGlobals, - } - - testSwiftProxyEmulatorGlobals.jrpcResponsePool = &sync.Pool{ - New: func() (bufAsInterface interface{}) { - var ( - bufAsByteSlice []byte - ) - - bufAsByteSlice = make([]byte, testJrpcResponseBufSize) - - bufAsInterface = bufAsByteSlice - - return - }, - } - - testSwiftProxyEmulatorGlobals.Add(1) - - go func() { - _ = testSwiftProxyEmulatorGlobals.httpServer.ListenAndServe() - - testSwiftProxyEmulatorGlobals.Done() - }() - - for { - infoResponse, err = http.Get("http://" + testSwiftProxyAddr + "/info") - if nil == err { - break - } - - time.Sleep(testDaemonStartPollInterval) - } - - if http.StatusOK != infoResponse.StatusCode { - t.Fatalf("GET /info from ServeHTTP() got unexpected status %s", infoResponse.Status) - } -} - -func stopSwiftProxyEmulator() { - var ( - err error - ) - - err = testSwiftProxyEmulatorGlobals.httpServer.Shutdown(context.Background()) - if nil != err { - testSwiftProxyEmulatorGlobals.t.Fatalf("testSwiftProxyEmulatorGlobals.httpServer.Shutdown() failed: %v", err) - } - - testSwiftProxyEmulatorGlobals.Wait() - - testSwiftProxyEmulatorGlobals.jrpcResponsePool = nil -} - -func (dummy *testSwiftProxyEmulatorGlobalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - // Handle the GET of/on info & AuthURL cases - - if http.MethodGet == request.Method { - switch request.URL.Path { - case "/info": - doInfo(responseWriter) - return - case "/auth/v1.0": - doAuth(responseWriter, request) - return - default: - // Fall through to normal processing - } - } - - // Reject unauthorized requests - - if request.Header.Get("X-Auth-Token") != testAuthToken { - responseWriter.WriteHeader(http.StatusUnauthorized) - return - } - - // Branch off to individual request method handlers - - switch request.Method { - case http.MethodGet: - doGET(responseWriter, request) - case http.MethodPut: - doPUT(responseWriter, request) - case "PROXYFS": - doRPC(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusMethodNotAllowed) - } -} - -func doInfo(authResponseWriter http.ResponseWriter) { - var ( - err error - getBuf []byte - noAuthResponse *http.Response - ) - - noAuthResponse, err = http.Get(testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL + "/info") - if nil != err { - testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info from ramswift failed: %v", err) - } - - if http.StatusOK != noAuthResponse.StatusCode { - testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info from ramswift returned bad status: %v", noAuthResponse.Status) - } - - getBuf, err = ioutil.ReadAll(noAuthResponse.Body) - if nil != err { - testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info returned unreadable Body: %v", err) - } - - err = noAuthResponse.Body.Close() - if nil != err { - testSwiftProxyEmulatorGlobals.t.Fatalf("GET of /info returned uncloseable Body: %v", err) - } - - authResponseWriter.WriteHeader(http.StatusOK) - _, _ = authResponseWriter.Write(getBuf) -} - -func doAuth(authResponseWriter http.ResponseWriter, authRequest *http.Request) { - if authRequest.Header.Get("X-Auth-User") != testAuthUser { - authResponseWriter.WriteHeader(http.StatusUnauthorized) - return - } - if authRequest.Header.Get("X-Auth-Key") != testAuthKey { - authResponseWriter.WriteHeader(http.StatusUnauthorized) - return - } - authResponseWriter.Header().Add("X-Auth-Token", testAuthToken) - authResponseWriter.Header().Add("X-Storage-Url", "http://"+testSwiftProxyAddr+"/v1/"+testAccountName) - authResponseWriter.WriteHeader(http.StatusOK) -} - -// doGET proxies the GET over to ramswift -// -func doGET(authResponseWriter http.ResponseWriter, authRequest *http.Request) { - var ( - contentRangeHeader string - contentTypeHeader string - err error - getBuf []byte - hostHeader string - noAuthPath string - noAuthRequest *http.Request - noAuthResponse *http.Response - noAuthStatusCode int - rangeHeader string - ) - - if !strings.HasPrefix(authRequest.URL.Path, "/proxyfs/"+testAccountName) { - authResponseWriter.WriteHeader(http.StatusNotFound) - return - } - - noAuthPath = strings.Replace(authRequest.URL.Path, "proxyfs", "v1", 1) - - noAuthRequest, err = http.NewRequest("GET", testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL+noAuthPath, nil) - if nil != err { - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - hostHeader = authRequest.Header.Get("Host") - if "" != hostHeader { - noAuthRequest.Header.Add("Host", hostHeader) - } - - rangeHeader = authRequest.Header.Get("Range") - if "" != rangeHeader { - noAuthRequest.Header.Add("Range", rangeHeader) - } - - noAuthResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(noAuthRequest) - if nil != err { - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - noAuthStatusCode = noAuthResponse.StatusCode - - if (http.StatusOK != noAuthStatusCode) && (http.StatusPartialContent != noAuthStatusCode) { - _ = noAuthResponse.Body.Close() - authResponseWriter.WriteHeader(noAuthStatusCode) - return - } - - getBuf, err = ioutil.ReadAll(noAuthResponse.Body) - if nil != err { - _ = noAuthResponse.Body.Close() - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - err = noAuthResponse.Body.Close() - if nil != err { - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - contentTypeHeader = noAuthResponse.Header.Get("Content-Type") - if "" != contentTypeHeader { - authResponseWriter.Header().Add("Content-Type", contentTypeHeader) - } - - contentRangeHeader = noAuthResponse.Header.Get("Content-Range") - if "" != contentRangeHeader { - authResponseWriter.Header().Add("Content-Range", contentRangeHeader) - } - - authResponseWriter.WriteHeader(noAuthStatusCode) - - _, _ = authResponseWriter.Write(getBuf) -} - -// doPUT proxies the GET over to ramswift -// -func doPUT(authResponseWriter http.ResponseWriter, authRequest *http.Request) { - var ( - err error - hostHeader string - noAuthPath string - noAuthRequest *http.Request - noAuthResponse *http.Response - ) - - if !strings.HasPrefix(authRequest.URL.Path, "/proxyfs/"+testAccountName) { - _ = authRequest.Body.Close() - authResponseWriter.WriteHeader(http.StatusNotFound) - return - } - - noAuthPath = strings.Replace(authRequest.URL.Path, "proxyfs", "v1", 1) - - noAuthRequest, err = http.NewRequest("PUT", testSwiftProxyEmulatorGlobals.ramswiftNoAuthURL+noAuthPath, authRequest.Body) - if nil != err { - _ = authRequest.Body.Close() - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - hostHeader = authRequest.Header.Get("Host") - if "" != hostHeader { - noAuthRequest.Header.Add("Host", hostHeader) - } - - noAuthResponse, err = testSwiftProxyEmulatorGlobals.httpClient.Do(noAuthRequest) - if nil != err { - _ = authRequest.Body.Close() - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - err = authRequest.Body.Close() - if nil != err { - authResponseWriter.WriteHeader(http.StatusBadRequest) - return - } - - authResponseWriter.WriteHeader(noAuthResponse.StatusCode) -} - -// doRPC proxies the payload as a JSON RPC request over to proxyfsd -// -func doRPC(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - jrpcResponseBuf []byte - jrpcResponseLen int - jrpcRequestBuf []byte - tcpConn *net.TCPConn - ) - - if !strings.HasPrefix(request.URL.Path, "/proxyfs/"+testAccountName) { - _ = request.Body.Close() - responseWriter.WriteHeader(http.StatusNotFound) - return - } - - jrpcRequestBuf, err = ioutil.ReadAll(request.Body) - _ = request.Body.Close() - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - if request.Header.Get("Content-Type") != "application/json" { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } - - tcpConn, err = net.DialTCP("tcp", nil, testSwiftProxyEmulatorGlobals.proxyfsdJrpcTCPAddr) - if nil != err { - responseWriter.WriteHeader(http.StatusServiceUnavailable) - return - } - - _, err = tcpConn.Write(jrpcRequestBuf) - if nil != err { - _ = tcpConn.Close() - responseWriter.WriteHeader(http.StatusServiceUnavailable) - return - } - - jrpcResponseBuf = testSwiftProxyEmulatorGlobals.jrpcResponsePool.Get().([]byte) - - jrpcResponseLen, err = tcpConn.Read(jrpcResponseBuf) - if nil != err { - _ = tcpConn.Close() - responseWriter.WriteHeader(http.StatusServiceUnavailable) - return - } - - err = tcpConn.Close() - if nil != err { - responseWriter.WriteHeader(http.StatusServiceUnavailable) - return - } - - responseWriter.Header().Add("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write(jrpcResponseBuf[:jrpcResponseLen]) - - testSwiftProxyEmulatorGlobals.jrpcResponsePool.Put(jrpcResponseBuf) -} diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go new file mode 100644 index 000000000..a9269f1c2 --- /dev/null +++ b/imgr/imgrpkg/api.go @@ -0,0 +1,29 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +import ( + "github.com/swiftstack/ProxyFS/conf" +) + +// Start is called to start serving +// +func Start(confMap conf.ConfMap) (err error) { + err = start(confMap) + return +} + +// Stop is called to stop serving +// +func Stop() (err error) { + err = stop() + return +} + +// Signal is called to interrupt the server for performing operations such as log rotation +// +func Signal() (err error) { + err = signal() + return +} diff --git a/imgr/imgrpkg/dummy.go b/imgr/imgrpkg/dummy.go index e1af09375..f32836d47 100644 --- a/imgr/imgrpkg/dummy.go +++ b/imgr/imgrpkg/dummy.go @@ -2,3 +2,19 @@ // SPDX-License-Identifier: Apache-2.0 package imgrpkg + +import ( + "github.com/swiftstack/ProxyFS/conf" +) + +func start(confMap conf.ConfMap) (err error) { + return nil // TODO +} + +func stop() (err error) { + return nil // TODO +} + +func signal() (err error) { + return nil // TODO +} From d00ef07a56e64c0afe194df64f562ad9309b79be Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 19 Feb 2021 10:30:57 -0800 Subject: [PATCH 045/160] Fix stale references to old repo location in imgr/imgrpkg/ --- imgr/imgrpkg/api.go | 2 +- imgr/imgrpkg/dummy.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index a9269f1c2..a21367544 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -4,7 +4,7 @@ package imgrpkg import ( - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) // Start is called to start serving diff --git a/imgr/imgrpkg/dummy.go b/imgr/imgrpkg/dummy.go index f32836d47..d59b4bfe8 100644 --- a/imgr/imgrpkg/dummy.go +++ b/imgr/imgrpkg/dummy.go @@ -4,7 +4,7 @@ package imgrpkg import ( - "github.com/swiftstack/ProxyFS/conf" + "github.com/NVIDIA/proxyfs/conf" ) func start(confMap conf.ConfMap) (err error) { From ac152b691a89204b1f5e76e5d437c9b162581792 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 22 Feb 2021 17:12:25 -0800 Subject: [PATCH 046/160] Re-worked Auth PlugIn to leverage Golang's pkg/plugin and be useable by both PFSAgent and ProxyFS --- GoPlugInMakefile | 40 +++++++++++++++ Makefile | 19 +++++++ iauth/Makefile | 6 +++ iauth/README.md | 28 +++++++++++ iauth/api.go | 46 +++++++++++++++++ iauth/dummy_test.go | 11 ++++ iauth/iauth-swift/.gitignore | 1 + iauth/iauth-swift/Makefile | 7 +++ iauth/iauth-swift/README.md | 48 ++++++++++++++++++ iauth/iauth-swift/iauth-swift.go | 86 ++++++++++++++++++++++++++++++++ iauth/iauth-swift/plugin_test.go | 64 ++++++++++++++++++++++++ 11 files changed, 356 insertions(+) create mode 100644 GoPlugInMakefile create mode 100644 iauth/Makefile create mode 100644 iauth/README.md create mode 100644 iauth/api.go create mode 100644 iauth/dummy_test.go create mode 100644 iauth/iauth-swift/.gitignore create mode 100644 iauth/iauth-swift/Makefile create mode 100644 iauth/iauth-swift/README.md create mode 100644 iauth/iauth-swift/iauth-swift.go create mode 100644 iauth/iauth-swift/plugin_test.go diff --git a/GoPlugInMakefile b/GoPlugInMakefile new file mode 100644 index 000000000..24d232818 --- /dev/null +++ b/GoPlugInMakefile @@ -0,0 +1,40 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +# GoPlugInMakefile is a template to be included by Go PlugIn Makefile's in this repo + +all: version fmt generate install test + +.PHONY: all bench clean cover fmt generate get install test + +bench: + go test -bench $(gosubdir) + +clean: + @set -e; \ + go clean -i $(gosubdir); \ + rm -f $(GOPATH)/bin/$(plugin); \ + for generatedfile in $(generatedfiles); do \ + rm -f $$generatedfile; \ + done + +cover: + go test $(gosubdir) -covermode=atomic -coverprofile=coverage.coverprofile + +fmt: + go fmt $(gosubdir) + +generate: + go generate $(gosubdir) + +install: + @set -e; \ + go build -buildmode=plugin $(gosubdir); \ + cp $(plugin) $(GOPATH)/bin/. + +test: + go test -vet all $(gosubdir) + +version: + @go version + diff --git a/Makefile b/Makefile index f7462c9d2..a530a31bf 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ gopkgsubdirs = \ halter \ headhunter \ httpserver \ + iauth \ inode \ jrpcfs \ liveness \ @@ -36,6 +37,9 @@ gopkgsubdirs = \ emswift/emswiftpkg \ imgr/imgrpkg +goplugindirs = \ + iauth/iauth-swift + gobinsubdirs = \ cleanproxyfs \ emswift \ @@ -125,6 +129,9 @@ clean: for gosubdir in $(gopkgsubdirs); do \ $(MAKE) --no-print-directory -C $$gosubdir clean; \ done; \ + for gosubdir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir clean; \ + done; \ for gosubdir in $(gobinsubdirs); do \ $(MAKE) --no-print-directory -C $$gosubdir clean; \ done @@ -140,6 +147,9 @@ fmt: for gosubdir in $(gopkgsubdirs); do \ $(MAKE) --no-print-directory -C $$gosubdir fmt; \ done; \ + for gosubdir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir fmt; \ + done; \ for gosubdir in $(gobinsubdirs); do \ $(MAKE) --no-print-directory -C $$gosubdir fmt; \ done @@ -149,6 +159,9 @@ generate: for gosubdir in $(gopkgsubdirs); do \ $(MAKE) --no-print-directory -C $$gosubdir generate; \ done; \ + for gosubdir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir generate; \ + done; \ for gosubdir in $(gobinsubdirs); do \ $(MAKE) --no-print-directory -C $$gosubdir generate; \ done @@ -158,6 +171,9 @@ install: for gosubdir in $(gopkgsubdirs); do \ $(MAKE) --no-print-directory -C $$gosubdir install; \ done; \ + for gosubdir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir install; \ + done; \ for gosubdir in $(gobinsubdirs); do \ $(MAKE) --no-print-directory -C $$gosubdir install; \ done @@ -183,6 +199,9 @@ test: for gosubdir in $(gopkgsubdirs); do \ $(MAKE) --no-print-directory -C $$gosubdir test; \ done; \ + for gosubdir in $(goplugindirs); do \ + $(MAKE) --no-print-directory -C $$gosubdir test; \ + done; \ for gosubdir in $(gobinsubdirs); do \ if [ $$gosubdir == "pfsagentd" ]; \ then \ diff --git a/iauth/Makefile b/iauth/Makefile new file mode 100644 index 000000000..d7a74479c --- /dev/null +++ b/iauth/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/iauth + +include ../GoMakefile diff --git a/iauth/README.md b/iauth/README.md new file mode 100644 index 000000000..e9730e191 --- /dev/null +++ b/iauth/README.md @@ -0,0 +1,28 @@ +# Authentication Plug-In + +Both ProxyFS itself and each PFSAgent client access Swift Accounts, +Containers, and Objects directly. PFSAgent, potentially residing +outside a Swift cluster's `trust domain`, will use normal +OpenStack Swift methods for such access. This access must be authorized +by means of obtaining an `AuthToken`. ProxyFS, if not configured +alongside a so-called `NoAuth` Swift Proxy, will also need the same. +Even in Swift clusters have a `NoAuth` Swift Proxy configured, ProxyFS +will, from time to time, validate PFSAgent client access by testing their +AuthToken locally as well. + +While many Swift clusters implement authorization via Swift Proxy +pipeline filters that honor the OpenStack Swift convention, some clusters +may require alternate authorization mechanisms. To support any such +authorization solution, a Golang Plug-In mechanism is employed. + +A standard OpenStack Swift Plug-In is provided (see subdirectory `iauth-swift`) +that may either be used for clusters honoring the OpenStack Swift authorization +convention or as a template for development of any particular authorization +solution. The only requirements are that: + +* The plug-in's location is provided this `iauth` package +* Credentials to be authorized are provided in a single string (possibly a JSON document) +* The plug-in returns both a Swift AuthToken and StorageURL (or an error) +* The StorageURL has been properly modified as necessary to ensure + * the proper transport is used (i.e. either "http" or "https") + * the version portion of the path has been set to "proxyfs" (rather than "v1") diff --git a/iauth/api.go b/iauth/api.go new file mode 100644 index 000000000..c8f3c0440 --- /dev/null +++ b/iauth/api.go @@ -0,0 +1,46 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iauth + +import ( + "fmt" + "plugin" +) + +// PerformAuth accepts a path to an Auth PlugIn and a string to pass to a func +// also named PerformAuth requesting it to perform the necessary authorization. +// +// The return from the Auth PlugIn's PerformAuth func is simply returned to the +// caller of this func. +// +func PerformAuth(authPlugInPath string, authInString string) (authToken string, storageURL string, err error) { + var ( + ok bool + performAuthAsFunc func(authInString string) (authToken string, storageURL string, err error) + performAuthAsSymbol plugin.Symbol + plugIn *plugin.Plugin + ) + + plugIn, err = plugin.Open(authPlugInPath) + if nil != err { + err = fmt.Errorf("plugin.Open(\"%s\") failed: %v", authPlugInPath, err) + return + } + + performAuthAsSymbol, err = plugIn.Lookup("PerformAuth") + if nil != err { + err = fmt.Errorf("plugIn[\"%s\"].Lookup(\"PerformAuth\") failed: %v", authPlugInPath, err) + return + } + + performAuthAsFunc, ok = performAuthAsSymbol.(func(authInString string) (authToken string, storageURL string, err error)) + if !ok { + err = fmt.Errorf("performAuthAsSymbol.(func(authInString string) (authToken string, storageURL string, err error)) returned !ok") + return + } + + authToken, storageURL, err = performAuthAsFunc(authInString) + + return +} diff --git a/iauth/dummy_test.go b/iauth/dummy_test.go new file mode 100644 index 000000000..5157b88c3 --- /dev/null +++ b/iauth/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iauth + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/iauth/iauth-swift/.gitignore b/iauth/iauth-swift/.gitignore new file mode 100644 index 000000000..140f8cf80 --- /dev/null +++ b/iauth/iauth-swift/.gitignore @@ -0,0 +1 @@ +*.so diff --git a/iauth/iauth-swift/Makefile b/iauth/iauth-swift/Makefile new file mode 100644 index 000000000..fabba3505 --- /dev/null +++ b/iauth/iauth-swift/Makefile @@ -0,0 +1,7 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/iauth/iauth-swift +plugin := iauth-swift.so + +include ../../GoPlugInMakefile diff --git a/iauth/iauth-swift/README.md b/iauth/iauth-swift/README.md new file mode 100644 index 000000000..5b76d603b --- /dev/null +++ b/iauth/iauth-swift/README.md @@ -0,0 +1,48 @@ +# OpenStack Swift Authorization PlugIn + +To provide a workable solution for those following standard Swift Authentication, +this plug-in instance may be employed. Otherwise, consider this implementation a +template for the desired instantiation of whatever Authentication mechanism is +employed. + +For the standard Swift Authentication instantiation, the value of `authInJSON` +is required to be a UTF-8 encoded JSON Document: + +``` +{ + "AuthURL" : "/auth/v1.0>", + "AuthUser" : "", + "AuthKey" : "", + "Account" : "" +} +``` + +There are three modifications to the Storage URL normally returned by a +standard Swift Authentication operation: + +* The `scheme` used to authenticate may be either `http` or `https`. In the + case of `https`, it is likely that some form of TLS termination prior + to reaching the Swift Proxy has rewritten the `scheme` to be `http`. In such + a case, the Storage URL returned will specify `http` as its scheme. Since + the client must continue to use `https` to reach the Swift Proxy for each + authenticated subsequent request, the plug-in will rewrite the scheme to + be `https`. Note that this is an incomplete solution in cases where standard + port numbers (i.e. `80` for `http` and `443` for `https`) are not assumed + (i.e. port numbers are specified in the URL). + +* The path portion of the Storage URL returned by the Swift Proxy will start + with `v1` to indicate the version of the OpenStack Swift protocol being used. + No `v2` has ever been defined, so this is a constant in normal requests. + To indicate the client, in the case of GETs and PUTs, has specified the + `physical` path (i.e. to LogSegments of a FileInode) rather than the + `virtual` path (i.e. the full path of the file in the file system), this + protocol version will be changed to `proxyfs`. This updated protocol verion + will also be used with the `PROXYFS` method used to pass a `mount` request, + via JSON RPC, over to the ProxyFS process serving the volume to be mounted. + +* The final element of the path portion of the Storage URL returned by the + Swift Proxy will typically be the Account associated with the specified + AuthUser (e.g. AuthUser `test` typically has a corresponding Account named + `AUTH_test`). The volume being accessed may, however be stored in a different + Account than this. As such, the account element of the path will be replaced + with the `Account` as requested. diff --git a/iauth/iauth-swift/iauth-swift.go b/iauth/iauth-swift/iauth-swift.go new file mode 100644 index 000000000..f92bedf6a --- /dev/null +++ b/iauth/iauth-swift/iauth-swift.go @@ -0,0 +1,86 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/NVIDIA/proxyfs/version" +) + +type authInStruct struct { + AuthURL string + AuthUser string + AuthKey string + Account string +} + +// PerformAuth accepts a JSON string, performs the OpenStack Swift Authorization +// process and returns the AuthToken and StorageURL that may be used to access +// the OpenStack Swift Account/Container/Objects. +// +// The format of authInJSON is determined by the Go JSON unmarshalling conventions +// of the AuthInStruct declared above. +// +func PerformAuth(authInJSON string) (authToken string, storageURL string, err error) { + var ( + authIn authInStruct + authRequest *http.Request + authResponse *http.Response + storageURLSplit []string + ) + + err = json.Unmarshal([]byte(authInJSON), &authIn) + if nil != err { + err = fmt.Errorf("json.Unmarshal(\"%s\",) failed: %v", authInJSON, err) + return + } + + authRequest, err = http.NewRequest("GET", authIn.AuthURL, nil) + if nil != err { + err = fmt.Errorf("http.NewRequest(\"GET\", \"%s\", nil) failed: %v", authIn.AuthURL, err) + return + } + + authRequest.Header.Add("X-Auth-User", authIn.AuthUser) + authRequest.Header.Add("X-Auth-Key", authIn.AuthKey) + + authRequest.Header.Add("User-Agent", "iauth-swift "+version.ProxyFSVersion) + + authResponse, err = http.DefaultClient.Do(authRequest) + if nil != err { + err = fmt.Errorf("http.DefaultClient.Do(authRequest) failed: %v", err) + return + } + + if http.StatusOK != authResponse.StatusCode { + err = fmt.Errorf("authResponse.Status unexpected: %v", authResponse.Status) + return + } + + authToken = authResponse.Header.Get("X-Auth-Token") + storageURL = authResponse.Header.Get("X-Storage-Url") + + if strings.HasPrefix(authIn.AuthURL, "https://") && strings.HasPrefix(storageURL, "http://") { + // We need to correct for the case where AuthURL starts with "https://"" + // but the Swift Proxy is behind a TLS terminating proxy. In this case, + // Swift Proxy auth will actually see an AuthURL starting with "http://"" + // and respond with a StorageURL starting with "http://"". + + storageURL = strings.Replace(storageURL, "http://", "https://", 1) + } + + storageURLSplit = strings.Split(storageURL, "/") + + storageURLSplit[3] = "proxyfs" + storageURLSplit[4] = authIn.Account + + storageURL = strings.Join(storageURLSplit, "/") + + err = nil + return +} diff --git a/iauth/iauth-swift/plugin_test.go b/iauth/iauth-swift/plugin_test.go new file mode 100644 index 000000000..020c2b9b4 --- /dev/null +++ b/iauth/iauth-swift/plugin_test.go @@ -0,0 +1,64 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/emswift/emswiftpkg" +) + +func TestSwiftAuth(t *testing.T) { + var ( + authInJSON string + authToken string + confMap conf.ConfMap + confStrings = []string{ + "EMSWIFT.AuthIPAddr=127.0.0.1", + "EMSWIFT.AuthTCPPort=9997", + "EMSWIFT.JRPCIPAddr=127.0.0.1", + "EMSWIFT.JRPCTCPPort=9998", + "EMSWIFT.NoAuthIPAddr=127.0.0.1", + "EMSWIFT.NoAuthTCPPort=9999", + "EMSWIFT.MaxAccountNameLength=256", + "EMSWIFT.MaxContainerNameLength=256", + "EMSWIFT.MaxObjectNameLength=1024", + "EMSWIFT.AccountListingLimit=10000", + "EMSWIFT.ContainerListingLimit=10000", + } + err error + storageURL string + ) + + confMap, err = conf.MakeConfMapFromStrings(confStrings) + if nil != err { + t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned unexpected error: %v", err) + } + + err = emswiftpkg.Start(confMap) + if nil != err { + t.Fatalf("emswiftpkg.Start(confMap) returned unexpected error: %v", err) + } + + authInJSON = "{" + + " \"AuthURL\" : \"http://127.0.0.1:9997/auth/v1.0\"," + + " \"AuthUser\" : \"test:tester\"," + + " \"AuthKey\" : \"testing\"," + + " \"Account\" : \"AUTH_test\"" + + "}" + + authToken, storageURL, err = PerformAuth(authInJSON) + if nil == err { + t.Logf("authToken: %s", authToken) + t.Logf("storageURL: %s", storageURL) + err = emswiftpkg.Stop() + if nil != err { + t.Fatalf("emswiftpkg.Stop() returned unexpected error: %v", err) + } + } else { + _ = emswiftpkg.Stop() + t.Fatalf("PerformAuth failed: %v", err) + } +} From 5d51e8523031e6f4ad21e2f400008bb20f27771b Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 1 Mar 2021 11:49:38 -0600 Subject: [PATCH 047/160] bump swift to ss-release-2.26.0.15 --- pfs_middleware/tox.ini | 2 +- saio/container/Dockerfile | 2 +- saio/vagrant_provision.sh | 2 +- sait/vagrant_provision.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pfs_middleware/tox.ini b/pfs_middleware/tox.ini index 4bd691bd1..5e3e7719f 100644 --- a/pfs_middleware/tox.ini +++ b/pfs_middleware/tox.ini @@ -7,7 +7,7 @@ usedevelop = True deps = lint: flake8 !lint: -r{toxinidir}/test-requirements.txt - release: git+https://github.com/NVIDIA/swift.git@ss-release-2.26.0.13 + release: git+https://github.com/NVIDIA/swift.git@ss-release-2.26.0.15 minver: http://tarballs.openstack.org/swift/swift-2.9.0.tar.gz master: http://tarballs.openstack.org/swift/swift-master.tar.gz commands = python -m unittest discover diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 5c5992937..898371ca3 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -3,7 +3,7 @@ FROM centos:7.4.1708 -ARG SwiftVersion=ss-release-2.26.0.13 +ARG SwiftVersion=ss-release-2.26.0.15 ARG GolangVersion=1.15.5 ARG ProxyFS_Version=stable diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index d991cb3c7..2b4d8f0ea 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -228,7 +228,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.26.0.13 +git checkout ss-release-2.26.0.15 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index f05fbebfd..eb87994a7 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -230,7 +230,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.26.0.13 +git checkout ss-release-2.26.0.15 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet From 2271ba6ff428df2b382d19d29bd09fff9fef0a4f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 2 Mar 2021 09:46:44 -0800 Subject: [PATCH 048/160] Remove errant purge of fileInode at end of pfsagentd's DoRead() --- pfsagentd/fission.go | 69 ++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/pfsagentd/fission.go b/pfsagentd/fission.go index 14302789f..b0e1b5cd0 100644 --- a/pfsagentd/fission.go +++ b/pfsagentd/fission.go @@ -168,44 +168,43 @@ func (dummy *globalsStruct) DoLookup(inHeader *fission.InHeader, lookupIn *fissi return } - fileInode = lockInodeIfExclusiveLeaseGranted(inode.InodeNumber(lookupPlusReply.InodeNumber)) - if nil != fileInode { - if nil == fileInode.cachedStat { - // Might as well cache the Stat portion of lookupPlusReply in fileInode.cachedStat - // - // Note: This is a convenient, and unlikely, optimization that won't be done in - // the SharedLease case... but that is also an unlikely case. - - fileInode.cachedStat = &jrpcfs.StatStruct{ - CTimeNs: lookupPlusReply.CTimeNs, - CRTimeNs: lookupPlusReply.CRTimeNs, - MTimeNs: lookupPlusReply.MTimeNs, - ATimeNs: lookupPlusReply.ATimeNs, - Size: lookupPlusReply.Size, - NumLinks: lookupPlusReply.NumLinks, - StatInodeNumber: lookupPlusReply.StatInodeNumber, - FileMode: lookupPlusReply.FileMode, - UserID: lookupPlusReply.UserID, - GroupID: lookupPlusReply.GroupID, - } - } else { - // Update lookupPlusReply from fileInode.cachedStat - - lookupPlusReply.CTimeNs = fileInode.cachedStat.CTimeNs - lookupPlusReply.CRTimeNs = fileInode.cachedStat.CRTimeNs - lookupPlusReply.MTimeNs = fileInode.cachedStat.MTimeNs - lookupPlusReply.ATimeNs = fileInode.cachedStat.ATimeNs - lookupPlusReply.Size = fileInode.cachedStat.Size - lookupPlusReply.NumLinks = fileInode.cachedStat.NumLinks - lookupPlusReply.StatInodeNumber = fileInode.cachedStat.StatInodeNumber - lookupPlusReply.FileMode = fileInode.cachedStat.FileMode - lookupPlusReply.UserID = fileInode.cachedStat.UserID - lookupPlusReply.GroupID = fileInode.cachedStat.GroupID + fileInode = lockInodeWithSharedLease(inode.InodeNumber(lookupPlusReply.InodeNumber)) + + if nil == fileInode.cachedStat { + // Might as well cache the Stat portion of lookupPlusReply in fileInode.cachedStat + // + // Note: This is a convenient, and unlikely, optimization that won't be done in + // the SharedLease case... but that is also an unlikely case. + + fileInode.cachedStat = &jrpcfs.StatStruct{ + CTimeNs: lookupPlusReply.CTimeNs, + CRTimeNs: lookupPlusReply.CRTimeNs, + MTimeNs: lookupPlusReply.MTimeNs, + ATimeNs: lookupPlusReply.ATimeNs, + Size: lookupPlusReply.Size, + NumLinks: lookupPlusReply.NumLinks, + StatInodeNumber: lookupPlusReply.StatInodeNumber, + FileMode: lookupPlusReply.FileMode, + UserID: lookupPlusReply.UserID, + GroupID: lookupPlusReply.GroupID, } + } else { + // Update lookupPlusReply from fileInode.cachedStat - fileInode.unlock(false) + lookupPlusReply.CTimeNs = fileInode.cachedStat.CTimeNs + lookupPlusReply.CRTimeNs = fileInode.cachedStat.CRTimeNs + lookupPlusReply.MTimeNs = fileInode.cachedStat.MTimeNs + lookupPlusReply.ATimeNs = fileInode.cachedStat.ATimeNs + lookupPlusReply.Size = fileInode.cachedStat.Size + lookupPlusReply.NumLinks = fileInode.cachedStat.NumLinks + lookupPlusReply.StatInodeNumber = fileInode.cachedStat.StatInodeNumber + lookupPlusReply.FileMode = fileInode.cachedStat.FileMode + lookupPlusReply.UserID = fileInode.cachedStat.UserID + lookupPlusReply.GroupID = fileInode.cachedStat.GroupID } + fileInode.unlock(false) + aTimeSec, aTimeNSec = nsToUnixTime(lookupPlusReply.ATimeNs) mTimeSec, mTimeNSec = nsToUnixTime(lookupPlusReply.MTimeNs) cTimeSec, cTimeNSec = nsToUnixTime(lookupPlusReply.CTimeNs) @@ -1355,7 +1354,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R } } - fileInode.unlock(true) + fileInode.unlock(false) _ = atomic.AddUint64(&globals.metrics.FUSE_DoRead_bytes, uint64(len(readOut.Data))) From 2d9438f371d4efaa04173fca390868b40448ceb8 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 3 Mar 2021 17:37:54 -0800 Subject: [PATCH 049/160] Back out incorrect change to pfsagentd/fission.go::DoLookup() --- pfsagentd/fission.go | 67 ++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/pfsagentd/fission.go b/pfsagentd/fission.go index b0e1b5cd0..a3eb1c9d5 100644 --- a/pfsagentd/fission.go +++ b/pfsagentd/fission.go @@ -168,43 +168,44 @@ func (dummy *globalsStruct) DoLookup(inHeader *fission.InHeader, lookupIn *fissi return } - fileInode = lockInodeWithSharedLease(inode.InodeNumber(lookupPlusReply.InodeNumber)) - - if nil == fileInode.cachedStat { - // Might as well cache the Stat portion of lookupPlusReply in fileInode.cachedStat - // - // Note: This is a convenient, and unlikely, optimization that won't be done in - // the SharedLease case... but that is also an unlikely case. - - fileInode.cachedStat = &jrpcfs.StatStruct{ - CTimeNs: lookupPlusReply.CTimeNs, - CRTimeNs: lookupPlusReply.CRTimeNs, - MTimeNs: lookupPlusReply.MTimeNs, - ATimeNs: lookupPlusReply.ATimeNs, - Size: lookupPlusReply.Size, - NumLinks: lookupPlusReply.NumLinks, - StatInodeNumber: lookupPlusReply.StatInodeNumber, - FileMode: lookupPlusReply.FileMode, - UserID: lookupPlusReply.UserID, - GroupID: lookupPlusReply.GroupID, + fileInode = lockInodeIfExclusiveLeaseGranted(inode.InodeNumber(lookupPlusReply.InodeNumber)) + if nil != fileInode { + if nil == fileInode.cachedStat { + // Might as well cache the Stat portion of lookupPlusReply in fileInode.cachedStat + // + // Note: This is a convenient, and unlikely, optimization that won't be done in + // the SharedLease case... but that is also an unlikely case. + + fileInode.cachedStat = &jrpcfs.StatStruct{ + CTimeNs: lookupPlusReply.CTimeNs, + CRTimeNs: lookupPlusReply.CRTimeNs, + MTimeNs: lookupPlusReply.MTimeNs, + ATimeNs: lookupPlusReply.ATimeNs, + Size: lookupPlusReply.Size, + NumLinks: lookupPlusReply.NumLinks, + StatInodeNumber: lookupPlusReply.StatInodeNumber, + FileMode: lookupPlusReply.FileMode, + UserID: lookupPlusReply.UserID, + GroupID: lookupPlusReply.GroupID, + } + } else { + // Update lookupPlusReply from fileInode.cachedStat + + lookupPlusReply.CTimeNs = fileInode.cachedStat.CTimeNs + lookupPlusReply.CRTimeNs = fileInode.cachedStat.CRTimeNs + lookupPlusReply.MTimeNs = fileInode.cachedStat.MTimeNs + lookupPlusReply.ATimeNs = fileInode.cachedStat.ATimeNs + lookupPlusReply.Size = fileInode.cachedStat.Size + lookupPlusReply.NumLinks = fileInode.cachedStat.NumLinks + lookupPlusReply.StatInodeNumber = fileInode.cachedStat.StatInodeNumber + lookupPlusReply.FileMode = fileInode.cachedStat.FileMode + lookupPlusReply.UserID = fileInode.cachedStat.UserID + lookupPlusReply.GroupID = fileInode.cachedStat.GroupID } - } else { - // Update lookupPlusReply from fileInode.cachedStat - lookupPlusReply.CTimeNs = fileInode.cachedStat.CTimeNs - lookupPlusReply.CRTimeNs = fileInode.cachedStat.CRTimeNs - lookupPlusReply.MTimeNs = fileInode.cachedStat.MTimeNs - lookupPlusReply.ATimeNs = fileInode.cachedStat.ATimeNs - lookupPlusReply.Size = fileInode.cachedStat.Size - lookupPlusReply.NumLinks = fileInode.cachedStat.NumLinks - lookupPlusReply.StatInodeNumber = fileInode.cachedStat.StatInodeNumber - lookupPlusReply.FileMode = fileInode.cachedStat.FileMode - lookupPlusReply.UserID = fileInode.cachedStat.UserID - lookupPlusReply.GroupID = fileInode.cachedStat.GroupID + fileInode.unlock(false) } - fileInode.unlock(false) - aTimeSec, aTimeNSec = nsToUnixTime(lookupPlusReply.ATimeNs) mTimeSec, mTimeNSec = nsToUnixTime(lookupPlusReply.MTimeNs) cTimeSec, cTimeNSec = nsToUnixTime(lookupPlusReply.CTimeNs) From 08ae1f8d3e318a1f94c0bf2bb9c1b2850a5cb9d4 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 3 Mar 2021 19:15:23 -0800 Subject: [PATCH 050/160] Avoid and panic on any rpcInterrupt delivered to PFSAgent (until they can be correctly supported) --- jrpcfs/config.go | 4 ++-- pfsagentd/lease.go | 2 ++ proxyfsd/default.conf | 4 ++-- proxyfsd/file_server.conf | 4 ++-- proxyfsd/file_server_mac_3_peers.conf | 4 ++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/jrpcfs/config.go b/jrpcfs/config.go index 31e96e341..1af2e1f9f 100644 --- a/jrpcfs/config.go +++ b/jrpcfs/config.go @@ -332,12 +332,12 @@ func (dummy *globalsStruct) ServeVolume(confMap conf.ConfMap, volumeName string) volume.activeLeaseEvictLowLimit, err = confMap.FetchOptionValueUint64("Volume:"+volumeName, "ActiveLeaseEvictLowLimit") if nil != err { logger.Infof("failed to get Volume:" + volumeName + ".ActiveLeaseEvictLowLimit from config file - defaulting to 5000") - volume.activeLeaseEvictLowLimit = 5000 + volume.activeLeaseEvictLowLimit = 500000 } volume.activeLeaseEvictHighLimit, err = confMap.FetchOptionValueUint64("Volume:"+volumeName, "ActiveLeaseEvictHighLimit") if nil != err { logger.Infof("failed to get Volume:" + volumeName + ".ActiveLeaseEvictHighLimit from config file - defaulting to 5010") - volume.activeLeaseEvictHighLimit = 5010 + volume.activeLeaseEvictHighLimit = 500010 } globals.volumeMap[volumeName] = volume diff --git a/pfsagentd/lease.go b/pfsagentd/lease.go index 359dc77b3..fe6212d61 100644 --- a/pfsagentd/lease.go +++ b/pfsagentd/lease.go @@ -31,7 +31,9 @@ func (dummy *globalsStruct) Interrupt(rpcInterruptBuf []byte) { case jrpcfs.RPCInterruptTypeUnmount: logFatalf("UNSUPPORTED: (*globalsStruct).Interrupt() received jrpcfs.RPCInterruptTypeUnmount") case jrpcfs.RPCInterruptTypeDemote: + logFatalf("UNSUPPORTED: (*globalsStruct).Interrupt() received jrpcfs.RPCInterruptTypeDemote") case jrpcfs.RPCInterruptTypeRelease: + logFatalf("UNSUPPORTED: (*globalsStruct).Interrupt() received jrpcfs.RPCInterruptTypeRelease") default: logFatalf("(*globalsStruct).Interrupt() received unknown rpcInterrupt.RPCInterruptType: %v", rpcInterrupt.RPCInterruptType) } diff --git a/proxyfsd/default.conf b/proxyfsd/default.conf index 5e750e60e..545ef4510 100644 --- a/proxyfsd/default.conf +++ b/proxyfsd/default.conf @@ -146,8 +146,8 @@ SMBBrowseable: true SMBStrictSync: yes SMBAuditLogging: false SMBEncryptionRequired: false -ActiveLeaseEvictLowLimit: 5000 -ActiveLeaseEvictHighLimit: 5010 +ActiveLeaseEvictLowLimit: 500000 +ActiveLeaseEvictHighLimit: 500010 [NFSClientMap:CommonVolumeNFSClient0] ClientPattern: * diff --git a/proxyfsd/file_server.conf b/proxyfsd/file_server.conf index 3db18e868..bad9d1c25 100644 --- a/proxyfsd/file_server.conf +++ b/proxyfsd/file_server.conf @@ -81,8 +81,8 @@ SMBBrowseable: true SMBStrictSync: yes SMBAuditLogging: false SMBEncryptionRequired: false -ActiveLeaseEvictLowLimit: 5000 -ActiveLeaseEvictHighLimit: 5010 +ActiveLeaseEvictLowLimit: 500000 +ActiveLeaseEvictHighLimit: 500010 [NFSClientMap:CommonVolumeNFSClient0] ClientPattern: * diff --git a/proxyfsd/file_server_mac_3_peers.conf b/proxyfsd/file_server_mac_3_peers.conf index c64ca229b..13c856806 100644 --- a/proxyfsd/file_server_mac_3_peers.conf +++ b/proxyfsd/file_server_mac_3_peers.conf @@ -81,8 +81,8 @@ SMBBrowseable: true SMBStrictSync: yes SMBAuditLogging: false SMBEncryptionRequired: false -ActiveLeaseEvictLowLimit: 5000 -ActiveLeaseEvictHighLimit: 5010 +ActiveLeaseEvictLowLimit: 500000 +ActiveLeaseEvictHighLimit: 500010 [NFSClientMap:CommonVolumeNFSClient0] ClientPattern: * From f1e512684b2d01240c7fb5ab900caab0b454903e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Thu, 11 Mar 2021 16:42:53 -0800 Subject: [PATCH 051/160] Update golang version in Runway --- ci/ansible/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/ansible/tasks/main.yml b/ci/ansible/tasks/main.yml index dd1658f0a..66152a72e 100644 --- a/ci/ansible/tasks/main.yml +++ b/ci/ansible/tasks/main.yml @@ -6,8 +6,8 @@ pkg_ver: etcd: "3.4.7" golang_versions: - current: "go1.13.6" - candidate: "go1.15.5" + current: "go1.15.5" + candidate: "go1.16.2" latest: "{{ lookup('url', 'https://golang.org/VERSION?m=text', split_lines=False) }}" active_golang_version: "{{ golang_version | default('current') }}" golang_versions_file: "/etc/golang_versions.json" From 28d4c29eb5a07d233e3686477ed96089df0f7cab Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 23 Feb 2021 11:33:23 -0800 Subject: [PATCH 052/160] Removed references to Samba in saio/Vagrantfile now that it is no longer supported --- saio/Vagrantfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/saio/Vagrantfile b/saio/Vagrantfile index 621fdd2af..a49d7299e 100644 --- a/saio/Vagrantfile +++ b/saio/Vagrantfile @@ -7,10 +7,9 @@ # 2) Though not required, GOPATH is assumed to be the ../../../../../ directory # 3) The directory on the VM Host will be /vagrant on the VM and be the path in GOPATH # 4) ProxyFS repo git clone'd to $GOPATH/src/github.com/NVIDIA/ -# 5) samba repo automatically staged in $GOPATH/src/github.com/NVIDIA/proxyfs/saio/ -# 6) Swift repos et. al. git clone'd to $GOPATH/src/github.com/NVIDIA/proxyfs/saio/ -# 7) ../Makefile will be ready to be executed after `cdpfs` inside the VM -# 8) As GOPATH is effectively shared between Host and VM, builds in the two environments +# 5) Swift repos et. al. git clone'd to $GOPATH/src/github.com/NVIDIA/proxyfs/saio/ +# 6) ../Makefile will be ready to be executed after `cdpfs` inside the VM +# 7) As GOPATH is effectively shared between Host and VM, builds in the two environments # will collide in the contents of the $GOPATH/bin (only executables, not libraries) Vagrant.configure(2) do |config| From f8688aca89946c586e084f2ecf0875929e9a2d50 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 23 Feb 2021 15:33:40 -0800 Subject: [PATCH 053/160] Evolved imgr/imgrpkg to match pattern in emswift/emswiftpkg --- imgr/imgr.conf | 3 -- imgr/imgrpkg/api.go | 12 +++++ imgr/{ => imgrpkg}/globals.go | 60 ++++++++++++------------- imgr/{ => imgrpkg}/http_server.go_HIDE | 0 imgr/imgrpkg/{dummy.go => impl.go} | 13 ++++-- imgr/{ => imgrpkg}/log.go | 2 +- imgr/main.go | 61 ++++++++++++++------------ 7 files changed, 85 insertions(+), 66 deletions(-) rename imgr/{ => imgrpkg}/globals.go (85%) rename imgr/{ => imgrpkg}/http_server.go_HIDE (100%) rename imgr/imgrpkg/{dummy.go => impl.go} (69%) rename imgr/{ => imgrpkg}/log.go (98%) diff --git a/imgr/imgr.conf b/imgr/imgr.conf index d471c93b3..90463d2e8 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -17,9 +17,6 @@ MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 -SwiftNoAuthIPAddr: 127.0.0.1 -SwiftNoAuthTCPPort: 8090 - SwiftRetryDelay: 1s SwiftRetryExpBackoff: 1.5 SwiftRetryLimit: 11 diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index a21367544..ba66f6a0e 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -27,3 +27,15 @@ func Signal() (err error) { err = signal() return } + +// LogWarnf is a wrapper around the internal logWarnf() func called by imgr/main.go::main() +// +func LogWarnf(format string, args ...interface{}) { + logWarnf(format, args...) +} + +// LogInfof is a wrapper around the internal logInfof() func called by imgr/main.go::main() +// +func LogInfof(format string, args ...interface{}) { + logInfof(format, args...) +} diff --git a/imgr/globals.go b/imgr/imgrpkg/globals.go similarity index 85% rename from imgr/globals.go rename to imgr/imgrpkg/globals.go index 9e7623f9f..e75da7475 100644 --- a/imgr/globals.go +++ b/imgr/imgrpkg/globals.go @@ -1,7 +1,7 @@ // Copyright (c) 2015-2021, NVIDIA CORPORATION. // SPDX-License-Identifier: Apache-2.0 -package main +package imgrpkg import ( "container/list" @@ -17,7 +17,7 @@ type configStruct struct { PrivateIPAddr string PublicIPAddr string JSONRPCPort uint16 // To be served only on PrivateIPAddr via TCP - RetryRPCPort uint16 // To be served only on PublicIPAddr via TLS + RetryRPCPort uint16 // To be served only on PublicIPAddr via TLS HTTPServerPort uint16 // To be served only on PrivateIPAddr via TCP RetryRPCTTLCompleted time.Duration @@ -29,9 +29,6 @@ type configStruct struct { LeaseInterruptInterval time.Duration LeaseInterruptLimit uint32 - SwiftNoAuthIPAddr string - SwiftNoAuthTCPPort uint16 - SwiftRetryDelay time.Duration SwiftRetryExpBackoff float64 SwiftRetryLimit uint32 @@ -60,17 +57,30 @@ type chunkedPutContextStruct struct { } type statsStruct struct { - SharedLeaseRequests bucketstats.BucketLog2Round - PromoteLeaseRequests bucketstats.BucketLog2Round - ExclusiveLeaseRequests bucketstats.BucketLog2Round - DemoteLeaseRequests bucketstats.BucketLog2Round - ReleaseLeaseRequests bucketstats.BucketLog2Round + MountUsecs bucketstats.BucketLog2Round + + SharedLeaseRequestUsecs bucketstats.BucketLog2Round + PromoteLeaseRequestUsecs bucketstats.BucketLog2Round + ExclusiveLeaseRequestUsecs bucketstats.BucketLog2Round + DemoteLeaseRequestUsecs bucketstats.BucketLog2Round + ReleaseLeaseRequestUsecs bucketstats.BucketLog2Round + GetInodeTableEntryUsecs bucketstats.BucketLog2Round + PutInodeTableEntriesUsecs bucketstats.BucketLog2Round + FetchNonceRangeUsecs bucketstats.BucketLog2Round + RenewMountUsecs bucketstats.BucketLog2Round + UnmountUsecs bucketstats.BucketLog2Round + + UnmountInterrupts bucketstats.Totaler DemoteLeaseInterrupts bucketstats.Totaler RevokeLeaseInterrupts bucketstats.Totaler - InodeGets bucketstats.BucketLog2Round - InodePuts bucketstats.BucketLog2Round + GetVolumeListUsecs bucketstats.BucketLog2Round + GetVolumeUsecs bucketstats.BucketLog2Round + PutVolumeUsecs bucketstats.BucketLog2Round + DeleteVolumeUsecs bucketstats.BucketLog2Round + + VolumeCheckpointUsecs bucketstats.BucketLog2Round } type globalsStruct struct { @@ -82,11 +92,7 @@ type globalsStruct struct { var globals globalsStruct -func initializeGlobals(confMap conf.ConfMap) { - var ( - err error - ) - +func initializeGlobals(confMap conf.ConfMap) (err error) { // Default logging related globals globals.config.LogFilePath = "" @@ -150,15 +156,6 @@ func initializeGlobals(confMap conf.ConfMap) { logFatal(err) } - globals.config.SwiftNoAuthIPAddr, err = confMap.FetchOptionValueString("IMGR", "SwiftNoAuthIPAddr") - if nil != err { - logFatal(err) - } - globals.config.SwiftNoAuthTCPPort, err = confMap.FetchOptionValueUint16("IMGR", "SwiftNoAuthTCPPort") - if nil != err { - logFatal(err) - } - globals.config.SwiftRetryDelay, err = confMap.FetchOptionValueDuration("IMGR", "SwiftRetryDelay") if nil != err { logFatal(err) @@ -198,9 +195,12 @@ func initializeGlobals(confMap conf.ConfMap) { globals.stats = &statsStruct{} bucketstats.Register("IMGR", "", globals.stats) + + err = nil + return } -func uninitializeGlobals() { +func uninitializeGlobals() (err error) { globals.config.PrivateIPAddr = "" globals.config.PublicIPAddr = "" globals.config.JSONRPCPort = 0 @@ -216,9 +216,6 @@ func uninitializeGlobals() { globals.config.LeaseInterruptInterval = time.Duration(0) globals.config.LeaseInterruptLimit = 0 - globals.config.SwiftNoAuthIPAddr = "" - globals.config.SwiftNoAuthTCPPort = 0 - globals.config.SwiftRetryDelay = time.Duration(0) globals.config.SwiftRetryExpBackoff = 0.0 globals.config.SwiftRetryLimit = 0 @@ -230,4 +227,7 @@ func uninitializeGlobals() { globals.config.TraceEnabled = false bucketstats.UnRegister("IMGR", "") + + err = nil + return } diff --git a/imgr/http_server.go_HIDE b/imgr/imgrpkg/http_server.go_HIDE similarity index 100% rename from imgr/http_server.go_HIDE rename to imgr/imgrpkg/http_server.go_HIDE diff --git a/imgr/imgrpkg/dummy.go b/imgr/imgrpkg/impl.go similarity index 69% rename from imgr/imgrpkg/dummy.go rename to imgr/imgrpkg/impl.go index d59b4bfe8..7d9ba4955 100644 --- a/imgr/imgrpkg/dummy.go +++ b/imgr/imgrpkg/impl.go @@ -8,13 +8,20 @@ import ( ) func start(confMap conf.ConfMap) (err error) { - return nil // TODO + err = initializeGlobals(confMap) + + return } func stop() (err error) { - return nil // TODO + err = uninitializeGlobals() + + return } func signal() (err error) { - return nil // TODO + logSIGHUP() + + err = nil + return } diff --git a/imgr/log.go b/imgr/imgrpkg/log.go similarity index 98% rename from imgr/log.go rename to imgr/imgrpkg/log.go index 3736ab567..cae5bc5b5 100644 --- a/imgr/log.go +++ b/imgr/imgrpkg/log.go @@ -1,7 +1,7 @@ // Copyright (c) 2015-2021, NVIDIA CORPORATION. // SPDX-License-Identifier: Apache-2.0 -package main +package imgrpkg import ( "fmt" diff --git a/imgr/main.go b/imgr/main.go index 25ebc728a..7c2508f83 100644 --- a/imgr/main.go +++ b/imgr/main.go @@ -4,13 +4,14 @@ package main import ( - "log" + "fmt" "os" "os/signal" "golang.org/x/sys/unix" "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/imgr/imgrpkg" ) func main() { @@ -22,59 +23,61 @@ func main() { ) if len(os.Args) < 2 { - log.Fatalf("no .conf file specified") + fmt.Fprintf(os.Stderr, "no .conf file specified\n") + os.Exit(1) } - // Parse arguments (at this point, logging goes only to the console) - - globals.logFile = nil - globals.config.LogFilePath = "" - globals.config.LogToConsole = true - confMap, err = conf.MakeConfMapFromFile(os.Args[1]) if nil != err { - log.Fatalf("failed to load config: %v", err) + fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err) + os.Exit(1) } err = confMap.UpdateFromStrings(os.Args[2:]) if nil != err { - log.Fatalf("failed to apply config overrides: %v", err) + fmt.Fprintf(os.Stderr, "failed to apply config overrides: %v\n", err) + os.Exit(1) + } + + // Start imgr + + err = imgrpkg.Start(confMap) + if nil != err { + fmt.Fprintf(os.Stderr, "imgrpkg.Start(confMap) failed: %v\n", err) + os.Exit(1) } - // Arm signal handler used to indicate termination and wait on it + imgrpkg.LogInfof("UP") + + // Arm signal handler used to indicate interruption/termination & wait on it // - // Note: signalled chan must be buffered to avoid race with window between + // Note: signal'd chan must be buffered to avoid race with window between // arming handler and blocking on the chan read signalChan = make(chan os.Signal, 1) signal.Notify(signalChan, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) - // Initialize globals - - initializeGlobals(confMap) - - // Indicate we are UP - - logInfof("UP") - - // Await any of specified signals or fission exit - for { signalReceived = <-signalChan - logInfof("Received signal: %v", signalReceived) if unix.SIGHUP == signalReceived { - logSIGHUP() + imgrpkg.LogInfof("Received SIGHUP") + err = imgrpkg.Signal() + if nil != err { + imgrpkg.LogWarnf("imgrpkg.Signal() failed: %v", err) + } } else { break } } - // Indicate we are DOWN + // Stop imgr - logInfof("DOWN") + imgrpkg.LogInfof("DOWN") - // Uninitialize globals - - uninitializeGlobals() + err = imgrpkg.Stop() + if nil != err { + fmt.Fprintf(os.Stderr, "imgrpkg.Stop() failed: %v\n", err) + os.Exit(1) + } } From 69fe8e3fb0ed2346d88aae45b95af42c7edd923c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 23 Feb 2021 15:54:38 -0800 Subject: [PATCH 054/160] Added templates for each of the three RPC/API servers in imgr --- imgr/imgrpkg/http-server.go | 12 ++++++++++++ imgr/imgrpkg/json-rpc.go | 12 ++++++++++++ imgr/imgrpkg/retry-rpc.go | 12 ++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 imgr/imgrpkg/http-server.go create mode 100644 imgr/imgrpkg/json-rpc.go create mode 100644 imgr/imgrpkg/retry-rpc.go diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go new file mode 100644 index 000000000..ec5c8ed8a --- /dev/null +++ b/imgr/imgrpkg/http-server.go @@ -0,0 +1,12 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +func startHTTPRPCServer() (err error) { + return nil // TODO +} + +func stopHTTPRPCServer() (err error) { + return nil // TODO +} diff --git a/imgr/imgrpkg/json-rpc.go b/imgr/imgrpkg/json-rpc.go new file mode 100644 index 000000000..42b821fb7 --- /dev/null +++ b/imgr/imgrpkg/json-rpc.go @@ -0,0 +1,12 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +func startJSONRPCServer() (err error) { + return nil // TODO +} + +func stopJSONRPCServer() (err error) { + return nil // TODO +} diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go new file mode 100644 index 000000000..a8e44ce77 --- /dev/null +++ b/imgr/imgrpkg/retry-rpc.go @@ -0,0 +1,12 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +func startRetryRPCServer() (err error) { + return nil // TODO +} + +func stopRetryRPCServer() (err error) { + return nil // TODO +} From 0fc92637ec053ba1e2848380def1bb469a13c5af Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 24 Feb 2021 10:08:13 -0800 Subject: [PATCH 055/160] Added startup templates for all imgr services... and beginnings of HTTPServer --- imgr/imgr.conf | 41 +-- imgr/imgrpkg/globals.go | 30 ++- imgr/imgrpkg/http-server.go | 172 +++++++++++- imgr/imgrpkg/http_server.go_HIDE | 431 ------------------------------- imgr/imgrpkg/impl.go | 43 +++ imgr/imgrpkg/inode.go | 22 ++ 6 files changed, 282 insertions(+), 457 deletions(-) delete mode 100644 imgr/imgrpkg/http_server.go_HIDE create mode 100644 imgr/imgrpkg/inode.go diff --git a/imgr/imgr.conf b/imgr/imgr.conf index 90463d2e8..68583ee02 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -2,27 +2,30 @@ # SPDX-License-Identifier: Apache-2.0 [IMGR] -PublicIPAddr: 0.0.0.0 -PrivateIPAddr: 0.0.0.0 -JSONRPCPort: 12345 -RetryRPCPort: 32356 -HTTPServerPort: 15346 +PublicIPAddr: 0.0.0.0 +PrivateIPAddr: 0.0.0.0 +JSONRPCPort: 12345 +RetryRPCPort: 32356 +HTTPServerPort: 15346 -RetryRPCTTLCompleted: 10m -RetryRPCAckTrim: 100ms -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s +RetryRPCTTLCompleted: 10m +RetryRPCAckTrim: 100ms +RetryRPCDeadlineIO: 60s +RetryRPCKeepAlivePeriod: 60s -MinLeaseDuration: 250ms -LeaseInterruptInterval: 250ms -LeaseInterruptLimit: 20 +MinLeaseDuration: 250ms +LeaseInterruptInterval: 250ms +LeaseInterruptLimit: 20 -SwiftRetryDelay: 1s -SwiftRetryExpBackoff: 1.5 -SwiftRetryLimit: 11 +SwiftRetryDelay: 1s +SwiftRetryExpBackoff: 1.5 +SwiftRetryLimit: 11 -SwiftConnectionPoolSize: 128 +SwiftConnectionPoolSize: 128 -LogFilePath: # imgr.log -LogToConsole: true # false -TraceEnabled: false +InodeTableCacheEvictLowLimit: 10000 +InodeTableCacheEvictHighLimit: 10010 + +LogFilePath: # imgr.log +LogToConsole: true # false +TraceEnabled: false diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index e75da7475..5d72f8181 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -5,10 +5,13 @@ package imgrpkg import ( "container/list" + "net/http" "os" "sync" "time" + "github.com/NVIDIA/sortedmap" + "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/conf" ) @@ -35,6 +38,9 @@ type configStruct struct { SwiftConnectionPoolSize uint32 + InodeTableCacheEvictLowLimit uint64 + InodeTableCacheEvictHighLimit uint64 + LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled LogToConsole bool TraceEnabled bool @@ -81,13 +87,19 @@ type statsStruct struct { DeleteVolumeUsecs bucketstats.BucketLog2Round VolumeCheckpointUsecs bucketstats.BucketLog2Round + + InodeTableCacheHits bucketstats.Totaler + InodeTableCacheMisses bucketstats.Totaler } type globalsStruct struct { sync.RWMutex - config configStruct - logFile *os.File // == nil if config.LogFilePath == "" - stats *statsStruct + config configStruct + logFile *os.File // == nil if config.LogFilePath == "" + inodeTableCache sortedmap.BPlusTreeCache + httpServer *http.Server + httpServerWG sync.WaitGroup + stats *statsStruct } var globals globalsStruct @@ -174,6 +186,15 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { logFatal(err) } + globals.config.InodeTableCacheEvictLowLimit, err = confMap.FetchOptionValueUint64("IMGR", "InodeTableCacheEvictLowLimit") + if nil != err { + logFatal(err) + } + globals.config.InodeTableCacheEvictHighLimit, err = confMap.FetchOptionValueUint64("IMGR", "InodeTableCacheEvictHighLimit") + if nil != err { + logFatal(err) + } + globals.config.LogFilePath, err = confMap.FetchOptionValueString("IMGR", "LogFilePath") if nil != err { err = confMap.VerifyOptionValueIsEmpty("IMGR", "LogFilePath") @@ -222,6 +243,9 @@ func uninitializeGlobals() (err error) { globals.config.SwiftConnectionPoolSize = 0 + globals.config.InodeTableCacheEvictLowLimit = 0 + globals.config.InodeTableCacheEvictHighLimit = 0 + globals.config.LogFilePath = "" globals.config.LogToConsole = false globals.config.TraceEnabled = false diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index ec5c8ed8a..ad56be612 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -3,10 +3,174 @@ package imgrpkg -func startHTTPRPCServer() (err error) { - return nil // TODO +import ( + "context" + "encoding/json" + "log" + "net" + "net/http" + "strconv" + "strings" + + "github.com/NVIDIA/proxyfs/bucketstats" +) + +func startHTTPServer() (err error) { + var ( + ipAddrTCPPort string + ) + + ipAddrTCPPort = net.JoinHostPort(globals.config.PrivateIPAddr, strconv.Itoa(int(globals.config.HTTPServerPort))) + + globals.httpServer = &http.Server{ + Addr: ipAddrTCPPort, + Handler: &globals, + } + + globals.httpServerWG.Add(1) + + go func() { + var ( + err error + ) + + err = globals.httpServer.ListenAndServe() + if http.ErrServerClosed != err { + log.Fatalf("httpServer.ListenAndServe() exited unexpectedly: %v", err) + } + + globals.httpServerWG.Done() + }() + + err = nil + return +} + +func stopHTTPServer() (err error) { + err = globals.httpServer.Shutdown(context.TODO()) + if nil == err { + globals.httpServerWG.Wait() + } + + return +} + +func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { + switch request.Method { + case http.MethodDelete: + globals.httpServerWG.Add(1) + go serveHTTPDelete(responseWriter, request) + case http.MethodGet: + globals.httpServerWG.Add(1) + serveHTTPGet(responseWriter, request) + case http.MethodPut: + globals.httpServerWG.Add(1) + serveHTTPPut(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func serveHTTPDelete(responseWriter http.ResponseWriter, request *http.Request) { + var ( + path string + ) + + path = strings.TrimRight(request.URL.Path, "/") + + switch { + case strings.HasPrefix(path, "/volume"): + serveHTTPDeleteOfVolume(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusNotFound) + } + + globals.httpServerWG.Done() +} + +func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, request *http.Request) { + responseWriter.WriteHeader(http.StatusNotImplemented) // TODO +} + +func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request) { + var ( + path string + ) + + path = strings.TrimRight(request.URL.Path, "/") + + switch { + case "/config" == path: + serveHTTPGetOfConfig(responseWriter, request) + case "/stats" == path: + serveHTTPGetOfStats(responseWriter, request) + case strings.HasPrefix(path, "/volume"): + serveHTTPGetOfVolume(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusNotFound) + } + + globals.httpServerWG.Done() +} + +func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request) { + responseWriter.WriteHeader(http.StatusNotImplemented) // TODO +} + +func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Request) { + var ( + confMapJSON []byte + err error + ) + + confMapJSON, err = json.Marshal(globals.config) + if nil != err { + logFatalf("json.Marshal(globals.config) failed: %v", err) + } + + responseWriter.Header().Set("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write(confMapJSON) + if nil != err { + logWarnf("responseWriter.Write(confMapJSON) failed: %v", err) + } +} + +func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + statsAsString string + ) + + statsAsString = bucketstats.SprintStats(bucketstats.StatFormatParsable1, "*", "*") + + responseWriter.Header().Set("Content-Type", "text/plain") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write([]byte(statsAsString)) + if nil != err { + logWarnf("responseWriter.Write([]byte(statsAsString)) failed: %v", err) + } +} + +func serveHTTPPut(responseWriter http.ResponseWriter, request *http.Request) { + var ( + path string + ) + + path = strings.TrimRight(request.URL.Path, "/") + + switch { + case strings.HasPrefix(path, "/volume"): + serveHTTPPutOfVolume(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusNotFound) + } + + globals.httpServerWG.Done() } -func stopHTTPRPCServer() (err error) { - return nil // TODO +func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Request) { + responseWriter.WriteHeader(http.StatusNotImplemented) // TODO } diff --git a/imgr/imgrpkg/http_server.go_HIDE b/imgr/imgrpkg/http_server.go_HIDE deleted file mode 100644 index faa8833dd..000000000 --- a/imgr/imgrpkg/http_server.go_HIDE +++ /dev/null @@ -1,431 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "container/list" - "context" - "encoding/json" - "fmt" - "log" - "net" - "net/http" - "net/http/pprof" - "reflect" - "runtime" - "strconv" - "strings" - "sync/atomic" - - "github.com/NVIDIA/sortedmap" - - "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/version" -) - -type leaseReportStruct struct { - MountID string - None []string // inode.InodeNumber in 16-digit Hex (no leading "0x") - SharedRequested []string - SharedGranted []string - SharedPromoting []string - SharedReleasing []string - ExclusiveRequested []string - ExclusiveGranted []string - ExclusiveDemoting []string - ExclusiveReleasing []string -} - -func serveHTTP() { - var ( - ipAddrTCPPort string - ) - - ipAddrTCPPort = net.JoinHostPort(globals.config.HTTPServerIPAddr, strconv.Itoa(int(globals.config.HTTPServerTCPPort))) - - globals.httpServer = &http.Server{ - Addr: ipAddrTCPPort, - Handler: &globals, - } - - globals.httpServerWG.Add(1) - - go func() { - var ( - err error - ) - - err = globals.httpServer.ListenAndServe() - if http.ErrServerClosed != err { - log.Fatalf("httpServer.ListenAndServe() exited unexpectedly: %v", err) - } - - globals.httpServerWG.Done() - }() -} - -func unserveHTTP() { - var ( - err error - ) - - err = globals.httpServer.Shutdown(context.TODO()) - if nil != err { - log.Fatalf("httpServer.Shutdown() returned with an error: %v", err) - } - - globals.httpServerWG.Wait() -} - -func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { - switch request.Method { - case http.MethodGet: - serveGet(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusMethodNotAllowed) - } -} - -func serveGet(responseWriter http.ResponseWriter, request *http.Request) { - var ( - path string - ) - - path = strings.TrimRight(request.URL.Path, "/") - - switch { - case "" == path: - serveGetOfIndexDotHTML(responseWriter, request) - case "/config" == path: - serveGetOfConfig(responseWriter, request) - case "/debug/pprof/cmdline" == path: - pprof.Cmdline(responseWriter, request) - case "/debug/pprof/profile" == path: - pprof.Profile(responseWriter, request) - case "/debug/pprof/symbol" == path: - pprof.Symbol(responseWriter, request) - case "/debug/pprof/trace" == path: - pprof.Trace(responseWriter, request) - case strings.HasPrefix(path, "/debug/pprof"): - pprof.Index(responseWriter, request) - case "index.html" == path: - serveGetOfIndexDotHTML(responseWriter, request) - case "/leases" == path: - serveGetOfLeases(responseWriter, request) - case "/metrics" == path: - serveGetOfMetrics(responseWriter, request) - case "/stats" == path: - serveGetOfStats(responseWriter, request) - case "/version" == path: - serveGetOfVersion(responseWriter, request) - default: - responseWriter.WriteHeader(http.StatusNotFound) - } -} - -func serveGetOfConfig(responseWriter http.ResponseWriter, request *http.Request) { - var ( - confMapJSON bytes.Buffer - confMapJSONPacked []byte - ok bool - paramList []string - sendPackedConfig bool - ) - - paramList, ok = request.URL.Query()["compact"] - if ok { - if 0 == len(paramList) { - sendPackedConfig = false - } else { - sendPackedConfig = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) - } - } else { - sendPackedConfig = false - } - - confMapJSONPacked, _ = json.Marshal(globals.config) - - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - if sendPackedConfig { - _, _ = responseWriter.Write(confMapJSONPacked) - } else { - json.Indent(&confMapJSON, confMapJSONPacked, "", "\t") - _, _ = responseWriter.Write(confMapJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } -} - -func serveGetOfIndexDotHTML(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.Header().Set("Content-Type", "text/html") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(fmt.Sprintf(indexDotHTMLTemplate, net.JoinHostPort(globals.config.HTTPServerIPAddr, strconv.Itoa(int(globals.config.HTTPServerTCPPort)))))) -} - -func serveGetOfLeases(responseWriter http.ResponseWriter, request *http.Request) { - var ( - fileInode *fileInodeStruct - leaseListElement *list.Element - leaseReport *leaseReportStruct - leaseReportJSON bytes.Buffer - leaseReportJSONPacked []byte - ok bool - paramList []string - sendPackedLeaseReport bool - ) - - leaseReport = &leaseReportStruct{ - MountID: fmt.Sprintf("%s", globals.mountID), - None: make([]string, 0), - SharedRequested: make([]string, 0), - SharedGranted: make([]string, 0), - SharedPromoting: make([]string, 0), - SharedReleasing: make([]string, 0), - ExclusiveRequested: make([]string, 0), - ExclusiveGranted: make([]string, 0), - ExclusiveDemoting: make([]string, 0), - ExclusiveReleasing: make([]string, 0), - } - - globals.Lock() - - for leaseListElement = globals.unleasedFileInodeCacheLRU.Front(); leaseListElement != nil; leaseListElement = leaseListElement.Next() { - fileInode = leaseListElement.Value.(*fileInodeStruct) - switch fileInode.leaseState { - case fileInodeLeaseStateNone: - leaseReport.None = append(leaseReport.None, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateSharedReleasing: - leaseReport.SharedReleasing = append(leaseReport.SharedReleasing, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateExclusiveReleasing: - leaseReport.ExclusiveReleasing = append(leaseReport.ExclusiveReleasing, fmt.Sprintf("%016X", fileInode.InodeNumber)) - default: - logFatalf("serveGetOfLeases() found unexpected fileInode.leaseState %v on globals.unleasedFileInodeCacheLRU", fileInode.leaseState) - } - } - - for leaseListElement = globals.sharedLeaseFileInodeCacheLRU.Front(); leaseListElement != nil; leaseListElement = leaseListElement.Next() { - fileInode = leaseListElement.Value.(*fileInodeStruct) - switch fileInode.leaseState { - case fileInodeLeaseStateSharedRequested: - leaseReport.SharedRequested = append(leaseReport.SharedRequested, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateSharedGranted: - leaseReport.SharedGranted = append(leaseReport.SharedGranted, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateExclusiveDemoting: - leaseReport.ExclusiveDemoting = append(leaseReport.ExclusiveDemoting, fmt.Sprintf("%016X", fileInode.InodeNumber)) - default: - logFatalf("serveGetOfLeases() found unexpected fileInode.leaseState %v on globals.sharedLeaseFileInodeCacheLRU", fileInode.leaseState) - } - } - - for leaseListElement = globals.exclusiveLeaseFileInodeCacheLRU.Front(); leaseListElement != nil; leaseListElement = leaseListElement.Next() { - fileInode = leaseListElement.Value.(*fileInodeStruct) - switch fileInode.leaseState { - case fileInodeLeaseStateSharedPromoting: - leaseReport.SharedPromoting = append(leaseReport.SharedPromoting, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateExclusiveRequested: - leaseReport.ExclusiveRequested = append(leaseReport.ExclusiveRequested, fmt.Sprintf("%016X", fileInode.InodeNumber)) - case fileInodeLeaseStateExclusiveGranted: - leaseReport.ExclusiveGranted = append(leaseReport.ExclusiveGranted, fmt.Sprintf("%016X", fileInode.InodeNumber)) - default: - logFatalf("serveGetOfLeases() found unexpected fileInode.leaseState %v on globals.exclusiveLeaseFileInodeCacheLRU", fileInode.leaseState) - } - } - - globals.Unlock() - - paramList, ok = request.URL.Query()["compact"] - if ok { - if 0 == len(paramList) { - sendPackedLeaseReport = false - } else { - sendPackedLeaseReport = !((paramList[0] == "") || (paramList[0] == "0") || (paramList[0] == "false")) - } - } else { - sendPackedLeaseReport = false - } - - leaseReportJSONPacked, _ = json.Marshal(leaseReport) - - responseWriter.Header().Set("Content-Type", "application/json") - responseWriter.WriteHeader(http.StatusOK) - - if sendPackedLeaseReport { - _, _ = responseWriter.Write(leaseReportJSONPacked) - } else { - json.Indent(&leaseReportJSON, leaseReportJSONPacked, "", "\t") - _, _ = responseWriter.Write(leaseReportJSON.Bytes()) - _, _ = responseWriter.Write([]byte("\n")) - } -} - -func serveGetOfMetrics(responseWriter http.ResponseWriter, request *http.Request) { - var ( - err error - format string - i int - keyAsKey sortedmap.Key - keyAsString string - line string - longestKeyAsString int - longestValueAsString int - memStats runtime.MemStats - metricsFieldName string - metricsFieldValuePtr *uint64 - metricsLLRB sortedmap.LLRBTree - metricsLLRBLen int - metricsStructValue reflect.Value - metricsValue reflect.Value - ok bool - pauseNsAccumulator uint64 - valueAsString string - valueAsValue sortedmap.Value - ) - - runtime.ReadMemStats(&memStats) - - metricsLLRB = sortedmap.NewLLRBTree(sortedmap.CompareString, nil) - - // General statistics. - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Alloc", memStats.Alloc) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_TotalAlloc", memStats.TotalAlloc) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Sys", memStats.Sys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Lookups", memStats.Lookups) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Mallocs", memStats.Mallocs) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_Frees", memStats.Frees) - - // Main allocation heap statistics. - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapAlloc", memStats.HeapAlloc) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapSys", memStats.HeapSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapIdle", memStats.HeapIdle) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapInuse", memStats.HeapInuse) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapReleased", memStats.HeapReleased) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_HeapObjects", memStats.HeapObjects) - - // Low-level fixed-size structure allocator statistics. - // Inuse is bytes used now. - // Sys is bytes obtained from system. - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_StackInuse", memStats.StackInuse) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_StackSys", memStats.StackSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MSpanInuse", memStats.MSpanInuse) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MSpanSys", memStats.MSpanSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MCacheInuse", memStats.MCacheInuse) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_MCacheSys", memStats.MCacheSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_BuckHashSys", memStats.BuckHashSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_GCSys", memStats.GCSys) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_OtherSys", memStats.OtherSys) - - // Garbage collector statistics (fixed portion). - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_LastGC", memStats.LastGC) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseTotalNs", memStats.PauseTotalNs) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_NumGC", uint64(memStats.NumGC)) - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_GCCPUPercentage", uint64(100.0*memStats.GCCPUFraction)) - - // Garbage collector statistics (go_runtime_MemStats_PauseAverageNs). - if 0 == memStats.NumGC { - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseAverageNs", 0) - } else { - pauseNsAccumulator = 0 - if memStats.NumGC < 255 { - for i = 0; i < int(memStats.NumGC); i++ { - pauseNsAccumulator += memStats.PauseNs[i] - } - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseAverageNs", pauseNsAccumulator/uint64(memStats.NumGC)) - } else { - for i = 0; i < 256; i++ { - pauseNsAccumulator += memStats.PauseNs[i] - } - insertInMetricsLLRB(metricsLLRB, "go_runtime_MemStats_PauseAverageNs", pauseNsAccumulator/256) - } - } - - // Add in locally generated metrics - - metricsStructValue = reflect.Indirect(reflect.ValueOf(globals.metrics)) - metricsValue = reflect.ValueOf(globals.metrics).Elem() - - for i = 0; i < metricsStructValue.NumField(); i++ { - metricsFieldName = metricsStructValue.Type().Field(i).Name - metricsFieldValuePtr = metricsValue.Field(i).Addr().Interface().(*uint64) - insertInMetricsLLRB(metricsLLRB, metricsFieldName, atomic.LoadUint64(metricsFieldValuePtr)) - } - - // Produce sorted and column-aligned response - - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - - metricsLLRBLen, err = metricsLLRB.Len() - if nil != err { - logFatalf("metricsLLRB.Len() failed: %v", err) - } - - longestKeyAsString = 0 - longestValueAsString = 0 - - for i = 0; i < metricsLLRBLen; i++ { - keyAsKey, valueAsValue, ok, err = metricsLLRB.GetByIndex(i) - if nil != err { - logFatalf("llrb.GetByIndex(%v) failed: %v", i, err) - } - if !ok { - logFatalf("llrb.GetByIndex(%v) returned ok == false", i) - } - keyAsString = keyAsKey.(string) - valueAsString = valueAsValue.(string) - if len(keyAsString) > longestKeyAsString { - longestKeyAsString = len(keyAsString) - } - if len(valueAsString) > longestValueAsString { - longestValueAsString = len(valueAsString) - } - } - - format = fmt.Sprintf("%%-%vs %%%vs\n", longestKeyAsString, longestValueAsString) - - for i = 0; i < metricsLLRBLen; i++ { - keyAsKey, valueAsValue, ok, err = metricsLLRB.GetByIndex(i) - if nil != err { - logFatalf("llrb.GetByIndex(%v) failed: %v", i, err) - } - if !ok { - logFatalf("llrb.GetByIndex(%v) returned ok == false", i) - } - keyAsString = keyAsKey.(string) - valueAsString = valueAsValue.(string) - line = fmt.Sprintf(format, keyAsString, valueAsString) - _, _ = responseWriter.Write([]byte(line)) - } -} - -func serveGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(bucketstats.SprintStats(bucketstats.StatFormatParsable1, "*", "*"))) -} - -func insertInMetricsLLRB(metricsLLRB sortedmap.LLRBTree, metricKey string, metricValueAsUint64 uint64) { - var ( - err error - metricValueAsString string - ok bool - ) - - metricValueAsString = fmt.Sprintf("%v", metricValueAsUint64) - - ok, err = metricsLLRB.Put(metricKey, metricValueAsString) - if nil != err { - logFatalf("metricsLLRB.Put(%v, %v) failed: %v", metricKey, metricValueAsString, err) - } - if !ok { - logFatalf("metricsLLRB.Put(%v, %v) returned ok == false", metricKey, metricValueAsString) - } -} - -func serveGetOfVersion(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.Header().Set("Content-Type", "text/plain") - responseWriter.WriteHeader(http.StatusOK) - _, _ = responseWriter.Write([]byte(version.ProxyFSVersion)) -} diff --git a/imgr/imgrpkg/impl.go b/imgr/imgrpkg/impl.go index 7d9ba4955..a30e1400e 100644 --- a/imgr/imgrpkg/impl.go +++ b/imgr/imgrpkg/impl.go @@ -9,11 +9,54 @@ import ( func start(confMap conf.ConfMap) (err error) { err = initializeGlobals(confMap) + if nil != err { + return + } + + err = startInodeTableManagement() + if nil != err { + return + } + + err = startRetryRPCServer() + if nil != err { + return + } + + err = startJSONRPCServer() + if nil != err { + return + } + + err = startHTTPServer() + if nil != err { + return + } return } func stop() (err error) { + err = stopHTTPServer() + if nil != err { + return + } + + err = stopJSONRPCServer() + if nil != err { + return + } + + err = stopRetryRPCServer() + if nil != err { + return + } + + err = stopInodeTableManagement() + if nil != err { + return + } + err = uninitializeGlobals() return diff --git a/imgr/imgrpkg/inode.go b/imgr/imgrpkg/inode.go new file mode 100644 index 000000000..b3fdd2689 --- /dev/null +++ b/imgr/imgrpkg/inode.go @@ -0,0 +1,22 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +import ( + "github.com/NVIDIA/sortedmap" +) + +func startInodeTableManagement() (err error) { + globals.inodeTableCache = sortedmap.NewBPlusTreeCache(globals.config.InodeTableCacheEvictLowLimit, globals.config.InodeTableCacheEvictLowLimit) + + err = nil + return +} + +func stopInodeTableManagement() (err error) { + globals.inodeTableCache = nil + + err = nil + return +} From d08b9f1792191afd75b759740ec59895b95365bd Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 11 Mar 2021 12:07:26 -0800 Subject: [PATCH 056/160] Added beginnings of new icert tool --- Makefile | 2 + icert/.gitignore | 1 + icert/Makefile | 6 + icert/dummy_test.go | 11 ++ icert/icertpkg/Makefile | 6 + icert/icertpkg/api.go | 61 +++++++++ icert/icertpkg/dummy_test.go | 11 ++ icert/icertpkg/impl.go | 240 +++++++++++++++++++++++++++++++++++ icert/main.go | 62 +++++++++ 9 files changed, 400 insertions(+) create mode 100644 icert/.gitignore create mode 100644 icert/Makefile create mode 100644 icert/dummy_test.go create mode 100644 icert/icertpkg/Makefile create mode 100644 icert/icertpkg/api.go create mode 100644 icert/icertpkg/dummy_test.go create mode 100644 icert/icertpkg/impl.go create mode 100644 icert/main.go diff --git a/Makefile b/Makefile index a530a31bf..7a06e31b9 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,7 @@ gopkgsubdirs = \ utils \ version \ emswift/emswiftpkg \ + icert/icertpkg imgr/imgrpkg goplugindirs = \ @@ -44,6 +45,7 @@ gobinsubdirs = \ cleanproxyfs \ emswift \ fsworkout \ + icert \ imgr \ inodeworkout \ pfs-crash \ diff --git a/icert/.gitignore b/icert/.gitignore new file mode 100644 index 000000000..cfaad7611 --- /dev/null +++ b/icert/.gitignore @@ -0,0 +1 @@ +*.pem diff --git a/icert/Makefile b/icert/Makefile new file mode 100644 index 000000000..a7eeb3655 --- /dev/null +++ b/icert/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/icert + +include ../GoMakefile diff --git a/icert/dummy_test.go b/icert/dummy_test.go new file mode 100644 index 000000000..d42991728 --- /dev/null +++ b/icert/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/icert/icertpkg/Makefile b/icert/icertpkg/Makefile new file mode 100644 index 000000000..e4a909021 --- /dev/null +++ b/icert/icertpkg/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/icert/icertpkg + +include ../../GoMakefile diff --git a/icert/icertpkg/api.go b/icert/icertpkg/api.go new file mode 100644 index 000000000..d0452cc0c --- /dev/null +++ b/icert/icertpkg/api.go @@ -0,0 +1,61 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +// Inspired by https://shaneutt.com/blog/golang-ca-and-signed-cert-go/ + +package icertpkg + +import ( + "time" +) + +const ( + // GenerateKeyAlgorithmEd25519 selects the Ed25519 signature algorithm. + // + GenerateKeyAlgorithmEd25519 = "ed25519" + + // GenerateKeyAlgorithmRSA selects the RSA signature algorithm. + // + GenerateKeyAlgorithmRSA = "rsa" + + // GenerateKeyAlgorithmRSABits is the number of bits that will be + // used if GenerateKeyAlgorithmRSA is selected. + // + GenerateKeyAlgorithmRSABits = 4096 + + // CertificateSerialNumberRandomBits is the number of bits that will + // be randomly generated for a certificate's SerialNumber. + // + CertificateSerialNumberRandomBits = 256 + + // GeneratedFilePerm is the permission bits that, after the application + // of umask, will specify the mode of the created cert|key files. + // + GeneratedFilePerm = 0644 +) + +// GenCACert is called to generate a Certificate Authority for the specified +// organization using the requested generateKeyAlgorithm who's validity lasts +// for the desired ttl starting from time.Now(). The resultant PEM-encoded +// CA Certificate is written to certFile. The PEM-encoded private key for +// the CA Certificate is written to keyFile. If certFile and keyFile are +// the same, both the CA Certificate and its private key will be written to +// the common file. +// +func GenCACert(organization string, generateKeyAlgorithm string, ttl time.Duration, certFile string, keyFile string) (err error) { + return genCACert(organization, generateKeyAlgorithm, ttl, certFile, keyFile) +} + +// GenCert is called to generate a Certificate for the specified organization +// and IP Address using the requested generateKeyAlgorithm who's validity +// lasts for the desired ttl starting from time.Now(). The Certificate will +// be signed by the CA Certificate specified via caCertFile and caKeyFile. +// The caCertFile and caKeyFile values may be identical. The resultant +// PEM-encoded Certificate is written to certFile. The PEM-encoded private +// key for the Certificate is written to keyFile. If certFile and keyFile +// are the same, both the Certificate and its private key will be written to +// the common file. +// +func GenCert(organization string, ipAddress string, generateKeyAlgorithm string, ttl time.Duration, caCertFile string, caKeyFile string, certFile string, keyFile string) (err error) { + return genCert(organization, ipAddress, generateKeyAlgorithm, ttl, caCertFile, caKeyFile, certFile, keyFile) +} diff --git a/icert/icertpkg/dummy_test.go b/icert/icertpkg/dummy_test.go new file mode 100644 index 000000000..a34dcad9d --- /dev/null +++ b/icert/icertpkg/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package icertpkg + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/icert/icertpkg/impl.go b/icert/icertpkg/impl.go new file mode 100644 index 000000000..c08c510d7 --- /dev/null +++ b/icert/icertpkg/impl.go @@ -0,0 +1,240 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package icertpkg + +import ( + "crypto" + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io/ioutil" + "math/big" + "net" + "time" +) + +func genCACert(organization string, generateKeyAlgorithm string, ttl time.Duration, certFile string, keyFile string) (err error) { + var ( + caX509Certificate []byte + caX509CertificateTemplate *x509.Certificate + certPEM []byte + ed25519PrivateKey ed25519.PrivateKey + ed25519PublicKey ed25519.PublicKey + keyPEM []byte + pkcs8PrivateKey []byte + rsaPrivateKey *rsa.PrivateKey + rsaPublicKey crypto.PublicKey + serialNumber *big.Int + serialNumberMax *big.Int + timeNow time.Time + ) + + serialNumberMax = big.NewInt(0) + _ = serialNumberMax.Exp(big.NewInt(2), big.NewInt(CertificateSerialNumberRandomBits), nil) + + serialNumber, err = rand.Int(rand.Reader, serialNumberMax) + if nil != err { + return + } + + timeNow = time.Now() + + caX509CertificateTemplate = &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{organization}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + NotBefore: timeNow, + NotAfter: timeNow.Add(ttl), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + switch generateKeyAlgorithm { + case GenerateKeyAlgorithmEd25519: + ed25519PublicKey, ed25519PrivateKey, err = ed25519.GenerateKey(rand.Reader) + if nil != err { + return + } + + caX509Certificate, err = x509.CreateCertificate(rand.Reader, caX509CertificateTemplate, caX509CertificateTemplate, ed25519PublicKey, ed25519PrivateKey) + if nil != err { + return + } + + pkcs8PrivateKey, err = x509.MarshalPKCS8PrivateKey(ed25519PrivateKey) + if nil != err { + return + } + case GenerateKeyAlgorithmRSA: + rsaPrivateKey, err = rsa.GenerateKey(rand.Reader, GenerateKeyAlgorithmRSABits) + if nil != err { + return + } + rsaPublicKey = rsaPrivateKey.Public() + + caX509Certificate, err = x509.CreateCertificate(rand.Reader, caX509CertificateTemplate, caX509CertificateTemplate, rsaPublicKey, rsaPrivateKey) + if nil != err { + return + } + + pkcs8PrivateKey, err = x509.MarshalPKCS8PrivateKey(rsaPrivateKey) + if nil != err { + return + } + default: + err = fmt.Errorf("generateKeyAlgorithm \"%s\" not supported... must be one of \"%s\" or \"%s\"", generateKeyAlgorithm, GenerateKeyAlgorithmEd25519, GenerateKeyAlgorithmRSA) + return + } + + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caX509Certificate}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8PrivateKey}) + + if certFile == keyFile { + err = ioutil.WriteFile(certFile, append(certPEM, keyPEM...), GeneratedFilePerm) + if nil != err { + return + } + } else { + err = ioutil.WriteFile(certFile, certPEM, GeneratedFilePerm) + if nil != err { + return + } + err = ioutil.WriteFile(keyFile, keyPEM, GeneratedFilePerm) + if nil != err { + return + } + } + + err = nil + return +} + +func genCert(organization string, ipAddress string, generateKeyAlgorithm string, ttl time.Duration, caCertFile string, caKeyFile string, certFile string, keyFile string) (err error) { + var ( + caTLSCertificate tls.Certificate + caX509Certificate *x509.Certificate + certPEM []byte + ed25519PrivateKey ed25519.PrivateKey + ed25519PublicKey ed25519.PublicKey + keyPEM []byte + pkcs8PrivateKey []byte + rsaPrivateKey *rsa.PrivateKey + rsaPublicKey crypto.PublicKey + serialNumber *big.Int + serialNumberMax *big.Int + timeNow time.Time + x509Certificate []byte + x509CertificateTemplate *x509.Certificate + ) + + serialNumberMax = big.NewInt(0) + _ = serialNumberMax.Exp(big.NewInt(2), big.NewInt(CertificateSerialNumberRandomBits), nil) + + serialNumber, err = rand.Int(rand.Reader, serialNumberMax) + if nil != err { + return + } + + timeNow = time.Now() + + x509CertificateTemplate = &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{organization}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + IPAddresses: []net.IP{net.ParseIP(ipAddress)}, + NotBefore: timeNow, + NotAfter: timeNow.Add(ttl), + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + } + + caTLSCertificate, err = tls.LoadX509KeyPair(caCertFile, caKeyFile) + if nil != err { + return + } + + caX509Certificate, err = x509.ParseCertificate(caTLSCertificate.Certificate[0]) + if nil != err { + return + } + + switch generateKeyAlgorithm { + case GenerateKeyAlgorithmEd25519: + ed25519PublicKey, ed25519PrivateKey, err = ed25519.GenerateKey(rand.Reader) + if nil != err { + return + } + + x509Certificate, err = x509.CreateCertificate(rand.Reader, x509CertificateTemplate, caX509Certificate, ed25519PublicKey, caTLSCertificate.PrivateKey) + if nil != err { + return + } + + pkcs8PrivateKey, err = x509.MarshalPKCS8PrivateKey(ed25519PrivateKey) + if nil != err { + return + } + case GenerateKeyAlgorithmRSA: + rsaPrivateKey, err = rsa.GenerateKey(rand.Reader, GenerateKeyAlgorithmRSABits) + if nil != err { + return + } + rsaPublicKey = rsaPrivateKey.Public() + + x509Certificate, err = x509.CreateCertificate(rand.Reader, x509CertificateTemplate, caX509Certificate, rsaPublicKey, caTLSCertificate.PrivateKey) + if nil != err { + return + } + + pkcs8PrivateKey, err = x509.MarshalPKCS8PrivateKey(rsaPrivateKey) + if nil != err { + return + } + default: + err = fmt.Errorf("generateKeyAlgorithm \"%s\" not supported... must be one of \"%s\" or \"%s\"", generateKeyAlgorithm, GenerateKeyAlgorithmEd25519, GenerateKeyAlgorithmRSA) + return + } + + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: x509Certificate}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8PrivateKey}) + + if certFile == keyFile { + err = ioutil.WriteFile(certFile, append(certPEM, keyPEM...), GeneratedFilePerm) + if nil != err { + return + } + } else { + err = ioutil.WriteFile(certFile, certPEM, GeneratedFilePerm) + if nil != err { + return + } + err = ioutil.WriteFile(keyFile, keyPEM, GeneratedFilePerm) + if nil != err { + return + } + } + + err = nil + return +} diff --git a/icert/main.go b/icert/main.go new file mode 100644 index 000000000..3728a82aa --- /dev/null +++ b/icert/main.go @@ -0,0 +1,62 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "time" + + "github.com/NVIDIA/proxyfs/icert/icertpkg" +) + +func main() { + var ( + err error + ) + + // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\") returned err: %v\n", err) + // os.Exit(1) + // } + // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\") returned err: %v\n", err) + // os.Exit(2) + // } + + err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_cert.pem", "./rsa_ca_key.pem") + if nil != err { + fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_cert.pem\",\"./rsa_ca_key.pem\") returned err: %v\n", err) + os.Exit(3) + } + // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\") returned err: %v\n", err) + // os.Exit(4) + // } + + // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem", "./ed25519_cert.pem", "./ed25519_key.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\",\"./ed25519_cert.pem\",\"./ed25519_key.pem\") returned err: %v\n", err) + // os.Exit(5) + // } + // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem", "./ed25519_combined.pem", "./ed25519_combined.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\",\"./ed25519_combined.pem\",\"./ed25519_combined.pem\") returned err: %v\n", err) + // os.Exit(6) + // } + + err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_cert.pem", "./rsa_ca_key.pem", "./rsa_cert.pem", "./rsa_key.pem") + if nil != err { + fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_cert.pem\",\"./rsa_ca_key.pem\",\"./rsa_cert.pem\",\"./rsa_key.pem\") returned err: %v\n", err) + os.Exit(7) + } + // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem", "./rsa_combined.pem", "./rsa_combined.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\",\"./rsa_combined.pem\",\"./rsa_combined.pem\") returned err: %v\n", err) + // os.Exit(8) + // } +} From 25ca537c1dc706ef5543f5ac5f2fb82417eb34ec Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 11 Mar 2021 12:46:38 -0800 Subject: [PATCH 057/160] icert now "works"... need to convert icert/main.go into icert/icertpkg/..._test.go --- icert/client/.gitignore | 1 + icert/client/client.go | 56 +++++++++++++++++++++++++++++++++++++ icert/main.go | 62 ++++++++++++++++++++--------------------- icert/server/.gitignore | 1 + icert/server/server.go | 58 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 31 deletions(-) create mode 100644 icert/client/.gitignore create mode 100644 icert/client/client.go create mode 100644 icert/server/.gitignore create mode 100644 icert/server/server.go diff --git a/icert/client/.gitignore b/icert/client/.gitignore new file mode 100644 index 000000000..b051c6c57 --- /dev/null +++ b/icert/client/.gitignore @@ -0,0 +1 @@ +client diff --git a/icert/client/client.go b/icert/client/client.go new file mode 100644 index 000000000..1c667dd64 --- /dev/null +++ b/icert/client/client.go @@ -0,0 +1,56 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" + "log" +) + +func main() { + log.SetFlags(log.Lshortfile) + + // caCertPEM, err := ioutil.ReadFile("../ed25519_ca_cert.pem") + // caCertPEM, err := ioutil.ReadFile("../ed25519_ca_combined.pem") + // caCertPEM, err := ioutil.ReadFile("../rsa_ca_cert.pem") + caCertPEM, err := ioutil.ReadFile("../rsa_ca_combined.pem") + if err != nil { + log.Println(err) + return + } + + rootCAs := x509.NewCertPool() + ok := rootCAs.AppendCertsFromPEM(caCertPEM) + if !ok { + err := fmt.Errorf("rootCAs.AppendCertsFromPEM(caCertPEM) returned !ok") + log.Println(err) + return + } + + tlsConfig := &tls.Config{ + RootCAs: rootCAs, + } + + conn, err := tls.Dial("tcp", "127.0.0.1:8443", tlsConfig) + if err != nil { + log.Println(err) + return + } + defer conn.Close() + + n, err := conn.Write([]byte("hello\n")) + if err != nil { + log.Println(n, err) + return + } + + buf := make([]byte, 100) + n, err = conn.Read(buf) + if err != nil { + log.Println(n, err) + return + } + + println(string(buf[:n])) +} diff --git a/icert/main.go b/icert/main.go index 3728a82aa..4edf51ca2 100644 --- a/icert/main.go +++ b/icert/main.go @@ -16,47 +16,47 @@ func main() { err error ) - // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\") returned err: %v\n", err) - // os.Exit(1) - // } - // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\") returned err: %v\n", err) - // os.Exit(2) - // } + err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem") + if nil != err { + fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\") returned err: %v\n", err) + os.Exit(1) + } + err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem") + if nil != err { + fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\") returned err: %v\n", err) + os.Exit(2) + } err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_cert.pem", "./rsa_ca_key.pem") if nil != err { fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_cert.pem\",\"./rsa_ca_key.pem\") returned err: %v\n", err) os.Exit(3) } - // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\") returned err: %v\n", err) - // os.Exit(4) - // } - - // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem", "./ed25519_cert.pem", "./ed25519_key.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\",\"./ed25519_cert.pem\",\"./ed25519_key.pem\") returned err: %v\n", err) - // os.Exit(5) - // } - // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem", "./ed25519_combined.pem", "./ed25519_combined.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\",\"./ed25519_combined.pem\",\"./ed25519_combined.pem\") returned err: %v\n", err) - // os.Exit(6) - // } + err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem") + if nil != err { + fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\") returned err: %v\n", err) + os.Exit(4) + } + + err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem", "./ed25519_cert.pem", "./ed25519_key.pem") + if nil != err { + fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\",\"./ed25519_cert.pem\",\"./ed25519_key.pem\") returned err: %v\n", err) + os.Exit(5) + } + err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem", "./ed25519_combined.pem", "./ed25519_combined.pem") + if nil != err { + fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\",\"./ed25519_combined.pem\",\"./ed25519_combined.pem\") returned err: %v\n", err) + os.Exit(6) + } err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_cert.pem", "./rsa_ca_key.pem", "./rsa_cert.pem", "./rsa_key.pem") if nil != err { fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_cert.pem\",\"./rsa_ca_key.pem\",\"./rsa_cert.pem\",\"./rsa_key.pem\") returned err: %v\n", err) os.Exit(7) } - // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem", "./rsa_combined.pem", "./rsa_combined.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\",\"./rsa_combined.pem\",\"./rsa_combined.pem\") returned err: %v\n", err) - // os.Exit(8) - // } + err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem", "./rsa_combined.pem", "./rsa_combined.pem") + if nil != err { + fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\",\"./rsa_combined.pem\",\"./rsa_combined.pem\") returned err: %v\n", err) + os.Exit(8) + } } diff --git a/icert/server/.gitignore b/icert/server/.gitignore new file mode 100644 index 000000000..254defddb --- /dev/null +++ b/icert/server/.gitignore @@ -0,0 +1 @@ +server diff --git a/icert/server/server.go b/icert/server/server.go new file mode 100644 index 000000000..f89876934 --- /dev/null +++ b/icert/server/server.go @@ -0,0 +1,58 @@ +package main + +import ( + "bufio" + "crypto/tls" + "log" + "net" +) + +func main() { + log.SetFlags(log.Lshortfile) + + // cer, err := tls.LoadX509KeyPair("../ed25519_cert.pem", "../ed25519_key.pem") + // cer, err := tls.LoadX509KeyPair("../ed25519_combined.pem", "../ed25519_combined.pem") + // cer, err := tls.LoadX509KeyPair("../rsa_cert.pem", "../rsa_key.pem") + cer, err := tls.LoadX509KeyPair("../rsa_combined.pem", "../rsa_combined.pem") + if err != nil { + log.Println(err) + return + } + + config := &tls.Config{Certificates: []tls.Certificate{cer}} + ln, err := tls.Listen("tcp", "127.0.0.1:8443", config) + if err != nil { + log.Println(err) + return + } + defer ln.Close() + + for { + conn, err := ln.Accept() + if err != nil { + log.Println(err) + continue + } + go handleConnection(conn) + } +} + +func handleConnection(conn net.Conn) { + defer conn.Close() + r := bufio.NewReader(conn) + for { + msg, err := r.ReadString('\n') + if err != nil { + log.Println(err) + return + } + + println(msg) + + n, err := conn.Write([]byte("world\n")) + if err != nil { + log.Println(n, err) + return + } + } +} From 889c6db935e49f4539abf146f8db517587254d6d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 11 Mar 2021 14:37:01 -0800 Subject: [PATCH 058/160] Enabled client example to leverage x509.SystemCertPool --- icert/client/client.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/icert/client/client.go b/icert/client/client.go index 1c667dd64..9c5a35c32 100644 --- a/icert/client/client.go +++ b/icert/client/client.go @@ -20,7 +20,13 @@ func main() { return } - rootCAs := x509.NewCertPool() + // rootCAs := x509.NewCertPool() + rootCAs, err := x509.SystemCertPool() + if err != nil { + log.Println(err) + return + } + ok := rootCAs.AppendCertsFromPEM(caCertPEM) if !ok { err := fmt.Errorf("rootCAs.AppendCertsFromPEM(caCertPEM) returned !ok") From 68b612bdf39f95ad92acc3244deea14af9a6fd4c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 11 Mar 2021 16:54:23 -0800 Subject: [PATCH 059/160] Finished icert/icertpkg (including api_test.go)... now back to the cmdline tool in icert... --- icert/client/.gitignore | 1 - icert/client/client.go | 62 ---------- icert/icertpkg/api.go | 12 +- icert/icertpkg/api_test.go | 228 +++++++++++++++++++++++++++++++++++ icert/icertpkg/dummy_test.go | 11 -- icert/icertpkg/impl.go | 10 +- icert/main.go | 102 ++++++++-------- icert/server/.gitignore | 1 - icert/server/server.go | 58 --------- 9 files changed, 286 insertions(+), 199 deletions(-) delete mode 100644 icert/client/.gitignore delete mode 100644 icert/client/client.go create mode 100644 icert/icertpkg/api_test.go delete mode 100644 icert/icertpkg/dummy_test.go delete mode 100644 icert/server/.gitignore delete mode 100644 icert/server/server.go diff --git a/icert/client/.gitignore b/icert/client/.gitignore deleted file mode 100644 index b051c6c57..000000000 --- a/icert/client/.gitignore +++ /dev/null @@ -1 +0,0 @@ -client diff --git a/icert/client/client.go b/icert/client/client.go deleted file mode 100644 index 9c5a35c32..000000000 --- a/icert/client/client.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "crypto/tls" - "crypto/x509" - "fmt" - "io/ioutil" - "log" -) - -func main() { - log.SetFlags(log.Lshortfile) - - // caCertPEM, err := ioutil.ReadFile("../ed25519_ca_cert.pem") - // caCertPEM, err := ioutil.ReadFile("../ed25519_ca_combined.pem") - // caCertPEM, err := ioutil.ReadFile("../rsa_ca_cert.pem") - caCertPEM, err := ioutil.ReadFile("../rsa_ca_combined.pem") - if err != nil { - log.Println(err) - return - } - - // rootCAs := x509.NewCertPool() - rootCAs, err := x509.SystemCertPool() - if err != nil { - log.Println(err) - return - } - - ok := rootCAs.AppendCertsFromPEM(caCertPEM) - if !ok { - err := fmt.Errorf("rootCAs.AppendCertsFromPEM(caCertPEM) returned !ok") - log.Println(err) - return - } - - tlsConfig := &tls.Config{ - RootCAs: rootCAs, - } - - conn, err := tls.Dial("tcp", "127.0.0.1:8443", tlsConfig) - if err != nil { - log.Println(err) - return - } - defer conn.Close() - - n, err := conn.Write([]byte("hello\n")) - if err != nil { - log.Println(n, err) - return - } - - buf := make([]byte, 100) - n, err = conn.Read(buf) - if err != nil { - log.Println(n, err) - return - } - - println(string(buf[:n])) -} diff --git a/icert/icertpkg/api.go b/icert/icertpkg/api.go index d0452cc0c..e1cd38385 100644 --- a/icert/icertpkg/api.go +++ b/icert/icertpkg/api.go @@ -46,16 +46,16 @@ func GenCACert(organization string, generateKeyAlgorithm string, ttl time.Durati return genCACert(organization, generateKeyAlgorithm, ttl, certFile, keyFile) } -// GenCert is called to generate a Certificate for the specified organization -// and IP Address using the requested generateKeyAlgorithm who's validity -// lasts for the desired ttl starting from time.Now(). The Certificate will -// be signed by the CA Certificate specified via caCertFile and caKeyFile. +// GenIPAddressCert is called to generate a Certificate for the specified +// organization and IP Address using the requested generateKeyAlgorithm who's +// validity lasts for the desired ttl starting from time.Now(). The Certificate +// will be signed by the CA Certificate specified via caCertFile and caKeyFile. // The caCertFile and caKeyFile values may be identical. The resultant // PEM-encoded Certificate is written to certFile. The PEM-encoded private // key for the Certificate is written to keyFile. If certFile and keyFile // are the same, both the Certificate and its private key will be written to // the common file. // -func GenCert(organization string, ipAddress string, generateKeyAlgorithm string, ttl time.Duration, caCertFile string, caKeyFile string, certFile string, keyFile string) (err error) { - return genCert(organization, ipAddress, generateKeyAlgorithm, ttl, caCertFile, caKeyFile, certFile, keyFile) +func GenIPAddressCert(organization string, ipAddress string, generateKeyAlgorithm string, ttl time.Duration, caCertFile string, caKeyFile string, certFile string, keyFile string) (err error) { + return genIPAddressCert(organization, ipAddress, generateKeyAlgorithm, ttl, caCertFile, caKeyFile, certFile, keyFile) } diff --git a/icert/icertpkg/api_test.go b/icert/icertpkg/api_test.go new file mode 100644 index 000000000..a8a995b31 --- /dev/null +++ b/icert/icertpkg/api_test.go @@ -0,0 +1,228 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package icertpkg + +import ( + "bufio" + "crypto/tls" + "crypto/x509" + "io" + "io/ioutil" + "net" + "os" + "path/filepath" + "sync" + "testing" + "time" +) + +const ( + testOrganizationCA = "Test Organization CA" + testOrganizationIPAddress = "Test Organization IP Address" + + testCertificateTTL = time.Hour + + testIPAddress = "127.0.0.1" + testTLSPort = "9443" + + testTempDirPattern = "icertpkg_*" + + testCACertPEMFileName = "ca_cert.pem" + testCAKeyPEMFileName = "ca_key.pem" + testCACombinedPEMFileName = "ca_combined.pem" + + testIPAddressCertPEMFileName = "ip_address_cert.pem" + testIPAddressKeyPEMFileName = "ip_address_key.pem" + testIPAddressCombinedPEMFileName = "ip_address_combined.pem" + + testClientMsg = "ping\n" + testServerMsg = "pong\n" +) + +func TestEd25519DistinctCertAndKeyFiles(t *testing.T) { + testAPI(t, GenerateKeyAlgorithmEd25519, true) +} +func TestEd25519CombinedCertAndKeyFile(t *testing.T) { + testAPI(t, GenerateKeyAlgorithmEd25519, false) +} +func TestRSADistinctCertAndKeyFiles(t *testing.T) { + testAPI(t, GenerateKeyAlgorithmRSA, true) +} +func TestRSACombinedCertAndKeyFile(t *testing.T) { + testAPI(t, GenerateKeyAlgorithmRSA, false) +} + +func testAPI(t *testing.T, generateKeyAlgorithm string, combined bool) { + var ( + caCertPEM []byte + caCertPemFilePath string + caCertPool *x509.CertPool + caKeyPemFilePath string + clientErr error + clientTLSConfig *tls.Config + clientWG sync.WaitGroup + err error + ipAddressCertPemFilePath string + ipAddressKeyPemFilePath string + ipAddressPort string + ok bool + tempDir string + serverErr error + serverNetListener net.Listener + serverTLSCertificate tls.Certificate + serverWG sync.WaitGroup + ) + + tempDir, err = ioutil.TempDir("", testTempDirPattern) + if nil != err { + t.Fatalf("ioutil.TempDir(\"\", \"%s\") failed: %v", testTempDirPattern, err) + } + defer func(t *testing.T, tempDir string) { + var ( + err error + ) + + err = os.RemoveAll(tempDir) + if nil != err { + t.Fatalf("os.RemoveAll(\"%s\") failed: %v", tempDir, err) + } + }(t, tempDir) + + if combined { + caCertPemFilePath = filepath.Join(tempDir, testCACombinedPEMFileName) + caKeyPemFilePath = caCertPemFilePath + ipAddressCertPemFilePath = filepath.Join(tempDir, testIPAddressCombinedPEMFileName) + ipAddressKeyPemFilePath = ipAddressCertPemFilePath + } else { + caCertPemFilePath = filepath.Join(tempDir, testCACertPEMFileName) + caKeyPemFilePath = filepath.Join(tempDir, testCAKeyPEMFileName) + ipAddressCertPemFilePath = filepath.Join(tempDir, testIPAddressCertPEMFileName) + ipAddressKeyPemFilePath = filepath.Join(tempDir, testIPAddressKeyPEMFileName) + } + + err = GenCACert(testOrganizationCA, generateKeyAlgorithm, testCertificateTTL, caCertPemFilePath, caKeyPemFilePath) + if nil != err { + t.Fatalf("GenCACert() failed: %v", err) + } + + err = GenIPAddressCert(testOrganizationIPAddress, testIPAddress, generateKeyAlgorithm, testCertificateTTL, caCertPemFilePath, caKeyPemFilePath, ipAddressCertPemFilePath, ipAddressKeyPemFilePath) + if nil != err { + t.Fatalf("GenCert() failed: %v", err) + } + + ipAddressPort = net.JoinHostPort(testIPAddress, testTLSPort) + + serverTLSCertificate, err = tls.LoadX509KeyPair(ipAddressCertPemFilePath, ipAddressKeyPemFilePath) + if nil != err { + t.Fatalf("tls.LoadX509KeyPair() failed: %v", err) + } + + serverNetListener, err = tls.Listen("tcp", ipAddressPort, &tls.Config{Certificates: []tls.Certificate{serverTLSCertificate}}) + if nil != err { + t.Fatalf("tls.Listen() failed: %v", err) + } + + caCertPEM, err = ioutil.ReadFile(caCertPemFilePath) + if nil != err { + t.Fatalf("ioutil.ReadFile(caCertPemFilePath) failed: %v", err) + } + + caCertPool = x509.NewCertPool() + ok = caCertPool.AppendCertsFromPEM(caCertPEM) + if !ok { + t.Fatalf("caCertPool.AppendCertsFromPEM(caCertPEM) returned !ok") + } + + clientTLSConfig = &tls.Config{RootCAs: caCertPool} + + serverWG.Add(1) + + go func() { + var ( + bufioReader *bufio.Reader + netConn net.Conn + receivedMsg string + ) + + netConn, serverErr = serverNetListener.Accept() + if nil != serverErr { + serverWG.Done() + return + } + + bufioReader = bufio.NewReader(netConn) + + for { + receivedMsg, serverErr = bufioReader.ReadString('\n') + if io.EOF == serverErr { + serverErr = nil + break + } else if nil != serverErr { + break + } else { + t.Logf("Server received %s", receivedMsg) + _, serverErr = netConn.Write([]byte(testServerMsg)) + if nil != serverErr { + break + } + } + } + + _ = netConn.Close() + + serverWG.Done() + }() + + clientWG.Add(1) + + go func() { + var ( + bufioReader *bufio.Reader + receivedMsg string + tlsConn *tls.Conn + ) + + tlsConn, clientErr = tls.Dial("tcp", ipAddressPort, clientTLSConfig) + if nil != clientErr { + clientWG.Done() + return + } + + _, clientErr = tlsConn.Write([]byte(testClientMsg)) + if nil != clientErr { + _ = tlsConn.Close() + clientWG.Done() + return + } + + bufioReader = bufio.NewReader(tlsConn) + + receivedMsg, serverErr = bufioReader.ReadString('\n') + if nil != err { + _ = tlsConn.Close() + clientWG.Done() + return + } + + t.Logf("Client received %s", receivedMsg) + + _ = tlsConn.Close() + + clientWG.Done() + }() + + clientWG.Wait() + + if nil != clientErr { + t.Fatalf("client failed to successfully tls.Dial(): %v", clientErr) + } + + _ = serverNetListener.Close() + + serverWG.Wait() + + if nil != serverErr { + t.Fatalf("server failed to successfully serverNetListener.Accept(): %v", serverErr) + } +} diff --git a/icert/icertpkg/dummy_test.go b/icert/icertpkg/dummy_test.go deleted file mode 100644 index a34dcad9d..000000000 --- a/icert/icertpkg/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package icertpkg - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/icert/icertpkg/impl.go b/icert/icertpkg/impl.go index c08c510d7..6bae0d8e7 100644 --- a/icert/icertpkg/impl.go +++ b/icert/icertpkg/impl.go @@ -123,7 +123,7 @@ func genCACert(organization string, generateKeyAlgorithm string, ttl time.Durati return } -func genCert(organization string, ipAddress string, generateKeyAlgorithm string, ttl time.Duration, caCertFile string, caKeyFile string, certFile string, keyFile string) (err error) { +func genIPAddressCert(organization string, ipAddress string, generateKeyAlgorithm string, ttl time.Duration, caCertFile string, caKeyFile string, ipAddressCertFile string, ipAddressKeyFile string) (err error) { var ( caTLSCertificate tls.Certificate caX509Certificate *x509.Certificate @@ -219,17 +219,17 @@ func genCert(organization string, ipAddress string, generateKeyAlgorithm string, certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: x509Certificate}) keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8PrivateKey}) - if certFile == keyFile { - err = ioutil.WriteFile(certFile, append(certPEM, keyPEM...), GeneratedFilePerm) + if ipAddressCertFile == ipAddressKeyFile { + err = ioutil.WriteFile(ipAddressCertFile, append(certPEM, keyPEM...), GeneratedFilePerm) if nil != err { return } } else { - err = ioutil.WriteFile(certFile, certPEM, GeneratedFilePerm) + err = ioutil.WriteFile(ipAddressCertFile, certPEM, GeneratedFilePerm) if nil != err { return } - err = ioutil.WriteFile(keyFile, keyPEM, GeneratedFilePerm) + err = ioutil.WriteFile(ipAddressKeyFile, keyPEM, GeneratedFilePerm) if nil != err { return } diff --git a/icert/main.go b/icert/main.go index 4edf51ca2..f9b8464d7 100644 --- a/icert/main.go +++ b/icert/main.go @@ -3,60 +3,52 @@ package main -import ( - "fmt" - "os" - "time" - - "github.com/NVIDIA/proxyfs/icert/icertpkg" -) - func main() { - var ( - err error - ) - - err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem") - if nil != err { - fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\") returned err: %v\n", err) - os.Exit(1) - } - err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem") - if nil != err { - fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\") returned err: %v\n", err) - os.Exit(2) - } - - err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_cert.pem", "./rsa_ca_key.pem") - if nil != err { - fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_cert.pem\",\"./rsa_ca_key.pem\") returned err: %v\n", err) - os.Exit(3) - } - err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem") - if nil != err { - fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\") returned err: %v\n", err) - os.Exit(4) - } - - err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem", "./ed25519_cert.pem", "./ed25519_key.pem") - if nil != err { - fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\",\"./ed25519_cert.pem\",\"./ed25519_key.pem\") returned err: %v\n", err) - os.Exit(5) - } - err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem", "./ed25519_combined.pem", "./ed25519_combined.pem") - if nil != err { - fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\",\"./ed25519_combined.pem\",\"./ed25519_combined.pem\") returned err: %v\n", err) - os.Exit(6) - } - - err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_cert.pem", "./rsa_ca_key.pem", "./rsa_cert.pem", "./rsa_key.pem") - if nil != err { - fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_cert.pem\",\"./rsa_ca_key.pem\",\"./rsa_cert.pem\",\"./rsa_key.pem\") returned err: %v\n", err) - os.Exit(7) - } - err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem", "./rsa_combined.pem", "./rsa_combined.pem") - if nil != err { - fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\",\"./rsa_combined.pem\",\"./rsa_combined.pem\") returned err: %v\n", err) - os.Exit(8) - } + // var ( + // err error + // ) + + // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\") returned err: %v\n", err) + // os.Exit(1) + // } + // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\") returned err: %v\n", err) + // os.Exit(2) + // } + + // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_cert.pem", "./rsa_ca_key.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_cert.pem\",\"./rsa_ca_key.pem\") returned err: %v\n", err) + // os.Exit(3) + // } + // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\") returned err: %v\n", err) + // os.Exit(4) + // } + + // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem", "./ed25519_cert.pem", "./ed25519_key.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\",\"./ed25519_cert.pem\",\"./ed25519_key.pem\") returned err: %v\n", err) + // os.Exit(5) + // } + // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem", "./ed25519_combined.pem", "./ed25519_combined.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\",\"./ed25519_combined.pem\",\"./ed25519_combined.pem\") returned err: %v\n", err) + // os.Exit(6) + // } + + // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_cert.pem", "./rsa_ca_key.pem", "./rsa_cert.pem", "./rsa_key.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_cert.pem\",\"./rsa_ca_key.pem\",\"./rsa_cert.pem\",\"./rsa_key.pem\") returned err: %v\n", err) + // os.Exit(7) + // } + // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem", "./rsa_combined.pem", "./rsa_combined.pem") + // if nil != err { + // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\",\"./rsa_combined.pem\",\"./rsa_combined.pem\") returned err: %v\n", err) + // os.Exit(8) + // } } diff --git a/icert/server/.gitignore b/icert/server/.gitignore deleted file mode 100644 index 254defddb..000000000 --- a/icert/server/.gitignore +++ /dev/null @@ -1 +0,0 @@ -server diff --git a/icert/server/server.go b/icert/server/server.go deleted file mode 100644 index f89876934..000000000 --- a/icert/server/server.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "bufio" - "crypto/tls" - "log" - "net" -) - -func main() { - log.SetFlags(log.Lshortfile) - - // cer, err := tls.LoadX509KeyPair("../ed25519_cert.pem", "../ed25519_key.pem") - // cer, err := tls.LoadX509KeyPair("../ed25519_combined.pem", "../ed25519_combined.pem") - // cer, err := tls.LoadX509KeyPair("../rsa_cert.pem", "../rsa_key.pem") - cer, err := tls.LoadX509KeyPair("../rsa_combined.pem", "../rsa_combined.pem") - if err != nil { - log.Println(err) - return - } - - config := &tls.Config{Certificates: []tls.Certificate{cer}} - ln, err := tls.Listen("tcp", "127.0.0.1:8443", config) - if err != nil { - log.Println(err) - return - } - defer ln.Close() - - for { - conn, err := ln.Accept() - if err != nil { - log.Println(err) - continue - } - go handleConnection(conn) - } -} - -func handleConnection(conn net.Conn) { - defer conn.Close() - r := bufio.NewReader(conn) - for { - msg, err := r.ReadString('\n') - if err != nil { - log.Println(err) - return - } - - println(msg) - - n, err := conn.Write([]byte("world\n")) - if err != nil { - log.Println(n, err) - return - } - } -} From baca505be2dca5a9d74c5d3309c6f1213be6af20 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 11 Mar 2021 17:17:26 -0800 Subject: [PATCH 060/160] Fix typo in Makefile around the recently added icert/icertpkg line in gopkgsubdirs --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7a06e31b9..5496b34ad 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ gopkgsubdirs = \ utils \ version \ emswift/emswiftpkg \ - icert/icertpkg + icert/icertpkg \ imgr/imgrpkg goplugindirs = \ From 75ae85a2e866eabef378aa07e57091ccc0517c90 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 12 Mar 2021 19:53:49 -0800 Subject: [PATCH 061/160] Re-worked icertpkg API to support DNS & IP Endpoints Also parsed flags for the icert program --- icert/icertpkg/api.go | 42 +++++----- icert/icertpkg/api_test.go | 95 ++++++++++++++------- icert/icertpkg/impl.go | 37 +++------ icert/main.go | 164 ++++++++++++++++++++++++++----------- 4 files changed, 216 insertions(+), 122 deletions(-) diff --git a/icert/icertpkg/api.go b/icert/icertpkg/api.go index e1cd38385..5e317dffd 100644 --- a/icert/icertpkg/api.go +++ b/icert/icertpkg/api.go @@ -6,6 +6,8 @@ package icertpkg import ( + "crypto/x509/pkix" + "net" "time" ) @@ -34,28 +36,28 @@ const ( GeneratedFilePerm = 0644 ) -// GenCACert is called to generate a Certificate Authority for the specified -// organization using the requested generateKeyAlgorithm who's validity lasts -// for the desired ttl starting from time.Now(). The resultant PEM-encoded -// CA Certificate is written to certFile. The PEM-encoded private key for -// the CA Certificate is written to keyFile. If certFile and keyFile are -// the same, both the CA Certificate and its private key will be written to -// the common file. +// GenCACert is called to generate a Certificate Authority using the requested +// generateKeyAlgorithm for the specified subject who's validity last for the +// desired ttl starting from time.Now(). The resultant PEM-encoded CA Certificate +// is written to certFile. The PEM-encoded private key for the CA Certificate is +// written to keyFile. If certFile and keyFile are the same, both the CA Certificate +// and its private key will be written to the common file. // -func GenCACert(organization string, generateKeyAlgorithm string, ttl time.Duration, certFile string, keyFile string) (err error) { - return genCACert(organization, generateKeyAlgorithm, ttl, certFile, keyFile) +func GenCACert(generateKeyAlgorithm string, subject pkix.Name, ttl time.Duration, certFile string, keyFile string) (err error) { + return genCACert(generateKeyAlgorithm, subject, ttl, certFile, keyFile) } -// GenIPAddressCert is called to generate a Certificate for the specified -// organization and IP Address using the requested generateKeyAlgorithm who's -// validity lasts for the desired ttl starting from time.Now(). The Certificate -// will be signed by the CA Certificate specified via caCertFile and caKeyFile. -// The caCertFile and caKeyFile values may be identical. The resultant -// PEM-encoded Certificate is written to certFile. The PEM-encoded private -// key for the Certificate is written to keyFile. If certFile and keyFile -// are the same, both the Certificate and its private key will be written to -// the common file. +// GenEndpointCert is called to generate a Certificate using the requested +// generateKeyAlgorithm for the specified subject who's validity lasts for +// the desired ttl starting from time.Now(). The endpoints for which the +// Certificate will apply will be the specified dnsNames and/or ipAddresses. +// The Certificate will be signed by the CA Certificate specified via caCertFile +// and caKeyFile. The caCertFile and caKeyFile values may be identical. The +// resultant PEM-encoded Certificate is written to certFile. The PEM-encoded +// private key for the Certificate is written to keyFile. If certFile and +// keyFile are the same, both the Certificate and its private key will be +// written to the common file. // -func GenIPAddressCert(organization string, ipAddress string, generateKeyAlgorithm string, ttl time.Duration, caCertFile string, caKeyFile string, certFile string, keyFile string) (err error) { - return genIPAddressCert(organization, ipAddress, generateKeyAlgorithm, ttl, caCertFile, caKeyFile, certFile, keyFile) +func GenEndpointCert(generateKeyAlgorithm string, subject pkix.Name, dnsNames []string, ipAddresses []net.IP, ttl time.Duration, caCertFile string, caKeyFile string, endpointCertFile string, endpointKeyFile string) (err error) { + return genEndpointCert(generateKeyAlgorithm, subject, dnsNames, ipAddresses, ttl, caCertFile, caKeyFile, endpointCertFile, endpointKeyFile) } diff --git a/icert/icertpkg/api_test.go b/icert/icertpkg/api_test.go index a8a995b31..07b322fd3 100644 --- a/icert/icertpkg/api_test.go +++ b/icert/icertpkg/api_test.go @@ -7,6 +7,7 @@ import ( "bufio" "crypto/tls" "crypto/x509" + "crypto/x509/pkix" "io" "io/ioutil" "net" @@ -18,13 +19,16 @@ import ( ) const ( - testOrganizationCA = "Test Organization CA" - testOrganizationIPAddress = "Test Organization IP Address" + testOrganizationCA = "Test Organization CA" + testOrganizationEndpoint = "Test Organization Endpoint" testCertificateTTL = time.Hour - testIPAddress = "127.0.0.1" - testTLSPort = "9443" + testV4DomainName = "localhost" + testV6DomainName = "localhost6" + testIPv4Address = "127.0.0.1" + testIPv6Address = "::1" + testTLSPort = "9443" testTempDirPattern = "icertpkg_*" @@ -55,23 +59,23 @@ func TestRSACombinedCertAndKeyFile(t *testing.T) { func testAPI(t *testing.T, generateKeyAlgorithm string, combined bool) { var ( - caCertPEM []byte - caCertPemFilePath string - caCertPool *x509.CertPool - caKeyPemFilePath string - clientErr error - clientTLSConfig *tls.Config - clientWG sync.WaitGroup - err error - ipAddressCertPemFilePath string - ipAddressKeyPemFilePath string - ipAddressPort string - ok bool - tempDir string - serverErr error - serverNetListener net.Listener - serverTLSCertificate tls.Certificate - serverWG sync.WaitGroup + caCertPEM []byte + caCertPemFilePath string + caCertPool *x509.CertPool + caKeyPemFilePath string + clientErr error + clientTLSConfig *tls.Config + clientWG sync.WaitGroup + endpointCertPemFilePath string + endpointKeyPemFilePath string + err error + ipAddressPort string + ok bool + tempDir string + serverErr error + serverNetListener net.Listener + serverTLSCertificate tls.Certificate + serverWG sync.WaitGroup ) tempDir, err = ioutil.TempDir("", testTempDirPattern) @@ -92,28 +96,59 @@ func testAPI(t *testing.T, generateKeyAlgorithm string, combined bool) { if combined { caCertPemFilePath = filepath.Join(tempDir, testCACombinedPEMFileName) caKeyPemFilePath = caCertPemFilePath - ipAddressCertPemFilePath = filepath.Join(tempDir, testIPAddressCombinedPEMFileName) - ipAddressKeyPemFilePath = ipAddressCertPemFilePath + endpointCertPemFilePath = filepath.Join(tempDir, testIPAddressCombinedPEMFileName) + endpointKeyPemFilePath = endpointCertPemFilePath } else { caCertPemFilePath = filepath.Join(tempDir, testCACertPEMFileName) caKeyPemFilePath = filepath.Join(tempDir, testCAKeyPEMFileName) - ipAddressCertPemFilePath = filepath.Join(tempDir, testIPAddressCertPEMFileName) - ipAddressKeyPemFilePath = filepath.Join(tempDir, testIPAddressKeyPEMFileName) + endpointCertPemFilePath = filepath.Join(tempDir, testIPAddressCertPEMFileName) + endpointKeyPemFilePath = filepath.Join(tempDir, testIPAddressKeyPEMFileName) } - err = GenCACert(testOrganizationCA, generateKeyAlgorithm, testCertificateTTL, caCertPemFilePath, caKeyPemFilePath) + err = GenCACert( + generateKeyAlgorithm, + pkix.Name{ + Organization: []string{testOrganizationCA}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + testCertificateTTL, + caCertPemFilePath, + caKeyPemFilePath) if nil != err { t.Fatalf("GenCACert() failed: %v", err) } - err = GenIPAddressCert(testOrganizationIPAddress, testIPAddress, generateKeyAlgorithm, testCertificateTTL, caCertPemFilePath, caKeyPemFilePath, ipAddressCertPemFilePath, ipAddressKeyPemFilePath) + err = genEndpointCert( + generateKeyAlgorithm, + pkix.Name{ + Organization: []string{testOrganizationEndpoint}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + []string{testV4DomainName, testV6DomainName}, + []net.IP{net.ParseIP(testIPv4Address), net.ParseIP(testIPv6Address)}, + testCertificateTTL, + caCertPemFilePath, + caKeyPemFilePath, + endpointCertPemFilePath, + endpointKeyPemFilePath) if nil != err { - t.Fatalf("GenCert() failed: %v", err) + t.Fatalf("genEndpointCert() failed: %v", err) } - ipAddressPort = net.JoinHostPort(testIPAddress, testTLSPort) + // ipAddressPort = net.JoinHostPort(testV4DomainName, testTLSPort) + // ipAddressPort = net.JoinHostPort(testV6DomainName, testTLSPort) + ipAddressPort = net.JoinHostPort(testIPv4Address, testTLSPort) + // ipAddressPort = net.JoinHostPort(testIPv6Address, testTLSPort) - serverTLSCertificate, err = tls.LoadX509KeyPair(ipAddressCertPemFilePath, ipAddressKeyPemFilePath) + serverTLSCertificate, err = tls.LoadX509KeyPair(endpointCertPemFilePath, endpointKeyPemFilePath) if nil != err { t.Fatalf("tls.LoadX509KeyPair() failed: %v", err) } diff --git a/icert/icertpkg/impl.go b/icert/icertpkg/impl.go index 6bae0d8e7..05e5f4a27 100644 --- a/icert/icertpkg/impl.go +++ b/icert/icertpkg/impl.go @@ -19,7 +19,7 @@ import ( "time" ) -func genCACert(organization string, generateKeyAlgorithm string, ttl time.Duration, certFile string, keyFile string) (err error) { +func genCACert(generateKeyAlgorithm string, subject pkix.Name, ttl time.Duration, certFile string, keyFile string) (err error) { var ( caX509Certificate []byte caX509CertificateTemplate *x509.Certificate @@ -46,15 +46,8 @@ func genCACert(organization string, generateKeyAlgorithm string, ttl time.Durati timeNow = time.Now() caX509CertificateTemplate = &x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - Organization: []string{organization}, - Country: []string{}, - Province: []string{}, - Locality: []string{}, - StreetAddress: []string{}, - PostalCode: []string{}, - }, + SerialNumber: serialNumber, + Subject: subject, NotBefore: timeNow, NotAfter: timeNow.Add(ttl), IsCA: true, @@ -123,7 +116,7 @@ func genCACert(organization string, generateKeyAlgorithm string, ttl time.Durati return } -func genIPAddressCert(organization string, ipAddress string, generateKeyAlgorithm string, ttl time.Duration, caCertFile string, caKeyFile string, ipAddressCertFile string, ipAddressKeyFile string) (err error) { +func genEndpointCert(generateKeyAlgorithm string, subject pkix.Name, dnsNames []string, ipAddresses []net.IP, ttl time.Duration, caCertFile string, caKeyFile string, endpointCertFile string, endpointKeyFile string) (err error) { var ( caTLSCertificate tls.Certificate caX509Certificate *x509.Certificate @@ -152,16 +145,10 @@ func genIPAddressCert(organization string, ipAddress string, generateKeyAlgorith timeNow = time.Now() x509CertificateTemplate = &x509.Certificate{ - SerialNumber: serialNumber, - Subject: pkix.Name{ - Organization: []string{organization}, - Country: []string{}, - Province: []string{}, - Locality: []string{}, - StreetAddress: []string{}, - PostalCode: []string{}, - }, - IPAddresses: []net.IP{net.ParseIP(ipAddress)}, + SerialNumber: serialNumber, + Subject: subject, + DNSNames: dnsNames, + IPAddresses: ipAddresses, NotBefore: timeNow, NotAfter: timeNow.Add(ttl), ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, @@ -219,17 +206,17 @@ func genIPAddressCert(organization string, ipAddress string, generateKeyAlgorith certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: x509Certificate}) keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8PrivateKey}) - if ipAddressCertFile == ipAddressKeyFile { - err = ioutil.WriteFile(ipAddressCertFile, append(certPEM, keyPEM...), GeneratedFilePerm) + if endpointCertFile == endpointKeyFile { + err = ioutil.WriteFile(endpointCertFile, append(certPEM, keyPEM...), GeneratedFilePerm) if nil != err { return } } else { - err = ioutil.WriteFile(ipAddressCertFile, certPEM, GeneratedFilePerm) + err = ioutil.WriteFile(endpointCertFile, certPEM, GeneratedFilePerm) if nil != err { return } - err = ioutil.WriteFile(ipAddressKeyFile, keyPEM, GeneratedFilePerm) + err = ioutil.WriteFile(endpointKeyFile, keyPEM, GeneratedFilePerm) if nil != err { return } diff --git a/icert/main.go b/icert/main.go index f9b8464d7..e528739cf 100644 --- a/icert/main.go +++ b/icert/main.go @@ -3,52 +3,122 @@ package main +import ( + "flag" + "fmt" + "os" + "time" + + "github.com/NVIDIA/proxyfs/icert/icertpkg" +) + +type stringSlice []string + +func (sS *stringSlice) String() (toReturn string) { + toReturn = fmt.Sprint(*sS) + return +} + +func (sS *stringSlice) Set(s string) (err error) { + *sS = append(*sS, s) + err = nil + return +} + +var ( + caFlag = flag.Bool("ca", false, "generated CA Certicate usable for signing Endpoint Certificates") + + generateKeyAlgorithmEd25519Flag = flag.Bool(icertpkg.GenerateKeyAlgorithmEd25519, false, "generate key via Ed25519") + generateKeyAlgorithmRSAFlag = flag.Bool(icertpkg.GenerateKeyAlgorithmRSA, false, "generate key via RSA") + + organizationFlag stringSlice + countryFlag stringSlice + provinceFlag stringSlice + localityFlag stringSlice + streetAddressFlag stringSlice + postalCodeFlag stringSlice + + ttlFlag = flag.Duration("ttl", time.Duration(0), "generated Certificate's time to live") + + dnsNamesFlag stringSlice + ipAddressesFlag stringSlice + + caCertPemFilePathFlag = flag.String("caCert", "", "path to CA Certificate") + caKeyPemFilePathFlag = flag.String("caKey", "", "path to CA Certificate's PrivateKey") + + endpointCertPemFilePathFlag = flag.String("cert", "", "path to Endpoint Certificate") + endpointKeyPemFilePathFlag = flag.String("key", "", "path to Endpoint Certificate's PrivateKey") +) + func main() { - // var ( - // err error - // ) - - // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\") returned err: %v\n", err) - // os.Exit(1) - // } - // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\") returned err: %v\n", err) - // os.Exit(2) - // } - - // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_cert.pem", "./rsa_ca_key.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_cert.pem\",\"./rsa_ca_key.pem\") returned err: %v\n", err) - // os.Exit(3) - // } - // err = icertpkg.GenCACert("Test Organization CA", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCACert(,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\") returned err: %v\n", err) - // os.Exit(4) - // } - - // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_cert.pem", "./ed25519_ca_key.pem", "./ed25519_cert.pem", "./ed25519_key.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_cert.pem\",\"./ed25519_ca_key.pem\",\"./ed25519_cert.pem\",\"./ed25519_key.pem\") returned err: %v\n", err) - // os.Exit(5) - // } - // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmEd25519, time.Duration(time.Hour), "./ed25519_ca_combined.pem", "./ed25519_ca_combined.pem", "./ed25519_combined.pem", "./ed25519_combined.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmEd25519,,\"./ed25519_ca_combined.pem\",\"./ed25519_ca_combined.pem\",\"./ed25519_combined.pem\",\"./ed25519_combined.pem\") returned err: %v\n", err) - // os.Exit(6) - // } - - // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_cert.pem", "./rsa_ca_key.pem", "./rsa_cert.pem", "./rsa_key.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_cert.pem\",\"./rsa_ca_key.pem\",\"./rsa_cert.pem\",\"./rsa_key.pem\") returned err: %v\n", err) - // os.Exit(7) - // } - // err = icertpkg.GenCert("Test Organization Endpoint", "127.0.0.1", icertpkg.GenerateKeyAlgorithmRSA, time.Duration(time.Hour), "./rsa_ca_combined.pem", "./rsa_ca_combined.pem", "./rsa_combined.pem", "./rsa_combined.pem") - // if nil != err { - // fmt.Printf("UNDO: icertpkg.GenCert(,,icertpkg.GenerateKeyAlgorithmRSA,,\"./rsa_ca_combined.pem\",\"./rsa_ca_combined.pem\",\"./rsa_combined.pem\",\"./rsa_combined.pem\") returned err: %v\n", err) - // os.Exit(8) - // } + flag.Var(&organizationFlag, "organization", "generated Certificate's Subject.Organization") + flag.Var(&countryFlag, "country", "generated Certificate's Subject.Country") + flag.Var(&provinceFlag, "province", "generated Certificate's Subject.Province") + flag.Var(&localityFlag, "locality", "generated Certificate's Subject.Locality") + flag.Var(&streetAddressFlag, "streetAddress", "generated Certificate's Subject.StreetAddress") + flag.Var(&postalCodeFlag, "postalCode", "generated Certificate's Subject.PostalCode") + + flag.Var(&dnsNamesFlag, "dns", "generated Certificate's DNS Name") + flag.Var(&ipAddressesFlag, "ip", "generated Certificate's IP Address") + + flag.Parse() + + fmt.Printf(" caFlag: %v\n", *caFlag) + fmt.Println() + fmt.Printf("generateKeyAlgorithmEd25519Flag: %v\n", *generateKeyAlgorithmEd25519Flag) + fmt.Printf(" generateKeyAlgorithmRSAFlag: %v\n", *generateKeyAlgorithmRSAFlag) + fmt.Println() + fmt.Printf(" organizationFlag: %v\n", organizationFlag) + fmt.Printf(" countryFlag: %v\n", countryFlag) + fmt.Printf(" provinceFlag: %v\n", provinceFlag) + fmt.Printf(" localityFlag: %v\n", localityFlag) + fmt.Printf(" streetAddressFlag: %v\n", streetAddressFlag) + fmt.Printf(" postalCodeFlag: %v\n", postalCodeFlag) + fmt.Println() + fmt.Printf(" ttlFlag: %v\n", *ttlFlag) + fmt.Println() + fmt.Printf(" dnsNamesFlag: %v\n", dnsNamesFlag) + fmt.Printf(" ipAddressesFlag: %v\n", ipAddressesFlag) + fmt.Println() + fmt.Printf(" caCertPemFilePathFlag: \"%v\"\n", *caCertPemFilePathFlag) + fmt.Printf(" caKeyPemFilePathFlag: \"%v\"\n", *caKeyPemFilePathFlag) + fmt.Println() + fmt.Printf(" endpointCertPemFilePathFlag: \"%v\"\n", *endpointCertPemFilePathFlag) + fmt.Printf(" endpointKeyPemFilePathFlag: \"%v\"\n", *endpointKeyPemFilePathFlag) + + if (*generateKeyAlgorithmEd25519Flag && *generateKeyAlgorithmRSAFlag) || + (!*generateKeyAlgorithmEd25519Flag && !*generateKeyAlgorithmRSAFlag) { + fmt.Printf("Precisely one of -%s or -%s must be specified\n", icertpkg.GenerateKeyAlgorithmEd25519, icertpkg.GenerateKeyAlgorithmRSA) + os.Exit(1) + } + + if time.Duration(0) == *ttlFlag { + fmt.Printf("A non-zero -ttl must be specified\n") + os.Exit(1) + } + + if ("" == *caCertPemFilePathFlag) || ("" == *caKeyPemFilePathFlag) { + fmt.Printf("Both -caCert and -caKey must be specified\n") + os.Exit(1) + } + + if *caFlag { + if (0 != len(dnsNamesFlag)) || (0 != len(ipAddressesFlag)) { + fmt.Printf("If -ca is specified, neither -dns nor -ip may be specified\n") + os.Exit(1) + } + if ("" != *endpointCertPemFilePathFlag) || ("" != *endpointKeyPemFilePathFlag) { + fmt.Printf("If -ca is specified, neither -cert nor -key may be specified\n") + os.Exit(1) + } + } else { + if (0 == len(dnsNamesFlag)) && (0 == len(ipAddressesFlag)) { + fmt.Printf("If -ca is not specified, at least one -dns or -ip must be specified\n") + os.Exit(1) + } + if ("" == *endpointCertPemFilePathFlag) || ("" == *endpointKeyPemFilePathFlag) { + fmt.Printf("If -ca is not specified, both -cert and -key must be specified\n") + os.Exit(1) + } + } } From 5317be14dfe47950943d691ad4371da4deb68e6f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 13 Mar 2021 10:31:32 -0800 Subject: [PATCH 062/160] Finished icert cli program --- icert/main.go | 144 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 100 insertions(+), 44 deletions(-) diff --git a/icert/main.go b/icert/main.go index e528739cf..da10e316f 100644 --- a/icert/main.go +++ b/icert/main.go @@ -4,8 +4,10 @@ package main import ( + "crypto/x509/pkix" "flag" "fmt" + "net" "os" "time" @@ -25,32 +27,39 @@ func (sS *stringSlice) Set(s string) (err error) { return } -var ( - caFlag = flag.Bool("ca", false, "generated CA Certicate usable for signing Endpoint Certificates") +func main() { + var ( + verboseFlag = flag.Bool("v", false, "verbose mode") - generateKeyAlgorithmEd25519Flag = flag.Bool(icertpkg.GenerateKeyAlgorithmEd25519, false, "generate key via Ed25519") - generateKeyAlgorithmRSAFlag = flag.Bool(icertpkg.GenerateKeyAlgorithmRSA, false, "generate key via RSA") + caFlag = flag.Bool("ca", false, "generated CA Certicate usable for signing Endpoint Certificates") - organizationFlag stringSlice - countryFlag stringSlice - provinceFlag stringSlice - localityFlag stringSlice - streetAddressFlag stringSlice - postalCodeFlag stringSlice + generateKeyAlgorithmEd25519Flag = flag.Bool(icertpkg.GenerateKeyAlgorithmEd25519, false, "generate key via Ed25519") + generateKeyAlgorithmRSAFlag = flag.Bool(icertpkg.GenerateKeyAlgorithmRSA, false, "generate key via RSA") - ttlFlag = flag.Duration("ttl", time.Duration(0), "generated Certificate's time to live") + organizationFlag stringSlice + countryFlag stringSlice + provinceFlag stringSlice + localityFlag stringSlice + streetAddressFlag stringSlice + postalCodeFlag stringSlice - dnsNamesFlag stringSlice - ipAddressesFlag stringSlice + ttlFlag = flag.Duration("ttl", time.Duration(0), "generated Certificate's time to live") - caCertPemFilePathFlag = flag.String("caCert", "", "path to CA Certificate") - caKeyPemFilePathFlag = flag.String("caKey", "", "path to CA Certificate's PrivateKey") + dnsNamesFlag stringSlice + ipAddressesFlag stringSlice - endpointCertPemFilePathFlag = flag.String("cert", "", "path to Endpoint Certificate") - endpointKeyPemFilePathFlag = flag.String("key", "", "path to Endpoint Certificate's PrivateKey") -) + caCertPemFilePathFlag = flag.String("caCert", "", "path to CA Certificate") + caKeyPemFilePathFlag = flag.String("caKey", "", "path to CA Certificate's PrivateKey") + + endpointCertPemFilePathFlag = flag.String("cert", "", "path to Endpoint Certificate") + endpointKeyPemFilePathFlag = flag.String("key", "", "path to Endpoint Certificate's PrivateKey") + + err error + generateKeyAlgorithm string + ipAddresses []net.IP + subject pkix.Name + ) -func main() { flag.Var(&organizationFlag, "organization", "generated Certificate's Subject.Organization") flag.Var(&countryFlag, "country", "generated Certificate's Subject.Country") flag.Var(&provinceFlag, "province", "generated Certificate's Subject.Province") @@ -63,31 +72,41 @@ func main() { flag.Parse() - fmt.Printf(" caFlag: %v\n", *caFlag) - fmt.Println() - fmt.Printf("generateKeyAlgorithmEd25519Flag: %v\n", *generateKeyAlgorithmEd25519Flag) - fmt.Printf(" generateKeyAlgorithmRSAFlag: %v\n", *generateKeyAlgorithmRSAFlag) - fmt.Println() - fmt.Printf(" organizationFlag: %v\n", organizationFlag) - fmt.Printf(" countryFlag: %v\n", countryFlag) - fmt.Printf(" provinceFlag: %v\n", provinceFlag) - fmt.Printf(" localityFlag: %v\n", localityFlag) - fmt.Printf(" streetAddressFlag: %v\n", streetAddressFlag) - fmt.Printf(" postalCodeFlag: %v\n", postalCodeFlag) - fmt.Println() - fmt.Printf(" ttlFlag: %v\n", *ttlFlag) - fmt.Println() - fmt.Printf(" dnsNamesFlag: %v\n", dnsNamesFlag) - fmt.Printf(" ipAddressesFlag: %v\n", ipAddressesFlag) - fmt.Println() - fmt.Printf(" caCertPemFilePathFlag: \"%v\"\n", *caCertPemFilePathFlag) - fmt.Printf(" caKeyPemFilePathFlag: \"%v\"\n", *caKeyPemFilePathFlag) - fmt.Println() - fmt.Printf(" endpointCertPemFilePathFlag: \"%v\"\n", *endpointCertPemFilePathFlag) - fmt.Printf(" endpointKeyPemFilePathFlag: \"%v\"\n", *endpointKeyPemFilePathFlag) - - if (*generateKeyAlgorithmEd25519Flag && *generateKeyAlgorithmRSAFlag) || - (!*generateKeyAlgorithmEd25519Flag && !*generateKeyAlgorithmRSAFlag) { + if *verboseFlag { + fmt.Printf(" caFlag: %v\n", *caFlag) + fmt.Println() + fmt.Printf("generateKeyAlgorithmEd25519Flag: %v\n", *generateKeyAlgorithmEd25519Flag) + fmt.Printf(" generateKeyAlgorithmRSAFlag: %v\n", *generateKeyAlgorithmRSAFlag) + fmt.Println() + fmt.Printf(" organizationFlag: %v\n", organizationFlag) + fmt.Printf(" countryFlag: %v\n", countryFlag) + fmt.Printf(" provinceFlag: %v\n", provinceFlag) + fmt.Printf(" localityFlag: %v\n", localityFlag) + fmt.Printf(" streetAddressFlag: %v\n", streetAddressFlag) + fmt.Printf(" postalCodeFlag: %v\n", postalCodeFlag) + fmt.Println() + fmt.Printf(" ttlFlag: %v\n", *ttlFlag) + fmt.Println() + fmt.Printf(" dnsNamesFlag: %v\n", dnsNamesFlag) + fmt.Printf(" ipAddressesFlag: %v\n", ipAddressesFlag) + fmt.Println() + fmt.Printf(" caCertPemFilePathFlag: \"%v\"\n", *caCertPemFilePathFlag) + fmt.Printf(" caKeyPemFilePathFlag: \"%v\"\n", *caKeyPemFilePathFlag) + fmt.Println() + fmt.Printf(" endpointCertPemFilePathFlag: \"%v\"\n", *endpointCertPemFilePathFlag) + fmt.Printf(" endpointKeyPemFilePathFlag: \"%v\"\n", *endpointKeyPemFilePathFlag) + } + + if *generateKeyAlgorithmEd25519Flag { + if *generateKeyAlgorithmRSAFlag { + fmt.Printf("Precisely one of -%s or -%s must be specified\n", icertpkg.GenerateKeyAlgorithmEd25519, icertpkg.GenerateKeyAlgorithmRSA) + os.Exit(1) + } + + generateKeyAlgorithm = icertpkg.GenerateKeyAlgorithmEd25519 + } else if *generateKeyAlgorithmRSAFlag { + generateKeyAlgorithm = icertpkg.GenerateKeyAlgorithmRSA + } else { fmt.Printf("Precisely one of -%s or -%s must be specified\n", icertpkg.GenerateKeyAlgorithmEd25519, icertpkg.GenerateKeyAlgorithmRSA) os.Exit(1) } @@ -121,4 +140,41 @@ func main() { os.Exit(1) } } + + subject = pkix.Name{ + Organization: organizationFlag, + Country: countryFlag, + Province: provinceFlag, + Locality: localityFlag, + StreetAddress: streetAddressFlag, + PostalCode: postalCodeFlag, + } + + if *caFlag { + err = icertpkg.GenCACert(generateKeyAlgorithm, subject, *ttlFlag, *caCertPemFilePathFlag, *caKeyPemFilePathFlag) + if nil != err { + fmt.Printf("icertpkg.GenCACert() failed: %v\n", err) + os.Exit(1) + } + + if *verboseFlag { + fmt.Printf("icertpkg.GenCACert() generated caCert: \"%s\" and caKey: \"%s\"\n", *caCertPemFilePathFlag, *caKeyPemFilePathFlag) + } + } else { + ipAddresses = make([]net.IP, 0, len(ipAddressesFlag)) + + for _, ipAddress := range ipAddressesFlag { + ipAddresses = append(ipAddresses, net.ParseIP(ipAddress)) + } + + err = icertpkg.GenEndpointCert(generateKeyAlgorithm, subject, dnsNamesFlag, ipAddresses, *ttlFlag, *caCertPemFilePathFlag, *caKeyPemFilePathFlag, *endpointCertPemFilePathFlag, *endpointKeyPemFilePathFlag) + if nil != err { + fmt.Printf("icertpkg.GenEndpointCert() failed: %v\n", err) + os.Exit(1) + } + + if *verboseFlag { + fmt.Printf("icertpkg.GenEndpointCert() generated cert: \"%s\" and key: \"%s\"\n", *endpointCertPemFilePathFlag, *endpointKeyPemFilePathFlag) + } + } } From fc1a9171d6ec936af9cf921da40ba21cf09a043e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 13 Mar 2021 10:49:28 -0800 Subject: [PATCH 063/160] Added icert README.md --- icert/README.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++++ icert/main.go | 14 ++++++------- 2 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 icert/README.md diff --git a/icert/README.md b/icert/README.md new file mode 100644 index 000000000..a3440b468 --- /dev/null +++ b/icert/README.md @@ -0,0 +1,56 @@ +# icert + +A simplification of tools like `openssl` to create either a CA Certificate +or an Endpoint Certificate using either the `Ed25519` or `RSA` key generation +method. + +## Usage +``` + -ca + generated CA Certicate usable for signing Endpoint Certificates + -caCert string + path to CA Certificate + -caKey string + path to CA Certificate's PrivateKey + -cert string + path to Endpoint Certificate + -country value + generated Certificate's Subject.Country + -dns value + generated Certificate's DNS Name + -ed25519 + generate key via Ed25519 + -ip value + generated Certificate's IP Address + -key string + path to Endpoint Certificate's PrivateKey + -locality value + generated Certificate's Subject.Locality + -organization value + generated Certificate's Subject.Organization + -postalCode value + generated Certificate's Subject.PostalCode + -province value + generated Certificate's Subject.Province + -rsa + generate key via RSA + -streetAddress value + generated Certificate's Subject.StreetAddress + -ttl duration + generated Certificate's time to live + -v verbose mode +``` + +Precisely one of `-ed25519` or `-rsa` must be specified. + +A `-ttl` must be specified. + +Both `-caCert` and `-caKey` must be specified. + +if `-ca` is specified: +* neither `-cert` nor `key` may be specified +* no `-dns` or `-ip` may be specified + +If `-ca` is not specified: +* both `-cert` and `-key` must be specified +* at least one `-dns` and/or one `-ip` must be specified diff --git a/icert/main.go b/icert/main.go index da10e316f..3a3f9dbf1 100644 --- a/icert/main.go +++ b/icert/main.go @@ -122,23 +122,23 @@ func main() { } if *caFlag { - if (0 != len(dnsNamesFlag)) || (0 != len(ipAddressesFlag)) { - fmt.Printf("If -ca is specified, neither -dns nor -ip may be specified\n") - os.Exit(1) - } if ("" != *endpointCertPemFilePathFlag) || ("" != *endpointKeyPemFilePathFlag) { fmt.Printf("If -ca is specified, neither -cert nor -key may be specified\n") os.Exit(1) } - } else { - if (0 == len(dnsNamesFlag)) && (0 == len(ipAddressesFlag)) { - fmt.Printf("If -ca is not specified, at least one -dns or -ip must be specified\n") + if (0 != len(dnsNamesFlag)) || (0 != len(ipAddressesFlag)) { + fmt.Printf("If -ca is specified, neither -dns nor -ip may be specified\n") os.Exit(1) } + } else { if ("" == *endpointCertPemFilePathFlag) || ("" == *endpointKeyPemFilePathFlag) { fmt.Printf("If -ca is not specified, both -cert and -key must be specified\n") os.Exit(1) } + if (0 == len(dnsNamesFlag)) && (0 == len(ipAddressesFlag)) { + fmt.Printf("If -ca is not specified, at least one -dns or -ip must be specified\n") + os.Exit(1) + } } subject = pkix.Name{ From 465c49067db7b47eb54f89d5f4ba5cb47413a9bf Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 15 Mar 2021 13:26:17 -0500 Subject: [PATCH 064/160] bump swift to ss-release-2.26.0.16 --- pfs_middleware/tox.ini | 2 +- saio/container/Dockerfile | 2 +- saio/vagrant_provision.sh | 2 +- sait/vagrant_provision.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pfs_middleware/tox.ini b/pfs_middleware/tox.ini index 5e3e7719f..69d951627 100644 --- a/pfs_middleware/tox.ini +++ b/pfs_middleware/tox.ini @@ -7,7 +7,7 @@ usedevelop = True deps = lint: flake8 !lint: -r{toxinidir}/test-requirements.txt - release: git+https://github.com/NVIDIA/swift.git@ss-release-2.26.0.15 + release: git+https://github.com/NVIDIA/swift.git@ss-release-2.26.0.16 minver: http://tarballs.openstack.org/swift/swift-2.9.0.tar.gz master: http://tarballs.openstack.org/swift/swift-master.tar.gz commands = python -m unittest discover diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 898371ca3..d77ac2052 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -3,7 +3,7 @@ FROM centos:7.4.1708 -ARG SwiftVersion=ss-release-2.26.0.15 +ARG SwiftVersion=ss-release-2.26.0.16 ARG GolangVersion=1.15.5 ARG ProxyFS_Version=stable diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index 2b4d8f0ea..91f6fa915 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -228,7 +228,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.26.0.15 +git checkout ss-release-2.26.0.16 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index eb87994a7..021562e2c 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -230,7 +230,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.26.0.15 +git checkout ss-release-2.26.0.16 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet From 5a233d745948966451fb85fdc7fb273b9ddb015c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Tue, 16 Mar 2021 16:52:30 -0700 Subject: [PATCH 065/160] Convert check_pinned_packages & pfs-swift-load-plot to py3 --- ci/ansible/check_pinned_packages | 47 +++++++++++++++++++++++++----- pfs-swift-load/pfs-swift-load-plot | 41 ++++++++++++++------------ 2 files changed, 63 insertions(+), 25 deletions(-) diff --git a/ci/ansible/check_pinned_packages b/ci/ansible/check_pinned_packages index b3d3358a6..c33bb3d1b 100755 --- a/ci/ansible/check_pinned_packages +++ b/ci/ansible/check_pinned_packages @@ -1,8 +1,37 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2015-2021, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 +""" +We decided to not pin packages anymore, but if we ever decide to do it again, this +should still work, as long as the file ci/ansible/chef_files/packages/rhel.json keeps +using the same conventions as when this script was created, which looks something like +this: + +{ + "proxyfs_packages": [ + ["json-c-devel", "0.11-4.el7_0"], + ["fuse", "2.9.3-5.el7"] + ], + "wireshark_packages": [ + ["wireshark"], + ["libcap"] + ], + "ssh_packages": [ + ["sshpass"] + ], + "gdb_packages": [ + ["gdb", "7.6.1-120.el7"], + ["yum-utils"] + ], + "utils_packages": [ + ["atop"], + ["vim-common"] + ] +} +""" + import json import os import subprocess @@ -10,7 +39,7 @@ import sys script_dir = os.path.abspath(os.path.dirname(__file__)) -json_path = os.path.join(script_dir, "chef_files", "packages", "rhel_ss.json") +json_path = os.path.join(script_dir, "chef_files", "packages", "rhel.json") with open(json_path, "r") as json_file: versions_data = json.load(json_file) @@ -24,16 +53,20 @@ for section, packages in versions_data.items(): try: yum_versions = subprocess.check_output( "repoquery {} --queryformat '%{{vr}}'".format(package), - shell=True, stderr=subprocess.STDOUT) + shell=True, + stderr=subprocess.STDOUT, + ) except subprocess.CalledProcessError as e: - print("! Error checking {}: {}".format( - package, e.output.strip())) + print("! Error checking {}: {}".format(package, e.output.strip())) everything_ok = False continue yum_versions = yum_versions.strip().split("\n") if pinned_version not in yum_versions: - print("{} not ok! pinned: {} | found in yum: {}".format( - package, pinned_version, ", ".join(yum_versions))) + print( + "{} not ok! pinned: {} | found in yum: {}".format( + package, pinned_version, ", ".join(yum_versions) + ) + ) everything_ok = False if everything_ok: diff --git a/pfs-swift-load/pfs-swift-load-plot b/pfs-swift-load/pfs-swift-load-plot index b1f371991..992c6f326 100755 --- a/pfs-swift-load/pfs-swift-load-plot +++ b/pfs-swift-load/pfs-swift-load-plot @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import argparse import itertools @@ -32,25 +32,31 @@ def plot(plot_position, x_axis_data, x_axis_label, warning_msg=None): ax.plot( x_axis_data, data_frame[worker][OBJECTS_PER_SECOND_COL], - color=color_iterator.next(), label=worker) + color=next(color_iterator), + label=worker, + ) ax.set_xlabel(x_axis_label, color="k") ax.set_ylabel(OBJECTS_PER_SECOND_TEXT, color="k") - ax.grid(color="tab:gray", linestyle="dashdot", linewidth=.4) + ax.grid(color="tab:gray", linestyle="dashdot", linewidth=0.4) plt.legend() if warning_msg: - plt.title(warning_msg, bbox=dict(facecolor='red', alpha=0.5)) + plt.title(warning_msg, bbox=dict(facecolor="red", alpha=0.5)) def parse_args(): - parser = argparse.ArgumentParser(description="Plot results from " - "pfs-swift-load") - parser.add_argument("-i", "--input", type=str, required=True, - help="Input CSV file") - parser.add_argument("-o", "--output", type=str, required=True, - help="Output PNG file") - parser.add_argument("-w", "--warning", type=str, required=False, - default=None, help="Warning message to print on the " - "PNG file") + parser = argparse.ArgumentParser(description="Plot results from " "pfs-swift-load") + parser.add_argument("-i", "--input", type=str, required=True, help="Input CSV file") + parser.add_argument( + "-o", "--output", type=str, required=True, help="Output PNG file" + ) + parser.add_argument( + "-w", + "--warning", + type=str, + required=False, + default=None, + help="Warning message to print on the " "PNG file", + ) args = parser.parse_args() options = { "input": args.input, @@ -68,8 +74,7 @@ if __name__ == "__main__": # We only have len(PLOT_COLORS) colors to use, and we don't want 2 workers # to be plotted with the same color... if len(workers) > len(PLOT_COLORS): - print("Too many workers: {}, max: {}".format(len(workers), - len(PLOT_COLORS))) + print("Too many workers: {}, max: {}".format(len(workers), len(PLOT_COLORS))) sys.exit(1) # Check how long are the intervals between samples @@ -78,13 +83,13 @@ if __name__ == "__main__": # account the fact that several threads could have been working in # parallel. Maybe we could add an option? for worker in workers: - data_frame[worker, OBJECTS_PER_SECOND_COL] = \ + data_frame[worker, OBJECTS_PER_SECOND_COL] = ( data_frame[worker][DELTA_COL] / interval + ) data_frame = data_frame.sort_index(axis=1) # Draw the chart(s) fig = plt.figure(figsize=FIG_SIZE) - plot(1, data_frame[worker][ELAPSED_TIME_COL], ELAPSED_TIME_TEXT, - options["warning"]) + plot(1, data_frame[worker][ELAPSED_TIME_COL], ELAPSED_TIME_TEXT, options["warning"]) plot(2, data_frame[worker][COMPLETED_COL], COMPLETED_TEXT) plt.savefig(options["output"]) From 11dc25703132be29fd16fd6bcd6875490472bf85 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 20 Mar 2021 10:45:02 -0700 Subject: [PATCH 066/160] Add benchmark for jrpcfs.RpcLease() --- jrpcfs/lease_test.go | 119 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 116 insertions(+), 3 deletions(-) diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index f0517c1de..8e0a3ac8c 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -25,7 +25,6 @@ const ( testRpcLeaseMultiNumInstances int = 5 testRpcLeaseSingleInodeNumber int64 = 1 testRpcLeaseSingleNumInstances int = 101 // Must be >= 4 - testRpcLeaseTimeFormatASDF = time.StampMilli testRpcLeaseTimeFormat = "15:04:05.000" ) @@ -46,6 +45,118 @@ type testRpcLeaseClientStruct struct { t *testing.T } +func BenchmarkRpcLease(b *testing.B) { + var ( + benchmarkIteration int + deadlineIO time.Duration + err error + keepAlivePeriod time.Duration + leaseReply *LeaseReply + leaseRequest *LeaseRequest + mountByAccountNameRequest *MountByAccountNameRequest + mountByAccountNameReply *MountByAccountNameReply + retryRPCClient *retryrpc.Client + retryrpcClientConfig *retryrpc.ClientConfig + server *Server + testRpcLeaseClient *testRpcLeaseClientStruct + unmountReply *Reply + unmountRequest *UnmountRequest + ) + + server = &Server{} + + mountByAccountNameRequest = &MountByAccountNameRequest{ + AccountName: testAccountName, + MountOptions: 0, + AuthUserID: 0, + AuthGroupID: 0, + } + mountByAccountNameReply = &MountByAccountNameReply{} + + err = server.RpcMountByAccountName(mountByAccountNameRequest, mountByAccountNameReply) + if nil != err { + b.Fatalf("server.RpcMountByAccountName(AccountName=\"%s\",) failed: %v", mountByAccountNameRequest.AccountName, err) + } + + deadlineIO, err = time.ParseDuration(testRpcLeaseRetryRPCDeadlineIO) + if nil != err { + b.Fatalf("time.ParseDuration(\"%s\") failed: %v", testRpcLeaseRetryRPCDeadlineIO, err) + } + keepAlivePeriod, err = time.ParseDuration(testRpcLeaseRetryRPCKeepAlivePeriod) + if nil != err { + b.Fatalf("time.ParseDuration(\"%s\") failed: %v", testRpcLeaseRetryRPCKeepAlivePeriod, err) + } + + retryrpcClientConfig = &retryrpc.ClientConfig{ + MyUniqueID: string(mountByAccountNameReply.MountID), + IPAddr: mountByAccountNameReply.RetryRPCPublicIPAddr, + Port: int(mountByAccountNameReply.RetryRPCPort), + RootCAx509CertificatePEM: mountByAccountNameReply.RootCAx509CertificatePEM, + Callbacks: testRpcLeaseClient, + DeadlineIO: deadlineIO, + KeepAlivePeriod: keepAlivePeriod, + } + + retryRPCClient, err = retryrpc.NewClient(retryrpcClientConfig) + if nil != err { + b.Fatalf("retryrpc.NewClient() failed: %v", err) + } + + b.ResetTimer() + + for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { + leaseRequest = &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeExclusive, + } + leaseReply = &LeaseReply{} + + err = retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) + if nil != err { + b.Fatalf("retryRPCClient.Send(\"RpcLease\",LeaseRequestTypeExclusive) failed: %v", err) + } + + if LeaseReplyTypeExclusive != leaseReply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", leaseReply.LeaseReplyType) + } + + leaseRequest = &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeRelease, + } + leaseReply = &LeaseReply{} + + err = retryRPCClient.Send("RpcLease", leaseRequest, leaseReply) + if nil != err { + b.Fatalf("retryRPCClient.Send(\"RpcLease\",LeaseRequestTypeRelease) failed: %v", err) + } + + if LeaseReplyTypeReleased != leaseReply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", leaseReply.LeaseReplyType) + } + } + + b.StopTimer() + + retryRPCClient.Close() + + unmountRequest = &UnmountRequest{ + MountID: mountByAccountNameReply.MountID, + } + unmountReply = &Reply{} + + err = server.RpcUnmount(unmountRequest, unmountReply) + if nil != err { + b.Fatalf("server.RpcUnmount(MountID=\"%s\",) failed: %v", unmountRequest.MountID, err) + } +} + func TestRpcLease(t *testing.T) { var ( instance int @@ -415,8 +526,6 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { leaseRequestType, ok = <-testRpcLeaseClient.chIn if ok { - time.Sleep(testRpcLeaseDelayBeforeSendingRequest) - leaseRequest = &LeaseRequest{ InodeHandle: InodeHandle{ MountID: mountByAccountNameReply.MountID, @@ -510,6 +619,10 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) sendLeaseRequest(leaseReques time.Sleep(testRpcLeaseDelayAfterSendingRequest) } +func (testRpcLeaseClient *testRpcLeaseClientStruct) sendLeaseRequestPromptly(leaseRequestType LeaseRequestType) { + testRpcLeaseClient.chIn <- leaseRequestType +} + func (testRpcLeaseClient *testRpcLeaseClientStruct) validateChOutValueIsLeaseReplyType(expectedLeaseReplyType LeaseReplyType) { var ( chOutValueAsInterface interface{} From a85504f534e4d7e5463f169271fd004f9611f054 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 22 Mar 2021 16:08:17 -0700 Subject: [PATCH 067/160] Revert optimization where uncontended Shared Lease Request is granted an Exclusive Lease --- jrpcfs/lease.go | 51 ++++++++---------- jrpcfs/lease_test.go | 126 ++++++++++++++++++++----------------------- 2 files changed, 78 insertions(+), 99 deletions(-) diff --git a/jrpcfs/lease.go b/jrpcfs/lease.go index 11acf3ac5..a15cf67c6 100644 --- a/jrpcfs/lease.go +++ b/jrpcfs/lease.go @@ -241,13 +241,13 @@ func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *lease leaseRequestOperation.mount.leaseRequestMap[inodeLease.InodeNumber] = leaseRequest switch inodeLease.leaseState { case inodeLeaseStateNone: - leaseRequest.requestState = leaseRequestStateExclusiveGranted - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - inodeLease.exclusiveHolder = leaseRequest + leaseRequest.requestState = leaseRequestStateSharedGranted + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) inodeLease.lastGrantTime = time.Now() inodeLease.longAgoTimer = time.NewTimer(globals.minLeaseDuration) leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, + LeaseReplyType: LeaseReplyTypeShared, } leaseRequest.replyChan <- leaseReply case inodeLeaseStateSharedGrantedRecently: @@ -850,32 +850,23 @@ func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *lease _ = inodeLease.requestedList.Remove(leaseRequestElement) if leaseRequestStateSharedRequested == leaseRequest.requestState { leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - if 0 == inodeLease.requestedList.Len() { - inodeLease.leaseState = inodeLeaseStateExclusiveGrantedRecently - leaseRequest.requestState = leaseRequestStateExclusiveGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeExclusive, - } - leaseRequest.replyChan <- leaseReply - } else { // 0 < inodeLease.requestedList.Len() - inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseReply = &LeaseReply{ - LeaseReplyType: LeaseReplyTypeShared, - } - leaseRequest.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - for nil != leaseRequestElement { - leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) - if leaseRequestStateSharedRequested == leaseRequest.requestState { - _ = inodeLease.requestedList.Remove(leaseRequestElement) - leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) - leaseRequest.requestState = leaseRequestStateSharedGranted - leaseRequest.replyChan <- leaseReply - leaseRequestElement = inodeLease.requestedList.Front() - } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState - leaseRequestElement = nil - } + inodeLease.leaseState = inodeLeaseStateSharedGrantedRecently + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseReply = &LeaseReply{ + LeaseReplyType: LeaseReplyTypeShared, + } + leaseRequest.replyChan <- leaseReply + leaseRequestElement = inodeLease.requestedList.Front() + for nil != leaseRequestElement { + leaseRequest = leaseRequestElement.Value.(*leaseRequestStruct) + if leaseRequestStateSharedRequested == leaseRequest.requestState { + _ = inodeLease.requestedList.Remove(leaseRequestElement) + leaseRequest.listElement = inodeLease.sharedHoldersList.PushBack(leaseRequest) + leaseRequest.requestState = leaseRequestStateSharedGranted + leaseRequest.replyChan <- leaseReply + leaseRequestElement = inodeLease.requestedList.Front() + } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState + leaseRequestElement = nil } } } else { // leaseRequestStateExclusiveRequested == leaseRequest.requestState diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index 8e0a3ac8c..033f1b2f3 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -189,10 +189,36 @@ func TestRpcLease(t *testing.T) { testRpcLeaseLogTestCase("1 Shared", true) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) + testRpcLeaseLogTestCase("2 Shared", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) + + testRpcLeaseLogTestCase("3 Shared", true) + + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) + testRpcLeaseLogTestCase("1 Exclusive", true) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) @@ -223,29 +249,10 @@ func TestRpcLease(t *testing.T) { testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseLogTestCase("1 Exclusive, 1 Shared (causing Demotion), then Promotion", true) + testRpcLeaseLogTestCase("1 Shared then 1 Exclusive leading to Release", true) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeDemote) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeDemoted, RPCInterruptTypeDemote) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypePromote) - testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypePromoted) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - - testRpcLeaseLogTestCase("1 Exclusive Demoted to Shared then 1 Exclusive", true) - - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeDemote) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeDemoted) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) @@ -254,40 +261,38 @@ func TestRpcLease(t *testing.T) { testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseLogTestCase("2 Shared", true) + testRpcLeaseLogTestCase("1 Exclusive then 1 Exclusive leading to Release", true) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeDemote) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseLogTestCase("3 Shared", true) + testRpcLeaseLogTestCase("2 Shared then Promotion leading to Release", true) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeDemote) testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypePromote) + testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) + testRpcLeaseClient[1].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypePromoted) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseLogTestCase("1 Exclusive, 2 Shared (leading to Demotion)", true) + testRpcLeaseLogTestCase("1 Exclusive then 2 Shared leading to Demotion", true) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeDemote) testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeDemoted, RPCInterruptTypeDemote) @@ -300,36 +305,28 @@ func TestRpcLease(t *testing.T) { testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseLogTestCase("1 Exclusive, 2 Shared (leading to Demotion), and another Exclusive", true) + testRpcLeaseLogTestCase("2 Shared then 1 Exclusive leading to Release", true) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) + testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeShared) testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[3].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) - testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeDemote) - testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeDemoted, RPCInterruptTypeDemote) + testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeExclusive) testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) - testRpcLeaseClient[2].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeRelease) testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) testRpcLeaseClient[0].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) testRpcLeaseClient[1].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeReleased, RPCInterruptTypeRelease) - testRpcLeaseClient[3].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[3].sendLeaseRequest(LeaseRequestTypeRelease) - testRpcLeaseClient[3].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) + testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) + testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeRelease) + testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) testRpcLeaseLogTestCase("2 Exclusives leading to Release that Expires", true) testRpcLeaseClient[0].sendLeaseRequest(LeaseRequestTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) testRpcLeaseClient[0].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) + testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeExclusive) testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) testRpcLeaseClient[0].validateChOutValueIsRPCInterruptType(RPCInterruptTypeRelease) @@ -341,12 +338,10 @@ func TestRpcLease(t *testing.T) { testRpcLeaseClient[0].alreadyUnmounted = true - testRpcLeaseLogTestCase("2 Shared, 2 Promotions (leading to one Release)", true) + testRpcLeaseLogTestCase("2 Shared then 2 Promotions leading to Release", true) testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeDemote) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeDemoted) + testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypePromote) @@ -359,16 +354,9 @@ func TestRpcLease(t *testing.T) { testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeRelease) testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeReleased) - testRpcLeaseLogTestCase(fmt.Sprintf("%v Shares (first one to be Demoted)", testRpcLeaseSingleNumInstances-1), false) + testRpcLeaseLogTestCase(fmt.Sprintf("%v Shared", testRpcLeaseSingleNumInstances-1), false) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyType(LeaseReplyTypeExclusive) - testRpcLeaseClient[2].sendLeaseRequest(LeaseRequestTypeShared) - testRpcLeaseClient[1].validateChOutValueIsRPCInterruptType(RPCInterruptTypeDemote) - testRpcLeaseClient[1].sendLeaseRequest(LeaseRequestTypeDemote) - testRpcLeaseClient[1].validateChOutValueIsLeaseReplyTypeIgnoringRPCInterruptType(LeaseReplyTypeDemoted, RPCInterruptTypeDemote) - testRpcLeaseClient[2].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) - for instance = 3; instance < testRpcLeaseSingleNumInstances; instance++ { + for instance = 1; instance < testRpcLeaseSingleNumInstances; instance++ { testRpcLeaseClient[instance].sendLeaseRequest(LeaseRequestTypeShared) testRpcLeaseClient[instance].validateChOutValueIsLeaseReplyType(LeaseReplyTypeShared) } From 845c5358d73b54efee92753799cb35e02090489e Mon Sep 17 00:00:00 2001 From: Bob Schatz Date: Wed, 17 Mar 2021 14:36:14 -0700 Subject: [PATCH 068/160] Generate unique client ID on server instead of on client Generate the unique client ID on the server instead of letting caller pass it. Basic algorithm, after client initially dials server it will ask the server for a unique ID before client sends new messages. On retransmit, client will send the saved client ID after redialing the server. --- retryrpc/api.go | 5 +- retryrpc/api_internal.go | 40 +++++++-- retryrpc/client.go | 170 +++++++++++++++++++++++++++++++++++++-- retryrpc/server.go | 107 +++++++++++++++--------- 4 files changed, 267 insertions(+), 55 deletions(-) diff --git a/retryrpc/api.go b/retryrpc/api.go index 7bec8b118..cac84df69 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -41,6 +41,7 @@ type Server struct { svrMap map[string]*methodArgs // Key: Method name ipaddr string // IP address server listens too port int // Port of server + clientIDNonce uint64 // Nonce used to create a unique client ID netListener net.Listener tlsListener net.Listener @@ -267,8 +268,6 @@ type clientState int const ( // INITIAL means the Client struct has just been created INITIAL clientState = iota + 1 - // DISCONNECTED means the Client has lost the connection to the server - DISCONNECTED // CONNECTED means the Client is connected to the server CONNECTED // RETRANSMITTING means a goroutine is in the middle of recovering @@ -355,8 +354,6 @@ func NewClient(config *ClientConfig) (client *Client, err error) { return nil, err } - bucketstats.Register("proxyfs.retryrpc", client.GetStatsGroupName(), &client.stats) - return client, err } diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index 2e25c1f4c..d8b43c867 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -121,7 +121,14 @@ const ( RPC MsgType = iota + 1 // Upcall represents an upcall from server to client Upcall - // PassID is the message sent by the client to identify itself to server + // AskMyUniqueID is the message sent by the client during it's initial connection + // to get it's unique client ID. + AskMyUniqueID + // ReturnUniqueID is the message sent by the server to client after initial connection. + // The server is returning the unique ID created for this client. + ReturnUniqueID + // PassID is the message sent by the client to identify itself to server. + // This is used when we are retransmitting. PassID ) @@ -147,13 +154,22 @@ type ioReply struct { } // internalSetIDRequest is the structure sent over the wire -// when the connection is first made. This is how the server -// learns the client ID +// when the connection existed, was broken and is being recreated +// by the client as a result of reDial(). This is how the server +// learns existing client ID. type internalSetIDRequest struct { Hdr ioHeader MyUniqueID []byte // Client unique ID as byte } +// internalINeedIDRequest is the structure sent over the wire +// when the connection is first made. This is how the client +// learns its client ID +type internalINeedIDRequest struct { + Hdr ioHeader + UniqueID []byte // Client unique ID as byte +} + type replyCtx struct { err error } @@ -232,6 +248,19 @@ func buildSetIDRequest(myUniqueID string) (isreq *internalSetIDRequest, err erro return } +func buildINeedIDRequest() (iinreq *internalINeedIDRequest, err error) { + iinreq = &internalINeedIDRequest{} + if err != nil { + return nil, err + } + iinreq.Hdr.Len = uint32(0) + iinreq.Hdr.Protocol = uint16(JSON) + iinreq.Hdr.Version = currentRetryVersion + iinreq.Hdr.Type = AskMyUniqueID + iinreq.Hdr.Magic = headerMagic + return +} + func getIO(genNum uint64, deadlineIO time.Duration, conn net.Conn) (buf []byte, msgType MsgType, err error) { if printDebugLogs { logger.Infof("conn: %v", conn) @@ -251,11 +280,12 @@ func getIO(genNum uint64, deadlineIO time.Duration, conn net.Conn) (buf []byte, return } + msgType = hdr.Type if hdr.Len == 0 { - err = fmt.Errorf("hdr.Len == 0") + // TODO - ????? + // err = fmt.Errorf("hdr.Len == 0") return } - msgType = hdr.Type // Now read the rest of the structure off the wire. var numBytes int diff --git a/retryrpc/client.go b/retryrpc/client.go index 068564efd..9235d9ed0 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -64,7 +64,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter connectionRetryDelay = ConnectionRetryInitialDelay for { - err = client.dial() + err = client.initialDial() if err == nil { break } @@ -81,6 +81,11 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter break } } + + // Now that we have a connection - we can setup bucketstats + if client.connection.state == CONNECTED { + bucketstats.Register("proxyfs.retryrpc", client.GetStatsGroupName(), &client.stats) + } } // Put request data into structure to be be marshaled into JSON @@ -88,7 +93,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter jreq.Params[0] = rpcRequest jreq.MyUniqueID = client.myUniqueID - if client.halting == true { + if client.halting { client.Unlock() e := fmt.Errorf("Calling retryrpc.Send() without dialing") logger.PanicfWithError(e, "") @@ -364,7 +369,7 @@ func (client *Client) retransmit(genNum uint64) { connectionRetryDelay = ConnectionRetryInitialDelay for { - err := client.dial() + err := client.reDial() // If we were able to connect then break - otherwise retry // after a delay if err == nil { @@ -380,7 +385,7 @@ func (client *Client) retransmit(genNum uint64) { connectionRetryDelay *= ConnectionRetryDelayMultiplier client.Lock() // While the lock was dropped we may be halting.... - if client.halting == true { + if client.halting { client.Unlock() return } @@ -395,7 +400,41 @@ func (client *Client) retransmit(genNum uint64) { client.Unlock() } -// Send myUniqueID to server +// Get myUniqueID from server. This is called when the client is +// creating this connection and wants a unique ID from the server. +// +// NOTE: Client lock is already held during this call. +func (client *Client) getMyUniqueID(tlsConn *tls.Conn) (err error) { + + // Setup ioreq to write structure on socket to server + iinreq, err := buildINeedIDRequest() + if err != nil { + e := fmt.Errorf("Client buildINeedIDRequest returned err: %v", err) + logger.PanicfWithError(e, "") + return err + } + + // We only have the header to send + client.connection.tlsConn.SetDeadline(time.Now().Add(client.deadlineIO)) + err = binary.Write(tlsConn, binary.BigEndian, iinreq.Hdr) + if err != nil { + return + } + + // Ask the server for the unique ID. + // + // TODO - return error back up.... close() the connection??? + // upper layer should do the redial + client.myUniqueID, err = client.readClientID(client.connection.genNum, tlsConn) + if err != nil { + // TODO - close() connection + } + + return +} + +// Send myUniqueID to server. This is called when the client already +// knows myUniqueID // // NOTE: Client lock is already held during this call. func (client *Client) sendMyInfo(tlsConn *tls.Conn) (err error) { @@ -415,7 +454,7 @@ func (client *Client) sendMyInfo(tlsConn *tls.Conn) (err error) { return } - // Send MyUniqueID + // We are an existinc client - send "MyUniqueID" client.connection.tlsConn.SetDeadline(time.Now().Add(client.deadlineIO)) bytesWritten, writeErr := tlsConn.Write(isreq.MyUniqueID) @@ -435,11 +474,16 @@ func (client *Client) sendMyInfo(tlsConn *tls.Conn) (err error) { return } -// dial sets up connection to server +// reDial sets up a connection to the server as a result of +// retransmit being called. +// +// This is different than initialDial() which gets called the +// first time send() is called. +// // It is assumed that the client lock is held. // // NOTE: Client lock is held -func (client *Client) dial() (err error) { +func (client *Client) reDial() (err error) { var entryState = client.connection.state client.connection.tlsConfig = &tls.Config{ @@ -480,6 +524,116 @@ func (client *Client) dial() (err error) { return } +// readClientID reads unique client ID response from server +// +// Client lock is held +// +// NOTE: Client lock is held +func (client *Client) readClientID(callingGenNum uint64, tlsConn *tls.Conn) (myUniqueID string, err error) { + var localCnt int + + for { + + // Wait reply from server + buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, tlsConn) + + // This must happen before checking error + if client.halting { + return + } + + // Ignore timeouts on idle connections while reading header + // + // We consider a connection to be idle if we have no outstanding requests when + // we get the timeout. Otherwise, we call retransmit. + if os.IsTimeout(getErr) && localCnt == 0 { + continue + } + + // TODO - don't loop forever - breakout after some many tries + + if getErr != nil { + + // TODO - if error initially, just close socket, return error and let initialDial() + // retry. close may already be done by upper layer... verify + /* + // If we had an error reading socket - call retransmit() and exit + // the goroutine. retransmit()/dial() will start another + // readReplies() goroutine. + client.retransmit(callingGenNum) + */ + return + } + + if msgType != ReturnUniqueID { + // TODO - panic??? + err = fmt.Errorf("CLIENT - invalid msgType: %v", msgType) + return + } + + fmt.Printf("CLIENT: read buf: %v\n", string(buf)) + myUniqueID = string(buf) + fmt.Printf("readClientID returned: %v\n", myUniqueID) + + return + } +} + +// initialDial does the initial dial of the server and retrieves the +// unique ID for this connection from the server. +// +// If the connection fails and retransmit() is called then we will +// redial with reDial() instead of initialDial(). +// +// The reason for the different dial function is that in one case +// we already know myUniqueId and in the other case we must get +// myUniqueId from the server. +// +// It is assumed that the client lock is held. +// +// NOTE: Client lock is held +func (client *Client) initialDial() (err error) { + var entryState = client.connection.state + + client.connection.tlsConfig = &tls.Config{ + RootCAs: client.connection.x509CertPool, + } + + // Now dial the server + d := &net.Dialer{KeepAlive: client.keepAlivePeriod} + tlsConn, dialErr := tls.DialWithDialer(d, "tcp", client.connection.hostPortStr, client.connection.tlsConfig) + if dialErr != nil { + err = fmt.Errorf("tls.Dial() failed: %v", dialErr) + return + } + + if client.connection.tlsConn != nil { + client.connection.tlsConn.Close() + client.connection.tlsConn = nil + } + + client.connection.tlsConn = tlsConn + client.connection.state = CONNECTED + client.connection.genNum++ + + // Ask server for my unique ID + // + // If this fails the dial will be retried. + err = client.getMyUniqueID(tlsConn) + if err != nil { + _ = client.connection.tlsConn.Close() + client.connection.tlsConn = nil + client.connection.state = entryState + return + } + + // Start readResponse goroutine to read responses from server + client.goroutineWG.Add(1) + go client.readReplies(client.connection.genNum, tlsConn) + + return +} + // Less tests whether the current item is less than the given argument. // // This must provide a strict weak ordering. diff --git a/retryrpc/server.go b/retryrpc/server.go index c44bd624e..f9283694d 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -21,7 +21,6 @@ import ( // Variable to control debug output var printDebugLogs bool = false -var debugPutGet bool = false // TODO - test if Register has been called??? @@ -55,6 +54,8 @@ func (server *Server) run() { elm := server.connections.PushBack(conn) server.connLock.Unlock() + // TODO - update comment per two cases initialDial() and reDial().... + // The first message sent on this socket by the client is the uniqueID. // // Read that first and create the relevant structures before calling @@ -179,6 +180,10 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b myConnCtx.activeRPCsWG.Done() } +// TODO - update this comment for initialDial() vs reDial() cases!!! +// should we rename function???? + +// // getClientIDAndWait reads the first message off the new connection. // The client sends its uniqueID before sending it's first RPC. // @@ -206,26 +211,65 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err return } - if msgType != PassID { - err = fmt.Errorf("Server expecting msgType PassID and received: %v", msgType) + if (msgType != PassID) && (msgType != AskMyUniqueID) { + err = fmt.Errorf("Server expecting msgType PassID or AskMyUniqueID and received: %v", msgType) return } - var connUniqueID string - err = json.Unmarshal(buf, &connUniqueID) - if err != nil { - return - } + if msgType == PassID { + var connUniqueID string + err = json.Unmarshal(buf, &connUniqueID) + if err != nil { + return + } + + // Check if this is the first time we have seen this client + server.Lock() + lci, ok := server.perClientInfo[connUniqueID] + if !ok { + // TODO - what??? + server.Unlock() + } else { + server.Unlock() + ci = lci + + // Wait for the serviceClient() goroutine from a prior connection to exit + // before proceeding. + ci.cCtx.Lock() + for !ci.cCtx.serviceClientExited { + ci.cCtx.cond.Wait() + } + ci.cCtx.Unlock() + + // Now wait for any outstanding RPCs to complete + ci.cCtx.activeRPCsWG.Wait() + + // Set cCtx back pointer to ci + ci.Lock() + cCtx.Lock() + cCtx.ci = ci + cCtx.Unlock() + + ci.cCtx = cCtx + ci.Unlock() + } + } else { + var ( + localIOR ioReply + ) - // Check if this is the first time we have seen this client - server.Lock() - lci, ok := server.perClientInfo[connUniqueID] - if !ok { // First time we have seen this client - c := &clientInfo{cCtx: cCtx, myUniqueID: connUniqueID} + + // Create a unique client ID and return to client + server.Lock() + server.clientIDNonce++ + newUniqueID := fmt.Sprintf("unqClnt-%v", server.clientIDNonce) + + // Setup new client data structures + c := &clientInfo{cCtx: cCtx, myUniqueID: newUniqueID} c.completedRequest = make(map[requestID]*completedEntry) c.completedRequestLRU = list.New() - server.perClientInfo[connUniqueID] = c + server.perClientInfo[newUniqueID] = c server.Unlock() ci = c cCtx.Lock() @@ -233,29 +277,15 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err cCtx.Unlock() bucketstats.Register("proxyfs.retryrpc", c.myUniqueID, &c.stats) - } else { - server.Unlock() - ci = lci - - // Wait for the serviceClient() goroutine from a prior connection to exit - // before proceeding. - ci.cCtx.Lock() - for ci.cCtx.serviceClientExited != true { - ci.cCtx.cond.Wait() - } - ci.cCtx.Unlock() - // Now wait for any outstanding RPCs to complete - ci.cCtx.activeRPCsWG.Wait() + // Build reply and send back to client + localIOR.JResult = []byte(newUniqueID) + setupHdrReply(&localIOR, ReturnUniqueID) - // Set cCtx back pointer to ci - ci.Lock() - cCtx.Lock() - cCtx.ci = ci - cCtx.Unlock() + // TODO - + fmt.Printf("SERVER: returning: %+v\n", localIOR) - ci.cCtx = cCtx - ci.Unlock() + server.returnResults(&localIOR, cCtx) } return ci, err @@ -266,7 +296,7 @@ func (server *Server) serviceClient(ci *clientInfo, cCtx *connCtx) { for { // Get RPC request buf, msgType, getErr := getIO(uint64(0), server.deadlineIO, cCtx.conn) - if os.IsTimeout(getErr) == false && getErr != nil { + if !os.IsTimeout(getErr) && getErr != nil { // Drop response on the floor. Client will either reconnect or // this response will age out of the queues. @@ -278,17 +308,18 @@ func (server *Server) serviceClient(ci *clientInfo, cCtx *connCtx) { } server.Lock() - if server.halting == true { + if server.halting { server.Unlock() return } - if os.IsTimeout(getErr) == true { + if os.IsTimeout(getErr) { server.Unlock() continue } if msgType != RPC { + // TODO - panic? log message.... fmt.Printf("serviceClient() received invalid msgType: %v - dropping\n", msgType) continue } @@ -442,7 +473,7 @@ func (server *Server) trimCompleted(t time.Time, long bool) { ci.Lock() ci.cCtx.Lock() - if ci.isEmpty() && ci.cCtx.serviceClientExited == true { + if ci.isEmpty() && ci.cCtx.serviceClientExited { bucketstats.UnRegister("proxyfs.retryrpc", ci.myUniqueID) delete(server.perClientInfo, key) logger.Infof("Trim - DELETE inactive clientInfo with ID: %v", ci.myUniqueID) From b0d5a397c6d69090d121948549eb857c6ce57ead Mon Sep 17 00:00:00 2001 From: Bob Schatz Date: Wed, 17 Mar 2021 16:05:42 -0700 Subject: [PATCH 069/160] Fix race condition and use proper client ID --- retryrpc/api.go | 5 +++++ retryrpc/client.go | 10 +++++----- retryrpc/stress_test.go | 2 +- retryrpc/upcall_test.go | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/retryrpc/api.go b/retryrpc/api.go index cac84df69..e93fde5ae 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -363,6 +363,11 @@ func (client *Client) Send(method string, request interface{}, reply interface{} return client.send(method, request, reply) } +// GetMyUniqueID returns the unique ID of the client +func (client *Client) GetMyUniqueID() string { + return client.myUniqueID +} + // GetStatsGroupName returns the bucketstats GroupName for this client func (client *Client) GetStatsGroupName() (s string) { diff --git a/retryrpc/client.go b/retryrpc/client.go index 9235d9ed0..4b9a3d14a 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -66,6 +66,11 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter for { err = client.initialDial() if err == nil { + + // Now that we have a connection - we can setup bucketstats + if client.connection.state == CONNECTED { + bucketstats.Register("proxyfs.retryrpc", client.GetStatsGroupName(), &client.stats) + } break } client.Unlock() @@ -81,11 +86,6 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter break } } - - // Now that we have a connection - we can setup bucketstats - if client.connection.state == CONNECTED { - bucketstats.Register("proxyfs.retryrpc", client.GetStatsGroupName(), &client.stats) - } } // Put request data into structure to be be marshaled into JSON diff --git a/retryrpc/stress_test.go b/retryrpc/stress_test.go index 953c3b2ca..93cd11f47 100644 --- a/retryrpc/stress_test.go +++ b/retryrpc/stress_test.go @@ -390,7 +390,7 @@ func pfsagent(t *testing.T, rrSvr *Server, ipAddr string, port int, agentID uint sendWg.Add(1) go func(z int, i int) { sendIt(t, client, z, sendCnt, &sendWg, &prevWg, agentID, method, i) - rrSvr.SendCallback(clientID, msg1) + rrSvr.SendCallback(client.GetMyUniqueID(), msg1) }(z, i) // Occasionally drop the connection to the server to diff --git a/retryrpc/upcall_test.go b/retryrpc/upcall_test.go index 349ab86ef..1d6a558d2 100644 --- a/retryrpc/upcall_test.go +++ b/retryrpc/upcall_test.go @@ -80,7 +80,7 @@ func testUpCall(t *testing.T) { // Send an upcall() from the server to the client and // verify that client method is called. var msg1 []byte = []byte("server msg back to client") - rrSvr.SendCallback(myUniqueClientID, msg1) + rrSvr.SendCallback(rrClnt.GetMyUniqueID(), msg1) // Wait until Interrupt() was called cb.Lock() From 5413138830c90c8e5e996eb4cef5d0a6f3d50a98 Mon Sep 17 00:00:00 2001 From: Bob Schatz Date: Fri, 19 Mar 2021 16:57:57 -0700 Subject: [PATCH 070/160] Add option for RPCs to request unique ID for client Also, fix race detector found race in HH as well as fixes from unit and stress testing --- headhunter/checkpoint.go | 3 +- retryrpc/api_internal.go | 12 +++----- retryrpc/client.go | 24 ++++----------- retryrpc/retryrpc_test.go | 56 ++++++++++++++++++++-------------- retryrpc/rpctest/ping.go | 15 ++++++++++ retryrpc/server.go | 28 ++++++++++------- retryrpc/svrmap.go | 63 ++++++++++++++++++++++++++++----------- 7 files changed, 123 insertions(+), 78 deletions(-) diff --git a/headhunter/checkpoint.go b/headhunter/checkpoint.go index de33bfb5d..d6ccbdcfd 100644 --- a/headhunter/checkpoint.go +++ b/headhunter/checkpoint.go @@ -3059,6 +3059,7 @@ func (volume *volumeStruct) checkpointDaemon() { bPlusTreeObjectCacheStats = globals.bPlusTreeObjectCache.Stats() createdDeletedObjectsCacheStats = globals.createdDeletedObjectsCache.Stats() + globals.Lock() inodeRecCacheHitsDelta = inodeRecCacheStats.CacheHits - globals.inodeRecCachePriorCacheHits inodeRecCacheMissesDelta = inodeRecCacheStats.CacheMisses - globals.inodeRecCachePriorCacheMisses @@ -3071,8 +3072,6 @@ func (volume *volumeStruct) checkpointDaemon() { createdDeletedObjectsCacheHitsDelta = createdDeletedObjectsCacheStats.CacheHits - globals.createdDeletedObjectsCachePriorCacheHits createdDeletedObjectsCacheMissesDelta = createdDeletedObjectsCacheStats.CacheMisses - globals.createdDeletedObjectsCachePriorCacheMisses - globals.Lock() - if 0 != inodeRecCacheHitsDelta { stats.IncrementOperationsBy(&stats.InodeRecCacheHits, inodeRecCacheHitsDelta) globals.inodeRecCachePriorCacheHits = inodeRecCacheStats.CacheHits diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index d8b43c867..379bce662 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -97,9 +97,10 @@ type pendingCtx struct { // methodArgs defines the method provided by the RPC server // as well as the request type and reply type arguments type methodArgs struct { - methodPtr *reflect.Method - request reflect.Type - reply reflect.Type + methodPtr *reflect.Method + passClientID bool + request reflect.Type + reply reflect.Type } // completedLRUEntry tracks time entry was completed for @@ -281,11 +282,6 @@ func getIO(genNum uint64, deadlineIO time.Duration, conn net.Conn) (buf []byte, } msgType = hdr.Type - if hdr.Len == 0 { - // TODO - ????? - // err = fmt.Errorf("hdr.Len == 0") - return - } // Now read the rest of the structure off the wire. var numBytes int diff --git a/retryrpc/client.go b/retryrpc/client.go index 4b9a3d14a..b5b99af12 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -422,12 +422,10 @@ func (client *Client) getMyUniqueID(tlsConn *tls.Conn) (err error) { } // Ask the server for the unique ID. - // - // TODO - return error back up.... close() the connection??? - // upper layer should do the redial + // If we error, just return it and let caller close the connection. client.myUniqueID, err = client.readClientID(client.connection.genNum, tlsConn) if err != nil { - // TODO - close() connection + return } return @@ -454,7 +452,7 @@ func (client *Client) sendMyInfo(tlsConn *tls.Conn) (err error) { return } - // We are an existinc client - send "MyUniqueID" + // We are an existing client - send "MyUniqueID" client.connection.tlsConn.SetDeadline(time.Now().Add(client.deadlineIO)) bytesWritten, writeErr := tlsConn.Write(isreq.MyUniqueID) @@ -552,29 +550,19 @@ func (client *Client) readClientID(callingGenNum uint64, tlsConn *tls.Conn) (myU // TODO - don't loop forever - breakout after some many tries + // Upper layer will close connection as a result of any error if getErr != nil { - - // TODO - if error initially, just close socket, return error and let initialDial() - // retry. close may already be done by upper layer... verify - /* - // If we had an error reading socket - call retransmit() and exit - // the goroutine. retransmit()/dial() will start another - // readReplies() goroutine. - client.retransmit(callingGenNum) - */ + err = getErr return } if msgType != ReturnUniqueID { - // TODO - panic??? err = fmt.Errorf("CLIENT - invalid msgType: %v", msgType) + logger.PanicfWithError(err, "") return } - fmt.Printf("CLIENT: read buf: %v\n", string(buf)) myUniqueID = string(buf) - fmt.Printf("readClientID returned: %v\n", myUniqueID) - return } } diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index fcfd81f78..873b8dc60 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -18,32 +18,12 @@ import ( // circular dependency if the test was in retryrpc. func TestRetryRPC(t *testing.T) { + testRegister(t) testServer(t) testBtree(t) testStatsAndBucketstats(t) } -type MyType struct { - field1 int -} - -type MyRequest struct { - Field1 int -} - -type MyResponse struct { - Error error -} - -func (m *MyType) ExportedFunction(request MyRequest, response *MyResponse) (err error) { - request.Field1 = 1 - return -} - -func (m *MyType) unexportedFunction(i int) { - m.field1 = i -} - func getNewServer(lt time.Duration, dontStartTrimmers bool) (rrSvr *Server, ip string, p int) { var ( ipaddr = "127.0.0.1" @@ -60,6 +40,28 @@ func getNewServer(lt time.Duration, dontStartTrimmers bool) (rrSvr *Server, ip s return } +// Test register function finds all expected functions +func testRegister(t *testing.T) { + assert := assert.New(t) + zero := 0 + assert.Equal(0, zero) + + // Create new rpctest server - needed for calling + // RPCs + myJrpcfs := rpctest.NewServer() + + rrSvr, _, _ := getNewServer(10*time.Second, false) + assert.NotNil(rrSvr) + + // Register the Server - sets up the methods supported by the + // server + err := rrSvr.Register(myJrpcfs) + assert.Nil(err) + + // Make sure we discovered the correct functions + assert.Equal(4, len(rrSvr.svrMap)) +} + // Test basic Server creation and deletion func testServer(t *testing.T) { assert := assert.New(t) @@ -100,13 +102,21 @@ func testServer(t *testing.T) { assert.Equal("pong 8 bytes", pingReply.Message) assert.Equal(1, rrSvr.CompletedCnt()) + // Send an RPC which expects the client ID and which should return success + pingRequest = &rpctest.PingReq{Message: "Ping Me!"} + pingReply = &rpctest.PingReply{} + sendErr = rrClnt.Send("RpcPingWithClientID", pingRequest, pingReply) + assert.Nil(sendErr) + assert.Equal("Client ID: unqClnt-1 pong 8 bytes", pingReply.Message) + assert.Equal(2, rrSvr.CompletedCnt()) + // Send an RPC which should return an error pingRequest = &rpctest.PingReq{Message: "Ping Me!"} pingReply = &rpctest.PingReply{} sendErr = rrClnt.Send("RpcPingWithError", pingRequest, pingReply) assert.NotNil(sendErr) - assert.Equal(2, rrSvr.CompletedCnt()) + assert.Equal(3, rrSvr.CompletedCnt()) // Send an RPC which should return an error pingRequest = &rpctest.PingReq{Message: "Ping Me!"} @@ -114,7 +124,7 @@ func testServer(t *testing.T) { sendErr = rrClnt.Send("RpcInvalidMethod", pingRequest, pingReply) assert.NotNil(sendErr) - assert.Equal(3, rrSvr.CompletedCnt()) + assert.Equal(4, rrSvr.CompletedCnt()) // Stop the client before exiting rrClnt.Close() diff --git a/retryrpc/rpctest/ping.go b/retryrpc/rpctest/ping.go index f008a1156..ca9078286 100644 --- a/retryrpc/rpctest/ping.go +++ b/retryrpc/rpctest/ping.go @@ -48,3 +48,18 @@ func (s *Server) RpcPingWithError(in *PingReq, reply *PingReply) (err error) { reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) return err } + +// RpcPingWithClientID returns an error +func (s *Server) RpcPingWithClientID(clientID string, in *PingReq, reply *PingReply) (err error) { + reply.Message = fmt.Sprintf("Client ID: %v pong %d bytes", clientID, len(in.Message)) + + return nil +} + +// RpcPingWithInvalidClientIDType is not a valid RPC +func (s *Server) RpcPingWithInvalidClientID(clientID int, in *PingReq, reply *PingReply) (err error) { + err = blunder.AddError(err, blunder.NotFoundError) + encodeErrno(&err) + reply.Message = fmt.Sprintf("client ID: %v pong %d bytes", clientID, len(in.Message)) + return err +} diff --git a/retryrpc/server.go b/retryrpc/server.go index f9283694d..2a6219fa0 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -143,7 +143,7 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b // be unmarshaled again to retrieve the parameters specific to // the RPC. startRPC := time.Now() - ior := server.callRPCAndFormatReply(buf, &jReq) + ior := server.callRPCAndFormatReply(buf, ci.myUniqueID, &jReq) ci.stats.RPCLenUsec.Add(uint64(time.Since(startRPC) / time.Microsecond)) ci.stats.RPCcompleted.Add(1) @@ -213,6 +213,7 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err if (msgType != PassID) && (msgType != AskMyUniqueID) { err = fmt.Errorf("Server expecting msgType PassID or AskMyUniqueID and received: %v", msgType) + logger.PanicfWithError(err, "") return } @@ -220,6 +221,7 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err var connUniqueID string err = json.Unmarshal(buf, &connUniqueID) if err != nil { + logger.PanicfWithError(err, "Unmarshal returned error") return } @@ -227,7 +229,8 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err server.Lock() lci, ok := server.perClientInfo[connUniqueID] if !ok { - // TODO - what??? + err = fmt.Errorf("Server - msgType PassID but can't find uniqueID in perClientInfo") + logger.PanicfWithError(err, "getClientIDAndWait() buf: %v connUniqueID: %v err: %+v", buf, connUniqueID, err) server.Unlock() } else { server.Unlock() @@ -282,9 +285,6 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err localIOR.JResult = []byte(newUniqueID) setupHdrReply(&localIOR, ReturnUniqueID) - // TODO - - fmt.Printf("SERVER: returning: %+v\n", localIOR) - server.returnResults(&localIOR, cCtx) } @@ -340,9 +340,12 @@ func (server *Server) serviceClient(ci *clientInfo, cCtx *connCtx) { } // callRPCAndMarshal calls the RPC and returns results to requestor -func (server *Server) callRPCAndFormatReply(buf []byte, jReq *jsonRequest) (ior *ioReply) { +func (server *Server) callRPCAndFormatReply(buf []byte, clientID string, jReq *jsonRequest) (ior *ioReply) { var ( - err error + err error + returnValues []reflect.Value + typOfReq reflect.Type + dummyReq interface{} ) // Setup the reply structure with common fields @@ -355,8 +358,8 @@ func (server *Server) callRPCAndFormatReply(buf []byte, jReq *jsonRequest) (ior // Another unmarshal of buf to find the parameters specific to // this RPC - typOfReq := ma.request.Elem() - dummyReq := reflect.New(typOfReq).Interface() + typOfReq = ma.request.Elem() + dummyReq = reflect.New(typOfReq).Interface() sReq := svrRequest{} sReq.Params[0] = dummyReq @@ -366,6 +369,7 @@ func (server *Server) callRPCAndFormatReply(buf []byte, jReq *jsonRequest) (ior return } req := reflect.ValueOf(dummyReq) + cid := reflect.ValueOf(clientID) // Create the reply structure typOfReply := ma.reply.Elem() @@ -373,7 +377,11 @@ func (server *Server) callRPCAndFormatReply(buf []byte, jReq *jsonRequest) (ior // Call the method function := ma.methodPtr.Func - returnValues := function.Call([]reflect.Value{server.receiver, req, myReply}) + if ma.passClientID { + returnValues = function.Call([]reflect.Value{server.receiver, cid, req, myReply}) + } else { + returnValues = function.Call([]reflect.Value{server.receiver, req, myReply}) + } // The return value for the method is an error. errInter := returnValues[0].Interface() diff --git a/retryrpc/svrmap.go b/retryrpc/svrmap.go index 229443de1..b858feb09 100644 --- a/retryrpc/svrmap.go +++ b/retryrpc/svrmap.go @@ -10,7 +10,9 @@ import ( "unicode/utf8" ) -var typeOfError = reflect.TypeOf((*error)(nil)).Elem() +var ( + typeOfError = reflect.TypeOf((*error)(nil)).Elem() +) // Find all methods for the type which can be exported. // Build svrMap listing methods available as well as their @@ -30,20 +32,9 @@ func (server *Server) buildSvrMap(typ reflect.Type) { continue } - if mtype.NumIn() != 3 { - continue - } - argType := mtype.In(1) - if !isExportedOrBuiltinType(argType) { - continue - } - - replyType := mtype.In(2) - if replyType.Kind() != reflect.Ptr { - continue - } - - if !isExportedOrBuiltinType(replyType) { + // Will have 3 arguments if pass request/reply + // Will have 4 arguments if expect clientID and request/reply + if (mtype.NumIn() != 3) && (mtype.NumIn() != 4) { continue } @@ -58,8 +49,46 @@ func (server *Server) buildSvrMap(typ reflect.Type) { // We save off the request type so we know how to unmarshal the request. // We use the reply type to allocate the reply struct and marshal the response. - ma := methodArgs{methodPtr: &method, request: argType, reply: replyType} - server.svrMap[mname] = &ma + if mtype.NumIn() == 3 { + argType := mtype.In(1) + if !isExportedOrBuiltinType(argType) { + continue + } + replyType := mtype.In(2) + if replyType.Kind() != reflect.Ptr { + continue + } + + if !isExportedOrBuiltinType(replyType) { + continue + } + + ma := methodArgs{methodPtr: &method, passClientID: false, request: argType, reply: replyType} + server.svrMap[mname] = &ma + } else if mtype.NumIn() == 4 { + argType := mtype.In(2) + if !isExportedOrBuiltinType(argType) { + continue + } + + // Check if first argument is string for clientID + clientIDType := mtype.In(1) + if clientIDType.Kind() != reflect.String { + continue + } + + replyType := mtype.In(3) + if replyType.Kind() != reflect.Ptr { + continue + } + + if !isExportedOrBuiltinType(replyType) { + continue + } + + ma := methodArgs{methodPtr: &method, passClientID: true, request: argType, reply: replyType} + server.svrMap[mname] = &ma + } } } From f1a8b35f3c64bce53955a94dda4f35a6eebcaf34 Mon Sep 17 00:00:00 2001 From: Bob Schatz Date: Mon, 22 Mar 2021 13:17:19 -0700 Subject: [PATCH 071/160] Convert uniqueID from string to uint64 --- retryrpc/api.go | 23 ++++++++++++----------- retryrpc/api_internal.go | 8 ++++---- retryrpc/client.go | 7 +++++-- retryrpc/retryrpc_test.go | 6 +++--- retryrpc/rpctest/ping.go | 2 +- retryrpc/server.go | 23 +++++++++++++++-------- retryrpc/svrmap.go | 4 ++-- retryrpc/upcall_test.go | 4 +--- 8 files changed, 43 insertions(+), 34 deletions(-) diff --git a/retryrpc/api.go b/retryrpc/api.go index e93fde5ae..c6bbf522f 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -18,6 +18,7 @@ import ( "fmt" "net" "reflect" + "strconv" "sync" "time" @@ -53,7 +54,7 @@ type Server struct { Creds *ServerCreds listenersWG sync.WaitGroup receiver reflect.Value // Package receiver being served - perClientInfo map[string]*clientInfo // Key: "clientID". Tracks clients + perClientInfo map[uint64]*clientInfo // Key: "clientID". Tracks clients completedTickerDone chan bool completedLongTicker *time.Ticker // Longer ~10 minute timer to trim completedShortTicker *time.Ticker // Shorter ~100ms timer to trim known completed @@ -83,7 +84,7 @@ func NewServer(config *ServerConfig) *Server { completedAckTrim: config.ShortTrim, deadlineIO: config.DeadlineIO, keepAlivePeriod: config.KeepAlivePeriod, dontStartTrimmers: config.dontStartTrimmers} server.svrMap = make(map[string]*methodArgs) - server.perClientInfo = make(map[string]*clientInfo) + server.perClientInfo = make(map[uint64]*clientInfo) server.completedTickerDone = make(chan bool) server.connections = list.New() @@ -176,7 +177,7 @@ func (server *Server) Run() { // // The message is "best effort" - if we fail to write on socket then the // message is silently dropped on floor. -func (server *Server) SendCallback(clientID string, msg []byte) { +func (server *Server) SendCallback(clientID uint64, msg []byte) { // TODO - what if client no longer in list of current clients? var ( @@ -229,7 +230,8 @@ func (server *Server) Close() { // Cleanup bucketstats so that unit tests can run for _, ci := range server.perClientInfo { ci.Lock() - bucketstats.UnRegister("proxyfs.retryrpc", ci.myUniqueID) + s10 := strconv.FormatInt(int64(ci.myUniqueID), 10) + bucketstats.UnRegister("proxyfs.retryrpc", s10) ci.Unlock() } @@ -293,7 +295,7 @@ type Client struct { // tick at mount and increment from there? // Handle reset of time? connection connectionTracker - myUniqueID string // Unique ID across all clients + myUniqueID uint64 // Unique ID across all clients cb interface{} // Callbacks to client deadlineIO time.Duration keepAlivePeriod time.Duration @@ -314,7 +316,6 @@ type ClientCallbacks interface { // ClientConfig is used to configure a retryrpc Client type ClientConfig struct { - MyUniqueID string IPAddr string // IP Address of Server Port int // Port of Server RootCAx509CertificatePEM []byte // Root certificate @@ -338,8 +339,8 @@ type ClientConfig struct { // starting point for requestID. func NewClient(config *ClientConfig) (client *Client, err error) { - client = &Client{myUniqueID: config.MyUniqueID, cb: config.Callbacks, - keepAlivePeriod: config.KeepAlivePeriod, deadlineIO: config.DeadlineIO} + client = &Client{cb: config.Callbacks, keepAlivePeriod: config.KeepAlivePeriod, + deadlineIO: config.DeadlineIO} portStr := fmt.Sprintf("%d", config.Port) client.connection.state = INITIAL client.connection.hostPortStr = net.JoinHostPort(config.IPAddr, portStr) @@ -364,14 +365,14 @@ func (client *Client) Send(method string, request interface{}, reply interface{} } // GetMyUniqueID returns the unique ID of the client -func (client *Client) GetMyUniqueID() string { +func (client *Client) GetMyUniqueID() uint64 { return client.myUniqueID } // GetStatsGroupName returns the bucketstats GroupName for this client func (client *Client) GetStatsGroupName() (s string) { - - return clientSideGroupPrefix + client.myUniqueID + s10 := strconv.FormatInt(int64(client.myUniqueID), 10) + return clientSideGroupPrefix + s10 } // Close gracefully shuts down the client diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index 379bce662..e76f44931 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -61,7 +61,7 @@ type statsInfo struct { type clientInfo struct { sync.Mutex cCtx *connCtx // Current connCtx for client - myUniqueID string // Unique ID of this client + myUniqueID uint64 // Unique ID of this client completedRequest map[requestID]*completedEntry // Key: "RequestID" completedRequestLRU *list.List // LRU used to remove completed request in ticker highestReplySeen requestID // Highest consectutive requestID client has seen @@ -185,7 +185,7 @@ type reqCtx struct { // jsonRequest is used to marshal an RPC request in/out of JSON type jsonRequest struct { - MyUniqueID string `json:"myuniqueid"` // ID of client + MyUniqueID uint64 `json:"myuniqueid"` // ID of client RequestID requestID `json:"requestid"` // ID of this request HighestReplySeen requestID `json:"highestReplySeen"` // Used to trim completedRequests on server Method string `json:"method"` @@ -194,7 +194,7 @@ type jsonRequest struct { // jsonReply is used to marshal an RPC response in/out of JSON type jsonReply struct { - MyUniqueID string `json:"myuniqueid"` // ID of client + MyUniqueID uint64 `json:"myuniqueid"` // ID of client RequestID requestID `json:"requestid"` // ID of this request ErrStr string `json:"errstr"` Result interface{} `json:"result"` @@ -235,7 +235,7 @@ func setupHdrReply(ioreply *ioReply, t MsgType) { return } -func buildSetIDRequest(myUniqueID string) (isreq *internalSetIDRequest, err error) { +func buildSetIDRequest(myUniqueID uint64) (isreq *internalSetIDRequest, err error) { isreq = &internalSetIDRequest{} isreq.MyUniqueID, err = json.Marshal(myUniqueID) if err != nil { diff --git a/retryrpc/client.go b/retryrpc/client.go index b5b99af12..00c6077f6 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -527,7 +527,7 @@ func (client *Client) reDial() (err error) { // Client lock is held // // NOTE: Client lock is held -func (client *Client) readClientID(callingGenNum uint64, tlsConn *tls.Conn) (myUniqueID string, err error) { +func (client *Client) readClientID(callingGenNum uint64, tlsConn *tls.Conn) (myUniqueID uint64, err error) { var localCnt int for { @@ -562,7 +562,10 @@ func (client *Client) readClientID(callingGenNum uint64, tlsConn *tls.Conn) (myU return } - myUniqueID = string(buf) + err = json.Unmarshal(buf, &myUniqueID) + if err != nil { + logger.PanicfWithError(err, "Unmarshal of buf: %v to myUniqueID failed with err: %v", buf, err) + } return } } diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index 873b8dc60..c21edf828 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -88,7 +88,7 @@ func testServer(t *testing.T) { rrSvr.Run() // Now - setup a client to send requests to the server - clientConfig := &ClientConfig{MyUniqueID: "client 1", IPAddr: ipaddr, Port: port, RootCAx509CertificatePEM: rrSvr.Creds.RootCAx509CertificatePEM, + clientConfig := &ClientConfig{IPAddr: ipaddr, Port: port, RootCAx509CertificatePEM: rrSvr.Creds.RootCAx509CertificatePEM, Callbacks: nil, DeadlineIO: 5 * time.Second} rrClnt, newErr := NewClient(clientConfig) assert.NotNil(rrClnt) @@ -107,7 +107,7 @@ func testServer(t *testing.T) { pingReply = &rpctest.PingReply{} sendErr = rrClnt.Send("RpcPingWithClientID", pingRequest, pingReply) assert.Nil(sendErr) - assert.Equal("Client ID: unqClnt-1 pong 8 bytes", pingReply.Message) + assert.Equal("Client ID: 1 pong 8 bytes", pingReply.Message) assert.Equal(2, rrSvr.CompletedCnt()) // Send an RPC which should return an error @@ -140,7 +140,7 @@ func testBtree(t *testing.T) { assert.NotNil(rrSvr) // Setup a client - we only will be targeting the btree - clientConfig := &ClientConfig{MyUniqueID: "client 1", IPAddr: ipaddr, Port: port, RootCAx509CertificatePEM: rrSvr.Creds.RootCAx509CertificatePEM, + clientConfig := &ClientConfig{IPAddr: ipaddr, Port: port, RootCAx509CertificatePEM: rrSvr.Creds.RootCAx509CertificatePEM, Callbacks: nil, DeadlineIO: 5 * time.Second} client, newErr := NewClient(clientConfig) assert.NotNil(client) diff --git a/retryrpc/rpctest/ping.go b/retryrpc/rpctest/ping.go index ca9078286..0d87bf8b5 100644 --- a/retryrpc/rpctest/ping.go +++ b/retryrpc/rpctest/ping.go @@ -50,7 +50,7 @@ func (s *Server) RpcPingWithError(in *PingReq, reply *PingReply) (err error) { } // RpcPingWithClientID returns an error -func (s *Server) RpcPingWithClientID(clientID string, in *PingReq, reply *PingReply) (err error) { +func (s *Server) RpcPingWithClientID(clientID uint64, in *PingReq, reply *PingReply) (err error) { reply.Message = fmt.Sprintf("Client ID: %v pong %d bytes", clientID, len(in.Message)) return nil diff --git a/retryrpc/server.go b/retryrpc/server.go index 2a6219fa0..22ee6e97a 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -11,6 +11,7 @@ import ( "net" "os" "reflect" + "strconv" "sync" "time" @@ -218,7 +219,7 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err } if msgType == PassID { - var connUniqueID string + var connUniqueID uint64 err = json.Unmarshal(buf, &connUniqueID) if err != nil { logger.PanicfWithError(err, "Unmarshal returned error") @@ -266,7 +267,7 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err // Create a unique client ID and return to client server.Lock() server.clientIDNonce++ - newUniqueID := fmt.Sprintf("unqClnt-%v", server.clientIDNonce) + newUniqueID := server.clientIDNonce // Setup new client data structures c := &clientInfo{cCtx: cCtx, myUniqueID: newUniqueID} @@ -279,10 +280,15 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err cCtx.ci = ci cCtx.Unlock() - bucketstats.Register("proxyfs.retryrpc", c.myUniqueID, &c.stats) + s10 := strconv.FormatInt(int64(newUniqueID), 10) + bucketstats.Register("proxyfs.retryrpc", s10, &c.stats) // Build reply and send back to client - localIOR.JResult = []byte(newUniqueID) + var e error + localIOR.JResult, e = json.Marshal(newUniqueID) + if e != nil { + logger.PanicfWithError(e, "Marshal of newUniqueID: %v failed with err: %v", newUniqueID, err) + } setupHdrReply(&localIOR, ReturnUniqueID) server.returnResults(&localIOR, cCtx) @@ -340,7 +346,7 @@ func (server *Server) serviceClient(ci *clientInfo, cCtx *connCtx) { } // callRPCAndMarshal calls the RPC and returns results to requestor -func (server *Server) callRPCAndFormatReply(buf []byte, clientID string, jReq *jsonRequest) (ior *ioReply) { +func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *jsonRequest) (ior *ioReply) { var ( err error returnValues []reflect.Value @@ -476,13 +482,14 @@ func (server *Server) trimCompleted(t time.Time, long bool) { // We can only do the check if we are currently holding the // lock. for e := l.Front(); e != nil; e = e.Next() { - key := e.Value.(string) + key := e.Value.(uint64) ci := server.perClientInfo[key] ci.Lock() ci.cCtx.Lock() if ci.isEmpty() && ci.cCtx.serviceClientExited { - bucketstats.UnRegister("proxyfs.retryrpc", ci.myUniqueID) + s10 := strconv.FormatInt(int64(ci.myUniqueID), 10) + bucketstats.UnRegister("proxyfs.retryrpc", s10) delete(server.perClientInfo, key) logger.Infof("Trim - DELETE inactive clientInfo with ID: %v", ci.myUniqueID) } @@ -503,7 +510,7 @@ func (server *Server) trimCompleted(t time.Time, long bool) { // on TTL or RequestID acknowledgement from client. // // NOTE: We assume Server Lock is held -func (server *Server) trimAClientBasedACK(uniqueID string, ci *clientInfo) (numItems int) { +func (server *Server) trimAClientBasedACK(uniqueID uint64, ci *clientInfo) (numItems int) { ci.Lock() diff --git a/retryrpc/svrmap.go b/retryrpc/svrmap.go index b858feb09..6d920c9d5 100644 --- a/retryrpc/svrmap.go +++ b/retryrpc/svrmap.go @@ -71,9 +71,9 @@ func (server *Server) buildSvrMap(typ reflect.Type) { continue } - // Check if first argument is string for clientID + // Check if first argument is uint64 for clientID clientIDType := mtype.In(1) - if clientIDType.Kind() != reflect.String { + if clientIDType.Kind() != reflect.Uint64 { continue } diff --git a/retryrpc/upcall_test.go b/retryrpc/upcall_test.go index 1d6a558d2..4f8ed6d2e 100644 --- a/retryrpc/upcall_test.go +++ b/retryrpc/upcall_test.go @@ -33,8 +33,6 @@ func (cb *MyClient) Interrupt(payload []byte) { // Test getting an upcallbasic Server creation and deletion func testUpCall(t *testing.T) { - var myUniqueClientID string = "my upcall client 1" - assert := assert.New(t) zero := 0 assert.Equal(0, zero) @@ -62,7 +60,7 @@ func testUpCall(t *testing.T) { cb := &MyClient{} cb.cond = sync.NewCond(&cb.Mutex) - clientConfig := &ClientConfig{MyUniqueID: myUniqueClientID, IPAddr: ipaddr, Port: port, + clientConfig := &ClientConfig{IPAddr: ipaddr, Port: port, RootCAx509CertificatePEM: rrSvr.Creds.RootCAx509CertificatePEM, Callbacks: cb, DeadlineIO: 5 * time.Second} rrClnt, newErr := NewClient(clientConfig) From 2478f87d54c78a371c01f1703765bef81b99efa3 Mon Sep 17 00:00:00 2001 From: Bob Schatz Date: Mon, 22 Mar 2021 13:18:35 -0700 Subject: [PATCH 072/160] Updates for string to uint64 --- retryrpc/stress_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/retryrpc/stress_test.go b/retryrpc/stress_test.go index 93cd11f47..fd673850c 100644 --- a/retryrpc/stress_test.go +++ b/retryrpc/stress_test.go @@ -339,8 +339,7 @@ func pfsagent(t *testing.T, rrSvr *Server, ipAddr string, port int, agentID uint cb := &stressMyClient{} cb.cond = sync.NewCond(&cb.Mutex) - clientID := fmt.Sprintf("client - %v", agentID) - clientConfig := &ClientConfig{MyUniqueID: clientID, IPAddr: ipAddr, Port: port, + clientConfig := &ClientConfig{IPAddr: ipAddr, Port: port, RootCAx509CertificatePEM: rootCAx509CertificatePEM, Callbacks: cb, DeadlineIO: 5 * time.Second} client, err := NewClient(clientConfig) if err != nil { From eb56d5f01a73a788c06b5290ac60f84d5f754c93 Mon Sep 17 00:00:00 2001 From: Bob Schatz Date: Tue, 23 Mar 2021 10:57:06 -0700 Subject: [PATCH 073/160] Change unique ID from string to uint64 --- jrpcfs/config.go | 1 + jrpcfs/filesystem.go | 11 ++++++----- jrpcfs/lease.go | 36 ++++++++++++++++++------------------ pfsagentd/request.go | 1 - 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/jrpcfs/config.go b/jrpcfs/config.go index 1af2e1f9f..34dd4ed8a 100644 --- a/jrpcfs/config.go +++ b/jrpcfs/config.go @@ -95,6 +95,7 @@ type mountStruct struct { volume *volumeStruct mountIDAsByteArray MountIDAsByteArray mountIDAsString MountIDAsString + retryRpcUniqueID uint64 acceptingLeaseRequests bool // also an indicator (when false) that mount is being unmounted leaseRequestMap map[inode.InodeNumber]*leaseRequestStruct // if present, there is an ongoing Lease Request for this inode.InodeNumber // if not present, there is no ongoing Lease Request for this inode.InodeNumber diff --git a/jrpcfs/filesystem.go b/jrpcfs/filesystem.go index ea6ab978c..a83e87996 100644 --- a/jrpcfs/filesystem.go +++ b/jrpcfs/filesystem.go @@ -1335,7 +1335,7 @@ func (s *Server) RpcMkdirPath(in *MkdirPathRequest, reply *Reply) (err error) { return } -func performMount(volumeHandle fs.VolumeHandle) (mountIDAsByteArray MountIDAsByteArray, mountIDAsString MountIDAsString, err error) { +func performMount(volumeHandle fs.VolumeHandle, clientID uint64) (mountIDAsByteArray MountIDAsByteArray, mountIDAsString MountIDAsString, err error) { var ( i int keepTrying bool @@ -1383,6 +1383,7 @@ func performMount(volumeHandle fs.VolumeHandle) (mountIDAsByteArray MountIDAsByt volume: volume, mountIDAsByteArray: mountIDAsByteArray, mountIDAsString: mountIDAsString, + retryRpcUniqueID: clientID, acceptingLeaseRequests: true, leaseRequestMap: make(map[inode.InodeNumber]*leaseRequestStruct), } @@ -1398,7 +1399,7 @@ func performMount(volumeHandle fs.VolumeHandle) (mountIDAsByteArray MountIDAsByt return } -func (s *Server) RpcMountByAccountName(in *MountByAccountNameRequest, reply *MountByAccountNameReply) (err error) { +func (s *Server) RpcMountByAccountName(clientID uint64, in *MountByAccountNameRequest, reply *MountByAccountNameReply) (err error) { enterGate() defer leaveGate() @@ -1408,7 +1409,7 @@ func (s *Server) RpcMountByAccountName(in *MountByAccountNameRequest, reply *Mou volumeHandle, err := fs.FetchVolumeHandleByAccountName(in.AccountName) if nil == err { - _, reply.MountID, err = performMount(volumeHandle) + _, reply.MountID, err = performMount(volumeHandle, clientID) if nil == err { reply.RootDirInodeNumber = int64(uint64(inode.RootDirInodeNumber)) reply.RetryRPCPublicIPAddr = globals.publicIPAddr @@ -1420,7 +1421,7 @@ func (s *Server) RpcMountByAccountName(in *MountByAccountNameRequest, reply *Mou return } -func (s *Server) RpcMountByVolumeName(in *MountByVolumeNameRequest, reply *MountByVolumeNameReply) (err error) { +func (s *Server) RpcMountByVolumeName(clientID uint64, in *MountByVolumeNameRequest, reply *MountByVolumeNameReply) (err error) { enterGate() defer leaveGate() @@ -1430,7 +1431,7 @@ func (s *Server) RpcMountByVolumeName(in *MountByVolumeNameRequest, reply *Mount volumeHandle, err := fs.FetchVolumeHandleByVolumeName(in.VolumeName) if nil == err { - _, reply.MountID, err = performMount(volumeHandle) + _, reply.MountID, err = performMount(volumeHandle, clientID) if nil == err { reply.RootDirInodeNumber = int64(uint64(inode.RootDirInodeNumber)) reply.RetryRPCPublicIPAddr = globals.publicIPAddr diff --git a/jrpcfs/lease.go b/jrpcfs/lease.go index a15cf67c6..c02177791 100644 --- a/jrpcfs/lease.go +++ b/jrpcfs/lease.go @@ -298,7 +298,7 @@ func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *lease logger.Fatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 1]", rpcInterrupt, err) } evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(inodeLease.demotingHolder.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(inodeLease.demotingHolder.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(inodeLease.demotingHolder.mount.retryRpcUniqueID, rpcInterruptBuf) inodeLease.lastInterruptTime = time.Now() inodeLease.interruptsSent = 1 inodeLease.interruptTimer = time.NewTimer(globals.leaseInterruptInterval) @@ -375,7 +375,7 @@ func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *lease leaseRequest.listElement = inodeLease.releasingHoldersList.PushBack(leaseRequest) leaseRequest.requestState = leaseRequestStateSharedReleasing evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) } inodeLease.lastInterruptTime = time.Now() inodeLease.interruptsSent = 1 @@ -458,7 +458,7 @@ func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *lease logger.Fatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 3]", rpcInterrupt, err) } evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(sharedHolderLeaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(sharedHolderLeaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(sharedHolderLeaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) } inodeLease.lastInterruptTime = time.Now() inodeLease.interruptsSent = 1 @@ -485,7 +485,7 @@ func (inodeLease *inodeLeaseStruct) handleOperation(leaseRequestOperation *lease logger.Fatalf("(*inodeLeaseStruct).handleOperation() unable to json.Marshal(rpcInterrupt: %#v): %v [case 4]", rpcInterrupt, err) } evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(inodeLease.exclusiveHolder.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(inodeLease.exclusiveHolder.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(inodeLease.exclusiveHolder.mount.retryRpcUniqueID, rpcInterruptBuf) inodeLease.exclusiveHolder = nil inodeLease.lastInterruptTime = time.Now() inodeLease.interruptsSent = 1 @@ -1056,7 +1056,7 @@ func (inodeLease *inodeLeaseStruct) handleLongAgoTimerPop() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) leaseRequestElement = inodeLease.sharedHoldersList.Front() } @@ -1092,7 +1092,7 @@ func (inodeLease *inodeLeaseStruct) handleLongAgoTimerPop() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(inodeLease.demotingHolder.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(inodeLease.demotingHolder.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(inodeLease.demotingHolder.mount.retryRpcUniqueID, rpcInterruptBuf) case leaseRequestStateExclusiveRequested: inodeLease.leaseState = inodeLeaseStateExclusiveReleasing @@ -1111,7 +1111,7 @@ func (inodeLease *inodeLeaseStruct) handleLongAgoTimerPop() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(inodeLease.exclusiveHolder.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(inodeLease.exclusiveHolder.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(inodeLease.exclusiveHolder.mount.retryRpcUniqueID, rpcInterruptBuf) inodeLease.exclusiveHolder = nil default: @@ -1357,7 +1357,7 @@ func (inodeLease *inodeLeaseStruct) handleInterruptTimerPop() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) leaseRequestElement = leaseRequestElement.Next() } @@ -1381,7 +1381,7 @@ func (inodeLease *inodeLeaseStruct) handleInterruptTimerPop() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) leaseRequestElement = leaseRequestElement.Next() } @@ -1405,7 +1405,7 @@ func (inodeLease *inodeLeaseStruct) handleInterruptTimerPop() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) case inodeLeaseStateExclusiveDemoting: if nil == inodeLease.demotingHolder { logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found empty demotingHolder [case 2]") @@ -1423,7 +1423,7 @@ func (inodeLease *inodeLeaseStruct) handleInterruptTimerPop() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(inodeLease.demotingHolder.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(inodeLease.demotingHolder.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(inodeLease.demotingHolder.mount.retryRpcUniqueID, rpcInterruptBuf) default: logger.Fatalf("(*inodeLeaseStruct).handleInterruptTimerPop() found unexpected leaseState: %v [case 2]", inodeLease.leaseState) } @@ -1495,7 +1495,7 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) inodeLease.leaseState = inodeLeaseStateSharedReleasing } @@ -1550,7 +1550,7 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) } inodeLease.leaseState = inodeLeaseStateSharedReleasing @@ -1579,7 +1579,7 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) inodeLease.leaseState = inodeLeaseStateExclusiveReleasing @@ -1648,7 +1648,7 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) inodeLease.leaseState = inodeLeaseStateExclusiveReleasing @@ -1833,7 +1833,7 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) leaseRequestElement = leaseRequestElement.Next() } @@ -1874,7 +1874,7 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) inodeLease.lastInterruptTime = time.Now() inodeLease.interruptsSent++ @@ -1913,7 +1913,7 @@ func (inodeLease *inodeLeaseStruct) handleStopChanClose() { evtlog.Record(evtlog.FormatLeaseInterrupt, inodeLease.volume.volumeName, string(leaseRequest.mount.mountIDAsString), uint64(rpcInterrupt.InodeNumber), uint32(rpcInterrupt.RPCInterruptType)) - globals.retryrpcSvr.SendCallback(string(leaseRequest.mount.mountIDAsString), rpcInterruptBuf) + globals.retryrpcSvr.SendCallback(leaseRequest.mount.retryRpcUniqueID, rpcInterruptBuf) inodeLease.lastInterruptTime = time.Now() inodeLease.interruptsSent++ diff --git a/pfsagentd/request.go b/pfsagentd/request.go index 8631dddc6..9525dd3c8 100644 --- a/pfsagentd/request.go +++ b/pfsagentd/request.go @@ -71,7 +71,6 @@ func doMountProxyFS() { globals.rootCAx509CertificatePEM = mountReply.RootCAx509CertificatePEM retryrpcConfig := &retryrpc.ClientConfig{ - MyUniqueID: string(globals.mountID), IPAddr: globals.retryRPCPublicIPAddr, Port: int(globals.retryRPCPort), RootCAx509CertificatePEM: globals.rootCAx509CertificatePEM, From 38d64def9ee376bdd524c4d19ff50c0f711d3200 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 23 Mar 2021 15:24:47 -0700 Subject: [PATCH 074/160] Add optional loading of PEM Cert/Key/CACert files in ProxyFS & PFSAgent --- CONFIGURING.md | 2 ++ .../sample-proxyfs-configuration/proxyfs.conf | 2 ++ jrpcfs/config.go | 36 +++++++++++++++++++ jrpcfs/middleware_test.go | 2 ++ pfsagentd/README.md | 1 + pfsagentd/globals.go | 18 ++++++++++ pfsagentd/pfsagent.conf | 1 + pfsagentd/setup_teardown_test.go | 3 ++ proxyfsd/default.conf | 2 ++ proxyfsd/rpc_server.conf | 2 ++ saio/container/proxyfs.conf | 2 ++ 11 files changed, 71 insertions(+) diff --git a/CONFIGURING.md b/CONFIGURING.md index e6c29853c..eb2d9a4fd 100644 --- a/CONFIGURING.md +++ b/CONFIGURING.md @@ -203,6 +203,8 @@ For each of the keys supported, the following table will list whether or not its | | RetryRPCAckTrim | No | 100ms | Yes | No | | | RetryRPCDeadlineIO | No | 60s | Yes | No | | | RetryRPCKeepAlivePeriod | No | 60s | Yes | No | +| | RetryRPCCertFilePath | No | "" | Yes | No | +| | RetryRPCKeyFilePath | No | "" | Yes | No | | | MinLeaseDuration | No | 250ms | Yes | No | | | LeaseInterruptInterval | No | 250ms | Yes | No | | | LeaseInterruptLimit | No | 20 | Yes | No | diff --git a/confgen/sample-proxyfs-configuration/proxyfs.conf b/confgen/sample-proxyfs-configuration/proxyfs.conf index faa4219a3..9ad6c9826 100644 --- a/confgen/sample-proxyfs-configuration/proxyfs.conf +++ b/confgen/sample-proxyfs-configuration/proxyfs.conf @@ -81,6 +81,8 @@ RetryRPCTTLCompleted: 10m RetryRPCAckTrim: 100ms RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s +RetryRPCCertFilePath: +RetryRPCKeyFilePath: MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 diff --git a/jrpcfs/config.go b/jrpcfs/config.go index 34dd4ed8a..7841b694e 100644 --- a/jrpcfs/config.go +++ b/jrpcfs/config.go @@ -6,6 +6,7 @@ package jrpcfs import ( "container/list" "fmt" + "io/ioutil" "net" "sync" "time" @@ -134,11 +135,16 @@ type globalsStruct struct { retryRPCAckTrim time.Duration retryRPCDeadlineIO time.Duration retryRPCKeepAlivePeriod time.Duration + retryRPCCertFilePath string + retryRPCKeyFilePath string minLeaseDuration time.Duration leaseInterruptInterval time.Duration leaseInterruptLimit uint32 dataPathLogging bool + retryRPCCertPEM []byte + retryRPCKeyPEM []byte + volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName mountMapByMountIDAsByteArray map[MountIDAsByteArray]*mountStruct // key == mountStruct.mountIDAsByteArray mountMapByMountIDAsString map[MountIDAsString]*mountStruct // key == mountStruct.mountIDAsString @@ -220,6 +226,32 @@ func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { logger.Infof("failed to get JSONRPCServer.RetryRPCKeepAlivePeriod from config file - defaulting to 60s") globals.retryRPCKeepAlivePeriod = 60 * time.Second } + globals.retryRPCCertFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCCertFilePath") + if nil == err { + globals.retryRPCCertPEM, err = ioutil.ReadFile(globals.retryRPCCertFilePath) + if nil != err { + logger.ErrorfWithError(err, "failed to load PEM-formatted [JSONRPCServer]RetryRPCCertFilePath [\"%s\"]: %v", globals.retryRPCCertFilePath, err) + return + } + } else { + logger.Infof("failed to get JSONRPCServer.RetryRPCCertFilePath from config file - defaulting to \"\"") + globals.retryRPCCertFilePath = "" + } + globals.retryRPCKeyFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCKeyFilePath") + if nil == err { + if "" == globals.retryRPCCertFilePath { + logger.Error("if [JSOPNRPCServer]RetryRPCKeyFilePath is specified, [JSOPNRPCServer]RetryRPCCertFilePath must be specified as well") + return + } + globals.retryRPCKeyPEM, err = ioutil.ReadFile(globals.retryRPCKeyFilePath) + if nil != err { + logger.ErrorfWithError(err, "failed to load PEM-formatted [JSONRPCServer]RetryRPCKeyFilePath [\"%s\"]: %v", globals.retryRPCKeyFilePath, err) + return + } + } else { + logger.Infof("failed to get JSONRPCServer.RetryRPCKeyFilePath from config file - defaulting to \"\"") + globals.retryRPCKeyFilePath = "" + } } else { logger.Infof("failed to get JSONRPCServer.RetryRPCPort from config file - skipping......") globals.retryRPCPort = 0 @@ -227,6 +259,10 @@ func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { globals.retryRPCAckTrim = time.Duration(0) globals.retryRPCDeadlineIO = time.Duration(0) globals.retryRPCKeepAlivePeriod = time.Duration(0) + globals.retryRPCCertFilePath = "" + globals.retryRPCKeyFilePath = "" + globals.retryRPCCertPEM = nil + globals.retryRPCKeyPEM = nil } // Set data path logging level to true, so that all trace logging is controlled by settings diff --git a/jrpcfs/middleware_test.go b/jrpcfs/middleware_test.go index 24f3b827f..209027a15 100644 --- a/jrpcfs/middleware_test.go +++ b/jrpcfs/middleware_test.go @@ -161,6 +161,8 @@ func testSetup() []func() { "JSONRPCServer.RetryRPCPort=32357", // ...and similarly here... "JSONRPCServer.RetryRPCTTLCompleted=10s", "JSONRPCServer.RetryRPCAckTrim=10ms", + "JSONRPCServer.RetryRPCCertFilePath=", + "JSONRPCServer.RetryRPCKeyFilePath=", "JSONRPCServer.DataPathLogging=false", "JSONRPCServer.MinLeaseDuration=100ms", "JSONRPCServer.LeaseInterruptInterval=100ms", diff --git a/pfsagentd/README.md b/pfsagentd/README.md index d50357b76..4444ba62b 100644 --- a/pfsagentd/README.md +++ b/pfsagentd/README.md @@ -77,6 +77,7 @@ FUSECongestionThreshhold: 0 FUSEMaxWrite: 131072 RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s +RetryRPCCACertFilePath: # Defaults to /dev/null ``` In the above example, some important fields are as follows: diff --git a/pfsagentd/globals.go b/pfsagentd/globals.go index 38995ceef..e832a3c31 100644 --- a/pfsagentd/globals.go +++ b/pfsagentd/globals.go @@ -7,6 +7,7 @@ import ( "container/list" "fmt" "io" + "io/ioutil" "log" "math" "net/http" @@ -73,6 +74,7 @@ type configStruct struct { FUSEMaxWrite uint32 RetryRPCDeadlineIO time.Duration RetryRPCKeepAlivePeriod time.Duration + RetryRPCCACertFilePath string } type retryDelayElementStruct struct { @@ -331,6 +333,7 @@ type globalsStruct struct { sync.Mutex config configStruct logFile *os.File // == nil if configStruct.LogFilePath == "" + retryRPCCACertPEM []byte retryRPCPublicIPAddr string retryRPCPort uint16 retryRPCClient *retryrpc.Client @@ -653,10 +656,24 @@ func initializeGlobals(confMap conf.ConfMap) { logFatal(err) } + globals.config.RetryRPCCACertFilePath, err = confMap.FetchOptionValueString("Agent", "RetryRPCCACertFilePath") + if nil != err { + globals.config.RetryRPCCACertFilePath = "" + } + configJSONified = utils.JSONify(globals.config, true) logInfof("\n%s", configJSONified) + if "" == globals.config.RetryRPCCACertFilePath { + globals.retryRPCCACertPEM = nil + } else { + globals.retryRPCCACertPEM, err = ioutil.ReadFile(globals.config.RetryRPCCACertFilePath) + if nil != err { + logFatal(err) + } + } + globals.entryValidSec, globals.entryValidNSec = nsToUnixTime(uint64(globals.config.EntryDuration)) globals.attrValidSec, globals.attrValidNSec = nsToUnixTime(uint64(globals.config.AttrDuration)) @@ -742,6 +759,7 @@ func uninitializeGlobals() { bucketstats.UnRegister("PFSAgent", "") globals.logFile = nil + globals.retryRPCCACertPEM = nil globals.retryRPCPublicIPAddr = "" globals.retryRPCPort = 0 globals.retryRPCClient = nil diff --git a/pfsagentd/pfsagent.conf b/pfsagentd/pfsagent.conf index a9ae574ea..588bde14f 100644 --- a/pfsagentd/pfsagent.conf +++ b/pfsagentd/pfsagent.conf @@ -43,3 +43,4 @@ FUSECongestionThreshhold: 0 FUSEMaxWrite: 131072 RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s +RetryRPCCACertFilePath: diff --git a/pfsagentd/setup_teardown_test.go b/pfsagentd/setup_teardown_test.go index 552391ca9..5329ac4c9 100644 --- a/pfsagentd/setup_teardown_test.go +++ b/pfsagentd/setup_teardown_test.go @@ -126,6 +126,7 @@ func testSetup(t *testing.T) { "Agent.FUSEMaxWrite=131072", // Linux max... 128KiB is good enough for testing "Agent.RetryRPCDeadlineIO=60s", "Agent.RetryRPCKeepAlivePeriod=60s", + "Agent.RetryRPCCACertFilePath=", "Stats.IPAddr=localhost", "Stats.UDPPort=54324", @@ -240,6 +241,8 @@ func testSetup(t *testing.T) { "JSONRPCServer.RetryRPCPort=54328", "JSONRPCServer.RetryRPCTTLCompleted=10s", "JSONRPCServer.RetryRPCAckTrim=10ms", + "JSONRPCServer.RetryRPCCertFilePath=", + "JSONRPCServer.RetryRPCKeyFilePath=", "JSONRPCServer.DataPathLogging=false", "JSONRPCServer.MinLeaseDuration=250ms", "JSONRPCServer.LeaseInterruptInterval=250ms", diff --git a/proxyfsd/default.conf b/proxyfsd/default.conf index 545ef4510..db7a27d2e 100644 --- a/proxyfsd/default.conf +++ b/proxyfsd/default.conf @@ -221,6 +221,8 @@ RetryRPCTTLCompleted: 10m RetryRPCAckTrim: 100ms RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s +RetryRPCCertFilePath: +RetryRPCKeyFilePath: MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 diff --git a/proxyfsd/rpc_server.conf b/proxyfsd/rpc_server.conf index b44f8531e..fe5ea3323 100644 --- a/proxyfsd/rpc_server.conf +++ b/proxyfsd/rpc_server.conf @@ -10,6 +10,8 @@ RetryRPCTTLCompleted: 10m RetryRPCAckTrim: 100ms RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s +RetryRPCCertFilePath: +RetryRPCKeyFilePath: MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 diff --git a/saio/container/proxyfs.conf b/saio/container/proxyfs.conf index d87ea9e5c..e2ed1596d 100644 --- a/saio/container/proxyfs.conf +++ b/saio/container/proxyfs.conf @@ -193,6 +193,8 @@ RetryRPCTTLCompleted: 10m RetryRPCAckTrim: 100ms RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s +RetryRPCCertFilePath: +RetryRPCKeyFilePath: MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 From 344bfbf659a5b70784746874b7b6d923a49ac74d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 23 Mar 2021 16:17:42 -0700 Subject: [PATCH 075/160] Updates for jrpcfs/lease_test.go Note: This change assumes passing a nil CACert to retryrpc.NewClient() selects TCP rather than TLS --- jrpcfs/lease_test.go | 90 ++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 49 deletions(-) diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index 033f1b2f3..68f33e392 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -57,27 +57,11 @@ func BenchmarkRpcLease(b *testing.B) { mountByAccountNameReply *MountByAccountNameReply retryRPCClient *retryrpc.Client retryrpcClientConfig *retryrpc.ClientConfig - server *Server testRpcLeaseClient *testRpcLeaseClientStruct unmountReply *Reply unmountRequest *UnmountRequest ) - server = &Server{} - - mountByAccountNameRequest = &MountByAccountNameRequest{ - AccountName: testAccountName, - MountOptions: 0, - AuthUserID: 0, - AuthGroupID: 0, - } - mountByAccountNameReply = &MountByAccountNameReply{} - - err = server.RpcMountByAccountName(mountByAccountNameRequest, mountByAccountNameReply) - if nil != err { - b.Fatalf("server.RpcMountByAccountName(AccountName=\"%s\",) failed: %v", mountByAccountNameRequest.AccountName, err) - } - deadlineIO, err = time.ParseDuration(testRpcLeaseRetryRPCDeadlineIO) if nil != err { b.Fatalf("time.ParseDuration(\"%s\") failed: %v", testRpcLeaseRetryRPCDeadlineIO, err) @@ -88,10 +72,9 @@ func BenchmarkRpcLease(b *testing.B) { } retryrpcClientConfig = &retryrpc.ClientConfig{ - MyUniqueID: string(mountByAccountNameReply.MountID), - IPAddr: mountByAccountNameReply.RetryRPCPublicIPAddr, - Port: int(mountByAccountNameReply.RetryRPCPort), - RootCAx509CertificatePEM: mountByAccountNameReply.RootCAx509CertificatePEM, + IPAddr: globals.publicIPAddr, + Port: int(globals.retryRPCPort), + RootCAx509CertificatePEM: nil, Callbacks: testRpcLeaseClient, DeadlineIO: deadlineIO, KeepAlivePeriod: keepAlivePeriod, @@ -102,6 +85,19 @@ func BenchmarkRpcLease(b *testing.B) { b.Fatalf("retryrpc.NewClient() failed: %v", err) } + mountByAccountNameRequest = &MountByAccountNameRequest{ + AccountName: testAccountName, + MountOptions: 0, + AuthUserID: 0, + AuthGroupID: 0, + } + mountByAccountNameReply = &MountByAccountNameReply{} + + err = retryRPCClient.Send("RpcMountByAccountName", mountByAccountNameRequest, mountByAccountNameReply) + if nil != err { + b.Fatalf("retryRPCClient.Send(\"RpcMountByAccountName\",,) failed: %v", err) + } + b.ResetTimer() for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { @@ -144,17 +140,17 @@ func BenchmarkRpcLease(b *testing.B) { b.StopTimer() - retryRPCClient.Close() - unmountRequest = &UnmountRequest{ MountID: mountByAccountNameReply.MountID, } unmountReply = &Reply{} - err = server.RpcUnmount(unmountRequest, unmountReply) + err = retryRPCClient.Send("RpcUnmount", unmountRequest, unmountReply) if nil != err { - b.Fatalf("server.RpcUnmount(MountID=\"%s\",) failed: %v", unmountRequest.MountID, err) + b.Fatalf("retryRPCClient.Send(\"RpcUnmount\",,) failed: %v", err) } + + retryRPCClient.Close() } func TestRpcLease(t *testing.T) { @@ -466,26 +462,10 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { ok bool retryRPCClient *retryrpc.Client retryrpcClientConfig *retryrpc.ClientConfig - server *Server unmountReply *Reply unmountRequest *UnmountRequest ) - server = &Server{} - - mountByAccountNameRequest = &MountByAccountNameRequest{ - AccountName: testAccountName, - MountOptions: 0, - AuthUserID: 0, - AuthGroupID: 0, - } - mountByAccountNameReply = &MountByAccountNameReply{} - - err = server.RpcMountByAccountName(mountByAccountNameRequest, mountByAccountNameReply) - if nil != err { - testRpcLeaseClient.Fatalf("server.RpcMountByAccountName(AccountName=\"%s\",) failed: %v", mountByAccountNameRequest.AccountName, err) - } - deadlineIO, err = time.ParseDuration(testRpcLeaseRetryRPCDeadlineIO) if nil != err { testRpcLeaseClient.Fatalf("time.ParseDuration(\"%s\") failed: %v", testRpcLeaseRetryRPCDeadlineIO, err) @@ -496,10 +476,9 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { } retryrpcClientConfig = &retryrpc.ClientConfig{ - MyUniqueID: string(mountByAccountNameReply.MountID), - IPAddr: mountByAccountNameReply.RetryRPCPublicIPAddr, - Port: int(mountByAccountNameReply.RetryRPCPort), - RootCAx509CertificatePEM: mountByAccountNameReply.RootCAx509CertificatePEM, + IPAddr: globals.publicIPAddr, + Port: int(globals.retryRPCPort), + RootCAx509CertificatePEM: nil, Callbacks: testRpcLeaseClient, DeadlineIO: deadlineIO, KeepAlivePeriod: keepAlivePeriod, @@ -510,6 +489,19 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { testRpcLeaseClient.Fatalf("retryrpc.NewClient() failed: %v", err) } + mountByAccountNameRequest = &MountByAccountNameRequest{ + AccountName: testAccountName, + MountOptions: 0, + AuthUserID: 0, + AuthGroupID: 0, + } + mountByAccountNameReply = &MountByAccountNameReply{} + + err = retryRPCClient.Send("RpcMountByAccountName", mountByAccountNameRequest, mountByAccountNameReply) + if nil != err { + testRpcLeaseClient.Fatalf("retryRPCClient.Send(\"RpcMountByAccountName\",,) failed: %v", err) + } + for { leaseRequestType, ok = <-testRpcLeaseClient.chIn @@ -534,24 +526,24 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { testRpcLeaseClient.chOut <- leaseReply.LeaseReplyType } else { - retryRPCClient.Close() - unmountRequest = &UnmountRequest{ MountID: mountByAccountNameReply.MountID, } unmountReply = &Reply{} - err = server.RpcUnmount(unmountRequest, unmountReply) + err = retryRPCClient.Send("RpcUnmount", unmountRequest, unmountReply) if testRpcLeaseClient.alreadyUnmounted { if nil == err { - testRpcLeaseClient.Fatalf("server.RpcUnmount(MountID=\"%s\",) should have failed", unmountRequest.MountID) + testRpcLeaseClient.Fatalf("retryRPCClient.Send(\"RpcUnmount\",,) should have failed") } } else { if nil != err { - testRpcLeaseClient.Fatalf("server.RpcUnmount(MountID=\"%s\",) failed: %v", unmountRequest.MountID, err) + testRpcLeaseClient.Fatalf("retryRPCClient.Send(\"RpcUnmount\",,) failed: %v", err) } } + retryRPCClient.Close() + testRpcLeaseClient.wg.Done() runtime.Goexit() From fd9a09da5c3761c7d8252b7c26e40d2a6cf893c9 Mon Sep 17 00:00:00 2001 From: Bob Schatz Date: Wed, 24 Mar 2021 11:09:23 -0700 Subject: [PATCH 076/160] Cleanup code --- retryrpc/retryrpc_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index c21edf828..aa58a78e9 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -29,8 +29,8 @@ func getNewServer(lt time.Duration, dontStartTrimmers bool) (rrSvr *Server, ip s ipaddr = "127.0.0.1" port = 24456 ) - config := &ServerConfig{LongTrim: lt, ShortTrim: 100 * time.Millisecond, IPAddr: "127.0.0.1", - Port: 24456, DeadlineIO: 5 * time.Second, dontStartTrimmers: dontStartTrimmers} + config := &ServerConfig{LongTrim: lt, ShortTrim: 100 * time.Millisecond, IPAddr: ipaddr, + Port: port, DeadlineIO: 5 * time.Second, dontStartTrimmers: dontStartTrimmers} // Create a new RetryRPC Server. Completed request will live on // completedRequests for 10 seconds. From b440913ffd14261fcb929b1c91e7d2da6058aa79 Mon Sep 17 00:00:00 2001 From: Bob Schatz Date: Wed, 24 Mar 2021 17:32:34 -0700 Subject: [PATCH 077/160] Used passed credentials and provide helper method to create in unit tests --- jrpcfs/config.go | 3 +-- jrpcfs/lease_test.go | 2 +- jrpcfs/retryrpc.go | 35 ++++++++++++++++++++++++----- retryrpc/api.go | 47 +++++++++++++++++++++++++++------------ retryrpc/api_internal.go | 2 +- retryrpc/retryrpc_test.go | 6 +++++ 6 files changed, 71 insertions(+), 24 deletions(-) diff --git a/jrpcfs/config.go b/jrpcfs/config.go index 7841b694e..0deb35022 100644 --- a/jrpcfs/config.go +++ b/jrpcfs/config.go @@ -306,8 +306,7 @@ func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { ioServerUp(globals.privateIPAddr, globals.fastPortString) // Init Retry RPC server - retryRPCServerUp(jserver, globals.publicIPAddr, globals.retryRPCPort, globals.retryRPCTTLCompleted, globals.retryRPCAckTrim, - globals.retryRPCDeadlineIO, globals.retryRPCKeepAlivePeriod) + retryRPCServerUp(jserver) err = nil return diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index 68f33e392..24eeaf83a 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -478,7 +478,7 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { retryrpcClientConfig = &retryrpc.ClientConfig{ IPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), - RootCAx509CertificatePEM: nil, + RootCAx509CertificatePEM: globals.retryrpcSvr.Creds.RootCAx509CertificatePEM, // Use same CA as server Callbacks: testRpcLeaseClient, DeadlineIO: deadlineIO, KeepAlivePeriod: keepAlivePeriod, diff --git a/jrpcfs/retryrpc.go b/jrpcfs/retryrpc.go index 850b43b33..b7c8b1dcc 100644 --- a/jrpcfs/retryrpc.go +++ b/jrpcfs/retryrpc.go @@ -4,15 +4,14 @@ package jrpcfs import ( - "time" + "crypto/tls" + "fmt" "github.com/NVIDIA/proxyfs/logger" "github.com/NVIDIA/proxyfs/retryrpc" ) -func retryRPCServerUp(jserver *Server, publicIPAddr string, retryRPCPort uint16, - retryRPCTTLCompleted time.Duration, retryRPCAckTrim time.Duration, - retryRPCDeadlineIO time.Duration, retryRPCKeepAlivePeriod time.Duration) { +func retryRPCServerUp(jserver *Server) { var err error @@ -21,8 +20,32 @@ func retryRPCServerUp(jserver *Server, publicIPAddr string, retryRPCPort uint16, } // Create a new RetryRPC Server. - retryConfig := &retryrpc.ServerConfig{LongTrim: retryRPCTTLCompleted, ShortTrim: retryRPCAckTrim, IPAddr: publicIPAddr, - Port: int(retryRPCPort), DeadlineIO: retryRPCDeadlineIO, KeepAlivePeriod: retryRPCKeepAlivePeriod} + retryConfig := &retryrpc.ServerConfig{LongTrim: globals.retryRPCTTLCompleted, ShortTrim: globals.retryRPCAckTrim, + IPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), DeadlineIO: globals.retryRPCDeadlineIO, + KeepAlivePeriod: globals.retryRPCKeepAlivePeriod} + + // Retrive credentials from globals if they are valid. + // + // If the credentials are invalid, create self-signed certifcate and log message + if globals.rootCAx509CertificatePEM == nil || globals.retryRPCCertPEM == nil || globals.retryRPCKeyPEM == nil { + var ( + constructErr error + ) + retryConfig.ServerCreds, constructErr = retryrpc.ConstructCreds(globals.publicIPAddr) + if constructErr != nil { + logger.PanicfWithError(constructErr, "Unable to create ServerCreds") + } + logger.Infof("Invalid credentials passed retryrpc - creating self-signed. Passed rootCAx509CertificatePEM: %v retryRPCCertPEM: %v retryRPCKeyPEM: %v", + globals.rootCAx509CertificatePEM, globals.retryRPCCertPEM, globals.retryRPCKeyPEM) + } else { + svrCert, tmpErr := tls.X509KeyPair(globals.retryRPCCertPEM, globals.retryRPCKeyPEM) + if tmpErr != nil { + pMsg := fmt.Errorf("tls.X509KeyPair failed with error: %v", tmpErr) + panic(pMsg) + + } + retryConfig.ServerCreds = &retryrpc.ServerCreds{RootCAx509CertificatePEM: globals.rootCAx509CertificatePEM, ServerTLSCertificate: svrCert} + } rrSvr := retryrpc.NewServer(retryConfig) diff --git a/retryrpc/api.go b/retryrpc/api.go index c6bbf522f..c365dde9d 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -31,7 +31,7 @@ import ( // server CA type ServerCreds struct { RootCAx509CertificatePEM []byte - serverTLSCertificate tls.Certificate + ServerTLSCertificate tls.Certificate } // Server tracks the state of the server @@ -72,6 +72,7 @@ type ServerConfig struct { Port int // Port that Server uses to listen DeadlineIO time.Duration // How long I/Os on sockets wait even if idle KeepAlivePeriod time.Duration // How frequently a KEEPALIVE is sent + ServerCreds *ServerCreds // Credentials dontStartTrimmers bool // Used for testing } @@ -80,23 +81,38 @@ func NewServer(config *ServerConfig) *Server { var ( err error ) + server := &Server{ipaddr: config.IPAddr, port: config.Port, completedLongTTL: config.LongTrim, completedAckTrim: config.ShortTrim, deadlineIO: config.DeadlineIO, - keepAlivePeriod: config.KeepAlivePeriod, dontStartTrimmers: config.dontStartTrimmers} + keepAlivePeriod: config.KeepAlivePeriod, dontStartTrimmers: config.dontStartTrimmers, + Creds: config.ServerCreds} server.svrMap = make(map[string]*methodArgs) server.perClientInfo = make(map[uint64]*clientInfo) server.completedTickerDone = make(chan bool) server.connections = list.New() - server.Creds, err = constructServerCreds(server.ipaddr) - if err != nil { - logger.Errorf("Construction of server credentials failed with err: %v", err) + if config.ServerCreds == nil { + logger.Errorf("config.ServerCreds is nil") panic(err) + } return server } +// ConstructCreds is used by unit tests to create self-signed credentials in the +// ServerConfig structure before passing the struct to NewServer +// +// Only useful for unit tests +func ConstructCreds(ipaddr string) (creds *ServerCreds, err error) { + creds, err = constructServerCreds(ipaddr) + if err != nil { + logger.Errorf("Construction of server credentials failed with err: %v", err) + panic(err) + } + return +} + // Register creates the map of server methods func (server *Server) Register(retrySvr interface{}) (err error) { @@ -111,7 +127,7 @@ func (server *Server) Start() (err error) { hostPortStr := net.JoinHostPort(server.ipaddr, portStr) tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{server.Creds.serverTLSCertificate}, + Certificates: []tls.Certificate{server.Creds.ServerTLSCertificate}, } listenConfig := &net.ListenConfig{KeepAlive: server.keepAlivePeriod} @@ -278,13 +294,12 @@ const ( ) type connectionTracker struct { - state clientState - genNum uint64 // Generation number of tlsConn - avoid racing recoveries - tlsConfig *tls.Config - tlsConn *tls.Conn // Our connection to the server - x509CertPool *x509.CertPool - rootCAx509CertificatePEM []byte - hostPortStr string + state clientState + genNum uint64 // Generation number of tlsConn - avoid racing recoveries + tlsConfig *tls.Config + tlsConn *tls.Conn // Our connection to the server + x509CertPool *x509.CertPool + hostPortStr string } // Client tracking structure @@ -341,14 +356,18 @@ func NewClient(config *ClientConfig) (client *Client, err error) { client = &Client{cb: config.Callbacks, keepAlivePeriod: config.KeepAlivePeriod, deadlineIO: config.DeadlineIO} + if config.RootCAx509CertificatePEM == nil { + err = fmt.Errorf("config.RootCAx509CertificatePEM cannot be nil") + return + } portStr := fmt.Sprintf("%d", config.Port) client.connection.state = INITIAL client.connection.hostPortStr = net.JoinHostPort(config.IPAddr, portStr) client.outstandingRequest = make(map[requestID]*reqCtx) - client.connection.x509CertPool = x509.NewCertPool() client.bt = btree.New(2) // Add cert for root CA to our pool + client.connection.x509CertPool = x509.NewCertPool() ok := client.connection.x509CertPool.AppendCertsFromPEM(config.RootCAx509CertificatePEM) if !ok { err = fmt.Errorf("x509CertPool.AppendCertsFromPEM() returned !ok") diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index e76f44931..16674fa6b 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -417,7 +417,7 @@ func constructServerCreds(serverIPAddrAsString string) (serverCreds *ServerCreds serverEd25519PrivateKeyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: serverEd25519PrivateKeyDER}) - serverCreds.serverTLSCertificate, err = tls.X509KeyPair(serverX509CertificatePEM, serverEd25519PrivateKeyPEM) + serverCreds.ServerTLSCertificate, err = tls.X509KeyPair(serverX509CertificatePEM, serverEd25519PrivateKeyPEM) return } diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index aa58a78e9..195830ebc 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/logger" "github.com/NVIDIA/proxyfs/retryrpc/rpctest" "github.com/stretchr/testify/assert" ) @@ -28,9 +29,14 @@ func getNewServer(lt time.Duration, dontStartTrimmers bool) (rrSvr *Server, ip s var ( ipaddr = "127.0.0.1" port = 24456 + err error ) config := &ServerConfig{LongTrim: lt, ShortTrim: 100 * time.Millisecond, IPAddr: ipaddr, Port: port, DeadlineIO: 5 * time.Second, dontStartTrimmers: dontStartTrimmers} + config.ServerCreds, err = ConstructCreds(ipaddr) + if err != nil { + logger.PanicfWithError(err, "Unable to create ServerCreds") + } // Create a new RetryRPC Server. Completed request will live on // completedRequests for 10 seconds. From 030258b2a74ac13c3c033fa7fc7b4bc30199f5be Mon Sep 17 00:00:00 2001 From: Bob Schatz Date: Wed, 24 Mar 2021 18:54:10 -0700 Subject: [PATCH 078/160] Cleanup comments --- jrpcfs/lease_test.go | 2 +- retryrpc/client.go | 58 +++++++++++++++++--------------------------- retryrpc/server.go | 32 ++++++++++++------------ 3 files changed, 40 insertions(+), 52 deletions(-) diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index 24eeaf83a..e37a05e3e 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -74,7 +74,7 @@ func BenchmarkRpcLease(b *testing.B) { retryrpcClientConfig = &retryrpc.ClientConfig{ IPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), - RootCAx509CertificatePEM: nil, + RootCAx509CertificatePEM: globals.retryrpcSvr.Creds.RootCAx509CertificatePEM, // Use same CA as server Callbacks: testRpcLeaseClient, DeadlineIO: deadlineIO, KeepAlivePeriod: keepAlivePeriod, diff --git a/retryrpc/client.go b/retryrpc/client.go index 00c6077f6..8ef287d3d 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -295,7 +295,7 @@ func (client *Client) readReplies(callingGenNum uint64, tlsConn *tls.Conn) { // // We consider a connection to be idle if we have no outstanding requests when // we get the timeout. Otherwise, we call retransmit. - if os.IsTimeout(getErr) == true && localCnt == 0 { + if os.IsTimeout(getErr) && localCnt == 0 { continue } @@ -354,7 +354,7 @@ func (client *Client) retransmit(genNum uint64) { return } - if client.halting == true { + if client.halting { client.Unlock() return } @@ -528,46 +528,32 @@ func (client *Client) reDial() (err error) { // // NOTE: Client lock is held func (client *Client) readClientID(callingGenNum uint64, tlsConn *tls.Conn) (myUniqueID uint64, err error) { - var localCnt int - - for { - // Wait reply from server - buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, tlsConn) + // Wait reply from server + buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, tlsConn) - // This must happen before checking error - if client.halting { - return - } - - // Ignore timeouts on idle connections while reading header - // - // We consider a connection to be idle if we have no outstanding requests when - // we get the timeout. Otherwise, we call retransmit. - if os.IsTimeout(getErr) && localCnt == 0 { - continue - } - - // TODO - don't loop forever - breakout after some many tries - - // Upper layer will close connection as a result of any error - if getErr != nil { - err = getErr - return - } + // This must happen before checking error + if client.halting { + return + } - if msgType != ReturnUniqueID { - err = fmt.Errorf("CLIENT - invalid msgType: %v", msgType) - logger.PanicfWithError(err, "") - return - } + // Upper layer will close connection as a result of any error + if getErr != nil { + err = getErr + return + } - err = json.Unmarshal(buf, &myUniqueID) - if err != nil { - logger.PanicfWithError(err, "Unmarshal of buf: %v to myUniqueID failed with err: %v", buf, err) - } + if msgType != ReturnUniqueID { + err = fmt.Errorf("CLIENT - invalid msgType: %v", msgType) + logger.PanicfWithError(err, "") return } + + err = json.Unmarshal(buf, &myUniqueID) + if err != nil { + logger.PanicfWithError(err, "Unmarshal of buf: %v to myUniqueID failed with err: %v", buf, err) + } + return } // initialDial does the initial dial of the server and retrieves the diff --git a/retryrpc/server.go b/retryrpc/server.go index 22ee6e97a..694375762 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -55,17 +55,18 @@ func (server *Server) run() { elm := server.connections.PushBack(conn) server.connLock.Unlock() - // TODO - update comment per two cases initialDial() and reDial().... - - // The first message sent on this socket by the client is the uniqueID. + // The server first calculates a uniqueID for this client and writes it back + // on the socket. // - // Read that first and create the relevant structures before calling + // Create the relevant data structures and write out the uniqueID before calling // serviceClient(). We do the cleanup in this routine because there are // race conditions with noisy clients repeatedly reconnecting. // // Those race conditions are resolved if we serialize the recovery. cCtx := &connCtx{conn: conn} cCtx.cond = sync.NewCond(&cCtx.Mutex) + + // TODO move getClientIDAndWait() in the goroutine... ci, err := server.getClientIDAndWait(cCtx) if err != nil { // Socket already had an error - just loop back @@ -186,7 +187,12 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b // // getClientIDAndWait reads the first message off the new connection. -// The client sends its uniqueID before sending it's first RPC. +// +// If the client is new, it will ask for a UniqueID. This routine will +// generate the new UniqueID and return it. +// +// If the client is existing, it will send it's UniqueID so that we +// can use that to look up it's data structures. // // Then getClientIDAndWait waits until any outstanding RPCs on a prior // connection have completed before proceeding. @@ -194,17 +200,12 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b // This avoids race conditions when there are cascading retransmits. // The algorithm is: // -// 1. Client sends UniqueID to server when the socket is first open +// 1. Client sends UniqueID to server when the connection is reestablished. // 2. After accepting new socket, the server waits for the UniqueID from // the client. -// 3. If this is the first connection from the client, the server creates a -// clientInfo structure for the client and proceeds to wait for RPCs. -// 4. If this is a client returning on a new socket, the server blocks +// 3. If this is a client returning on a new socket, the server blocks // until all outstanding RPCs and related goroutines have completed for the // client on the previous connection. -// 5. Additionally, the server blocks on accepting new connections -// (which could be yet another reconnect for the same client) until the -// previous connection has closed down. func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err error) { buf, msgType, getErr := getIO(uint64(0), server.deadlineIO, cCtx.conn) if getErr != nil { @@ -219,6 +220,8 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err } if msgType == PassID { + + // Returning client var connUniqueID uint64 err = json.Unmarshal(buf, &connUniqueID) if err != nil { @@ -226,7 +229,6 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err return } - // Check if this is the first time we have seen this client server.Lock() lci, ok := server.perClientInfo[connUniqueID] if !ok { @@ -325,8 +327,8 @@ func (server *Server) serviceClient(ci *clientInfo, cCtx *connCtx) { } if msgType != RPC { - // TODO - panic? log message.... - fmt.Printf("serviceClient() received invalid msgType: %v - dropping\n", msgType) + err := fmt.Errorf("serviceClient() received invalid msgType: %v - dropping", msgType) + logger.PanicfWithError(err, "") continue } From 638cf16887a869516e2e7ddd5997b0f2b35483e5 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 25 Mar 2021 09:27:55 -0700 Subject: [PATCH 079/160] Updated icert/icertpkg to work with both []byte and file path (string) certs and keys --- icert/icertpkg/api.go | 34 ++++++----- icert/icertpkg/api_test.go | 83 +++++++++++++++------------ icert/icertpkg/impl.go | 114 ++++++++++++++++++++++++------------- 3 files changed, 142 insertions(+), 89 deletions(-) diff --git a/icert/icertpkg/api.go b/icert/icertpkg/api.go index 5e317dffd..40617e0c5 100644 --- a/icert/icertpkg/api.go +++ b/icert/icertpkg/api.go @@ -38,26 +38,32 @@ const ( // GenCACert is called to generate a Certificate Authority using the requested // generateKeyAlgorithm for the specified subject who's validity last for the -// desired ttl starting from time.Now(). The resultant PEM-encoded CA Certificate -// is written to certFile. The PEM-encoded private key for the CA Certificate is -// written to keyFile. If certFile and keyFile are the same, both the CA Certificate -// and its private key will be written to the common file. +// desired ttl starting from time.Now(). The resultant PEM-encoded certPEMBlock +// and keyPEMBlock are returned. Optionally, certPEMBlock will be written to +// certFile (if not "") and/or keyPEMBlock will be written to keyFile (if not ""). +// If certFile and keyFile are the same, both the CA Certificate and its Private +// Key will be written to the common file. // -func GenCACert(generateKeyAlgorithm string, subject pkix.Name, ttl time.Duration, certFile string, keyFile string) (err error) { - return genCACert(generateKeyAlgorithm, subject, ttl, certFile, keyFile) +func GenCACert(generateKeyAlgorithm string, subject pkix.Name, ttl time.Duration, certFile string, keyFile string) (certPEMBlock []byte, keyPEMBlock []byte, err error) { + certPEMBlock, keyPEMBlock, err = genCACert(generateKeyAlgorithm, subject, ttl, certFile, keyFile) + return } // GenEndpointCert is called to generate a Certificate using the requested // generateKeyAlgorithm for the specified subject who's validity lasts for // the desired ttl starting from time.Now(). The endpoints for which the // Certificate will apply will be the specified dnsNames and/or ipAddresses. -// The Certificate will be signed by the CA Certificate specified via caCertFile -// and caKeyFile. The caCertFile and caKeyFile values may be identical. The -// resultant PEM-encoded Certificate is written to certFile. The PEM-encoded -// private key for the Certificate is written to keyFile. If certFile and -// keyFile are the same, both the Certificate and its private key will be -// written to the common file. +// The Certificate will be signed by the CA Certificate specified via +// caCert and caKey. Note that caCert and caKey may either be PEMBlock +// byte slices ([]byte) or file path strings. If caCert and caKey are both +// strings, they may refer to a common file. The resulted PEM-encoded +// endpoingCertPEMBlock and endpointKeyPEMBlock are returned. Optionally, +// endpoingCertPEMBlock will be written to endpointCertFile (if not "") +// and/or endpointKeyPEMBlock will be written to endpointKeyFile (if not ""). +// If endpointCertFile and endpointKeyFile are the same, both the Endpoint +// Certificate and its Private Key will be written to the common file. // -func GenEndpointCert(generateKeyAlgorithm string, subject pkix.Name, dnsNames []string, ipAddresses []net.IP, ttl time.Duration, caCertFile string, caKeyFile string, endpointCertFile string, endpointKeyFile string) (err error) { - return genEndpointCert(generateKeyAlgorithm, subject, dnsNames, ipAddresses, ttl, caCertFile, caKeyFile, endpointCertFile, endpointKeyFile) +func GenEndpointCert(generateKeyAlgorithm string, subject pkix.Name, dnsNames []string, ipAddresses []net.IP, ttl time.Duration, caCert interface{}, caKey interface{}, endpointCertFile string, endpointKeyFile string) (endpointCertPEMBlock []byte, endpointKeyPEMBlock []byte, err error) { + endpointCertPEMBlock, endpointKeyPEMBlock, err = genEndpointCert(generateKeyAlgorithm, subject, dnsNames, ipAddresses, ttl, caCert, caKey, endpointCertFile, endpointKeyFile) + return } diff --git a/icert/icertpkg/api_test.go b/icert/icertpkg/api_test.go index 07b322fd3..129ade989 100644 --- a/icert/icertpkg/api_test.go +++ b/icert/icertpkg/api_test.go @@ -59,23 +59,26 @@ func TestRSACombinedCertAndKeyFile(t *testing.T) { func testAPI(t *testing.T, generateKeyAlgorithm string, combined bool) { var ( - caCertPEM []byte - caCertPemFilePath string - caCertPool *x509.CertPool - caKeyPemFilePath string - clientErr error - clientTLSConfig *tls.Config - clientWG sync.WaitGroup - endpointCertPemFilePath string - endpointKeyPemFilePath string - err error - ipAddressPort string - ok bool - tempDir string - serverErr error - serverNetListener net.Listener - serverTLSCertificate tls.Certificate - serverWG sync.WaitGroup + caCertPEM []byte + caCertPEMFilePath string + caCertPool *x509.CertPool + // caKeyPEM []byte + caKeyPEMFilePath string + clientErr error + clientTLSConfig *tls.Config + clientWG sync.WaitGroup + // endpointCertPEM []byte + endpointCertPEMFilePath string + // endpointKeyPEM []byte + endpointKeyPEMFilePath string + err error + ipAddressPort string + ok bool + tempDir string + serverErr error + serverNetListener net.Listener + serverTLSCertificate tls.Certificate + serverWG sync.WaitGroup ) tempDir, err = ioutil.TempDir("", testTempDirPattern) @@ -94,18 +97,19 @@ func testAPI(t *testing.T, generateKeyAlgorithm string, combined bool) { }(t, tempDir) if combined { - caCertPemFilePath = filepath.Join(tempDir, testCACombinedPEMFileName) - caKeyPemFilePath = caCertPemFilePath - endpointCertPemFilePath = filepath.Join(tempDir, testIPAddressCombinedPEMFileName) - endpointKeyPemFilePath = endpointCertPemFilePath + caCertPEMFilePath = filepath.Join(tempDir, testCACombinedPEMFileName) + caKeyPEMFilePath = caCertPEMFilePath + endpointCertPEMFilePath = filepath.Join(tempDir, testIPAddressCombinedPEMFileName) + endpointKeyPEMFilePath = endpointCertPEMFilePath } else { - caCertPemFilePath = filepath.Join(tempDir, testCACertPEMFileName) - caKeyPemFilePath = filepath.Join(tempDir, testCAKeyPEMFileName) - endpointCertPemFilePath = filepath.Join(tempDir, testIPAddressCertPEMFileName) - endpointKeyPemFilePath = filepath.Join(tempDir, testIPAddressKeyPEMFileName) + caCertPEMFilePath = filepath.Join(tempDir, testCACertPEMFileName) + caKeyPEMFilePath = filepath.Join(tempDir, testCAKeyPEMFileName) + endpointCertPEMFilePath = filepath.Join(tempDir, testIPAddressCertPEMFileName) + endpointKeyPEMFilePath = filepath.Join(tempDir, testIPAddressKeyPEMFileName) } - err = GenCACert( + _, _, err = GenCACert( + // caCertPEM, caKeyPEM, err = GenCACert( generateKeyAlgorithm, pkix.Name{ Organization: []string{testOrganizationCA}, @@ -116,13 +120,14 @@ func testAPI(t *testing.T, generateKeyAlgorithm string, combined bool) { PostalCode: []string{}, }, testCertificateTTL, - caCertPemFilePath, - caKeyPemFilePath) + caCertPEMFilePath, + caKeyPEMFilePath) if nil != err { t.Fatalf("GenCACert() failed: %v", err) } - err = genEndpointCert( + _, _, err = genEndpointCert( + // endpointCertPEM, endpointKeyPEM, err = genEndpointCert( generateKeyAlgorithm, pkix.Name{ Organization: []string{testOrganizationEndpoint}, @@ -135,10 +140,12 @@ func testAPI(t *testing.T, generateKeyAlgorithm string, combined bool) { []string{testV4DomainName, testV6DomainName}, []net.IP{net.ParseIP(testIPv4Address), net.ParseIP(testIPv6Address)}, testCertificateTTL, - caCertPemFilePath, - caKeyPemFilePath, - endpointCertPemFilePath, - endpointKeyPemFilePath) + caCertPEMFilePath, + caKeyPEMFilePath, + // caCertPEM, + // caKeyPEM, + endpointCertPEMFilePath, + endpointKeyPEMFilePath) if nil != err { t.Fatalf("genEndpointCert() failed: %v", err) } @@ -148,19 +155,23 @@ func testAPI(t *testing.T, generateKeyAlgorithm string, combined bool) { ipAddressPort = net.JoinHostPort(testIPv4Address, testTLSPort) // ipAddressPort = net.JoinHostPort(testIPv6Address, testTLSPort) - serverTLSCertificate, err = tls.LoadX509KeyPair(endpointCertPemFilePath, endpointKeyPemFilePath) + serverTLSCertificate, err = tls.LoadX509KeyPair(endpointCertPEMFilePath, endpointKeyPEMFilePath) if nil != err { t.Fatalf("tls.LoadX509KeyPair() failed: %v", err) } + // serverTLSCertificate, err = tls.X509KeyPair(endpointCertPEM, endpointKeyPEM) + // if nil != err { + // t.Fatalf("tls.LoadX509KeyPair() failed: %v", err) + // } serverNetListener, err = tls.Listen("tcp", ipAddressPort, &tls.Config{Certificates: []tls.Certificate{serverTLSCertificate}}) if nil != err { t.Fatalf("tls.Listen() failed: %v", err) } - caCertPEM, err = ioutil.ReadFile(caCertPemFilePath) + caCertPEM, err = ioutil.ReadFile(caCertPEMFilePath) if nil != err { - t.Fatalf("ioutil.ReadFile(caCertPemFilePath) failed: %v", err) + t.Fatalf("ioutil.ReadFile(caCertPEMFilePath) failed: %v", err) } caCertPool = x509.NewCertPool() diff --git a/icert/icertpkg/impl.go b/icert/icertpkg/impl.go index 05e5f4a27..cc08753b0 100644 --- a/icert/icertpkg/impl.go +++ b/icert/icertpkg/impl.go @@ -19,14 +19,12 @@ import ( "time" ) -func genCACert(generateKeyAlgorithm string, subject pkix.Name, ttl time.Duration, certFile string, keyFile string) (err error) { +func genCACert(generateKeyAlgorithm string, subject pkix.Name, ttl time.Duration, certFile string, keyFile string) (certPEMBlock []byte, keyPEMBlock []byte, err error) { var ( caX509Certificate []byte caX509CertificateTemplate *x509.Certificate - certPEM []byte ed25519PrivateKey ed25519.PrivateKey ed25519PublicKey ed25519.PublicKey - keyPEM []byte pkcs8PrivateKey []byte rsaPrivateKey *rsa.PrivateKey rsaPublicKey crypto.PublicKey @@ -93,22 +91,28 @@ func genCACert(generateKeyAlgorithm string, subject pkix.Name, ttl time.Duration return } - certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caX509Certificate}) - keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8PrivateKey}) + certPEMBlock = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caX509Certificate}) + keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8PrivateKey}) - if certFile == keyFile { - err = ioutil.WriteFile(certFile, append(certPEM, keyPEM...), GeneratedFilePerm) - if nil != err { - return - } - } else { - err = ioutil.WriteFile(certFile, certPEM, GeneratedFilePerm) - if nil != err { - return - } - err = ioutil.WriteFile(keyFile, keyPEM, GeneratedFilePerm) - if nil != err { - return + if ("" != certFile) || ("" != keyFile) { + if certFile == keyFile { + err = ioutil.WriteFile(certFile, append(certPEMBlock, keyPEMBlock...), GeneratedFilePerm) + if nil != err { + return + } + } else { + if "" != certFile { + err = ioutil.WriteFile(certFile, certPEMBlock, GeneratedFilePerm) + if nil != err { + return + } + } + if "" != keyFile { + err = ioutil.WriteFile(keyFile, keyPEMBlock, GeneratedFilePerm) + if nil != err { + return + } + } } } @@ -116,14 +120,17 @@ func genCACert(generateKeyAlgorithm string, subject pkix.Name, ttl time.Duration return } -func genEndpointCert(generateKeyAlgorithm string, subject pkix.Name, dnsNames []string, ipAddresses []net.IP, ttl time.Duration, caCertFile string, caKeyFile string, endpointCertFile string, endpointKeyFile string) (err error) { +func genEndpointCert(generateKeyAlgorithm string, subject pkix.Name, dnsNames []string, ipAddresses []net.IP, ttl time.Duration, caCert interface{}, caKey interface{}, endpointCertFile string, endpointKeyFile string) (endpointCertPEMBlock []byte, endpointKeyPEMBlock []byte, err error) { var ( + caCertAsByteSlice []byte + caCertAsString string + caKeyAsByteSlice []byte + caKeyAsString string caTLSCertificate tls.Certificate caX509Certificate *x509.Certificate - certPEM []byte ed25519PrivateKey ed25519.PrivateKey ed25519PublicKey ed25519.PublicKey - keyPEM []byte + ok bool pkcs8PrivateKey []byte rsaPrivateKey *rsa.PrivateKey rsaPublicKey crypto.PublicKey @@ -156,9 +163,32 @@ func genEndpointCert(generateKeyAlgorithm string, subject pkix.Name, dnsNames [] BasicConstraintsValid: true, } - caTLSCertificate, err = tls.LoadX509KeyPair(caCertFile, caKeyFile) - if nil != err { - return + caCertAsByteSlice, ok = caCert.([]byte) + if ok { + caKeyAsByteSlice, ok = caKey.([]byte) + if !ok { + err = fmt.Errorf("if caCert is a []byte, caKey must also be a []byte") + return + } + caTLSCertificate, err = tls.X509KeyPair(caCertAsByteSlice, caKeyAsByteSlice) + if nil != err { + return + } + } else { + caCertAsString, ok = caCert.(string) + if !ok { + err = fmt.Errorf("caCert must be either a []byte or a string") + return + } + caKeyAsString, ok = caKey.(string) + if !ok { + err = fmt.Errorf("if caCert is a string, caKey must also be a string") + return + } + caTLSCertificate, err = tls.LoadX509KeyPair(caCertAsString, caKeyAsString) + if nil != err { + return + } } caX509Certificate, err = x509.ParseCertificate(caTLSCertificate.Certificate[0]) @@ -203,22 +233,28 @@ func genEndpointCert(generateKeyAlgorithm string, subject pkix.Name, dnsNames [] return } - certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: x509Certificate}) - keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8PrivateKey}) + endpointCertPEMBlock = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: x509Certificate}) + endpointKeyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: pkcs8PrivateKey}) - if endpointCertFile == endpointKeyFile { - err = ioutil.WriteFile(endpointCertFile, append(certPEM, keyPEM...), GeneratedFilePerm) - if nil != err { - return - } - } else { - err = ioutil.WriteFile(endpointCertFile, certPEM, GeneratedFilePerm) - if nil != err { - return - } - err = ioutil.WriteFile(endpointKeyFile, keyPEM, GeneratedFilePerm) - if nil != err { - return + if ("" != endpointCertFile) || ("" != endpointKeyFile) { + if endpointCertFile == endpointKeyFile { + err = ioutil.WriteFile(endpointCertFile, append(endpointCertPEMBlock, endpointKeyPEMBlock...), GeneratedFilePerm) + if nil != err { + return + } + } else { + if "" != endpointCertFile { + err = ioutil.WriteFile(endpointCertFile, endpointCertPEMBlock, GeneratedFilePerm) + if nil != err { + return + } + } + if "" != endpointKeyFile { + err = ioutil.WriteFile(endpointKeyFile, endpointKeyPEMBlock, GeneratedFilePerm) + if nil != err { + return + } + } } } From 64da5b2405dd92c6952db7002d1fca5aea0039a3 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 25 Mar 2021 09:33:36 -0700 Subject: [PATCH 080/160] Oops... forgot to update the icert program that calls icertpkg to ignore the []byte returns... --- icert/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icert/main.go b/icert/main.go index 3a3f9dbf1..bc27f2b7f 100644 --- a/icert/main.go +++ b/icert/main.go @@ -151,7 +151,7 @@ func main() { } if *caFlag { - err = icertpkg.GenCACert(generateKeyAlgorithm, subject, *ttlFlag, *caCertPemFilePathFlag, *caKeyPemFilePathFlag) + _, _, err = icertpkg.GenCACert(generateKeyAlgorithm, subject, *ttlFlag, *caCertPemFilePathFlag, *caKeyPemFilePathFlag) if nil != err { fmt.Printf("icertpkg.GenCACert() failed: %v\n", err) os.Exit(1) @@ -167,7 +167,7 @@ func main() { ipAddresses = append(ipAddresses, net.ParseIP(ipAddress)) } - err = icertpkg.GenEndpointCert(generateKeyAlgorithm, subject, dnsNamesFlag, ipAddresses, *ttlFlag, *caCertPemFilePathFlag, *caKeyPemFilePathFlag, *endpointCertPemFilePathFlag, *endpointKeyPemFilePathFlag) + _, _, err = icertpkg.GenEndpointCert(generateKeyAlgorithm, subject, dnsNamesFlag, ipAddresses, *ttlFlag, *caCertPemFilePathFlag, *caKeyPemFilePathFlag, *endpointCertPemFilePathFlag, *endpointKeyPemFilePathFlag) if nil != err { fmt.Printf("icertpkg.GenEndpointCert() failed: %v\n", err) os.Exit(1) From 23f5fe0c3db320a9b21ff0921194254c8f4a2360 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 25 Mar 2021 13:53:44 -0700 Subject: [PATCH 081/160] Reworked test retryrpc to use icert/icertpkg ...deprecated internal code previously used to self-sign certs using the old mechanism... --- icert/icertpkg/api_test.go | 6 +- retryrpc/api.go | 52 ++++----------- retryrpc/api_internal.go | 128 ------------------------------------- retryrpc/retryrpc_test.go | 127 +++++++++++++++++++++++++++++------- retryrpc/stress_test.go | 35 +++++----- retryrpc/upcall_test.go | 14 ++-- 6 files changed, 148 insertions(+), 214 deletions(-) diff --git a/icert/icertpkg/api_test.go b/icert/icertpkg/api_test.go index 129ade989..1900e141e 100644 --- a/icert/icertpkg/api_test.go +++ b/icert/icertpkg/api_test.go @@ -126,8 +126,8 @@ func testAPI(t *testing.T, generateKeyAlgorithm string, combined bool) { t.Fatalf("GenCACert() failed: %v", err) } - _, _, err = genEndpointCert( - // endpointCertPEM, endpointKeyPEM, err = genEndpointCert( + _, _, err = GenEndpointCert( + // endpointCertPEM, endpointKeyPEM, err = GenEndpointCert( generateKeyAlgorithm, pkix.Name{ Organization: []string{testOrganizationEndpoint}, @@ -161,7 +161,7 @@ func testAPI(t *testing.T, generateKeyAlgorithm string, combined bool) { } // serverTLSCertificate, err = tls.X509KeyPair(endpointCertPEM, endpointKeyPEM) // if nil != err { - // t.Fatalf("tls.LoadX509KeyPair() failed: %v", err) + // t.Fatalf("tls.X509KeyPair() failed: %v", err) // } serverNetListener, err = tls.Listen("tcp", ipAddressPort, &tls.Config{Certificates: []tls.Certificate{serverTLSCertificate}}) diff --git a/retryrpc/api.go b/retryrpc/api.go index c365dde9d..7546dde0e 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -27,13 +27,6 @@ import ( "github.com/google/btree" ) -// ServerCreds tracks the root CA and the -// server CA -type ServerCreds struct { - RootCAx509CertificatePEM []byte - ServerTLSCertificate tls.Certificate -} - // Server tracks the state of the server type Server struct { sync.Mutex @@ -51,7 +44,7 @@ type Server struct { connLock sync.Mutex connections *list.List connWG sync.WaitGroup - Creds *ServerCreds + tlsCertificate tls.Certificate listenersWG sync.WaitGroup receiver reflect.Value // Package receiver being served perClientInfo map[uint64]*clientInfo // Key: "clientID". Tracks clients @@ -66,53 +59,30 @@ type Server struct { // ServerConfig is used to configure a retryrpc Server type ServerConfig struct { - LongTrim time.Duration // How long the results of an RPC are stored on a Server before removed - ShortTrim time.Duration // How frequently completed and ACKed RPCs results are removed from Server - IPAddr string // IP Address that Server uses to listen - Port int // Port that Server uses to listen - DeadlineIO time.Duration // How long I/Os on sockets wait even if idle - KeepAlivePeriod time.Duration // How frequently a KEEPALIVE is sent - ServerCreds *ServerCreds // Credentials - dontStartTrimmers bool // Used for testing + LongTrim time.Duration // How long the results of an RPC are stored on a Server before removed + ShortTrim time.Duration // How frequently completed and ACKed RPCs results are removed from Server + IPAddr string // IP Address that Server uses to listen + Port int // Port that Server uses to listen + DeadlineIO time.Duration // How long I/Os on sockets wait even if idle + KeepAlivePeriod time.Duration // How frequently a KEEPALIVE is sent + TLSCertificate tls.Certificate // TLS Certificate to present to Clients + dontStartTrimmers bool // Used for testing } // NewServer creates the Server object func NewServer(config *ServerConfig) *Server { - var ( - err error - ) - server := &Server{ipaddr: config.IPAddr, port: config.Port, completedLongTTL: config.LongTrim, completedAckTrim: config.ShortTrim, deadlineIO: config.DeadlineIO, keepAlivePeriod: config.KeepAlivePeriod, dontStartTrimmers: config.dontStartTrimmers, - Creds: config.ServerCreds} + tlsCertificate: config.TLSCertificate} server.svrMap = make(map[string]*methodArgs) server.perClientInfo = make(map[uint64]*clientInfo) server.completedTickerDone = make(chan bool) server.connections = list.New() - if config.ServerCreds == nil { - logger.Errorf("config.ServerCreds is nil") - panic(err) - - } - return server } -// ConstructCreds is used by unit tests to create self-signed credentials in the -// ServerConfig structure before passing the struct to NewServer -// -// Only useful for unit tests -func ConstructCreds(ipaddr string) (creds *ServerCreds, err error) { - creds, err = constructServerCreds(ipaddr) - if err != nil { - logger.Errorf("Construction of server credentials failed with err: %v", err) - panic(err) - } - return -} - // Register creates the map of server methods func (server *Server) Register(retrySvr interface{}) (err error) { @@ -127,7 +97,7 @@ func (server *Server) Start() (err error) { hostPortStr := net.JoinHostPort(server.ipaddr, portStr) tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{server.Creds.ServerTLSCertificate}, + Certificates: []tls.Certificate{server.tlsCertificate}, } listenConfig := &net.ListenConfig{KeepAlive: server.keepAlivePeriod} diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index 16674fa6b..97f503130 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -7,17 +7,10 @@ package retryrpc import ( "container/list" - "crypto/ed25519" - "crypto/rand" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" "encoding/binary" "encoding/json" - "encoding/pem" "fmt" "io" - "math/big" "net" "reflect" "sync" @@ -300,124 +293,3 @@ func getIO(genNum uint64, deadlineIO time.Duration, conn net.Conn) (buf []byte, return } - -// constructServerCreds will generate root CA cert and server cert -// -// It is assumed that this is called on the "server" process and -// the caller will provide a mechanism to pass -// serverCreds.rootCAx509CertificatePEMkeys to the "clients". -func constructServerCreds(serverIPAddrAsString string) (serverCreds *ServerCreds, err error) { - var ( - commonX509NotAfter time.Time - commonX509NotBefore time.Time - rootCAEd25519PrivateKey ed25519.PrivateKey - rootCAEd25519PublicKey ed25519.PublicKey - rootCAx509CertificateDER []byte - rootCAx509CertificateTemplate *x509.Certificate - rootCAx509SerialNumber *big.Int - serverEd25519PrivateKey ed25519.PrivateKey - serverEd25519PrivateKeyDER []byte - serverEd25519PrivateKeyPEM []byte - serverEd25519PublicKey ed25519.PublicKey - serverX509CertificateDER []byte - serverX509CertificatePEM []byte - serverX509CertificateTemplate *x509.Certificate - serverX509SerialNumber *big.Int - timeNow time.Time - ) - - serverCreds = &ServerCreds{} - - timeNow = time.Now() - - // TODO - what should the length of this be? What if we want to eject a client - // from the server? How would that work? - // - // Do we even want the root CA at all? - commonX509NotBefore = time.Date(timeNow.Year()-1, time.January, 1, 0, 0, 0, 0, timeNow.Location()) - commonX509NotAfter = time.Date(timeNow.Year()+99, time.January, 1, 0, 0, 0, 0, timeNow.Location()) - - rootCAx509SerialNumber, err = rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - err = fmt.Errorf("rand.Int() [1] failed: %v", err) - return - } - - rootCAx509CertificateTemplate = &x509.Certificate{ - SerialNumber: rootCAx509SerialNumber, - Subject: pkix.Name{ - Organization: []string{"CA Organization"}, - CommonName: "Root CA", - }, - NotBefore: commonX509NotBefore, - NotAfter: commonX509NotAfter, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - IsCA: true, - } - - // Generate public and private key - rootCAEd25519PublicKey, rootCAEd25519PrivateKey, err = ed25519.GenerateKey(nil) - if err != nil { - err = fmt.Errorf("ed25519.GenerateKey() [1] failed: %v", err) - return - } - - // Create the certificate with the keys - rootCAx509CertificateDER, err = x509.CreateCertificate(rand.Reader, - rootCAx509CertificateTemplate, rootCAx509CertificateTemplate, rootCAEd25519PublicKey, rootCAEd25519PrivateKey) - if err != nil { - err = fmt.Errorf("x509.CreateCertificate() [1] failed: %v", err) - return - } - - serverCreds.RootCAx509CertificatePEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: rootCAx509CertificateDER}) - - serverX509SerialNumber, err = rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) - if err != nil { - err = fmt.Errorf("rand.Int() [2] failed: %v", err) - return - } - - serverX509CertificateTemplate = &x509.Certificate{ - SerialNumber: serverX509SerialNumber, - Subject: pkix.Name{ - Organization: []string{"Server Organization"}, - CommonName: "Server", - }, - NotBefore: commonX509NotBefore, - NotAfter: commonX509NotAfter, - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - IPAddresses: []net.IP{net.ParseIP(serverIPAddrAsString)}, - } - - // Generate the server public/private keys - serverEd25519PublicKey, serverEd25519PrivateKey, err = ed25519.GenerateKey(nil) - if err != nil { - err = fmt.Errorf("ed25519.GenerateKey() [2] failed: %v", err) - return - } - - // Create the server certificate with the server public/private keys - serverX509CertificateDER, err = x509.CreateCertificate(rand.Reader, serverX509CertificateTemplate, rootCAx509CertificateTemplate, serverEd25519PublicKey, rootCAEd25519PrivateKey) - if err != nil { - err = fmt.Errorf("x509.CreateCertificate() [2] failed: %v", err) - return - } - - serverX509CertificatePEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverX509CertificateDER}) - - serverEd25519PrivateKeyDER, err = x509.MarshalPKCS8PrivateKey(serverEd25519PrivateKey) - if err != nil { - err = fmt.Errorf("x509.MarshalPKCS8PrivateKey() failed: %v", err) - return - } - - serverEd25519PrivateKeyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: serverEd25519PrivateKeyDER}) - - serverCreds.ServerTLSCertificate, err = tls.X509KeyPair(serverX509CertificatePEM, serverEd25519PrivateKeyPEM) - - return -} diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index 195830ebc..263550906 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -4,45 +4,114 @@ package retryrpc import ( + "crypto/tls" + "crypto/x509/pkix" + "net" "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/icert/icertpkg" "github.com/NVIDIA/proxyfs/retryrpc/rpctest" - "github.com/stretchr/testify/assert" ) +const ( + testIPAddr = "127.0.0.1" + testPort = 24456 +) + +type testTLSCertsStruct struct { + caCertPEMBlock []byte + caKeyPEMBlock []byte + endpointCertPEMBlock []byte + endpointKeyPEMBlock []byte + endpointTLSCert tls.Certificate +} + +var testTLSCerts *testTLSCertsStruct + +// Utility function to initialize testTLSCerts +func testTLSCertsAllocate(t *testing.T) { + var ( + err error + ) + + testTLSCerts = &testTLSCertsStruct{} + + testTLSCerts.caCertPEMBlock, testTLSCerts.caKeyPEMBlock, err = icertpkg.GenCACert( + icertpkg.GenerateKeyAlgorithmEd25519, + pkix.Name{ + Organization: []string{"Test Organization CA"}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + time.Hour, + "", + "") + if nil != err { + t.Fatalf("icertpkg.GenCACert() failed: %v", err) + } + + testTLSCerts.endpointCertPEMBlock, testTLSCerts.endpointKeyPEMBlock, err = icertpkg.GenEndpointCert( + icertpkg.GenerateKeyAlgorithmEd25519, + pkix.Name{ + Organization: []string{"Test Organization Endpoint"}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + []string{}, + []net.IP{net.ParseIP("127.0.0.1")}, + time.Hour, + testTLSCerts.caCertPEMBlock, + testTLSCerts.caKeyPEMBlock, + "", + "") + if nil != err { + t.Fatalf("icertpkg.genEndpointCert() failed: %v", err) + } + + testTLSCerts.endpointTLSCert, err = tls.X509KeyPair(testTLSCerts.endpointCertPEMBlock, testTLSCerts.endpointKeyPEMBlock) + if nil != err { + t.Fatalf("tls.LoadX509KeyPair() failed: %v", err) + } +} + // Test basic retryrpc primitives // // This unit test exists here since it uses jrpcfs which would be a // circular dependency if the test was in retryrpc. func TestRetryRPC(t *testing.T) { - + testTLSCertsAllocate(t) testRegister(t) testServer(t) testBtree(t) testStatsAndBucketstats(t) } -func getNewServer(lt time.Duration, dontStartTrimmers bool) (rrSvr *Server, ip string, p int) { - var ( - ipaddr = "127.0.0.1" - port = 24456 - err error - ) - config := &ServerConfig{LongTrim: lt, ShortTrim: 100 * time.Millisecond, IPAddr: ipaddr, - Port: port, DeadlineIO: 5 * time.Second, dontStartTrimmers: dontStartTrimmers} - config.ServerCreds, err = ConstructCreds(ipaddr) - if err != nil { - logger.PanicfWithError(err, "Unable to create ServerCreds") +func getNewServer(lt time.Duration, dontStartTrimmers bool) (rrSvr *Server) { + config := &ServerConfig{ + LongTrim: lt, + ShortTrim: 100 * time.Millisecond, + IPAddr: testIPAddr, + Port: testPort, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + TLSCertificate: testTLSCerts.endpointTLSCert, + dontStartTrimmers: dontStartTrimmers, } // Create a new RetryRPC Server. Completed request will live on // completedRequests for 10 seconds. rrSvr = NewServer(config) - ip = ipaddr - p = port + return } @@ -56,7 +125,7 @@ func testRegister(t *testing.T) { // RPCs myJrpcfs := rpctest.NewServer() - rrSvr, _, _ := getNewServer(10*time.Second, false) + rrSvr := getNewServer(10*time.Second, false) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -78,7 +147,7 @@ func testServer(t *testing.T) { // RPCs myJrpcfs := rpctest.NewServer() - rrSvr, ipaddr, port := getNewServer(10*time.Second, false) + rrSvr := getNewServer(10*time.Second, false) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -94,8 +163,14 @@ func testServer(t *testing.T) { rrSvr.Run() // Now - setup a client to send requests to the server - clientConfig := &ClientConfig{IPAddr: ipaddr, Port: port, RootCAx509CertificatePEM: rrSvr.Creds.RootCAx509CertificatePEM, - Callbacks: nil, DeadlineIO: 5 * time.Second} + clientConfig := &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, + Callbacks: nil, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } rrClnt, newErr := NewClient(clientConfig) assert.NotNil(rrClnt) assert.Nil(newErr) @@ -142,12 +217,18 @@ func testServer(t *testing.T) { func testBtree(t *testing.T) { assert := assert.New(t) - rrSvr, ipaddr, port := getNewServer(10*time.Second, false) + rrSvr := getNewServer(10*time.Second, false) assert.NotNil(rrSvr) // Setup a client - we only will be targeting the btree - clientConfig := &ClientConfig{IPAddr: ipaddr, Port: port, RootCAx509CertificatePEM: rrSvr.Creds.RootCAx509CertificatePEM, - Callbacks: nil, DeadlineIO: 5 * time.Second} + clientConfig := &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, + Callbacks: nil, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } client, newErr := NewClient(clientConfig) assert.NotNil(client) assert.Nil(newErr) diff --git a/retryrpc/stress_test.go b/retryrpc/stress_test.go index fd673850c..639ce38be 100644 --- a/retryrpc/stress_test.go +++ b/retryrpc/stress_test.go @@ -28,6 +28,7 @@ func TestStress(t *testing.T) { go http.ListenAndServe("localhost:12123", nil) */ + testTLSCertsAllocate(t) testLoop(t) testLoopClientAckTrim(t) testLoopTTLTrim(t) @@ -47,7 +48,7 @@ func testLoop(t *testing.T) { // RPCs myJrpcfs := rpctest.NewServer() - rrSvr, ipAddr, port := getNewServer(65*time.Second, false) + rrSvr := getNewServer(65*time.Second, false) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -63,7 +64,7 @@ func testLoop(t *testing.T) { rrSvr.Run() // Start up the agents - parallelAgentSenders(t, rrSvr, ipAddr, port, agentCount, "RpcPing", sendCount, rrSvr.Creds.RootCAx509CertificatePEM) + parallelAgentSenders(t, rrSvr, agentCount, "RpcPing", sendCount) rrSvr.Close() } @@ -87,7 +88,7 @@ func testLoopClientAckTrim(t *testing.T) { myJrpcfs := rpctest.NewServer() whenTTL := 10 * time.Millisecond - rrSvr, ipAddr, port := getNewServer(whenTTL, true) + rrSvr := getNewServer(whenTTL, true) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -103,7 +104,7 @@ func testLoopClientAckTrim(t *testing.T) { rrSvr.Run() // Start up the agents - parallelAgentSenders(t, rrSvr, ipAddr, port, agentCount, "RpcPing", sendCount, rrSvr.Creds.RootCAx509CertificatePEM) + parallelAgentSenders(t, rrSvr, agentCount, "RpcPing", sendCount) // Now for both trimmers to run tm := time.Now() @@ -149,7 +150,7 @@ func testLoopTTLTrim(t *testing.T) { myJrpcfs := rpctest.NewServer() whenTTL := 10 * time.Millisecond - rrSvr, ipAddr, port := getNewServer(whenTTL, true) + rrSvr := getNewServer(whenTTL, true) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -165,7 +166,7 @@ func testLoopTTLTrim(t *testing.T) { rrSvr.Run() // Start up the agents - parallelAgentSenders(t, rrSvr, ipAddr, port, agentCount, "RpcPing", sendCount, rrSvr.Creds.RootCAx509CertificatePEM) + parallelAgentSenders(t, rrSvr, agentCount, "RpcPing", sendCount) // Use the TTL trimmer to remove all messages after guaranteeing we are // past time when they should be removed @@ -199,7 +200,7 @@ func testSendLargeRPC(t *testing.T) { myJrpcfs := rpctest.NewServer() whenTTL := 10 * time.Millisecond - rrSvr, ipAddr, port := getNewServer(whenTTL, true) + rrSvr := getNewServer(whenTTL, true) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -215,7 +216,7 @@ func testSendLargeRPC(t *testing.T) { rrSvr.Run() // Start up the agents - parallelAgentSenders(t, rrSvr, ipAddr, port, agentCount, "RpcPingLarge", sendCount, rrSvr.Creds.RootCAx509CertificatePEM) + parallelAgentSenders(t, rrSvr, agentCount, "RpcPingLarge", sendCount) // Now for both trimmers to run tm := time.Now() @@ -333,14 +334,19 @@ func (cb *stressMyClient) Interrupt(payload []byte) { } // Represents a pfsagent - sepearate client -func pfsagent(t *testing.T, rrSvr *Server, ipAddr string, port int, agentID uint64, method string, - agentWg *sync.WaitGroup, sendCnt int, rootCAx509CertificatePEM []byte) { +func pfsagent(t *testing.T, rrSvr *Server, agentID uint64, method string, agentWg *sync.WaitGroup, sendCnt int) { defer agentWg.Done() cb := &stressMyClient{} cb.cond = sync.NewCond(&cb.Mutex) - clientConfig := &ClientConfig{IPAddr: ipAddr, Port: port, - RootCAx509CertificatePEM: rootCAx509CertificatePEM, Callbacks: cb, DeadlineIO: 5 * time.Second} + clientConfig := &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, + Callbacks: cb, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } client, err := NewClient(clientConfig) if err != nil { fmt.Printf("Dial() failed with err: %v\n", err) @@ -403,8 +409,7 @@ func pfsagent(t *testing.T, rrSvr *Server, ipAddr string, port int, agentID uint } // Start a bunch of "pfsagents" in parallel -func parallelAgentSenders(t *testing.T, rrSrv *Server, ipAddr string, port int, agentCnt int, - method string, sendCnt int, rootCAx509CertificatePEM []byte) { +func parallelAgentSenders(t *testing.T, rrSrv *Server, agentCnt int, method string, sendCnt int) { var agentWg sync.WaitGroup @@ -418,7 +423,7 @@ func parallelAgentSenders(t *testing.T, rrSrv *Server, ipAddr string, port int, agentID = clientSeed + uint64(i) agentWg.Add(1) - go pfsagent(t, rrSrv, ipAddr, port, agentID, method, &agentWg, sendCnt, rootCAx509CertificatePEM) + go pfsagent(t, rrSrv, agentID, method, &agentWg, sendCnt) } agentWg.Wait() } diff --git a/retryrpc/upcall_test.go b/retryrpc/upcall_test.go index 4f8ed6d2e..b362ebff8 100644 --- a/retryrpc/upcall_test.go +++ b/retryrpc/upcall_test.go @@ -14,6 +14,7 @@ import ( // Test Upcall() functionality func TestUpCall(t *testing.T) { + testTLSCertsAllocate(t) testUpCall(t) } @@ -41,7 +42,7 @@ func testUpCall(t *testing.T) { // RPCs myJrpcfs := rpctest.NewServer() - rrSvr, ipaddr, port := getNewServer(10*time.Second, false) + rrSvr := getNewServer(10*time.Second, false) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -60,9 +61,14 @@ func testUpCall(t *testing.T) { cb := &MyClient{} cb.cond = sync.NewCond(&cb.Mutex) - clientConfig := &ClientConfig{IPAddr: ipaddr, Port: port, - RootCAx509CertificatePEM: rrSvr.Creds.RootCAx509CertificatePEM, Callbacks: cb, - DeadlineIO: 5 * time.Second} + clientConfig := &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, + Callbacks: cb, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } rrClnt, newErr := NewClient(clientConfig) assert.NotNil(rrClnt) assert.Nil(newErr) From 08749b7be094c5d7829310196e6ef1f01579b0b4 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 25 Mar 2021 15:26:40 -0700 Subject: [PATCH 082/160] Changes to jrpcfs to match new retryrpc semantics complete Note: This currently specifies TCP, not TLS, for retryrpc... benchmark results slightly optimistic due to this --- jrpcfs/api.go | 18 ++++++++---------- jrpcfs/config.go | 42 +++++++++++++++++++++++++++--------------- jrpcfs/filesystem.go | 2 -- jrpcfs/lease_test.go | 4 ++-- jrpcfs/retryrpc.go | 42 +++++++++++------------------------------- 5 files changed, 48 insertions(+), 60 deletions(-) diff --git a/jrpcfs/api.go b/jrpcfs/api.go index 5b31debfa..1bb6dc9d1 100644 --- a/jrpcfs/api.go +++ b/jrpcfs/api.go @@ -395,11 +395,10 @@ type MountByAccountNameRequest struct { // MountByAccountNameReply is the reply object for RpcMountByAccountName. type MountByAccountNameReply struct { - MountID MountIDAsString - RootDirInodeNumber int64 - RetryRPCPublicIPAddr string - RetryRPCPort uint16 - RootCAx509CertificatePEM []byte + MountID MountIDAsString + RootDirInodeNumber int64 + RetryRPCPublicIPAddr string + RetryRPCPort uint16 } // MountByVolumeNameRequest is the request object for RpcMountByVolumeName. @@ -412,11 +411,10 @@ type MountByVolumeNameRequest struct { // MountByVolumeNameReply is the reply object for RpcMountByVolumeName. type MountByVolumeNameReply struct { - MountID MountIDAsString - RootDirInodeNumber int64 - RetryRPCPublicIPAddr string - RetryRPCPort uint16 - RootCAx509CertificatePEM []byte + MountID MountIDAsString + RootDirInodeNumber int64 + RetryRPCPublicIPAddr string + RetryRPCPort uint16 } // UnmountRequest is the request object for RpcUnmount. diff --git a/jrpcfs/config.go b/jrpcfs/config.go index 0deb35022..f10362dcc 100644 --- a/jrpcfs/config.go +++ b/jrpcfs/config.go @@ -5,6 +5,7 @@ package jrpcfs import ( "container/list" + "crypto/tls" "fmt" "io/ioutil" "net" @@ -145,13 +146,14 @@ type globalsStruct struct { retryRPCCertPEM []byte retryRPCKeyPEM []byte + retryRPCCertificate tls.Certificate + volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName mountMapByMountIDAsByteArray map[MountIDAsByteArray]*mountStruct // key == mountStruct.mountIDAsByteArray mountMapByMountIDAsString map[MountIDAsString]*mountStruct // key == mountStruct.mountIDAsString // RetryRPC server - retryrpcSvr *retryrpc.Server - rootCAx509CertificatePEM []byte + retryrpcSvr *retryrpc.Server // Connection list and listener list to close during shutdown: halting bool @@ -227,30 +229,39 @@ func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { globals.retryRPCKeepAlivePeriod = 60 * time.Second } globals.retryRPCCertFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCCertFilePath") - if nil == err { + if (nil == err) && ("" != globals.retryRPCCertFilePath) { + globals.retryRPCKeyFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCKeyFilePath") + if (nil != err) || ("" != globals.retryRPCKeyFilePath) { + logger.Error("if [JSOPNRPCServer]RetryRPCCertFilePath is specified, [JSOPNRPCServer]RetryRPCKeyFilePath must be specified as well") + return + } globals.retryRPCCertPEM, err = ioutil.ReadFile(globals.retryRPCCertFilePath) if nil != err { - logger.ErrorfWithError(err, "failed to load PEM-formatted [JSONRPCServer]RetryRPCCertFilePath [\"%s\"]: %v", globals.retryRPCCertFilePath, err) + logger.ErrorfWithError(err, "failed to load PEM-formatted [JSONRPCServer]RetryRPCCertFilePath [\"%s\"]", globals.retryRPCCertFilePath) return } - } else { - logger.Infof("failed to get JSONRPCServer.RetryRPCCertFilePath from config file - defaulting to \"\"") - globals.retryRPCCertFilePath = "" - } - globals.retryRPCKeyFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCKeyFilePath") - if nil == err { - if "" == globals.retryRPCCertFilePath { - logger.Error("if [JSOPNRPCServer]RetryRPCKeyFilePath is specified, [JSOPNRPCServer]RetryRPCCertFilePath must be specified as well") + globals.retryRPCKeyPEM, err = ioutil.ReadFile(globals.retryRPCKeyFilePath) + if nil != err { + logger.ErrorfWithError(err, "failed to load PEM-formatted [JSONRPCServer]retryRPCKeyFilePath [\"%s\"]", globals.retryRPCKeyFilePath) return } - globals.retryRPCKeyPEM, err = ioutil.ReadFile(globals.retryRPCKeyFilePath) + globals.retryRPCCertificate, err = tls.X509KeyPair(globals.retryRPCCertPEM, globals.retryRPCKeyPEM) if nil != err { - logger.ErrorfWithError(err, "failed to load PEM-formatted [JSONRPCServer]RetryRPCKeyFilePath [\"%s\"]: %v", globals.retryRPCKeyFilePath, err) + logger.ErrorfWithError(err, "tls.LoadX509KeyPair(\"%s\", \"%s\") failed", globals.retryRPCCertFilePath, globals.retryRPCKeyFilePath) return } } else { - logger.Infof("failed to get JSONRPCServer.RetryRPCKeyFilePath from config file - defaulting to \"\"") + globals.retryRPCKeyFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCKeyFilePath") + if (nil == err) && ("" != globals.retryRPCKeyFilePath) { + logger.Error("if [JSOPNRPCServer]RetryRPCKeyFilePath is specified, [JSOPNRPCServer]RetryRPCCertFilePath must be specified as well") + return + } + logger.Infof("failed to get JSONRPCServer.RetryRPC{Cert|Key}FilePath from config file - defaulting to \"\"") + globals.retryRPCCertFilePath = "" globals.retryRPCKeyFilePath = "" + globals.retryRPCCertPEM = nil + globals.retryRPCKeyPEM = nil + globals.retryRPCCertificate = tls.Certificate{} } } else { logger.Infof("failed to get JSONRPCServer.RetryRPCPort from config file - skipping......") @@ -263,6 +274,7 @@ func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { globals.retryRPCKeyFilePath = "" globals.retryRPCCertPEM = nil globals.retryRPCKeyPEM = nil + globals.retryRPCCertificate = tls.Certificate{} } // Set data path logging level to true, so that all trace logging is controlled by settings diff --git a/jrpcfs/filesystem.go b/jrpcfs/filesystem.go index a83e87996..029472b35 100644 --- a/jrpcfs/filesystem.go +++ b/jrpcfs/filesystem.go @@ -1414,7 +1414,6 @@ func (s *Server) RpcMountByAccountName(clientID uint64, in *MountByAccountNameRe reply.RootDirInodeNumber = int64(uint64(inode.RootDirInodeNumber)) reply.RetryRPCPublicIPAddr = globals.publicIPAddr reply.RetryRPCPort = globals.retryRPCPort - reply.RootCAx509CertificatePEM = globals.rootCAx509CertificatePEM } } @@ -1436,7 +1435,6 @@ func (s *Server) RpcMountByVolumeName(clientID uint64, in *MountByVolumeNameRequ reply.RootDirInodeNumber = int64(uint64(inode.RootDirInodeNumber)) reply.RetryRPCPublicIPAddr = globals.publicIPAddr reply.RetryRPCPort = globals.retryRPCPort - reply.RootCAx509CertificatePEM = globals.rootCAx509CertificatePEM } } diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index e37a05e3e..68f33e392 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -74,7 +74,7 @@ func BenchmarkRpcLease(b *testing.B) { retryrpcClientConfig = &retryrpc.ClientConfig{ IPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), - RootCAx509CertificatePEM: globals.retryrpcSvr.Creds.RootCAx509CertificatePEM, // Use same CA as server + RootCAx509CertificatePEM: nil, Callbacks: testRpcLeaseClient, DeadlineIO: deadlineIO, KeepAlivePeriod: keepAlivePeriod, @@ -478,7 +478,7 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { retryrpcClientConfig = &retryrpc.ClientConfig{ IPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), - RootCAx509CertificatePEM: globals.retryrpcSvr.Creds.RootCAx509CertificatePEM, // Use same CA as server + RootCAx509CertificatePEM: nil, Callbacks: testRpcLeaseClient, DeadlineIO: deadlineIO, KeepAlivePeriod: keepAlivePeriod, diff --git a/jrpcfs/retryrpc.go b/jrpcfs/retryrpc.go index b7c8b1dcc..429f473d6 100644 --- a/jrpcfs/retryrpc.go +++ b/jrpcfs/retryrpc.go @@ -4,47 +4,28 @@ package jrpcfs import ( - "crypto/tls" - "fmt" - "github.com/NVIDIA/proxyfs/logger" "github.com/NVIDIA/proxyfs/retryrpc" ) func retryRPCServerUp(jserver *Server) { - - var err error + var ( + err error + ) if globals.retryRPCPort == 0 { return } // Create a new RetryRPC Server. - retryConfig := &retryrpc.ServerConfig{LongTrim: globals.retryRPCTTLCompleted, ShortTrim: globals.retryRPCAckTrim, - IPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), DeadlineIO: globals.retryRPCDeadlineIO, - KeepAlivePeriod: globals.retryRPCKeepAlivePeriod} - - // Retrive credentials from globals if they are valid. - // - // If the credentials are invalid, create self-signed certifcate and log message - if globals.rootCAx509CertificatePEM == nil || globals.retryRPCCertPEM == nil || globals.retryRPCKeyPEM == nil { - var ( - constructErr error - ) - retryConfig.ServerCreds, constructErr = retryrpc.ConstructCreds(globals.publicIPAddr) - if constructErr != nil { - logger.PanicfWithError(constructErr, "Unable to create ServerCreds") - } - logger.Infof("Invalid credentials passed retryrpc - creating self-signed. Passed rootCAx509CertificatePEM: %v retryRPCCertPEM: %v retryRPCKeyPEM: %v", - globals.rootCAx509CertificatePEM, globals.retryRPCCertPEM, globals.retryRPCKeyPEM) - } else { - svrCert, tmpErr := tls.X509KeyPair(globals.retryRPCCertPEM, globals.retryRPCKeyPEM) - if tmpErr != nil { - pMsg := fmt.Errorf("tls.X509KeyPair failed with error: %v", tmpErr) - panic(pMsg) - - } - retryConfig.ServerCreds = &retryrpc.ServerCreds{RootCAx509CertificatePEM: globals.rootCAx509CertificatePEM, ServerTLSCertificate: svrCert} + retryConfig := &retryrpc.ServerConfig{ + LongTrim: globals.retryRPCTTLCompleted, + ShortTrim: globals.retryRPCAckTrim, + IPAddr: globals.publicIPAddr, + Port: int(globals.retryRPCPort), + DeadlineIO: globals.retryRPCDeadlineIO, + KeepAlivePeriod: globals.retryRPCKeepAlivePeriod, + TLSCertificate: globals.retryRPCCertificate, } rrSvr := retryrpc.NewServer(retryConfig) @@ -65,7 +46,6 @@ func retryRPCServerUp(jserver *Server) { globals.connLock.Lock() globals.retryrpcSvr = rrSvr - globals.rootCAx509CertificatePEM = rrSvr.Creds.RootCAx509CertificatePEM globals.connLock.Unlock() // Tell retryrpc server to start accepting requests From 169da7843c906101f80d5f380deb9ec72541963b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 25 Mar 2021 16:27:10 -0700 Subject: [PATCH 083/160] Finished enabling TCP retryrpc connections on the server side ...still have to finish the client side... and then make test jrpcfs should work --- retryrpc/api.go | 21 +++++++++++++++------ retryrpc/server.go | 24 ++++++++++++++++-------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/retryrpc/api.go b/retryrpc/api.go index 7546dde0e..2b147259a 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -36,8 +36,8 @@ type Server struct { ipaddr string // IP address server listens too port int // Port of server clientIDNonce uint64 // Nonce used to create a unique client ID - netListener net.Listener - tlsListener net.Listener + netListener net.Listener // Accepts on this to skip TLS upgrade + tlsListener net.Listener // Accepts on this to use TLS upgrade halting bool goroutineWG sync.WaitGroup // Used to track outstanding goroutines @@ -107,7 +107,9 @@ func (server *Server) Start() (err error) { return } - server.tlsListener = tls.NewListener(server.netListener, tlsConfig) + if 0 != len(server.tlsCertificate.Certificate) { + server.tlsListener = tls.NewListener(server.netListener, tlsConfig) + } server.listenersWG.Add(1) @@ -194,9 +196,16 @@ func (server *Server) Close() { server.halting = true server.Unlock() - err := server.tlsListener.Close() - if err != nil { - logger.Errorf("server.tlsListener.Close() returned err: %v", err) + if 0 == len(server.tlsCertificate.Certificate) { + err := server.netListener.Close() + if err != nil { + logger.Errorf("server.netListener.Close() returned err: %v", err) + } + } else { + err := server.tlsListener.Close() + if err != nil { + logger.Errorf("server.tlsListener.Close() returned err: %v", err) + } } server.listenersWG.Wait() diff --git a/retryrpc/server.go b/retryrpc/server.go index 694375762..d4f8666dd 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -26,7 +26,6 @@ var printDebugLogs bool = false // TODO - test if Register has been called??? func (server *Server) closeClient(myConn net.Conn, myElm *list.Element) { - server.connLock.Lock() server.connections.Remove(myElm) @@ -38,9 +37,18 @@ func (server *Server) closeClient(myConn net.Conn, myElm *list.Element) { } func (server *Server) run() { + var ( + conn net.Conn + err error + ) + defer server.goroutineWG.Done() for { - conn, err := server.tlsListener.Accept() + if 0 == len(server.tlsCertificate.Certificate) { + conn, err = server.netListener.Accept() + } else { + conn, err = server.tlsListener.Accept() + } if err != nil { if !server.halting { logger.ErrorfWithError(err, "net.Accept failed for Retry RPC listener") @@ -80,19 +88,19 @@ func (server *Server) run() { } server.goroutineWG.Add(1) - go func(myConn net.Conn, myElm *list.Element) { + go func(myCi *clientInfo, myCCtx *connCtx, myConn net.Conn, myElm *list.Element) { defer server.goroutineWG.Done() - logger.Infof("Servicing client: %v address: %v", ci.myUniqueID, myConn.RemoteAddr()) - server.serviceClient(ci, cCtx) + logger.Infof("Servicing client: %v address: %v", myCi.myUniqueID, myConn.RemoteAddr()) + server.serviceClient(myCi, myCCtx) - logger.Infof("Closing client: %v address: %v", ci.myUniqueID, myConn.RemoteAddr()) - server.closeClient(conn, elm) + logger.Infof("Closing client: %v address: %v", myCi.myUniqueID, myConn.RemoteAddr()) + server.closeClient(myConn, myElm) // The clientInfo for this client will first be trimmed and then later // deleted from the list of server.perClientInfo by the TTL trimmer. - }(conn, elm) + }(ci, cCtx, conn, elm) } } From 74dce353e19236a37fe311b484301ee5fcbfdb13 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 26 Mar 2021 11:09:51 -0700 Subject: [PATCH 084/160] Got jrpcfs tests to use icert-constructed TLS certs with retryrpc Also separated out TestMain() et. al. into jrpcfs/setup_teardown.go --- jrpcfs/config.go | 2 +- jrpcfs/lease_test.go | 4 +- jrpcfs/middleware_test.go | 375 ---------------------------- jrpcfs/setup_teardown_test.go | 458 ++++++++++++++++++++++++++++++++++ retryrpc/api.go | 33 ++- 5 files changed, 477 insertions(+), 395 deletions(-) create mode 100644 jrpcfs/setup_teardown_test.go diff --git a/jrpcfs/config.go b/jrpcfs/config.go index f10362dcc..e12dfa747 100644 --- a/jrpcfs/config.go +++ b/jrpcfs/config.go @@ -231,7 +231,7 @@ func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) { globals.retryRPCCertFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCCertFilePath") if (nil == err) && ("" != globals.retryRPCCertFilePath) { globals.retryRPCKeyFilePath, err = confMap.FetchOptionValueString("JSONRPCServer", "RetryRPCKeyFilePath") - if (nil != err) || ("" != globals.retryRPCKeyFilePath) { + if (nil != err) || ("" == globals.retryRPCKeyFilePath) { logger.Error("if [JSOPNRPCServer]RetryRPCCertFilePath is specified, [JSOPNRPCServer]RetryRPCKeyFilePath must be specified as well") return } diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index 68f33e392..953d9c60a 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -74,7 +74,7 @@ func BenchmarkRpcLease(b *testing.B) { retryrpcClientConfig = &retryrpc.ClientConfig{ IPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), - RootCAx509CertificatePEM: nil, + RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, Callbacks: testRpcLeaseClient, DeadlineIO: deadlineIO, KeepAlivePeriod: keepAlivePeriod, @@ -478,7 +478,7 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { retryrpcClientConfig = &retryrpc.ClientConfig{ IPAddr: globals.publicIPAddr, Port: int(globals.retryRPCPort), - RootCAx509CertificatePEM: nil, + RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, Callbacks: testRpcLeaseClient, DeadlineIO: deadlineIO, KeepAlivePeriod: keepAlivePeriod, diff --git a/jrpcfs/middleware_test.go b/jrpcfs/middleware_test.go index 209027a15..f0242340d 100644 --- a/jrpcfs/middleware_test.go +++ b/jrpcfs/middleware_test.go @@ -5,238 +5,17 @@ package jrpcfs import ( "fmt" - "io/ioutil" - "os" "strings" - "sync" "testing" "time" - "golang.org/x/sys/unix" - "github.com/NVIDIA/proxyfs/blunder" - "github.com/NVIDIA/proxyfs/conf" "github.com/NVIDIA/proxyfs/fs" "github.com/NVIDIA/proxyfs/inode" - "github.com/NVIDIA/proxyfs/logger" - "github.com/NVIDIA/proxyfs/ramswift" "github.com/NVIDIA/proxyfs/swiftclient" - "github.com/NVIDIA/proxyfs/transitions" "github.com/stretchr/testify/assert" ) -// Shorthand for our testing debug log id; global to the package -var test_debug = logger.DbgTesting - -const testVer = "/v1/" -const testAccountName = "AN_account" -const testContainerName = "test_container" -const testVerAccountName = testVer + testAccountName -const testVerAccountContainerName = testVerAccountName + "/" + testContainerName -const testAccountName2 = "AN_account2" - -const testRequireSlashesInPathsToProperlySort = false - -func testSetup() []func() { - var ( - cleanupFuncs []func() - cleanupTempDir func() - confStrings []string - doneChan chan bool - err error - signalHandlerIsArmedWG sync.WaitGroup - tempDir string - testConfMap conf.ConfMap - ) - - cleanupFuncs = make([]func(), 0) - - confStrings = []string{ - "Stats.IPAddr=localhost", - "Stats.UDPPort=52184", - "Stats.BufferLength=100", - "Stats.MaxLatency=1s", - "FSGlobals.VolumeGroupList=JrpcfsTestVolumeGroup", - "FSGlobals.CheckpointHeaderConsensusAttempts=5", - "FSGlobals.MountRetryLimit=6", - "FSGlobals.MountRetryDelay=1s", - "FSGlobals.MountRetryExpBackoff=2", - "FSGlobals.LogCheckpointHeaderPosts=true", - "FSGlobals.TryLockBackoffMin=10ms", - "FSGlobals.TryLockBackoffMax=50ms", - "FSGlobals.TryLockSerializationThreshhold=5", - "FSGlobals.SymlinkMax=8", - "FSGlobals.CoalesceElementChunkSize=16", - "FSGlobals.InodeRecCacheEvictLowLimit=10000", - "FSGlobals.InodeRecCacheEvictHighLimit=10010", - "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", - "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", - "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", - "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", - "FSGlobals.DirEntryCacheEvictLowLimit=10000", - "FSGlobals.DirEntryCacheEvictHighLimit=10010", - "FSGlobals.FileExtentMapEvictLowLimit=10000", - "FSGlobals.FileExtentMapEvictHighLimit=10010", - "FSGlobals.EtcdEnabled=false", - "SwiftClient.NoAuthIPAddr=127.0.0.1", - "SwiftClient.NoAuthTCPPort=35262", - "SwiftClient.Timeout=10s", - "SwiftClient.RetryLimit=3", - "SwiftClient.RetryLimitObject=3", - "SwiftClient.RetryDelay=10ms", - "SwiftClient.RetryDelayObject=10ms", - "SwiftClient.RetryExpBackoff=1.2", - "SwiftClient.RetryExpBackoffObject=2.0", - "SwiftClient.ChunkedConnectionPoolSize=64", - "SwiftClient.NonChunkedConnectionPoolSize=32", - "RamSwiftInfo.MaxAccountNameLength=256", - "RamSwiftInfo.MaxContainerNameLength=256", - "RamSwiftInfo.MaxObjectNameLength=256", - "RamSwiftInfo.AccountListingLimit=10000", - "RamSwiftInfo.ContainerListingLimit=10000", - "Peer:Peer0.PublicIPAddr=127.0.0.1", - "Peer:Peer0.PrivateIPAddr=127.0.0.1", - "Peer:Peer0.ReadCacheQuotaFraction=0.20", - "Cluster.Peers=Peer0", - "Cluster.WhoAmI=Peer0", - "Volume:SomeVolume.FSID=1", - "Volume:SomeVolume.AccountName=" + testAccountName, - "Volume:SomeVolume.AutoFormat=true", - "Volume:SomeVolume.CheckpointContainerName=.__checkpoint__", - "Volume:SomeVolume.CheckpointContainerStoragePolicy=gold", - "Volume:SomeVolume.CheckpointInterval=10s", - "Volume:SomeVolume.DefaultPhysicalContainerLayout=SomeContainerLayout", - "Volume:SomeVolume.MaxFlushSize=10027008", - "Volume:SomeVolume.MaxFlushTime=2s", - "Volume:SomeVolume.FileDefragmentChunkSize=10027008", - "Volume:SomeVolume.FileDefragmentChunkDelay=2ms", - "Volume:SomeVolume.NonceValuesToReserve=100", - "Volume:SomeVolume.MaxEntriesPerDirNode=32", - "Volume:SomeVolume.MaxExtentsPerFileNode=32", - "Volume:SomeVolume.MaxInodesPerMetadataNode=32", - "Volume:SomeVolume.MaxLogSegmentsPerMetadataNode=64", - "Volume:SomeVolume.MaxDirFileNodesPerMetadataNode=16", - "Volume:SomeVolume.MaxBytesInodeCache=100000", - "Volume:SomeVolume.InodeCacheEvictInterval=1s", - "Volume:SomeVolume.ActiveLeaseEvictLowLimit=2", - "Volume:SomeVolume.ActiveLeaseEvictHighLimit=4", - "Volume:SomeVolume2.FSID=2", - "Volume:SomeVolume2.AccountName=" + testAccountName2, - "Volume:SomeVolume2.AutoFormat=true", - "Volume:SomeVolume2.CheckpointContainerName=.__checkpoint__", - "Volume:SomeVolume2.CheckpointContainerStoragePolicy=gold", - "Volume:SomeVolume2.CheckpointInterval=10s", - "Volume:SomeVolume2.DefaultPhysicalContainerLayout=SomeContainerLayout2", - "Volume:SomeVolume2.MaxFlushSize=10027008", - "Volume:SomeVolume2.MaxFlushTime=2s", - "Volume:SomeVolume2.FileDefragmentChunkSize=10027008", - "Volume:SomeVolume2.FileDefragmentChunkDelay=2ms", - "Volume:SomeVolume2.NonceValuesToReserve=100", - "Volume:SomeVolume2.MaxEntriesPerDirNode=32", - "Volume:SomeVolume2.MaxExtentsPerFileNode=32", - "Volume:SomeVolume2.MaxInodesPerMetadataNode=32", - "Volume:SomeVolume2.MaxLogSegmentsPerMetadataNode=64", - "Volume:SomeVolume2.MaxDirFileNodesPerMetadataNode=16", - "Volume:SomeVolume2.MaxBytesInodeCache=100000", - "Volume:SomeVolume2.InodeCacheEvictInterval=1s", - "Volume:SomeVolume2.ActiveLeaseEvictLowLimit=5000", - "Volume:SomeVolume2.ActiveLeaseEvictHighLimit=5010", - "VolumeGroup:JrpcfsTestVolumeGroup.VolumeList=SomeVolume,SomeVolume2", - "VolumeGroup:JrpcfsTestVolumeGroup.VirtualIPAddr=", - "VolumeGroup:JrpcfsTestVolumeGroup.PrimaryPeer=Peer0", - "VolumeGroup:JrpcfsTestVolumeGroup.ReadCacheLineSize=1000000", - "VolumeGroup:JrpcfsTestVolumeGroup.ReadCacheWeight=100", - "PhysicalContainerLayout:SomeContainerLayout.ContainerStoragePolicy=silver", - "PhysicalContainerLayout:SomeContainerLayout.ContainerNamePrefix=kittens", - "PhysicalContainerLayout:SomeContainerLayout.ContainersPerPeer=10", - "PhysicalContainerLayout:SomeContainerLayout.MaxObjectsPerContainer=1000000", - "PhysicalContainerLayout:SomeContainerLayout2.ContainerStoragePolicy=silver", - "PhysicalContainerLayout:SomeContainerLayout2.ContainerNamePrefix=puppies", - "PhysicalContainerLayout:SomeContainerLayout2.ContainersPerPeer=10", - "PhysicalContainerLayout:SomeContainerLayout2.MaxObjectsPerContainer=1000000", - "Logging.LogFilePath=/dev/null", - "Logging.LogToConsole=false", - "JSONRPCServer.TCPPort=12346", // 12346 instead of 12345 so that test can run if proxyfsd is already running - "JSONRPCServer.FastTCPPort=32346", // ...and similarly here... - "JSONRPCServer.RetryRPCPort=32357", // ...and similarly here... - "JSONRPCServer.RetryRPCTTLCompleted=10s", - "JSONRPCServer.RetryRPCAckTrim=10ms", - "JSONRPCServer.RetryRPCCertFilePath=", - "JSONRPCServer.RetryRPCKeyFilePath=", - "JSONRPCServer.DataPathLogging=false", - "JSONRPCServer.MinLeaseDuration=100ms", - "JSONRPCServer.LeaseInterruptInterval=100ms", - "JSONRPCServer.LeaseInterruptLimit=5", - } - - tempDir, err = ioutil.TempDir("", "jrpcfs_test") - if nil != err { - panic(fmt.Sprintf("failed in testSetup: %v", err)) - } - cleanupTempDir = func() { - _ = os.RemoveAll(tempDir) - } - cleanupFuncs = append(cleanupFuncs, cleanupTempDir) - - testConfMap, err = conf.MakeConfMapFromStrings(confStrings) - if nil != err { - panic(fmt.Sprintf("failed in testSetup: %v", err)) - } - - signalHandlerIsArmedWG.Add(1) - doneChan = make(chan bool) - go ramswift.Daemon("/dev/null", confStrings, &signalHandlerIsArmedWG, doneChan, unix.SIGTERM) - - signalHandlerIsArmedWG.Wait() - - err = transitions.Up(testConfMap) - if nil != err { - panic(fmt.Sprintf("transitions.Up() failed: %v", err)) - } - - return cleanupFuncs -} - -func fsStatPath(accountName string, path string) fs.Stat { - _, _, _, _, volumeHandle, err := parseVirtPath(accountName) - if err != nil { - panic(err) - } - ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, path) - if err != nil { - panic(err) - } - stats, err := volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino) - if err != nil { - panic(err) - } - return stats -} - -func fsMkDir(volumeHandle fs.VolumeHandle, parentDirInode inode.InodeNumber, newDirName string) (createdInode inode.InodeNumber) { - createdInode, err := volumeHandle.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDirInode, newDirName, inode.PosixModePerm) - if err != nil { - panic(fmt.Sprintf("failed to create %v: %v", newDirName, err)) - } - return -} - -func fsCreateFile(volumeHandle fs.VolumeHandle, parentDirInode inode.InodeNumber, newFileName string) (createdInode inode.InodeNumber) { - createdInode, err := volumeHandle.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDirInode, newFileName, inode.PosixModePerm) - if err != nil { - panic(fmt.Sprintf("failed to create file %v: %v", newFileName, err)) - } - return -} - -func fsCreateSymlink(volumeHandle fs.VolumeHandle, parentDirInode inode.InodeNumber, symlinkName string, symlinkTarget string) { - _, err := volumeHandle.Symlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDirInode, symlinkName, symlinkTarget) - if err != nil { - panic(fmt.Sprintf("failed to create symlink %s -> %s: %v", symlinkName, symlinkTarget, err)) - } - return -} - func middlewareCreateContainer(t *testing.T, server *Server, fullPathContainer string, expectedError blunder.FsError) { assert := assert.New(t) @@ -282,160 +61,6 @@ func middlewarePutLocation(t *testing.T, server *Server, newPutPath string, expe return putLocationReply.PhysPath } -func makeSomeFilesAndSuch() { - // we should have enough stuff up now that we can actually make - // some files and directories and such - volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") - if nil != err { - panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) - } - - cInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c") - cNestedInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c-nested") - fsCreateSymlink(volumeHandle, inode.RootDirInodeNumber, "c-symlink", "c") - - err = volumeHandle.MiddlewarePost("", "c", []byte("metadata for c"), []byte{}) - if err != nil { - panic(err) - } - _ = fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c-no-metadata") - _ = fsMkDir(volumeHandle, cInode, "empty-directory") - - readmeInode := fsCreateFile(volumeHandle, cInode, "README") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, readmeInode, 0, []byte("who am I kidding? nobody reads these."), nil) - err = volumeHandle.MiddlewarePost("", "c/README", []byte("metadata for c/README"), []byte{}) - if err != nil { - panic(err) - } - - animalsInode := fsMkDir(volumeHandle, cInode, "animals") - files := map[string]string{ - "dog.txt": "dog goes woof", - "cat.txt": "cat goes meow", - "bird.txt": "bird goes tweet", - "mouse.txt": "mouse goes squeak", - "cow.txt": "cow goes moo", - "frog.txt": "frog goes croak", - "elephant.txt": "elephant goes toot", - } - for fileName, fileContents := range files { - fileInode := fsCreateFile(volumeHandle, animalsInode, fileName) - - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileContents), nil) - if err != nil { - panic(fmt.Sprintf("failed to write file %s: %v", fileName, err)) - } - } - - plantsInode := fsMkDir(volumeHandle, cInode, "plants") - ino := fsCreateFile(volumeHandle, cInode, "plants-README") - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, 0, []byte("nah"), nil) - if err != nil { - panic(fmt.Sprintf("failed to write file plants-README: %v", err)) - } - - files = map[string]string{ - // Random contents of varying lengths. - "aloe.txt": "skiameter-interlope", - "banana.txt": "ring ring ring ring ring ring ring bananaphone", - "cherry.txt": "archegonium-nonresidentiary", - "eggplant.txt": "bowk-unruled", - } - - for fileName, fileContents := range files { - fileInode := fsCreateFile(volumeHandle, plantsInode, fileName) - - _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileContents), nil) - if err != nil { - panic(fmt.Sprintf("failed to write file %s: %v", fileName, err)) - } - } - - fsCreateSymlink(volumeHandle, cInode, "plants-symlink", "plants") - fsCreateSymlink(volumeHandle, plantsInode, "eggplant.txt-symlink", "eggplant.txt") - - // Put some deeply nested things in c-nested. This listing is a - // shortened version of a real directory tree that exposed a bug. - fsCreateFile(volumeHandle, cNestedInode, ".DS_Store") - dotGitInode := fsMkDir(volumeHandle, cNestedInode, ".git") - fsCreateFile(volumeHandle, dotGitInode, ".DS_Store") - fsCreateFile(volumeHandle, dotGitInode, "COMMIT_EDITMSG") - fsCreateFile(volumeHandle, dotGitInode, "FETCH_HEAD") - fsCreateFile(volumeHandle, dotGitInode, "HEAD") - fsCreateFile(volumeHandle, dotGitInode, "ORIG_HEAD") - fsCreateFile(volumeHandle, dotGitInode, "index") - dotGitHooks := fsMkDir(volumeHandle, dotGitInode, "hooks") - fsCreateFile(volumeHandle, dotGitHooks, ".DS_Store") - fsCreateFile(volumeHandle, dotGitHooks, "applypatch-msg.sample") - fsCreateFile(volumeHandle, dotGitHooks, "commit-msg.sample") - dotGitLogs := fsMkDir(volumeHandle, dotGitInode, "logs") - fsCreateFile(volumeHandle, dotGitLogs, ".DS_Store") - fsCreateFile(volumeHandle, dotGitLogs, "HEAD") - dotGitLogsRefs := fsMkDir(volumeHandle, dotGitLogs, "refs") - fsCreateFile(volumeHandle, dotGitLogsRefs, ".DS_Store") - fsCreateFile(volumeHandle, dotGitLogsRefs, "stash") - dotGitLogsRefsHeads := fsMkDir(volumeHandle, dotGitLogsRefs, "heads") - fsCreateFile(volumeHandle, dotGitLogsRefsHeads, ".DS_Store") - fsCreateFile(volumeHandle, dotGitLogsRefsHeads, "development") - fsCreateFile(volumeHandle, dotGitLogsRefsHeads, "stable") - - aInode := fsMkDir(volumeHandle, cNestedInode, "a") - fsCreateFile(volumeHandle, aInode, "b-1") - fsCreateFile(volumeHandle, aInode, "b-2") - abInode := fsMkDir(volumeHandle, aInode, "b") - fsCreateFile(volumeHandle, abInode, "c-1") - fsCreateFile(volumeHandle, abInode, "c-2") - abcInode := fsMkDir(volumeHandle, abInode, "c") - fsCreateFile(volumeHandle, abcInode, "d-1") - fsCreateFile(volumeHandle, abcInode, "d-2") - - // SomeVolume2 is set up for testing account listings - volumeHandle2, err := fs.FetchVolumeHandleByVolumeName("SomeVolume2") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "alpha") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "bravo") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "charlie") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "delta") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "echo") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "foxtrot") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "golf") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "hotel") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "india") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "juliet") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "kilo") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "lima") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "mancy") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "november") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "oscar") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "papa") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "quebec") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "romeo") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "sierra") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "tango") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "uniform") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "victor") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "whiskey") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "xray") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "yankee") - _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "zulu") - _ = fsCreateFile(volumeHandle2, inode.RootDirInodeNumber, "alice.txt") - _ = fsCreateFile(volumeHandle2, inode.RootDirInodeNumber, "bob.txt") - _ = fsCreateFile(volumeHandle2, inode.RootDirInodeNumber, "carol.txt") -} - -func TestMain(m *testing.M) { - //setup, run, teardown, exit - cleanupFuncs := testSetup() - makeSomeFilesAndSuch() - - verdict := m.Run() - - for _, cleanupFunc := range cleanupFuncs { - cleanupFunc() - } - - os.Exit(verdict) -} - func TestRpcHead(t *testing.T) { s := &Server{} diff --git a/jrpcfs/setup_teardown_test.go b/jrpcfs/setup_teardown_test.go new file mode 100644 index 000000000..53793b02a --- /dev/null +++ b/jrpcfs/setup_teardown_test.go @@ -0,0 +1,458 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package jrpcfs + +import ( + "crypto/tls" + "crypto/x509/pkix" + "fmt" + "io/ioutil" + "net" + "os" + "sync" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/fs" + "github.com/NVIDIA/proxyfs/icert/icertpkg" + "github.com/NVIDIA/proxyfs/inode" + "github.com/NVIDIA/proxyfs/logger" + "github.com/NVIDIA/proxyfs/ramswift" + "github.com/NVIDIA/proxyfs/transitions" +) + +// Shorthand for our testing debug log id; global to the package +var test_debug = logger.DbgTesting + +const testVer = "/v1/" +const testAccountName = "AN_account" +const testContainerName = "test_container" +const testVerAccountName = testVer + testAccountName +const testVerAccountContainerName = testVerAccountName + "/" + testContainerName +const testAccountName2 = "AN_account2" + +const testRequireSlashesInPathsToProperlySort = false + +type testTLSCertsStruct struct { + caCertPEMBlock []byte + caKeyPEMBlock []byte + endpointCertPEMBlock []byte + endpointKeyPEMBlock []byte + endpointTLSCert tls.Certificate + caCertFile string + caKeyFile string + endpointCertFile string + endpointKeyFile string +} + +var testTLSCerts *testTLSCertsStruct + +func TestMain(m *testing.M) { + //setup, run, teardown, exit + + cleanupFuncs := testSetup() + + makeSomeFilesAndSuch() + + verdict := m.Run() + + for _, cleanupFunc := range cleanupFuncs { + cleanupFunc() + } + + os.Exit(verdict) +} + +func testSetup() []func() { + var ( + cleanupFuncs []func() + cleanupTempDir func() + confStrings []string + doneChan chan bool + err error + signalHandlerIsArmedWG sync.WaitGroup + tempDir string + testConfMap conf.ConfMap + ) + + cleanupFuncs = make([]func(), 0) + + tempDir, err = ioutil.TempDir("", "jrpcfs_test") + if nil != err { + panic(fmt.Sprintf("failed in testSetup: %v", err)) + } + cleanupTempDir = func() { + _ = os.RemoveAll(tempDir) + } + cleanupFuncs = append(cleanupFuncs, cleanupTempDir) + + testTLSCerts = &testTLSCertsStruct{ + caCertFile: tempDir + "/caCertFile", + caKeyFile: tempDir + "/caKeyFile", + endpointCertFile: tempDir + "/endpoingCertFile", + endpointKeyFile: tempDir + "/endpointKeyFile", + } + + testTLSCerts.caCertPEMBlock, testTLSCerts.caKeyPEMBlock, err = icertpkg.GenCACert( + icertpkg.GenerateKeyAlgorithmEd25519, + pkix.Name{ + Organization: []string{"Test Organization CA"}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + time.Hour, + testTLSCerts.caCertFile, + testTLSCerts.caKeyFile) + if nil != err { + panic(fmt.Errorf("icertpkg.GenCACert() failed: %v", err)) + } + + testTLSCerts.endpointCertPEMBlock, testTLSCerts.endpointKeyPEMBlock, err = icertpkg.GenEndpointCert( + icertpkg.GenerateKeyAlgorithmEd25519, + pkix.Name{ + Organization: []string{"Test Organization Endpoint"}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + []string{}, + []net.IP{net.ParseIP("127.0.0.1")}, + time.Hour, + testTLSCerts.caCertPEMBlock, + testTLSCerts.caKeyPEMBlock, + testTLSCerts.endpointCertFile, + testTLSCerts.endpointKeyFile) + if nil != err { + panic(fmt.Errorf("icertpkg.genEndpointCert() failed: %v", err)) + } + + testTLSCerts.endpointTLSCert, err = tls.X509KeyPair(testTLSCerts.endpointCertPEMBlock, testTLSCerts.endpointKeyPEMBlock) + if nil != err { + panic(fmt.Errorf("tls.LoadX509KeyPair() failed: %v", err)) + } + + confStrings = []string{ + "Stats.IPAddr=localhost", + "Stats.UDPPort=52184", + "Stats.BufferLength=100", + "Stats.MaxLatency=1s", + "FSGlobals.VolumeGroupList=JrpcfsTestVolumeGroup", + "FSGlobals.CheckpointHeaderConsensusAttempts=5", + "FSGlobals.MountRetryLimit=6", + "FSGlobals.MountRetryDelay=1s", + "FSGlobals.MountRetryExpBackoff=2", + "FSGlobals.LogCheckpointHeaderPosts=true", + "FSGlobals.TryLockBackoffMin=10ms", + "FSGlobals.TryLockBackoffMax=50ms", + "FSGlobals.TryLockSerializationThreshhold=5", + "FSGlobals.SymlinkMax=8", + "FSGlobals.CoalesceElementChunkSize=16", + "FSGlobals.InodeRecCacheEvictLowLimit=10000", + "FSGlobals.InodeRecCacheEvictHighLimit=10010", + "FSGlobals.LogSegmentRecCacheEvictLowLimit=10000", + "FSGlobals.LogSegmentRecCacheEvictHighLimit=10010", + "FSGlobals.BPlusTreeObjectCacheEvictLowLimit=10000", + "FSGlobals.BPlusTreeObjectCacheEvictHighLimit=10010", + "FSGlobals.DirEntryCacheEvictLowLimit=10000", + "FSGlobals.DirEntryCacheEvictHighLimit=10010", + "FSGlobals.FileExtentMapEvictLowLimit=10000", + "FSGlobals.FileExtentMapEvictHighLimit=10010", + "FSGlobals.EtcdEnabled=false", + "SwiftClient.NoAuthIPAddr=127.0.0.1", + "SwiftClient.NoAuthTCPPort=35262", + "SwiftClient.Timeout=10s", + "SwiftClient.RetryLimit=3", + "SwiftClient.RetryLimitObject=3", + "SwiftClient.RetryDelay=10ms", + "SwiftClient.RetryDelayObject=10ms", + "SwiftClient.RetryExpBackoff=1.2", + "SwiftClient.RetryExpBackoffObject=2.0", + "SwiftClient.ChunkedConnectionPoolSize=64", + "SwiftClient.NonChunkedConnectionPoolSize=32", + "RamSwiftInfo.MaxAccountNameLength=256", + "RamSwiftInfo.MaxContainerNameLength=256", + "RamSwiftInfo.MaxObjectNameLength=256", + "RamSwiftInfo.AccountListingLimit=10000", + "RamSwiftInfo.ContainerListingLimit=10000", + "Peer:Peer0.PublicIPAddr=127.0.0.1", + "Peer:Peer0.PrivateIPAddr=127.0.0.1", + "Peer:Peer0.ReadCacheQuotaFraction=0.20", + "Cluster.Peers=Peer0", + "Cluster.WhoAmI=Peer0", + "Volume:SomeVolume.FSID=1", + "Volume:SomeVolume.AccountName=" + testAccountName, + "Volume:SomeVolume.AutoFormat=true", + "Volume:SomeVolume.CheckpointContainerName=.__checkpoint__", + "Volume:SomeVolume.CheckpointContainerStoragePolicy=gold", + "Volume:SomeVolume.CheckpointInterval=10s", + "Volume:SomeVolume.DefaultPhysicalContainerLayout=SomeContainerLayout", + "Volume:SomeVolume.MaxFlushSize=10027008", + "Volume:SomeVolume.MaxFlushTime=2s", + "Volume:SomeVolume.FileDefragmentChunkSize=10027008", + "Volume:SomeVolume.FileDefragmentChunkDelay=2ms", + "Volume:SomeVolume.NonceValuesToReserve=100", + "Volume:SomeVolume.MaxEntriesPerDirNode=32", + "Volume:SomeVolume.MaxExtentsPerFileNode=32", + "Volume:SomeVolume.MaxInodesPerMetadataNode=32", + "Volume:SomeVolume.MaxLogSegmentsPerMetadataNode=64", + "Volume:SomeVolume.MaxDirFileNodesPerMetadataNode=16", + "Volume:SomeVolume.MaxBytesInodeCache=100000", + "Volume:SomeVolume.InodeCacheEvictInterval=1s", + "Volume:SomeVolume.ActiveLeaseEvictLowLimit=2", + "Volume:SomeVolume.ActiveLeaseEvictHighLimit=4", + "Volume:SomeVolume2.FSID=2", + "Volume:SomeVolume2.AccountName=" + testAccountName2, + "Volume:SomeVolume2.AutoFormat=true", + "Volume:SomeVolume2.CheckpointContainerName=.__checkpoint__", + "Volume:SomeVolume2.CheckpointContainerStoragePolicy=gold", + "Volume:SomeVolume2.CheckpointInterval=10s", + "Volume:SomeVolume2.DefaultPhysicalContainerLayout=SomeContainerLayout2", + "Volume:SomeVolume2.MaxFlushSize=10027008", + "Volume:SomeVolume2.MaxFlushTime=2s", + "Volume:SomeVolume2.FileDefragmentChunkSize=10027008", + "Volume:SomeVolume2.FileDefragmentChunkDelay=2ms", + "Volume:SomeVolume2.NonceValuesToReserve=100", + "Volume:SomeVolume2.MaxEntriesPerDirNode=32", + "Volume:SomeVolume2.MaxExtentsPerFileNode=32", + "Volume:SomeVolume2.MaxInodesPerMetadataNode=32", + "Volume:SomeVolume2.MaxLogSegmentsPerMetadataNode=64", + "Volume:SomeVolume2.MaxDirFileNodesPerMetadataNode=16", + "Volume:SomeVolume2.MaxBytesInodeCache=100000", + "Volume:SomeVolume2.InodeCacheEvictInterval=1s", + "Volume:SomeVolume2.ActiveLeaseEvictLowLimit=5000", + "Volume:SomeVolume2.ActiveLeaseEvictHighLimit=5010", + "VolumeGroup:JrpcfsTestVolumeGroup.VolumeList=SomeVolume,SomeVolume2", + "VolumeGroup:JrpcfsTestVolumeGroup.VirtualIPAddr=", + "VolumeGroup:JrpcfsTestVolumeGroup.PrimaryPeer=Peer0", + "VolumeGroup:JrpcfsTestVolumeGroup.ReadCacheLineSize=1000000", + "VolumeGroup:JrpcfsTestVolumeGroup.ReadCacheWeight=100", + "PhysicalContainerLayout:SomeContainerLayout.ContainerStoragePolicy=silver", + "PhysicalContainerLayout:SomeContainerLayout.ContainerNamePrefix=kittens", + "PhysicalContainerLayout:SomeContainerLayout.ContainersPerPeer=10", + "PhysicalContainerLayout:SomeContainerLayout.MaxObjectsPerContainer=1000000", + "PhysicalContainerLayout:SomeContainerLayout2.ContainerStoragePolicy=silver", + "PhysicalContainerLayout:SomeContainerLayout2.ContainerNamePrefix=puppies", + "PhysicalContainerLayout:SomeContainerLayout2.ContainersPerPeer=10", + "PhysicalContainerLayout:SomeContainerLayout2.MaxObjectsPerContainer=1000000", + "Logging.LogFilePath=/dev/null", + "Logging.LogToConsole=false", + "JSONRPCServer.TCPPort=12346", // 12346 instead of 12345 so that test can run if proxyfsd is already running + "JSONRPCServer.FastTCPPort=32346", // ...and similarly here... + "JSONRPCServer.RetryRPCPort=32357", // ...and similarly here... + "JSONRPCServer.RetryRPCTTLCompleted=10s", + "JSONRPCServer.RetryRPCAckTrim=10ms", + "JSONRPCServer.RetryRPCCertFilePath=" + testTLSCerts.endpointCertFile, + "JSONRPCServer.RetryRPCKeyFilePath=" + testTLSCerts.endpointKeyFile, + "JSONRPCServer.DataPathLogging=false", + "JSONRPCServer.MinLeaseDuration=100ms", + "JSONRPCServer.LeaseInterruptInterval=100ms", + "JSONRPCServer.LeaseInterruptLimit=5", + } + + testConfMap, err = conf.MakeConfMapFromStrings(confStrings) + if nil != err { + panic(fmt.Sprintf("failed in testSetup: %v", err)) + } + + signalHandlerIsArmedWG.Add(1) + doneChan = make(chan bool) + go ramswift.Daemon("/dev/null", confStrings, &signalHandlerIsArmedWG, doneChan, unix.SIGTERM) + + signalHandlerIsArmedWG.Wait() + + err = transitions.Up(testConfMap) + if nil != err { + panic(fmt.Sprintf("transitions.Up() failed: %v", err)) + } + + return cleanupFuncs +} + +func makeSomeFilesAndSuch() { + // we should have enough stuff up now that we can actually make + // some files and directories and such + volumeHandle, err := fs.FetchVolumeHandleByVolumeName("SomeVolume") + if nil != err { + panic(fmt.Sprintf("failed to mount SomeVolume: %v", err)) + } + + cInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c") + cNestedInode := fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c-nested") + fsCreateSymlink(volumeHandle, inode.RootDirInodeNumber, "c-symlink", "c") + + err = volumeHandle.MiddlewarePost("", "c", []byte("metadata for c"), []byte{}) + if err != nil { + panic(err) + } + _ = fsMkDir(volumeHandle, inode.RootDirInodeNumber, "c-no-metadata") + _ = fsMkDir(volumeHandle, cInode, "empty-directory") + + readmeInode := fsCreateFile(volumeHandle, cInode, "README") + _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, readmeInode, 0, []byte("who am I kidding? nobody reads these."), nil) + err = volumeHandle.MiddlewarePost("", "c/README", []byte("metadata for c/README"), []byte{}) + if err != nil { + panic(err) + } + + animalsInode := fsMkDir(volumeHandle, cInode, "animals") + files := map[string]string{ + "dog.txt": "dog goes woof", + "cat.txt": "cat goes meow", + "bird.txt": "bird goes tweet", + "mouse.txt": "mouse goes squeak", + "cow.txt": "cow goes moo", + "frog.txt": "frog goes croak", + "elephant.txt": "elephant goes toot", + } + for fileName, fileContents := range files { + fileInode := fsCreateFile(volumeHandle, animalsInode, fileName) + + _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileContents), nil) + if err != nil { + panic(fmt.Sprintf("failed to write file %s: %v", fileName, err)) + } + } + + plantsInode := fsMkDir(volumeHandle, cInode, "plants") + ino := fsCreateFile(volumeHandle, cInode, "plants-README") + _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino, 0, []byte("nah"), nil) + if err != nil { + panic(fmt.Sprintf("failed to write file plants-README: %v", err)) + } + + files = map[string]string{ + // Random contents of varying lengths. + "aloe.txt": "skiameter-interlope", + "banana.txt": "ring ring ring ring ring ring ring bananaphone", + "cherry.txt": "archegonium-nonresidentiary", + "eggplant.txt": "bowk-unruled", + } + + for fileName, fileContents := range files { + fileInode := fsCreateFile(volumeHandle, plantsInode, fileName) + + _, err = volumeHandle.Write(inode.InodeRootUserID, inode.InodeGroupID(0), nil, fileInode, 0, []byte(fileContents), nil) + if err != nil { + panic(fmt.Sprintf("failed to write file %s: %v", fileName, err)) + } + } + + fsCreateSymlink(volumeHandle, cInode, "plants-symlink", "plants") + fsCreateSymlink(volumeHandle, plantsInode, "eggplant.txt-symlink", "eggplant.txt") + + // Put some deeply nested things in c-nested. This listing is a + // shortened version of a real directory tree that exposed a bug. + fsCreateFile(volumeHandle, cNestedInode, ".DS_Store") + dotGitInode := fsMkDir(volumeHandle, cNestedInode, ".git") + fsCreateFile(volumeHandle, dotGitInode, ".DS_Store") + fsCreateFile(volumeHandle, dotGitInode, "COMMIT_EDITMSG") + fsCreateFile(volumeHandle, dotGitInode, "FETCH_HEAD") + fsCreateFile(volumeHandle, dotGitInode, "HEAD") + fsCreateFile(volumeHandle, dotGitInode, "ORIG_HEAD") + fsCreateFile(volumeHandle, dotGitInode, "index") + dotGitHooks := fsMkDir(volumeHandle, dotGitInode, "hooks") + fsCreateFile(volumeHandle, dotGitHooks, ".DS_Store") + fsCreateFile(volumeHandle, dotGitHooks, "applypatch-msg.sample") + fsCreateFile(volumeHandle, dotGitHooks, "commit-msg.sample") + dotGitLogs := fsMkDir(volumeHandle, dotGitInode, "logs") + fsCreateFile(volumeHandle, dotGitLogs, ".DS_Store") + fsCreateFile(volumeHandle, dotGitLogs, "HEAD") + dotGitLogsRefs := fsMkDir(volumeHandle, dotGitLogs, "refs") + fsCreateFile(volumeHandle, dotGitLogsRefs, ".DS_Store") + fsCreateFile(volumeHandle, dotGitLogsRefs, "stash") + dotGitLogsRefsHeads := fsMkDir(volumeHandle, dotGitLogsRefs, "heads") + fsCreateFile(volumeHandle, dotGitLogsRefsHeads, ".DS_Store") + fsCreateFile(volumeHandle, dotGitLogsRefsHeads, "development") + fsCreateFile(volumeHandle, dotGitLogsRefsHeads, "stable") + + aInode := fsMkDir(volumeHandle, cNestedInode, "a") + fsCreateFile(volumeHandle, aInode, "b-1") + fsCreateFile(volumeHandle, aInode, "b-2") + abInode := fsMkDir(volumeHandle, aInode, "b") + fsCreateFile(volumeHandle, abInode, "c-1") + fsCreateFile(volumeHandle, abInode, "c-2") + abcInode := fsMkDir(volumeHandle, abInode, "c") + fsCreateFile(volumeHandle, abcInode, "d-1") + fsCreateFile(volumeHandle, abcInode, "d-2") + + // SomeVolume2 is set up for testing account listings + volumeHandle2, err := fs.FetchVolumeHandleByVolumeName("SomeVolume2") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "alpha") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "bravo") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "charlie") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "delta") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "echo") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "foxtrot") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "golf") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "hotel") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "india") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "juliet") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "kilo") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "lima") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "mancy") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "november") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "oscar") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "papa") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "quebec") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "romeo") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "sierra") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "tango") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "uniform") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "victor") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "whiskey") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "xray") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "yankee") + _ = fsMkDir(volumeHandle2, inode.RootDirInodeNumber, "zulu") + _ = fsCreateFile(volumeHandle2, inode.RootDirInodeNumber, "alice.txt") + _ = fsCreateFile(volumeHandle2, inode.RootDirInodeNumber, "bob.txt") + _ = fsCreateFile(volumeHandle2, inode.RootDirInodeNumber, "carol.txt") +} + +func fsStatPath(accountName string, path string) fs.Stat { + _, _, _, _, volumeHandle, err := parseVirtPath(accountName) + if err != nil { + panic(err) + } + ino, err := volumeHandle.LookupPath(inode.InodeRootUserID, inode.InodeGroupID(0), nil, path) + if err != nil { + panic(err) + } + stats, err := volumeHandle.Getstat(inode.InodeRootUserID, inode.InodeGroupID(0), nil, ino) + if err != nil { + panic(err) + } + return stats +} + +func fsMkDir(volumeHandle fs.VolumeHandle, parentDirInode inode.InodeNumber, newDirName string) (createdInode inode.InodeNumber) { + createdInode, err := volumeHandle.Mkdir(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDirInode, newDirName, inode.PosixModePerm) + if err != nil { + panic(fmt.Sprintf("failed to create %v: %v", newDirName, err)) + } + return +} + +func fsCreateFile(volumeHandle fs.VolumeHandle, parentDirInode inode.InodeNumber, newFileName string) (createdInode inode.InodeNumber) { + createdInode, err := volumeHandle.Create(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDirInode, newFileName, inode.PosixModePerm) + if err != nil { + panic(fmt.Sprintf("failed to create file %v: %v", newFileName, err)) + } + return +} + +func fsCreateSymlink(volumeHandle fs.VolumeHandle, parentDirInode inode.InodeNumber, symlinkName string, symlinkTarget string) { + _, err := volumeHandle.Symlink(inode.InodeRootUserID, inode.InodeGroupID(0), nil, parentDirInode, symlinkName, symlinkTarget) + if err != nil { + panic(fmt.Sprintf("failed to create symlink %s -> %s: %v", symlinkName, symlinkTarget, err)) + } + return +} diff --git a/retryrpc/api.go b/retryrpc/api.go index 2b147259a..1a949e2f1 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -65,7 +65,7 @@ type ServerConfig struct { Port int // Port that Server uses to listen DeadlineIO time.Duration // How long I/Os on sockets wait even if idle KeepAlivePeriod time.Duration // How frequently a KEEPALIVE is sent - TLSCertificate tls.Certificate // TLS Certificate to present to Clients + TLSCertificate tls.Certificate // TLS Certificate to present to Clients (or tls.Certificate{} if using TCP) dontStartTrimmers bool // Used for testing } @@ -93,8 +93,7 @@ func (server *Server) Register(retrySvr interface{}) (err error) { // Start listener func (server *Server) Start() (err error) { - portStr := fmt.Sprintf("%d", server.port) - hostPortStr := net.JoinHostPort(server.ipaddr, portStr) + hostPortStr := net.JoinHostPort(server.ipaddr, fmt.Sprintf("%d", server.port)) tlsConfig := &tls.Config{ Certificates: []tls.Certificate{server.tlsCertificate}, @@ -276,8 +275,8 @@ type connectionTracker struct { state clientState genNum uint64 // Generation number of tlsConn - avoid racing recoveries tlsConfig *tls.Config - tlsConn *tls.Conn // Our connection to the server - x509CertPool *x509.CertPool + tlsConn *tls.Conn // Our connection to the server + x509CertPool *x509.CertPool // If nil, use TCP; if !nil, use TLS hostPortStr string } @@ -335,22 +334,22 @@ func NewClient(config *ClientConfig) (client *Client, err error) { client = &Client{cb: config.Callbacks, keepAlivePeriod: config.KeepAlivePeriod, deadlineIO: config.DeadlineIO} - if config.RootCAx509CertificatePEM == nil { - err = fmt.Errorf("config.RootCAx509CertificatePEM cannot be nil") - return - } - portStr := fmt.Sprintf("%d", config.Port) + client.connection.state = INITIAL - client.connection.hostPortStr = net.JoinHostPort(config.IPAddr, portStr) + client.connection.hostPortStr = net.JoinHostPort(config.IPAddr, fmt.Sprintf("%d", config.Port)) client.outstandingRequest = make(map[requestID]*reqCtx) client.bt = btree.New(2) - // Add cert for root CA to our pool - client.connection.x509CertPool = x509.NewCertPool() - ok := client.connection.x509CertPool.AppendCertsFromPEM(config.RootCAx509CertificatePEM) - if !ok { - err = fmt.Errorf("x509CertPool.AppendCertsFromPEM() returned !ok") - return nil, err + if nil == config.RootCAx509CertificatePEM { + client.connection.x509CertPool = nil + } else { + // Add cert for root CA to our pool + client.connection.x509CertPool = x509.NewCertPool() + ok := client.connection.x509CertPool.AppendCertsFromPEM(config.RootCAx509CertificatePEM) + if !ok { + err = fmt.Errorf("x509CertPool.AppendCertsFromPEM() returned !ok") + return nil, err + } } return client, err From cf7a9cb45cd205e2ad19a99cb16764c323b3c3da Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 26 Mar 2021 14:50:40 -0700 Subject: [PATCH 085/160] Nearly all remaining changes to allow retryrpc to optionally use TCP instead of TLS ...api_internal.go::getIO() is the lone holdout... and it is used by both client.go and server.go --- retryrpc/api.go | 33 ++++++++---- retryrpc/client.go | 123 ++++++++++++++++++++++++++++++++------------- 2 files changed, 111 insertions(+), 45 deletions(-) diff --git a/retryrpc/api.go b/retryrpc/api.go index 1a949e2f1..252c9f41a 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -272,12 +272,14 @@ const ( ) type connectionTracker struct { - state clientState - genNum uint64 // Generation number of tlsConn - avoid racing recoveries - tlsConfig *tls.Config - tlsConn *tls.Conn // Our connection to the server + state clientState // + genNum uint64 // Generation number of tlsConn - avoid racing recoveries + useTLS bool // + netConn *tls.Conn // Our TCP connection to the server + tlsConn *tls.Conn // Our TLS connection to the server + tlsConfig *tls.Config // x509CertPool *x509.CertPool // If nil, use TCP; if !nil, use TLS - hostPortStr string + hostPortStr string // } // Client tracking structure @@ -287,7 +289,7 @@ type Client struct { currentRequestID requestID // Last request ID - start from clock // tick at mount and increment from there? // Handle reset of time? - connection connectionTracker + connection *connectionTracker myUniqueID uint64 // Unique ID across all clients cb interface{} // Callbacks to client deadlineIO time.Duration @@ -332,17 +334,26 @@ type ClientConfig struct { // starting point for requestID. func NewClient(config *ClientConfig) (client *Client, err error) { - client = &Client{cb: config.Callbacks, keepAlivePeriod: config.KeepAlivePeriod, - deadlineIO: config.DeadlineIO} + client = &Client{ + connection: &connectionTracker{ + state: INITIAL, + hostPortStr: net.JoinHostPort(config.IPAddr, fmt.Sprintf("%d", config.Port)), + }, + cb: config.Callbacks, + keepAlivePeriod: config.KeepAlivePeriod, + deadlineIO: config.DeadlineIO, + } - client.connection.state = INITIAL - client.connection.hostPortStr = net.JoinHostPort(config.IPAddr, fmt.Sprintf("%d", config.Port)) client.outstandingRequest = make(map[requestID]*reqCtx) client.bt = btree.New(2) if nil == config.RootCAx509CertificatePEM { + client.connection.useTLS = false + client.connection.tlsConn = nil client.connection.x509CertPool = nil } else { + client.connection.useTLS = true + client.connection.netConn = nil // Add cert for root CA to our pool client.connection.x509CertPool = x509.NewCertPool() ok := client.connection.x509CertPool.AppendCertsFromPEM(config.RootCAx509CertificatePEM) @@ -380,7 +391,7 @@ func (client *Client) Close() { client.halting = true if client.connection.state == CONNECTED { client.connection.state = INITIAL - client.connection.tlsConn.Close() + client.connection.Close() } client.Unlock() diff --git a/retryrpc/client.go b/retryrpc/client.go index 8ef287d3d..d5750e21d 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "encoding/json" "fmt" + "io" "net" "os" "time" @@ -168,8 +169,8 @@ func (client *Client) sendToServer(crID requestID, ctx *reqCtx, queue bool) { } // Send header - client.connection.tlsConn.SetDeadline(time.Now().Add(client.deadlineIO)) - err := binary.Write(client.connection.tlsConn, binary.BigEndian, ctx.ioreq.Hdr) + client.connection.SetDeadline(time.Now().Add(client.deadlineIO)) + err := binary.Write(client.connection.ioWriter(), binary.BigEndian, ctx.ioreq.Hdr) if err != nil { genNum := ctx.genNum client.Unlock() @@ -181,8 +182,8 @@ func (client *Client) sendToServer(crID requestID, ctx *reqCtx, queue bool) { } // Send JSON request - client.connection.tlsConn.SetDeadline(time.Now().Add(client.deadlineIO)) - bytesWritten, writeErr := client.connection.tlsConn.Write(ctx.ioreq.JReq) + client.connection.SetDeadline(time.Now().Add(client.deadlineIO)) + bytesWritten, writeErr := client.connection.Write(ctx.ioreq.JReq) if (bytesWritten != len(ctx.ioreq.JReq)) || (writeErr != nil) { /* TODO - log message? @@ -273,14 +274,14 @@ func (client *Client) notifyReply(buf []byte, genNum uint64) { // // As soon as it reads a complete response, it launches a goroutine to process // the response and notify the blocked Send(). -func (client *Client) readReplies(callingGenNum uint64, tlsConn *tls.Conn) { +func (client *Client) readReplies(callingGenNum uint64) { defer client.goroutineWG.Done() var localCnt int for { // Wait reply from server - buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, tlsConn) + buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, client.connection.tlsConn) // This must happen before checking error client.Lock() @@ -361,7 +362,7 @@ func (client *Client) retransmit(genNum uint64) { // We are the first goroutine to notice the error on the // socket - close the connection and start trying to reconnect. - _ = client.connection.tlsConn.Close() + _ = client.connection.Close() client.connection.state = RETRANSMITTING client.stats.RetransmitsStarted.Add(1) @@ -404,7 +405,7 @@ func (client *Client) retransmit(genNum uint64) { // creating this connection and wants a unique ID from the server. // // NOTE: Client lock is already held during this call. -func (client *Client) getMyUniqueID(tlsConn *tls.Conn) (err error) { +func (client *Client) getMyUniqueID() (err error) { // Setup ioreq to write structure on socket to server iinreq, err := buildINeedIDRequest() @@ -415,15 +416,15 @@ func (client *Client) getMyUniqueID(tlsConn *tls.Conn) (err error) { } // We only have the header to send - client.connection.tlsConn.SetDeadline(time.Now().Add(client.deadlineIO)) - err = binary.Write(tlsConn, binary.BigEndian, iinreq.Hdr) + client.connection.SetDeadline(time.Now().Add(client.deadlineIO)) + err = binary.Write(client.connection.ioWriter(), binary.BigEndian, iinreq.Hdr) if err != nil { return } // Ask the server for the unique ID. // If we error, just return it and let caller close the connection. - client.myUniqueID, err = client.readClientID(client.connection.genNum, tlsConn) + client.myUniqueID, err = client.readClientID(client.connection.genNum) if err != nil { return } @@ -435,7 +436,7 @@ func (client *Client) getMyUniqueID(tlsConn *tls.Conn) (err error) { // knows myUniqueID // // NOTE: Client lock is already held during this call. -func (client *Client) sendMyInfo(tlsConn *tls.Conn) (err error) { +func (client *Client) sendMyInfo() (err error) { // Setup ioreq to write structure on socket to server isreq, err := buildSetIDRequest(client.myUniqueID) @@ -446,15 +447,15 @@ func (client *Client) sendMyInfo(tlsConn *tls.Conn) (err error) { } // Send header - client.connection.tlsConn.SetDeadline(time.Now().Add(client.deadlineIO)) - err = binary.Write(tlsConn, binary.BigEndian, isreq.Hdr) + client.connection.SetDeadline(time.Now().Add(client.deadlineIO)) + err = binary.Write(client.connection.ioWriter(), binary.BigEndian, isreq.Hdr) if err != nil { return } // We are an existing client - send "MyUniqueID" - client.connection.tlsConn.SetDeadline(time.Now().Add(client.deadlineIO)) - bytesWritten, writeErr := tlsConn.Write(isreq.MyUniqueID) + client.connection.SetDeadline(time.Now().Add(client.deadlineIO)) + bytesWritten, writeErr := client.connection.Write(isreq.MyUniqueID) if uint32(bytesWritten) != isreq.Hdr.Len { e := fmt.Errorf("sendMyInfo length incorrect") @@ -496,10 +497,7 @@ func (client *Client) reDial() (err error) { return } - if client.connection.tlsConn != nil { - client.connection.tlsConn.Close() - client.connection.tlsConn = nil - } + client.connection.CloseIfOpen() client.connection.tlsConn = tlsConn client.connection.state = CONNECTED @@ -507,17 +505,16 @@ func (client *Client) reDial() (err error) { // Send myUniqueID to server. If this fails the dial will // be retried. - err = client.sendMyInfo(tlsConn) + err = client.sendMyInfo() if err != nil { - _ = client.connection.tlsConn.Close() - client.connection.tlsConn = nil + _ = client.connection.Close() client.connection.state = entryState return } // Start readResponse goroutine to read responses from server client.goroutineWG.Add(1) - go client.readReplies(client.connection.genNum, tlsConn) + go client.readReplies(client.connection.genNum) return } @@ -527,10 +524,10 @@ func (client *Client) reDial() (err error) { // Client lock is held // // NOTE: Client lock is held -func (client *Client) readClientID(callingGenNum uint64, tlsConn *tls.Conn) (myUniqueID uint64, err error) { +func (client *Client) readClientID(callingGenNum uint64) (myUniqueID uint64, err error) { // Wait reply from server - buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, tlsConn) + buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, client.connection.tlsConn) // This must happen before checking error if client.halting { @@ -584,10 +581,7 @@ func (client *Client) initialDial() (err error) { return } - if client.connection.tlsConn != nil { - client.connection.tlsConn.Close() - client.connection.tlsConn = nil - } + client.connection.CloseIfOpen() client.connection.tlsConn = tlsConn client.connection.state = CONNECTED @@ -596,17 +590,16 @@ func (client *Client) initialDial() (err error) { // Ask server for my unique ID // // If this fails the dial will be retried. - err = client.getMyUniqueID(tlsConn) + err = client.getMyUniqueID() if err != nil { - _ = client.connection.tlsConn.Close() - client.connection.tlsConn = nil + _ = client.connection.Close() client.connection.state = entryState return } // Start readResponse goroutine to read responses from server client.goroutineWG.Add(1) - go client.readReplies(client.connection.genNum, tlsConn) + go client.readReplies(client.connection.genNum) return } @@ -676,3 +669,65 @@ func (client *Client) updateHighestConsecutiveNum(crID requestID) { client.setHighestConsecutive() client.Unlock() } + +// Below are wrapper func's on connectionTracker structs that perform +// the underlying/requested operation on the netConn or tlsConn based +// on the value of useTLS. + +func (cT *connectionTracker) Close() (err error) { + if cT.useTLS { + err = cT.tlsConn.Close() + cT.tlsConn = nil + } else { + err = cT.netConn.Close() + cT.netConn = nil + } + + return +} + +func (cT *connectionTracker) CloseIfOpen() (err error) { + if cT.useTLS { + if nil != cT.tlsConn { + err = cT.tlsConn.Close() + cT.tlsConn = nil + } + } else { + if nil != cT.netConn { + err = cT.netConn.Close() + cT.netConn = nil + } + } + + return +} + +func (cT *connectionTracker) SetDeadline(t time.Time) (err error) { + if cT.useTLS { + err = cT.tlsConn.SetDeadline(t) + } else { + err = cT.netConn.SetDeadline(t) + } + + return +} + +func (cT *connectionTracker) Write(b []byte) (n int, err error) { + if cT.useTLS { + n, err = cT.tlsConn.Write(b) + } else { + n, err = cT.netConn.Write(b) + } + + return +} + +func (cT *connectionTracker) ioWriter() (w io.Writer) { + if cT.useTLS { + w = cT.tlsConn + } else { + w = cT.netConn + } + + return +} From 65cb3346cf87780eed2a326de3e7b55bf2bd12c7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 26 Mar 2021 16:07:14 -0700 Subject: [PATCH 086/160] Remaining TCP (non-TLS) support added (and tested) to retryrpc --- retryrpc/api.go | 2 +- retryrpc/client.go | 74 +++++++++++++++++------ retryrpc/retryrpc_test.go | 123 +++++++++++++++++++++++++++----------- retryrpc/stress_test.go | 86 +++++++++++++++++--------- retryrpc/upcall_test.go | 42 +++++++++---- 5 files changed, 236 insertions(+), 91 deletions(-) diff --git a/retryrpc/api.go b/retryrpc/api.go index 252c9f41a..ca2c54b54 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -275,7 +275,7 @@ type connectionTracker struct { state clientState // genNum uint64 // Generation number of tlsConn - avoid racing recoveries useTLS bool // - netConn *tls.Conn // Our TCP connection to the server + netConn net.Conn // Our TCP connection to the server tlsConn *tls.Conn // Our TLS connection to the server tlsConfig *tls.Config // x509CertPool *x509.CertPool // If nil, use TCP; if !nil, use TLS diff --git a/retryrpc/client.go b/retryrpc/client.go index d5750e21d..7d10919e5 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -281,7 +281,7 @@ func (client *Client) readReplies(callingGenNum uint64) { for { // Wait reply from server - buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, client.connection.tlsConn) + buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, client.connection.castToNetConn()) // This must happen before checking error client.Lock() @@ -490,16 +490,30 @@ func (client *Client) reDial() (err error) { } // Now dial the server - d := &net.Dialer{KeepAlive: client.keepAlivePeriod} - tlsConn, dialErr := tls.DialWithDialer(d, "tcp", client.connection.hostPortStr, client.connection.tlsConfig) - if dialErr != nil { - err = fmt.Errorf("tls.Dial() failed: %v", dialErr) - return - } - client.connection.CloseIfOpen() + if client.connection.useTLS { + d := &net.Dialer{KeepAlive: client.keepAlivePeriod} + tlsConn, dialErr := tls.DialWithDialer(d, "tcp", client.connection.hostPortStr, client.connection.tlsConfig) + if dialErr != nil { + err = fmt.Errorf("tls.Dial() failed: %v", dialErr) + return + } + + client.connection.CloseIfOpen() + + client.connection.tlsConn = tlsConn + } else { + netConn, dialErr := net.Dial("tcp", client.connection.hostPortStr) + if dialErr != nil { + err = fmt.Errorf("net.Dial() failed: %v", dialErr) + return + } + + client.connection.CloseIfOpen() + + client.connection.netConn = netConn + } - client.connection.tlsConn = tlsConn client.connection.state = CONNECTED client.connection.genNum++ @@ -527,7 +541,7 @@ func (client *Client) reDial() (err error) { func (client *Client) readClientID(callingGenNum uint64) (myUniqueID uint64, err error) { // Wait reply from server - buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, client.connection.tlsConn) + buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, client.connection.castToNetConn()) // This must happen before checking error if client.halting { @@ -574,16 +588,30 @@ func (client *Client) initialDial() (err error) { } // Now dial the server - d := &net.Dialer{KeepAlive: client.keepAlivePeriod} - tlsConn, dialErr := tls.DialWithDialer(d, "tcp", client.connection.hostPortStr, client.connection.tlsConfig) - if dialErr != nil { - err = fmt.Errorf("tls.Dial() failed: %v", dialErr) - return - } - client.connection.CloseIfOpen() + if client.connection.useTLS { + d := &net.Dialer{KeepAlive: client.keepAlivePeriod} + tlsConn, dialErr := tls.DialWithDialer(d, "tcp", client.connection.hostPortStr, client.connection.tlsConfig) + if dialErr != nil { + err = fmt.Errorf("tls.Dial() failed: %v", dialErr) + return + } + + client.connection.CloseIfOpen() + + client.connection.tlsConn = tlsConn + } else { + netConn, dialErr := net.Dial("tcp", client.connection.hostPortStr) + if dialErr != nil { + err = fmt.Errorf("net.Dial() failed: %v", dialErr) + return + } + + client.connection.CloseIfOpen() + + client.connection.netConn = netConn + } - client.connection.tlsConn = tlsConn client.connection.state = CONNECTED client.connection.genNum++ @@ -722,6 +750,16 @@ func (cT *connectionTracker) Write(b []byte) (n int, err error) { return } +func (cT *connectionTracker) castToNetConn() (nC net.Conn) { + if cT.useTLS { + nC = cT.tlsConn + } else { + nC = cT.netConn + } + + return +} + func (cT *connectionTracker) ioWriter() (w io.Writer) { if cT.useTLS { w = cT.tlsConn diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index 263550906..95a34cb4c 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -88,24 +88,49 @@ func testTLSCertsAllocate(t *testing.T) { // // This unit test exists here since it uses jrpcfs which would be a // circular dependency if the test was in retryrpc. -func TestRetryRPC(t *testing.T) { +func TestTCPRetryRPC(t *testing.T) { + testTLSCerts = nil + testRegister(t, false) + testServer(t, false) + testBtree(t, false) + testStatsAndBucketstats(t) +} + +func TestTLSRetryRPC(t *testing.T) { testTLSCertsAllocate(t) - testRegister(t) - testServer(t) - testBtree(t) + testRegister(t, true) + testServer(t, true) + testBtree(t, true) testStatsAndBucketstats(t) } -func getNewServer(lt time.Duration, dontStartTrimmers bool) (rrSvr *Server) { - config := &ServerConfig{ - LongTrim: lt, - ShortTrim: 100 * time.Millisecond, - IPAddr: testIPAddr, - Port: testPort, - DeadlineIO: 60 * time.Second, - KeepAlivePeriod: 60 * time.Second, - TLSCertificate: testTLSCerts.endpointTLSCert, - dontStartTrimmers: dontStartTrimmers, +func getNewServer(lt time.Duration, dontStartTrimmers bool, useTLS bool) (rrSvr *Server) { + var ( + config *ServerConfig + ) + + if useTLS { + config = &ServerConfig{ + LongTrim: lt, + ShortTrim: 100 * time.Millisecond, + IPAddr: testIPAddr, + Port: testPort, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + TLSCertificate: testTLSCerts.endpointTLSCert, + dontStartTrimmers: dontStartTrimmers, + } + } else { + config = &ServerConfig{ + LongTrim: lt, + ShortTrim: 100 * time.Millisecond, + IPAddr: testIPAddr, + Port: testPort, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + TLSCertificate: tls.Certificate{}, + dontStartTrimmers: dontStartTrimmers, + } } // Create a new RetryRPC Server. Completed request will live on @@ -116,7 +141,7 @@ func getNewServer(lt time.Duration, dontStartTrimmers bool) (rrSvr *Server) { } // Test register function finds all expected functions -func testRegister(t *testing.T) { +func testRegister(t *testing.T, useTLS bool) { assert := assert.New(t) zero := 0 assert.Equal(0, zero) @@ -125,7 +150,7 @@ func testRegister(t *testing.T) { // RPCs myJrpcfs := rpctest.NewServer() - rrSvr := getNewServer(10*time.Second, false) + rrSvr := getNewServer(10*time.Second, false, useTLS) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -138,7 +163,11 @@ func testRegister(t *testing.T) { } // Test basic Server creation and deletion -func testServer(t *testing.T) { +func testServer(t *testing.T, useTLS bool) { + var ( + clientConfig *ClientConfig + ) + assert := assert.New(t) zero := 0 assert.Equal(0, zero) @@ -147,7 +176,7 @@ func testServer(t *testing.T) { // RPCs myJrpcfs := rpctest.NewServer() - rrSvr := getNewServer(10*time.Second, false) + rrSvr := getNewServer(10*time.Second, false, useTLS) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -163,13 +192,24 @@ func testServer(t *testing.T) { rrSvr.Run() // Now - setup a client to send requests to the server - clientConfig := &ClientConfig{ - IPAddr: testIPAddr, - Port: testPort, - RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, - Callbacks: nil, - DeadlineIO: 60 * time.Second, - KeepAlivePeriod: 60 * time.Second, + if useTLS { + clientConfig = &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, + Callbacks: nil, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } + } else { + clientConfig = &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: nil, + Callbacks: nil, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } } rrClnt, newErr := NewClient(clientConfig) assert.NotNil(rrClnt) @@ -214,20 +254,35 @@ func testServer(t *testing.T) { rrSvr.Close() } -func testBtree(t *testing.T) { +func testBtree(t *testing.T, useTLS bool) { + var ( + clientConfig *ClientConfig + ) + assert := assert.New(t) - rrSvr := getNewServer(10*time.Second, false) + rrSvr := getNewServer(10*time.Second, false, useTLS) assert.NotNil(rrSvr) // Setup a client - we only will be targeting the btree - clientConfig := &ClientConfig{ - IPAddr: testIPAddr, - Port: testPort, - RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, - Callbacks: nil, - DeadlineIO: 60 * time.Second, - KeepAlivePeriod: 60 * time.Second, + if useTLS { + clientConfig = &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, + Callbacks: nil, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } + } else { + clientConfig = &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: nil, + Callbacks: nil, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } } client, newErr := NewClient(clientConfig) assert.NotNil(client) diff --git a/retryrpc/stress_test.go b/retryrpc/stress_test.go index 639ce38be..94ada0dbb 100644 --- a/retryrpc/stress_test.go +++ b/retryrpc/stress_test.go @@ -18,7 +18,24 @@ import ( "github.com/stretchr/testify/assert" ) -func TestStress(t *testing.T) { +func TestTCPStress(t *testing.T) { + + /* + * DEBUG - used to debug memory leaks + * Run " go tool pprof http://localhost:12123/debug/pprof/heap" + * to look at memory inuse + // Start the ws that listens for pprof requests + go http.ListenAndServe("localhost:12123", nil) + */ + + testTLSCerts = nil + testLoop(t, false) + testLoopClientAckTrim(t, false) + testLoopTTLTrim(t, false) + testSendLargeRPC(t, false) +} + +func TestTLSStress(t *testing.T) { /* * DEBUG - used to debug memory leaks @@ -29,13 +46,13 @@ func TestStress(t *testing.T) { */ testTLSCertsAllocate(t) - testLoop(t) - testLoopClientAckTrim(t) - testLoopTTLTrim(t) - testSendLargeRPC(t) + testLoop(t, true) + testLoopClientAckTrim(t, true) + testLoopTTLTrim(t, true) + testSendLargeRPC(t, true) } -func testLoop(t *testing.T) { +func testLoop(t *testing.T, useTLS bool) { var ( agentCount = 15 sendCount = 250 @@ -48,7 +65,7 @@ func testLoop(t *testing.T) { // RPCs myJrpcfs := rpctest.NewServer() - rrSvr := getNewServer(65*time.Second, false) + rrSvr := getNewServer(65*time.Second, false, useTLS) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -64,7 +81,7 @@ func testLoop(t *testing.T) { rrSvr.Run() // Start up the agents - parallelAgentSenders(t, rrSvr, agentCount, "RpcPing", sendCount) + parallelAgentSenders(t, rrSvr, agentCount, "RpcPing", sendCount, useTLS) rrSvr.Close() } @@ -74,7 +91,7 @@ func testLoop(t *testing.T) { // on the client code saying "this is the highest consecutive sqn we have // seen". Then the server can throw away messages up to and including the // highest consecutive sqn. -func testLoopClientAckTrim(t *testing.T) { +func testLoopClientAckTrim(t *testing.T, useTLS bool) { var ( agentCount = 15 sendCount = 250 @@ -88,7 +105,7 @@ func testLoopClientAckTrim(t *testing.T) { myJrpcfs := rpctest.NewServer() whenTTL := 10 * time.Millisecond - rrSvr := getNewServer(whenTTL, true) + rrSvr := getNewServer(whenTTL, true, useTLS) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -104,7 +121,7 @@ func testLoopClientAckTrim(t *testing.T) { rrSvr.Run() // Start up the agents - parallelAgentSenders(t, rrSvr, agentCount, "RpcPing", sendCount) + parallelAgentSenders(t, rrSvr, agentCount, "RpcPing", sendCount, useTLS) // Now for both trimmers to run tm := time.Now() @@ -136,7 +153,7 @@ func testLoopClientAckTrim(t *testing.T) { rrSvr.Close() } -func testLoopTTLTrim(t *testing.T) { +func testLoopTTLTrim(t *testing.T, useTLS bool) { var ( agentCount = 15 sendCount = 250 @@ -150,7 +167,7 @@ func testLoopTTLTrim(t *testing.T) { myJrpcfs := rpctest.NewServer() whenTTL := 10 * time.Millisecond - rrSvr := getNewServer(whenTTL, true) + rrSvr := getNewServer(whenTTL, true, useTLS) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -166,7 +183,7 @@ func testLoopTTLTrim(t *testing.T) { rrSvr.Run() // Start up the agents - parallelAgentSenders(t, rrSvr, agentCount, "RpcPing", sendCount) + parallelAgentSenders(t, rrSvr, agentCount, "RpcPing", sendCount, useTLS) // Use the TTL trimmer to remove all messages after guaranteeing we are // past time when they should be removed @@ -186,7 +203,7 @@ func testLoopTTLTrim(t *testing.T) { rrSvr.Close() } -func testSendLargeRPC(t *testing.T) { +func testSendLargeRPC(t *testing.T, useTLS bool) { var ( agentCount = 15 sendCount = 250 @@ -200,7 +217,7 @@ func testSendLargeRPC(t *testing.T) { myJrpcfs := rpctest.NewServer() whenTTL := 10 * time.Millisecond - rrSvr := getNewServer(whenTTL, true) + rrSvr := getNewServer(whenTTL, true, useTLS) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -216,7 +233,7 @@ func testSendLargeRPC(t *testing.T) { rrSvr.Run() // Start up the agents - parallelAgentSenders(t, rrSvr, agentCount, "RpcPingLarge", sendCount) + parallelAgentSenders(t, rrSvr, agentCount, "RpcPingLarge", sendCount, useTLS) // Now for both trimmers to run tm := time.Now() @@ -334,18 +351,33 @@ func (cb *stressMyClient) Interrupt(payload []byte) { } // Represents a pfsagent - sepearate client -func pfsagent(t *testing.T, rrSvr *Server, agentID uint64, method string, agentWg *sync.WaitGroup, sendCnt int) { +func pfsagent(t *testing.T, rrSvr *Server, agentID uint64, method string, agentWg *sync.WaitGroup, sendCnt int, useTLS bool) { + var ( + clientConfig *ClientConfig + ) + defer agentWg.Done() cb := &stressMyClient{} cb.cond = sync.NewCond(&cb.Mutex) - clientConfig := &ClientConfig{ - IPAddr: testIPAddr, - Port: testPort, - RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, - Callbacks: cb, - DeadlineIO: 60 * time.Second, - KeepAlivePeriod: 60 * time.Second, + if useTLS { + clientConfig = &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, + Callbacks: cb, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } + } else { + clientConfig = &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: nil, + Callbacks: cb, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } } client, err := NewClient(clientConfig) if err != nil { @@ -409,7 +441,7 @@ func pfsagent(t *testing.T, rrSvr *Server, agentID uint64, method string, agentW } // Start a bunch of "pfsagents" in parallel -func parallelAgentSenders(t *testing.T, rrSrv *Server, agentCnt int, method string, sendCnt int) { +func parallelAgentSenders(t *testing.T, rrSrv *Server, agentCnt int, method string, sendCnt int, useTLS bool) { var agentWg sync.WaitGroup @@ -423,7 +455,7 @@ func parallelAgentSenders(t *testing.T, rrSrv *Server, agentCnt int, method stri agentID = clientSeed + uint64(i) agentWg.Add(1) - go pfsagent(t, rrSrv, agentID, method, &agentWg, sendCnt) + go pfsagent(t, rrSrv, agentID, method, &agentWg, sendCnt, useTLS) } agentWg.Wait() } diff --git a/retryrpc/upcall_test.go b/retryrpc/upcall_test.go index b362ebff8..884b53f9b 100644 --- a/retryrpc/upcall_test.go +++ b/retryrpc/upcall_test.go @@ -13,9 +13,14 @@ import ( ) // Test Upcall() functionality -func TestUpCall(t *testing.T) { +func TestTCPUpCall(t *testing.T) { + testTLSCerts = nil + testUpCall(t, false) +} + +func TestTLSUpCall(t *testing.T) { testTLSCertsAllocate(t) - testUpCall(t) + testUpCall(t, true) } type MyClient struct { @@ -33,7 +38,11 @@ func (cb *MyClient) Interrupt(payload []byte) { } // Test getting an upcallbasic Server creation and deletion -func testUpCall(t *testing.T) { +func testUpCall(t *testing.T, useTLS bool) { + var ( + clientConfig *ClientConfig + ) + assert := assert.New(t) zero := 0 assert.Equal(0, zero) @@ -42,7 +51,7 @@ func testUpCall(t *testing.T) { // RPCs myJrpcfs := rpctest.NewServer() - rrSvr := getNewServer(10*time.Second, false) + rrSvr := getNewServer(10*time.Second, false, useTLS) assert.NotNil(rrSvr) // Register the Server - sets up the methods supported by the @@ -61,13 +70,24 @@ func testUpCall(t *testing.T) { cb := &MyClient{} cb.cond = sync.NewCond(&cb.Mutex) - clientConfig := &ClientConfig{ - IPAddr: testIPAddr, - Port: testPort, - RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, - Callbacks: cb, - DeadlineIO: 60 * time.Second, - KeepAlivePeriod: 60 * time.Second, + if useTLS { + clientConfig = &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: testTLSCerts.caCertPEMBlock, + Callbacks: cb, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } + } else { + clientConfig = &ClientConfig{ + IPAddr: testIPAddr, + Port: testPort, + RootCAx509CertificatePEM: nil, + Callbacks: cb, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } } rrClnt, newErr := NewClient(clientConfig) assert.NotNil(rrClnt) From 67e6081b7f7779724322351d48231bed4e109d57 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 26 Mar 2021 17:07:22 -0700 Subject: [PATCH 087/160] Got PFSAgent working with the new retryrpc behavior Note that the .conf file has some additions for now... --- jrpcfs/api.go | 22 +++++------------- jrpcfs/config.go | 1 + jrpcfs/filesystem.go | 17 ++++---------- jrpcfs/lease_test.go | 12 ++++------ pfsagentd/README.md | 2 ++ pfsagentd/globals.go | 9 ++------ pfsagentd/pfsagent.conf | 2 ++ pfsagentd/request.go | 38 ++++++++++++++------------------ pfsagentd/setup_teardown_test.go | 5 ++++- retryrpc/api.go | 2 +- 10 files changed, 43 insertions(+), 67 deletions(-) diff --git a/jrpcfs/api.go b/jrpcfs/api.go index 1bb6dc9d1..244bd5f4b 100644 --- a/jrpcfs/api.go +++ b/jrpcfs/api.go @@ -387,34 +387,24 @@ type MkdirPathRequest struct { // MountByAccountNameRequest is the request object for RpcMountByAccountName. type MountByAccountNameRequest struct { - AccountName string - MountOptions uint64 - AuthUserID uint64 - AuthGroupID uint64 + AccountName string + AuthToken string } // MountByAccountNameReply is the reply object for RpcMountByAccountName. type MountByAccountNameReply struct { - MountID MountIDAsString - RootDirInodeNumber int64 - RetryRPCPublicIPAddr string - RetryRPCPort uint16 + MountID MountIDAsString } // MountByVolumeNameRequest is the request object for RpcMountByVolumeName. type MountByVolumeNameRequest struct { - VolumeName string - MountOptions uint64 - AuthUserID uint64 - AuthGroupID uint64 + VolumeName string + AuthToken string } // MountByVolumeNameReply is the reply object for RpcMountByVolumeName. type MountByVolumeNameReply struct { - MountID MountIDAsString - RootDirInodeNumber int64 - RetryRPCPublicIPAddr string - RetryRPCPort uint16 + MountID MountIDAsString } // UnmountRequest is the request object for RpcUnmount. diff --git a/jrpcfs/config.go b/jrpcfs/config.go index e12dfa747..f405e817d 100644 --- a/jrpcfs/config.go +++ b/jrpcfs/config.go @@ -97,6 +97,7 @@ type mountStruct struct { volume *volumeStruct mountIDAsByteArray MountIDAsByteArray mountIDAsString MountIDAsString + authToken string retryRpcUniqueID uint64 acceptingLeaseRequests bool // also an indicator (when false) that mount is being unmounted leaseRequestMap map[inode.InodeNumber]*leaseRequestStruct // if present, there is an ongoing Lease Request for this inode.InodeNumber diff --git a/jrpcfs/filesystem.go b/jrpcfs/filesystem.go index 029472b35..deec4598a 100644 --- a/jrpcfs/filesystem.go +++ b/jrpcfs/filesystem.go @@ -1335,7 +1335,7 @@ func (s *Server) RpcMkdirPath(in *MkdirPathRequest, reply *Reply) (err error) { return } -func performMount(volumeHandle fs.VolumeHandle, clientID uint64) (mountIDAsByteArray MountIDAsByteArray, mountIDAsString MountIDAsString, err error) { +func performMount(volumeHandle fs.VolumeHandle, authToken string, clientID uint64) (mountIDAsByteArray MountIDAsByteArray, mountIDAsString MountIDAsString, err error) { var ( i int keepTrying bool @@ -1383,6 +1383,7 @@ func performMount(volumeHandle fs.VolumeHandle, clientID uint64) (mountIDAsByteA volume: volume, mountIDAsByteArray: mountIDAsByteArray, mountIDAsString: mountIDAsString, + authToken: authToken, retryRpcUniqueID: clientID, acceptingLeaseRequests: true, leaseRequestMap: make(map[inode.InodeNumber]*leaseRequestStruct), @@ -1409,12 +1410,7 @@ func (s *Server) RpcMountByAccountName(clientID uint64, in *MountByAccountNameRe volumeHandle, err := fs.FetchVolumeHandleByAccountName(in.AccountName) if nil == err { - _, reply.MountID, err = performMount(volumeHandle, clientID) - if nil == err { - reply.RootDirInodeNumber = int64(uint64(inode.RootDirInodeNumber)) - reply.RetryRPCPublicIPAddr = globals.publicIPAddr - reply.RetryRPCPort = globals.retryRPCPort - } + _, reply.MountID, err = performMount(volumeHandle, in.AuthToken, clientID) } return @@ -1430,12 +1426,7 @@ func (s *Server) RpcMountByVolumeName(clientID uint64, in *MountByVolumeNameRequ volumeHandle, err := fs.FetchVolumeHandleByVolumeName(in.VolumeName) if nil == err { - _, reply.MountID, err = performMount(volumeHandle, clientID) - if nil == err { - reply.RootDirInodeNumber = int64(uint64(inode.RootDirInodeNumber)) - reply.RetryRPCPublicIPAddr = globals.publicIPAddr - reply.RetryRPCPort = globals.retryRPCPort - } + _, reply.MountID, err = performMount(volumeHandle, in.AuthToken, clientID) } return diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index 953d9c60a..bc751a88b 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -86,10 +86,8 @@ func BenchmarkRpcLease(b *testing.B) { } mountByAccountNameRequest = &MountByAccountNameRequest{ - AccountName: testAccountName, - MountOptions: 0, - AuthUserID: 0, - AuthGroupID: 0, + AccountName: testAccountName, + AuthToken: "", } mountByAccountNameReply = &MountByAccountNameReply{} @@ -490,10 +488,8 @@ func (testRpcLeaseClient *testRpcLeaseClientStruct) instanceGoroutine() { } mountByAccountNameRequest = &MountByAccountNameRequest{ - AccountName: testAccountName, - MountOptions: 0, - AuthUserID: 0, - AuthGroupID: 0, + AccountName: testAccountName, + AuthToken: "", } mountByAccountNameReply = &MountByAccountNameReply{} diff --git a/pfsagentd/README.md b/pfsagentd/README.md index 4444ba62b..85852d523 100644 --- a/pfsagentd/README.md +++ b/pfsagentd/README.md @@ -75,6 +75,8 @@ ReaddirMaxEntries: 1024 FUSEMaxBackground: 100 FUSECongestionThreshhold: 0 FUSEMaxWrite: 131072 +RetryRPCPublicIPAddr: 127.0.0.1 +RetryRPCPort: 32356 RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s RetryRPCCACertFilePath: # Defaults to /dev/null diff --git a/pfsagentd/globals.go b/pfsagentd/globals.go index e832a3c31..4080c1fc5 100644 --- a/pfsagentd/globals.go +++ b/pfsagentd/globals.go @@ -72,6 +72,8 @@ type configStruct struct { FUSEMaxBackground uint16 FUSECongestionThreshhold uint16 FUSEMaxWrite uint32 + RetryRPCPublicIPAddr string + RetryRPCPort uint16 RetryRPCDeadlineIO time.Duration RetryRPCKeepAlivePeriod time.Duration RetryRPCCACertFilePath string @@ -334,10 +336,7 @@ type globalsStruct struct { config configStruct logFile *os.File // == nil if configStruct.LogFilePath == "" retryRPCCACertPEM []byte - retryRPCPublicIPAddr string - retryRPCPort uint16 retryRPCClient *retryrpc.Client - rootCAx509CertificatePEM []byte entryValidSec uint64 entryValidNSec uint32 attrValidSec uint64 @@ -351,7 +350,6 @@ type globalsStruct struct { swiftAuthToken string // Protected by swiftAuthWaitGroup swiftStorageURL string // Protected by swiftAuthWaitGroup mountID jrpcfs.MountIDAsString - rootDirInodeNumber uint64 fissionErrChan chan error fissionVolume fission.Volume fuseConn *fuse.Conn @@ -760,10 +758,7 @@ func uninitializeGlobals() { globals.logFile = nil globals.retryRPCCACertPEM = nil - globals.retryRPCPublicIPAddr = "" - globals.retryRPCPort = 0 globals.retryRPCClient = nil - globals.rootCAx509CertificatePEM = []byte{} globals.entryValidSec = 0 globals.entryValidNSec = 0 globals.attrValidSec = 0 diff --git a/pfsagentd/pfsagent.conf b/pfsagentd/pfsagent.conf index 588bde14f..dcefa6b98 100644 --- a/pfsagentd/pfsagent.conf +++ b/pfsagentd/pfsagent.conf @@ -41,6 +41,8 @@ ReaddirMaxEntries: 1024 FUSEMaxBackground: 100 FUSECongestionThreshhold: 0 FUSEMaxWrite: 131072 +RetryRPCPublicIPAddr: 127.0.0.1 +RetryRPCPort: 32356 RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s RetryRPCCACertFilePath: diff --git a/pfsagentd/request.go b/pfsagentd/request.go index 9525dd3c8..8a211b602 100644 --- a/pfsagentd/request.go +++ b/pfsagentd/request.go @@ -41,6 +41,20 @@ func doMountProxyFS() { swiftStorageURLSplit []string ) + retryrpcConfig := &retryrpc.ClientConfig{ + IPAddr: globals.config.RetryRPCPublicIPAddr, + Port: int(globals.config.RetryRPCPort), + RootCAx509CertificatePEM: globals.retryRPCCACertPEM, + Callbacks: &globals, + DeadlineIO: globals.config.RetryRPCDeadlineIO, + KeepAlivePeriod: globals.config.RetryRPCKeepAlivePeriod, + } + + globals.retryRPCClient, err = retryrpc.NewClient(retryrpcConfig) + if nil != err { + logFatalf("unable to retryRPCClient.NewClient(%v,%v): Volume: %s err: %v", globals.config.RetryRPCPublicIPAddr, globals.config.RetryRPCPort, globals.config.FUSEVolumeName, accountName, err) + } + swiftStorageURL = fetchStorageURL() if "" == swiftStorageURL { logFatalf("unable to fetchStorageURL()") @@ -51,37 +65,19 @@ func doMountProxyFS() { accountName = swiftStorageURLSplit[4] mountRequest = &jrpcfs.MountByAccountNameRequest{ - AccountName: accountName, - MountOptions: 0, - AuthUserID: 0, - AuthGroupID: 0, + AccountName: accountName, + AuthToken: fetchAuthToken(), } mountReply = &jrpcfs.MountByAccountNameReply{} - err = doJRPCRequest("Server.RpcMountByAccountName", mountRequest, mountReply) + err = globals.retryRPCClient.Send("RpcMountByAccountName", mountRequest, mountReply) if nil != err { logFatalf("unable to mount Volume %s (Account: %s): %v", globals.config.FUSEVolumeName, accountName, err) } globals.mountID = mountReply.MountID - globals.rootDirInodeNumber = uint64(mountReply.RootDirInodeNumber) - globals.retryRPCPublicIPAddr = mountReply.RetryRPCPublicIPAddr - globals.retryRPCPort = mountReply.RetryRPCPort - globals.rootCAx509CertificatePEM = mountReply.RootCAx509CertificatePEM - retryrpcConfig := &retryrpc.ClientConfig{ - IPAddr: globals.retryRPCPublicIPAddr, - Port: int(globals.retryRPCPort), - RootCAx509CertificatePEM: globals.rootCAx509CertificatePEM, - Callbacks: &globals, - DeadlineIO: globals.config.RetryRPCDeadlineIO, - KeepAlivePeriod: globals.config.RetryRPCKeepAlivePeriod, - } - globals.retryRPCClient, err = retryrpc.NewClient(retryrpcConfig) - if nil != err { - logFatalf("unable to retryRPCClient.NewClient(%v,%v): Volume: %s (Account: %s) err: %v", globals.retryRPCPublicIPAddr, globals.retryRPCPort, globals.config.FUSEVolumeName, accountName, err) - } } func doUnmountProxyFS() { diff --git a/pfsagentd/setup_teardown_test.go b/pfsagentd/setup_teardown_test.go index 5329ac4c9..3f9f68a16 100644 --- a/pfsagentd/setup_teardown_test.go +++ b/pfsagentd/setup_teardown_test.go @@ -29,6 +29,7 @@ const ( testSwiftNoAuthIPAddr = "127.0.0.1" testSwiftNoAuthPort = "38090" testSwiftProxyAddr = "localhost:38080" + testRetryRPCPort = "54328" ) type testDaemonGlobalsStruct struct { @@ -124,6 +125,8 @@ func testSetup(t *testing.T) { "Agent.FUSEMaxBackground=100", "Agent.FUSECongestionThreshhold=0", "Agent.FUSEMaxWrite=131072", // Linux max... 128KiB is good enough for testing + "Agent.RetryRPCPublicIPAddr=" + testProxyFSDaemonIPAddr, + "Agent.RetryRPCPort=" + testRetryRPCPort, "Agent.RetryRPCDeadlineIO=60s", "Agent.RetryRPCKeepAlivePeriod=60s", "Agent.RetryRPCCACertFilePath=", @@ -238,7 +241,7 @@ func testSetup(t *testing.T) { "JSONRPCServer.TCPPort=54326", "JSONRPCServer.FastTCPPort=54327", - "JSONRPCServer.RetryRPCPort=54328", + "JSONRPCServer.RetryRPCPort=" + testRetryRPCPort, "JSONRPCServer.RetryRPCTTLCompleted=10s", "JSONRPCServer.RetryRPCAckTrim=10ms", "JSONRPCServer.RetryRPCCertFilePath=", diff --git a/retryrpc/api.go b/retryrpc/api.go index ca2c54b54..aefbff5d8 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -313,7 +313,7 @@ type ClientCallbacks interface { type ClientConfig struct { IPAddr string // IP Address of Server Port int // Port of Server - RootCAx509CertificatePEM []byte // Root certificate + RootCAx509CertificatePEM []byte // If TLS...Root certificate; If TCP... nil Callbacks interface{} // Structure implementing ClientCallbacks DeadlineIO time.Duration // How long I/Os on sockets wait even if idle KeepAlivePeriod time.Duration // How frequently a KEEPALIVE is sent From 06e54e06e9bac7f2f4c52337798765a444ef020c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 26 Mar 2021 17:12:32 -0700 Subject: [PATCH 088/160] Deprecated the now obsolete pfs-jrpc tool --- Makefile | 1 - pfs-jrpc/Makefile | 6 - pfs-jrpc/dummy_test.go | 11 - pfs-jrpc/main.go | 532 ----------------------------------------- pfs-jrpc/pfs-jrpc.conf | 6 - pfsagentd/request.go | 2 +- 6 files changed, 1 insertion(+), 557 deletions(-) delete mode 100644 pfs-jrpc/Makefile delete mode 100644 pfs-jrpc/dummy_test.go delete mode 100644 pfs-jrpc/main.go delete mode 100644 pfs-jrpc/pfs-jrpc.conf diff --git a/Makefile b/Makefile index 5496b34ad..d65c93082 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,6 @@ gobinsubdirs = \ inodeworkout \ pfs-crash \ pfs-fsck \ - pfs-jrpc \ pfs-restart-test \ pfs-stress \ pfs-swift-load \ diff --git a/pfs-jrpc/Makefile b/pfs-jrpc/Makefile deleted file mode 100644 index 741445ad4..000000000 --- a/pfs-jrpc/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/pfs-jrpc - -include ../GoMakefile diff --git a/pfs-jrpc/dummy_test.go b/pfs-jrpc/dummy_test.go deleted file mode 100644 index d42991728..000000000 --- a/pfs-jrpc/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/pfs-jrpc/main.go b/pfs-jrpc/main.go deleted file mode 100644 index 2bf7d548b..000000000 --- a/pfs-jrpc/main.go +++ /dev/null @@ -1,532 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "strings" - - "github.com/NVIDIA/proxyfs/conf" - "github.com/NVIDIA/proxyfs/jrpcfs" -) - -type jrpcRequestStruct struct { - JSONrpc string `json:"jsonrpc"` - Method string `json:"method"` - ID uint64 `json:"id"` - Params [1]interface{} `json:"params"` -} - -type jrpcRequestEmptyParamStruct struct{} - -type jrpcResponseIDAndErrorStruct struct { - ID uint64 `json:"id"` - Error string `json:"error"` -} - -type jrpcResponseNoErrorStruct struct { - ID uint64 `json:"id"` - Result interface{} `json:"result"` -} - -type envStruct struct { - AuthToken string - StorageURL string - LastMessageID uint64 - MountID jrpcfs.MountIDAsString -} - -var ( - authKey string - authURL string - authUser string - envPath string - verbose bool - - env *envStruct -) - -func main() { - var ( - args []string - cmd string - confFilePath string - confMap conf.ConfMap - err error - ) - - // Parse arguments - - args = os.Args[1:] - - if 2 > len(args) { - doHelp() - log.Fatalf("Must specify a .conf and a command") - } - - confFilePath = args[0] - - confMap, err = conf.MakeConfMapFromFile(confFilePath) - if nil != err { - log.Fatalf("Failed to load %s: %v", confFilePath, err) - } - - authURL, err = confMap.FetchOptionValueString("JRPCTool", "AuthURL") - if nil != err { - log.Fatalf("Failed to parse %s value for JRPCTool.AuthURL: %v", confFilePath, err) - } - if !strings.HasPrefix(strings.ToLower(authURL), "http:") && !strings.HasPrefix(strings.ToLower(authURL), "https:") { - log.Fatalf("JRPCTool.AuthURL (\"%s\") must start with either \"http:\" or \"https:\"", authURL) - } - authUser, err = confMap.FetchOptionValueString("JRPCTool", "AuthUser") - if nil != err { - log.Fatalf("Failed to parse %s value for JRPCTool.AuthUser: %v", confFilePath, err) - } - authKey, err = confMap.FetchOptionValueString("JRPCTool", "AuthKey") - if nil != err { - log.Fatalf("Failed to parse %s value for JRPCTool.AuthKey: %v", confFilePath, err) - } - envPath, err = confMap.FetchOptionValueString("JRPCTool", "EnvPath") - if nil != err { - log.Fatalf("Failed to parse %s value for JRPCTool.EnvPath: %v", confFilePath, err) - } - verbose, err = confMap.FetchOptionValueBool("JRPCTool", "Verbose") - if nil != err { - log.Fatalf("Failed to parse %s value for JRPCTool.Verbose: %v", confFilePath, err) - } - - cmd = args[1] - - switch cmd { - case "a": - doAuth() - case "m": - doMount() - case "r": - doJRPC(args[2:]...) - case "c": - doClean() - default: - doHelp() - log.Fatalf("Could not understand command \"%s\"", cmd) - } -} - -func doHelp() { - log.Printf("pfs-jrpc - commandline ProxyFS JSON RPC tool") - log.Printf("") - log.Printf("Usage: pfs-jrpc <.conf> a") - log.Printf(" pfs-jrpc <.conf> m") - log.Printf(" pfs-jrpc <.conf> r Server.RpcXXX ['\"Key\":[,\"Key\":]*']") - log.Printf(" pfs-jrpc <.conf> c") - log.Printf("") - log.Printf("Commands:") - log.Printf("") - log.Printf(" a pass AuthUser/AuthKey to AuthURL to fetch AuthToken/StorageURL") - log.Printf(" m perform Server.RpcMountByAccountName of AuthUser's Account") - log.Printf(" r Server.RpcXXX perform selected Server.RpcXXX passing JSON RPC params (if any)") - log.Printf(" c clean up EnvPath") - log.Printf("") - log.Printf("Example:") - log.Printf("") - log.Printf("$ cat pfs-jrpc.conf") - log.Printf("") - log.Printf("[JRPCTool]") - log.Printf("AuthURL: http://localhost:8080/auth/v1.0") - log.Printf("AuthUser: test:tester") - log.Printf("AuthKey: testing") - log.Printf("EnvPath: ./pfs-jrpc.env") - log.Printf("Verbose: true") - log.Printf("") - log.Printf("$ pfs-jrpc pfs-jrpc.conf a") - log.Printf("") - log.Printf("Auth Request:") - log.Printf(" httpRequest.URL: http://localhost:8080/auth/v1.0") - log.Printf(" httpRequest.Header: map[X-Auth-Key:[testing] X-Auth-User:[test:tester]]") - log.Printf("Auth Response:") - log.Printf(" env.AuthToken: AUTH_tk928c2374f62c4de3bfffe4c62bce2e5f") - log.Printf(" env.StorageURL: http://localhost:8080/v1/AUTH_test") - log.Printf("") - log.Printf("$ pfs-jrpc pfs-jrpc.conf m") - log.Printf("") - log.Printf("jrpcRequestBuf:") - log.Printf("{") - log.Printf(" \"jsonrpc\": \"2.0\",") - log.Printf(" \"method\": \"Server.RpcMountByAccountName\",") - log.Printf(" \"id\": 1,") - log.Printf(" \"params\": [") - log.Printf(" {") - log.Printf(" \"AccountName\": \"AUTH_test\",") - log.Printf(" \"MountOptions\": 0,") - log.Printf(" \"AuthUserID\": 0,") - log.Printf(" \"AuthGroupID\": 0") - log.Printf(" }") - log.Printf(" ]") - log.Printf("}") - log.Printf("httpResponseBody:") - log.Printf("{") - log.Printf(" \"error\": null,") - log.Printf(" \"id\": 1,") - log.Printf(" \"result\": {") - log.Printf(" \"RootCAx509CertificatePEM\": \"LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJVVENDQVFPZ0F3SUJBZ0lSQVB3eGNWNGtmOGk5RkxhODViQlg0amt3QlFZREsyVndNQ3d4R0RBV0JnTlYKQkFvVEQwTkJJRTl5WjJGdWFYcGhkR2x2YmpFUU1BNEdBMVVFQXhNSFVtOXZkQ0JEUVRBZ0Z3MHhPVEF4TURFdwpNREF3TURCYUdBOHlNVEU1TURFd01UQXdNREF3TUZvd0xERVlNQllHQTFVRUNoTVBRMEVnVDNKbllXNXBlbUYwCmFXOXVNUkF3RGdZRFZRUURFd2RTYjI5MElFTkJNQ293QlFZREsyVndBeUVBUjlrZ1ZNaFpoaHpDMTJCa0RUMkQKUHE3OTJoRThEVVRLd0ZvbjNKcVhwbTZqT0RBMk1BNEdBMVVkRHdFQi93UUVBd0lDaERBVEJnTlZIU1VFRERBSwpCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQVVHQXl0bGNBTkJBT0RQWUl4aVRoZ3RhY3l2Clg0SlhINGNDWGxWU3g4NUk4MjA0YkN2Zy8xeVorL3VNWXFVNGtDUW14ejZxM0h0eDh5aEV2YlpQa1V0QkI5b3cKWkVyaE93TT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=\",") - log.Printf(" \"RetryRPCPublicIPAddr\": \"0.0.0.0\",") - log.Printf(" \"RootDirInodeNumber\": 1,") - log.Printf(" \"RetryRPCPort\": 32356,") - log.Printf(" \"MountID\": \"W3g9ItrLBkdYCoSlKQjSnA==\"") - log.Printf(" }") - log.Printf("}") - log.Printf("") - log.Printf("$ pfs-jrpc pfs-jrpc.conf r Server.RpcPing '\"Message\":\"Hi\"'") - log.Printf("") - log.Printf("jrpcRequestBuf:") - log.Printf("{") - log.Printf(" \"jsonrpc\": \"2.0\",") - log.Printf(" \"method\": \"Server.RpcPing\",") - log.Printf(" \"id\": 2,") - log.Printf(" \"params\": [") - log.Printf(" {") - log.Printf(" \"MountID\": \"W3g9ItrLBkdYCoSlKQjSnA==\",") - log.Printf(" \"Message\": \"Hi\"") - log.Printf(" }") - log.Printf(" ]") - log.Printf("}") - log.Printf("httpResponseBody:") - log.Printf("{") - log.Printf(" \"error\": null,") - log.Printf(" \"id\": 2,") - log.Printf(" \"result\": {") - log.Printf(" \"Message\": \"pong 2 bytes\"") - log.Printf(" }") - log.Printf("}") - log.Printf("") - log.Printf("$ pfs-jrpc pfs-jrpc.conf c") - log.Printf("") -} - -func doAuth() { - var ( - err error - httpClient *http.Client - httpRequest *http.Request - httpResponse *http.Response - ) - - httpRequest, err = http.NewRequest("GET", authURL, nil) - if nil != err { - log.Fatalf("Failed to create GET of authURL==\"%s\": %v", authURL, err) - } - - httpRequest.Header["X-Auth-User"] = []string{authUser} - httpRequest.Header["X-Auth-Key"] = []string{authKey} - - if verbose { - fmt.Printf("Auth Request:\n") - fmt.Println(" httpRequest.URL: ", httpRequest.URL) - fmt.Println(" httpRequest.Header:", httpRequest.Header) - } - - httpClient = &http.Client{} - - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - log.Fatalf("Failed to issue GET of authURL==\"%s\": %v", authURL, err) - } - if http.StatusOK != httpResponse.StatusCode { - log.Fatalf("Received unexpected HTTP Status for Get of authURL==\"%s\": %s", authURL, httpResponse.Status) - } - - env = &envStruct{ - AuthToken: httpResponse.Header.Get("X-Auth-Token"), - StorageURL: httpResponse.Header.Get("X-Storage-Url"), - LastMessageID: 0, - MountID: "", - } - - if verbose { - fmt.Printf("Auth Response:\n") - fmt.Printf(" env.AuthToken: %s\n", env.AuthToken) - fmt.Printf(" env.StorageURL: %s\n", env.StorageURL) - } - - if strings.HasPrefix(env.StorageURL, "http:") && strings.HasPrefix(httpRequest.URL.String(), "https:") { - env.StorageURL = strings.Replace(env.StorageURL, "http:", "https:", 1) - - if verbose { - fmt.Printf("Auth Response (proxy-corrected):\n") - fmt.Printf(" env.AuthToken: %s\n", env.AuthToken) - fmt.Printf(" env.StorageURL: %s\n", env.StorageURL) - } - } - - writeEnv() -} - -func doMount() { - var ( - accountName string - err error - httpClient *http.Client - httpRequest *http.Request - httpResponse *http.Response - httpResponseBody []byte - httpResponseBodyBytesBuffer bytes.Buffer - jrpcRequest *jrpcRequestStruct - jrpcRequestBuf []byte - jrpcRequestBytesBuffer bytes.Buffer - jrpcResponseIDAndError *jrpcResponseIDAndErrorStruct - jrpcResponseNoError *jrpcResponseNoErrorStruct - mountReply *jrpcfs.MountByAccountNameReply - mountRequest *jrpcfs.MountByAccountNameRequest - storageURLSplit []string - thisMessageID uint64 - ) - - readEnv() - - thisMessageID = env.LastMessageID + 1 - env.LastMessageID = thisMessageID - - writeEnv() - - storageURLSplit = strings.Split(env.StorageURL, "/") - if 0 == len(storageURLSplit) { - log.Fatalf("Attempt to compute accountName from strings.Split(env.StorageURL, \"/\") failed") - } - - accountName = storageURLSplit[len(storageURLSplit)-1] - - mountRequest = &jrpcfs.MountByAccountNameRequest{ - AccountName: accountName, - MountOptions: 0, - AuthUserID: 0, - AuthGroupID: 0, - } - - jrpcRequest = &jrpcRequestStruct{ - JSONrpc: "2.0", - Method: "Server.RpcMountByAccountName", - ID: thisMessageID, - Params: [1]interface{}{mountRequest}, - } - - jrpcRequestBuf, err = json.Marshal(jrpcRequest) - if nil != err { - log.Fatalf("Attempt to marshal jrpcRequest(mount) failed: %v", err) - } - - if verbose { - _ = json.Indent(&jrpcRequestBytesBuffer, jrpcRequestBuf, "", " ") - fmt.Printf("jrpcRequestBuf:\n%s\n", jrpcRequestBytesBuffer.Bytes()) - } - - httpRequest, err = http.NewRequest("PROXYFS", env.StorageURL, bytes.NewReader(jrpcRequestBuf)) - if nil != err { - log.Fatalf("Failed to create httpRequest for mount: %v", err) - } - - httpRequest.Header["X-Auth-Token"] = []string{env.AuthToken} - httpRequest.Header["Content-Type"] = []string{"application/json"} - - httpClient = &http.Client{} - - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - log.Fatalf("Failed to issue PROXYFS Mount: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - log.Fatalf("Received unexpected HTTP Status for PROXYFS Mount: %s", httpResponse.Status) - } - - httpResponseBody, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - log.Fatalf("Failed to read httpResponse.Body: %v", err) - } - err = httpResponse.Body.Close() - if nil != err { - log.Fatalf("Failed to close httpResponse.Body: %v", err) - } - - if verbose { - _ = json.Indent(&httpResponseBodyBytesBuffer, httpResponseBody, "", " ") - fmt.Printf("httpResponseBody:\n%s\n", httpResponseBodyBytesBuffer.Bytes()) - } - - jrpcResponseIDAndError = &jrpcResponseIDAndErrorStruct{} - - err = json.Unmarshal(httpResponseBody, jrpcResponseIDAndError) - if nil != err { - log.Fatalf("Failed to json.Unmarshal(httpResponseBody) [Case 1]: %v", err) - } - if thisMessageID != jrpcResponseIDAndError.ID { - log.Fatalf("Got unexpected MessageID in httpResponseBody [Case 1]") - } - if "" != jrpcResponseIDAndError.Error { - log.Fatalf("Got JRPC Failure on PROXYFS Mount: %s", jrpcResponseIDAndError.Error) - } - - mountReply = &jrpcfs.MountByAccountNameReply{} - jrpcResponseNoError = &jrpcResponseNoErrorStruct{Result: mountReply} - - err = json.Unmarshal(httpResponseBody, jrpcResponseNoError) - if nil != err { - log.Fatalf("Failed to json.Unmarshal(httpResponseBody) [Case 2]: %v", err) - } - if thisMessageID != jrpcResponseIDAndError.ID { - log.Fatalf("Got unexpected MessageID in httpResponseBody [Case 2]") - } - - env.MountID = mountReply.MountID - - writeEnv() -} - -func doJRPC(s ...string) { - var ( - arbitraryRequestMethod string - arbitraryRequestParam string - err error - httpClient *http.Client - httpRequest *http.Request - httpResponse *http.Response - httpResponseBody []byte - httpResponseBodyBytesBuffer bytes.Buffer - jrpcRequest *jrpcRequestStruct - jrpcRequestBuf []byte - jrpcRequestBytesBuffer bytes.Buffer - jrpcResponseIDAndError *jrpcResponseIDAndErrorStruct - thisMessageID uint64 - ) - - readEnv() - - if "" == env.MountID { - log.Fatalf("Attempt to issue JSON RPC to unmounted volume") - } - - thisMessageID = env.LastMessageID + 1 - env.LastMessageID = thisMessageID - - writeEnv() - - arbitraryRequestMethod = s[0] - - switch len(s) { - case 1: - arbitraryRequestParam = "\"MountID\":\"" + string(env.MountID) + "\"" - case 2: - arbitraryRequestParam = "\"MountID\":\"" + string(env.MountID) + "\"," + s[1] - default: - log.Fatalf("JSON RPC must be either [Method] or [Method Param]... not %v", s) - } - - jrpcRequest = &jrpcRequestStruct{ - JSONrpc: "2.0", - Method: arbitraryRequestMethod, - ID: thisMessageID, - Params: [1]interface{}{&jrpcRequestEmptyParamStruct{}}, - } - - jrpcRequestBuf, err = json.Marshal(jrpcRequest) - if nil != err { - log.Fatalf("Attempt to marshal jrpcRequest failed: %v", err) - } - - jrpcRequestBuf = append(jrpcRequestBuf[:len(jrpcRequestBuf)-3], arbitraryRequestParam...) - jrpcRequestBuf = append(jrpcRequestBuf, "}]}"...) - - if verbose { - _ = json.Indent(&jrpcRequestBytesBuffer, jrpcRequestBuf, "", " ") - fmt.Printf("jrpcRequestBuf:\n%s\n", jrpcRequestBytesBuffer.Bytes()) - } - - httpRequest, err = http.NewRequest("PROXYFS", env.StorageURL, bytes.NewReader(jrpcRequestBuf)) - if nil != err { - log.Fatalf("Failed to create httpRequest: %v", err) - } - - httpRequest.Header["X-Auth-Token"] = []string{env.AuthToken} - httpRequest.Header["Content-Type"] = []string{"application/json"} - - httpClient = &http.Client{} - - httpResponse, err = httpClient.Do(httpRequest) - if nil != err { - log.Fatalf("Failed to issue PROXYFS: %v", err) - } - if http.StatusOK != httpResponse.StatusCode { - log.Fatalf("Received unexpected HTTP Status: %s", httpResponse.Status) - } - - httpResponseBody, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - log.Fatalf("Failed to read httpResponse.Body: %v", err) - } - err = httpResponse.Body.Close() - if nil != err { - log.Fatalf("Failed to close httpResponse.Body: %v", err) - } - - if verbose { - _ = json.Indent(&httpResponseBodyBytesBuffer, httpResponseBody, "", " ") - fmt.Printf("httpResponseBody:\n%s\n", httpResponseBodyBytesBuffer.Bytes()) - } - - jrpcResponseIDAndError = &jrpcResponseIDAndErrorStruct{} - - err = json.Unmarshal(httpResponseBody, jrpcResponseIDAndError) - if nil != err { - log.Fatalf("Failed to json.Unmarshal(httpResponseBody) [Case 2]: %v", err) - } - if thisMessageID != jrpcResponseIDAndError.ID { - log.Fatalf("Got unexpected MessageID in httpResponseBody [Case 2]") - } - if "" != jrpcResponseIDAndError.Error { - log.Fatalf("Got JRPC Failure: %s", jrpcResponseIDAndError.Error) - } -} - -func doClean() { - _ = os.RemoveAll(envPath) -} - -func writeEnv() { - var ( - envBuf []byte - err error - ) - - envBuf, err = json.Marshal(env) - if nil != err { - log.Fatalf("Failed to json.Marshal(env): %v", err) - } - - err = ioutil.WriteFile(envPath, envBuf, 0644) - if nil != err { - log.Fatalf("Failed to persist ENV to envPath==\"%s\": %v", envPath, err) - } -} - -func readEnv() { - var ( - envBuf []byte - err error - ) - - env = &envStruct{} - envBuf, err = ioutil.ReadFile(envPath) - if nil != err { - log.Fatalf("Failed to recover ENV from envPath==\"%s\": %v", envPath, err) - } - - err = json.Unmarshal(envBuf, env) - if nil != err { - log.Fatalf("Failed to json.Unmarshal(envBuf): %v", err) - } -} diff --git a/pfs-jrpc/pfs-jrpc.conf b/pfs-jrpc/pfs-jrpc.conf deleted file mode 100644 index ed1aaccbc..000000000 --- a/pfs-jrpc/pfs-jrpc.conf +++ /dev/null @@ -1,6 +0,0 @@ -[JRPCTool] -AuthURL: http://localhost:8080/auth/v1.0 -AuthUser: test:tester -AuthKey: testing -EnvPath: ./pfs-jrpc.env -Verbose: true diff --git a/pfsagentd/request.go b/pfsagentd/request.go index 8a211b602..e79ca5a8c 100644 --- a/pfsagentd/request.go +++ b/pfsagentd/request.go @@ -97,7 +97,7 @@ func doUnmountProxyFS() { unmountReply = &jrpcfs.Reply{} - err = doJRPCRequest("Server.RpcUnmount", unmountRequest, unmountReply) + err = globals.retryRPCClient.Send("RpcUnmount", unmountRequest, unmountReply) if nil != err { logFatalf("unable to unmount Volume %s: %v", globals.config.FUSEVolumeName, err) } From 368fdf86925afa76fa92effe8891e68bce805d3b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 26 Mar 2021 17:20:59 -0700 Subject: [PATCH 089/160] Forgot to actually read RetryRPCPublicIPAddr and RetryRPCPort from PFSAgent's .conf file --- pfsagentd/globals.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pfsagentd/globals.go b/pfsagentd/globals.go index 4080c1fc5..9f5becc87 100644 --- a/pfsagentd/globals.go +++ b/pfsagentd/globals.go @@ -644,6 +644,16 @@ func initializeGlobals(confMap conf.ConfMap) { logFatal(err) } + globals.config.RetryRPCPublicIPAddr, err = confMap.FetchOptionValueString("Agent", "RetryRPCPublicIPAddr") + if nil != err { + logFatal(err) + } + + globals.config.RetryRPCPort, err = confMap.FetchOptionValueUint16("Agent", "RetryRPCPort") + if nil != err { + logFatal(err) + } + globals.config.RetryRPCDeadlineIO, err = confMap.FetchOptionValueDuration("Agent", "RetryRPCDeadlineIO") if nil != err { logFatal(err) From ee3d57771ccbc5139c14f6b295899707363b36f2 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 27 Mar 2021 11:55:53 -0700 Subject: [PATCH 090/160] Got rid of unnecessary retryrpc/rpctest subdirectory --- retryrpc/api.go | 1 - retryrpc/client.go | 1 + retryrpc/ping_test.go | 75 +++++++++++++++++++++++++++++++++++++++ retryrpc/retryrpc_test.go | 35 +++++++++--------- retryrpc/rpctest/Makefile | 6 ---- retryrpc/rpctest/api.go | 27 -------------- retryrpc/rpctest/ping.go | 65 --------------------------------- retryrpc/stress_test.go | 33 ++++++++--------- retryrpc/svrmap.go | 1 - retryrpc/upcall_test.go | 12 +++---- 10 files changed, 111 insertions(+), 145 deletions(-) create mode 100644 retryrpc/ping_test.go delete mode 100644 retryrpc/rpctest/Makefile delete mode 100644 retryrpc/rpctest/api.go delete mode 100644 retryrpc/rpctest/ping.go diff --git a/retryrpc/api.go b/retryrpc/api.go index aefbff5d8..4e696bcbf 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -85,7 +85,6 @@ func NewServer(config *ServerConfig) *Server { // Register creates the map of server methods func (server *Server) Register(retrySvr interface{}) (err error) { - // Find all the methods associated with retrySvr and put into serviceMap server.receiver = reflect.ValueOf(retrySvr) return server.register(retrySvr) diff --git a/retryrpc/client.go b/retryrpc/client.go index 7d10919e5..568006eeb 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -56,6 +56,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter connectionRetryDelay time.Duration crID requestID ) + client.stats.SendCalled.Add(1) client.Lock() diff --git a/retryrpc/ping_test.go b/retryrpc/ping_test.go new file mode 100644 index 000000000..8108e2ded --- /dev/null +++ b/retryrpc/ping_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package retryrpc + +import ( + "bytes" + "fmt" + + "github.com/NVIDIA/proxyfs/blunder" +) + +// TestPingServer is a struct with pointer receivers implementing RpcTestPing*() +type TestPingServer struct{} + +// TestPingReq is the request object for RpcTestPing*() +type TestPingReq struct { + Message string +} + +// TestPingReply is the response object for RpcTestPing*() +type TestPingReply struct { + Message string +} + +// Simple ping for testing the RPC layer +func testEncodeErrno(e *error) { + if *e != nil { + *e = fmt.Errorf("errno: %d", blunder.Errno(*e)) + } +} + +// RpcTestPing simply does a len on the message path and returns the result +func (s *TestPingServer) RpcTestPing(in *TestPingReq, reply *TestPingReply) (err error) { + reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) + return nil +} + +var largeStr string = "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + +// RpcTestPingLarge simply does a len on the message path and returns the result +// along with a larger buffer. +func (s *TestPingServer) RpcTestPingLarge(in *TestPingReq, reply *TestPingReply) (err error) { + buf := bytes.Buffer{} + p := fmt.Sprintf("pong %d bytes", len(in.Message)) + buf.WriteString(p) + for i := 0; i < 1000; i++ { + buf.WriteString(largeStr) + } + reply.Message = fmt.Sprintf("%v", buf.String()) + return nil +} + +// RpcTestPingWithError returns an error +func (s *TestPingServer) RpcTestPingWithError(in *TestPingReq, reply *TestPingReply) (err error) { + err = blunder.AddError(err, blunder.NotFoundError) + testEncodeErrno(&err) + reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) + return err +} + +// RpcTestPingWithClientID simply does a len on the message path and returns the clientID & result +func (s *TestPingServer) RpcTestPingWithClientID(clientID uint64, in *TestPingReq, reply *TestPingReply) (err error) { + reply.Message = fmt.Sprintf("Client ID: %v pong %d bytes", clientID, len(in.Message)) + return nil +} + +// RpcTestPingWithInvalidClientID is not a valid RPC +// Note: Currently unused +func (s *TestPingServer) RpcTestPingWithInvalidClientID(clientID int, in *TestPingReq, reply *TestPingReply) (err error) { + err = blunder.AddError(err, blunder.NotFoundError) + testEncodeErrno(&err) + reply.Message = fmt.Sprintf("client ID: %v pong %d bytes", clientID, len(in.Message)) + return err +} diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index 95a34cb4c..5e2b4b37a 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -14,7 +14,6 @@ import ( "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/icert/icertpkg" - "github.com/NVIDIA/proxyfs/retryrpc/rpctest" ) const ( @@ -146,9 +145,8 @@ func testRegister(t *testing.T, useTLS bool) { zero := 0 assert.Equal(0, zero) - // Create new rpctest server - needed for calling - // RPCs - myJrpcfs := rpctest.NewServer() + // Create new TestPingServer - needed for calling RPCs + myJrpcfs := &TestPingServer{} rrSvr := getNewServer(10*time.Second, false, useTLS) assert.NotNil(rrSvr) @@ -172,9 +170,8 @@ func testServer(t *testing.T, useTLS bool) { zero := 0 assert.Equal(0, zero) - // Create new rpctest server - needed for calling - // RPCs - myJrpcfs := rpctest.NewServer() + // Create new TestPingServer - needed for calling RPCs + myJrpcfs := &TestPingServer{} rrSvr := getNewServer(10*time.Second, false, useTLS) assert.NotNil(rrSvr) @@ -216,33 +213,33 @@ func testServer(t *testing.T, useTLS bool) { assert.Nil(newErr) // Send an RPC which should return success - pingRequest := &rpctest.PingReq{Message: "Ping Me!"} - pingReply := &rpctest.PingReply{} - sendErr := rrClnt.Send("RpcPing", pingRequest, pingReply) + pingRequest := &TestPingReq{Message: "Ping Me!"} + pingReply := &TestPingReply{} + sendErr := rrClnt.Send("RpcTestPing", pingRequest, pingReply) assert.Nil(sendErr) assert.Equal("pong 8 bytes", pingReply.Message) assert.Equal(1, rrSvr.CompletedCnt()) // Send an RPC which expects the client ID and which should return success - pingRequest = &rpctest.PingReq{Message: "Ping Me!"} - pingReply = &rpctest.PingReply{} - sendErr = rrClnt.Send("RpcPingWithClientID", pingRequest, pingReply) + pingRequest = &TestPingReq{Message: "Ping Me!"} + pingReply = &TestPingReply{} + sendErr = rrClnt.Send("RpcTestPingWithClientID", pingRequest, pingReply) assert.Nil(sendErr) assert.Equal("Client ID: 1 pong 8 bytes", pingReply.Message) assert.Equal(2, rrSvr.CompletedCnt()) // Send an RPC which should return an error - pingRequest = &rpctest.PingReq{Message: "Ping Me!"} - pingReply = &rpctest.PingReply{} - sendErr = rrClnt.Send("RpcPingWithError", pingRequest, pingReply) + pingRequest = &TestPingReq{Message: "Ping Me!"} + pingReply = &TestPingReply{} + sendErr = rrClnt.Send("RpcTestPingWithError", pingRequest, pingReply) assert.NotNil(sendErr) assert.Equal(3, rrSvr.CompletedCnt()) // Send an RPC which should return an error - pingRequest = &rpctest.PingReq{Message: "Ping Me!"} - pingReply = &rpctest.PingReply{} - sendErr = rrClnt.Send("RpcInvalidMethod", pingRequest, pingReply) + pingRequest = &TestPingReq{Message: "Ping Me!"} + pingReply = &TestPingReply{} + sendErr = rrClnt.Send("RpcTestInvalidMethod", pingRequest, pingReply) assert.NotNil(sendErr) assert.Equal(4, rrSvr.CompletedCnt()) diff --git a/retryrpc/rpctest/Makefile b/retryrpc/rpctest/Makefile deleted file mode 100644 index 85d4ff7ee..000000000 --- a/retryrpc/rpctest/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015-2021, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -gosubdir := github.com/NVIDIA/proxyfs/retryrpc/rpctest - -include ../../GoMakefile diff --git a/retryrpc/rpctest/api.go b/retryrpc/rpctest/api.go deleted file mode 100644 index 7e2ddf2cb..000000000 --- a/retryrpc/rpctest/api.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package rpctest - -// rcptest is a set of RPC methods used to test retryrpc - -import () - -// PingReq is the request object for RpcPing -type PingReq struct { - Message string -} - -// PingReply is the response object for RpcPutLocation -type PingReply struct { - Message string -} - -type Server struct{} - -// NewServer creates the Server object and returns the pointer -func NewServer() *Server { - s := Server{} - - return &s -} diff --git a/retryrpc/rpctest/ping.go b/retryrpc/rpctest/ping.go deleted file mode 100644 index 0d87bf8b5..000000000 --- a/retryrpc/rpctest/ping.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package rpctest - -// Simple ping for testing the RPC layer -import ( - "bytes" - "fmt" - - "github.com/NVIDIA/proxyfs/blunder" -) - -func encodeErrno(e *error) { - if *e != nil { - *e = fmt.Errorf("errno: %d", blunder.Errno(*e)) - } -} - -// RpcPing simply does a len on the message path and returns the result -func (s *Server) RpcPing(in *PingReq, reply *PingReply) (err error) { - - reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) - return nil -} - -var largeStr string = "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" - -// RpcPingLarge simply does a len on the message path and returns the result -// along with a larger buffer. -func (s *Server) RpcPingLarge(in *PingReq, reply *PingReply) (err error) { - - buf := bytes.Buffer{} - p := fmt.Sprintf("pong %d bytes", len(in.Message)) - buf.WriteString(p) - for i := 0; i < 1000; i++ { - buf.WriteString(largeStr) - } - - reply.Message = fmt.Sprintf("%v", buf.String()) - return nil -} - -// RpcPingWithError returns an error -func (s *Server) RpcPingWithError(in *PingReq, reply *PingReply) (err error) { - err = blunder.AddError(err, blunder.NotFoundError) - encodeErrno(&err) - reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) - return err -} - -// RpcPingWithClientID returns an error -func (s *Server) RpcPingWithClientID(clientID uint64, in *PingReq, reply *PingReply) (err error) { - reply.Message = fmt.Sprintf("Client ID: %v pong %d bytes", clientID, len(in.Message)) - - return nil -} - -// RpcPingWithInvalidClientIDType is not a valid RPC -func (s *Server) RpcPingWithInvalidClientID(clientID int, in *PingReq, reply *PingReply) (err error) { - err = blunder.AddError(err, blunder.NotFoundError) - encodeErrno(&err) - reply.Message = fmt.Sprintf("client ID: %v pong %d bytes", clientID, len(in.Message)) - return err -} diff --git a/retryrpc/stress_test.go b/retryrpc/stress_test.go index 94ada0dbb..4b3eefae6 100644 --- a/retryrpc/stress_test.go +++ b/retryrpc/stress_test.go @@ -14,7 +14,6 @@ import ( _ "net/http/pprof" */ - "github.com/NVIDIA/proxyfs/retryrpc/rpctest" "github.com/stretchr/testify/assert" ) @@ -61,9 +60,8 @@ func testLoop(t *testing.T, useTLS bool) { zero := 0 assert.Equal(0, zero) - // Create new rpctest server - needed for calling - // RPCs - myJrpcfs := rpctest.NewServer() + // Create new TestPingServer - needed for calling RPCs + myJrpcfs := &TestPingServer{} rrSvr := getNewServer(65*time.Second, false, useTLS) assert.NotNil(rrSvr) @@ -100,9 +98,8 @@ func testLoopClientAckTrim(t *testing.T, useTLS bool) { zero := 0 assert.Equal(0, zero) - // Create new rpctest server - needed for calling - // RPCs - myJrpcfs := rpctest.NewServer() + // Create new TestPingServer - needed for calling RPCs + myJrpcfs := &TestPingServer{} whenTTL := 10 * time.Millisecond rrSvr := getNewServer(whenTTL, true, useTLS) @@ -162,9 +159,8 @@ func testLoopTTLTrim(t *testing.T, useTLS bool) { zero := 0 assert.Equal(0, zero) - // Create new rpctest server - needed for calling - // RPCs - myJrpcfs := rpctest.NewServer() + // Create new TestPingServer - needed for calling RPCs + myJrpcfs := &TestPingServer{} whenTTL := 10 * time.Millisecond rrSvr := getNewServer(whenTTL, true, useTLS) @@ -212,9 +208,8 @@ func testSendLargeRPC(t *testing.T, useTLS bool) { zero := 0 assert.Equal(0, zero) - // Create new rpctest server - needed for calling - // RPCs - myJrpcfs := rpctest.NewServer() + // Create new TestPingServer - needed for calling RPCs + myJrpcfs := &TestPingServer{} whenTTL := 10 * time.Millisecond rrSvr := getNewServer(whenTTL, true, useTLS) @@ -288,10 +283,10 @@ func cntNotTrimmed(server *Server) (numItems int) { func ping(t *testing.T, client *Client, i int, agentID uint64, assert *assert.Assertions) { // Send a ping RPC and print the results msg := fmt.Sprintf("Ping Me - %v", i) - pingRequest := &rpctest.PingReq{Message: msg} - pingReply := &rpctest.PingReply{} + pingRequest := &TestPingReq{Message: msg} + pingReply := &TestPingReply{} expectedReply := fmt.Sprintf("pong %d bytes", len(msg)) - err := client.Send("RpcPing", pingRequest, pingReply) + err := client.Send("RpcTestPing", pingRequest, pingReply) assert.Nil(err, "client.Send() returned an error") if expectedReply != pingReply.Message { fmt.Printf(" client - AGENTID: %v\n", agentID) @@ -307,9 +302,9 @@ func ping(t *testing.T, client *Client, i int, agentID uint64, assert *assert.As func pingLarge(t *testing.T, client *Client, i int, agentID uint64, assert *assert.Assertions) { // Send a ping RPC and print the results msg := fmt.Sprintf("Ping Me - %v", i) - pingRequest := &rpctest.PingReq{Message: msg} - pingReply := &rpctest.PingReply{} - err := client.Send("RpcPingLarge", pingRequest, pingReply) + pingRequest := &TestPingReq{Message: msg} + pingReply := &TestPingReply{} + err := client.Send("RpcTestPingLarge", pingRequest, pingReply) assert.Nil(err, "client.Send() returned an error") } diff --git a/retryrpc/svrmap.go b/retryrpc/svrmap.go index 6d920c9d5..810140542 100644 --- a/retryrpc/svrmap.go +++ b/retryrpc/svrmap.go @@ -107,7 +107,6 @@ func isMethodExported(name string) bool { // Figure out what methods we can have as RPCs and build the // service map func (server *Server) register(retrySvr interface{}) (err error) { - // Find all the methods associated with retrySvr and put into serviceMap typ := reflect.TypeOf(retrySvr) rcvr := reflect.ValueOf(retrySvr) diff --git a/retryrpc/upcall_test.go b/retryrpc/upcall_test.go index 884b53f9b..7f091db7c 100644 --- a/retryrpc/upcall_test.go +++ b/retryrpc/upcall_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/NVIDIA/proxyfs/retryrpc/rpctest" "github.com/stretchr/testify/assert" ) @@ -47,9 +46,8 @@ func testUpCall(t *testing.T, useTLS bool) { zero := 0 assert.Equal(0, zero) - // Create new rpctest server - needed for calling - // RPCs - myJrpcfs := rpctest.NewServer() + // Create new TestPingServer - needed for calling RPCs + myJrpcfs := &TestPingServer{} rrSvr := getNewServer(10*time.Second, false, useTLS) assert.NotNil(rrSvr) @@ -94,9 +92,9 @@ func testUpCall(t *testing.T, useTLS bool) { assert.Nil(newErr) // Send an RPC which should return success - pingRequest := &rpctest.PingReq{Message: "Ping Me!"} - pingReply := &rpctest.PingReply{} - sendErr := rrClnt.Send("RpcPing", pingRequest, pingReply) + pingRequest := &TestPingReq{Message: "Ping Me!"} + pingReply := &TestPingReply{} + sendErr := rrClnt.Send("RpcTestPing", pingRequest, pingReply) assert.Nil(sendErr) assert.Equal("pong 8 bytes", pingReply.Message) assert.Equal(1, rrSvr.CompletedCnt()) From b6cc83a75b7550b96e48f2fa49027a25039270e6 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 29 Mar 2021 10:55:10 -0700 Subject: [PATCH 091/160] Safely grab copy of {tls|net}Conn in retryrpc client.go readReplies() loop ...resolves race with (*Client).Close() nil'ing out the {tls|net}Conn field in Client.connectionTracker --- go.mod | 18 ++- go.sum | 254 +++++++++++++++++++++++++++++++++++++++ retryrpc/api_internal.go | 6 +- retryrpc/client.go | 29 +++-- 4 files changed, 292 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 347655b46..3ab68a413 100644 --- a/go.mod +++ b/go.mod @@ -12,17 +12,25 @@ require ( github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 + github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/go-errors/errors v1.0.0 // indirect github.com/gogo/protobuf v1.2.2-0.20190611061853-dadb62585089 // indirect github.com/google/btree v1.0.0 - github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f // indirect + github.com/google/uuid v1.2.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/mitchellh/mapstructure v1.4.1 // indirect + github.com/ory/go-acc v0.2.6 // indirect + github.com/pborman/uuid v1.2.1 // indirect + github.com/pelletier/go-toml v1.8.1 // indirect github.com/sirupsen/logrus v1.2.0 - github.com/stretchr/testify v1.3.0 + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cobra v1.1.3 // indirect + github.com/stretchr/testify v1.5.1 go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 go.uber.org/multierr v1.5.0 // indirect go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f // indirect golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 - golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 - golang.org/x/text v0.3.2 // indirect - google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601 // indirect + golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 + golang.org/x/text v0.3.5 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect ) diff --git a/go.sum b/go.sum index ee33bdbfb..9601e4c5a 100644 --- a/go.sum +++ b/go.sum @@ -1,50 +1,88 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898 h1:SC+c6A1qTFstO9qmB86mPV2IpYme/2ZoEQ0hrP+wo+Q= bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a h1:kC/haCyNLEPaV++f9Ro7w5MyW5KclVUNrTG7JhmSp/M= github.com/NVIDIA/cstruct v0.0.0-20210203201140-f4db46040b9a/go.mod h1:Ffet7HOf/prPpmqMxpWFXEimd/5YHt9MKO/murMNKos= github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0 h1:GQZP8OZqvkcMt7l7QAdtfa17PDUqiamz64df2DWNCDM= github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0/go.mod h1:YzbQsUqGZ4BCnhIotOEc2VtCTW9Ue6rJrLMfEwQrEvk= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec h1:SO7/dGH1jwQTdsaXHQR87UsFGXMIeAPtuT1YlTsD+Iw= github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec/go.mod h1:UobtfEnrrugMbdhaJNcTXPmrHDNJlbyznqkDFDyOcpM= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/ansel1/merry v1.0.1 h1:NEqlG+pyb6XKY55oRdGStyMkQ35YucmgFVG2tpybaP8= github.com/ansel1/merry v1.0.1/go.mod h1:qBYMZz+rgzOfZ3QACcTK7hE0bBMxvC0sKCok2t7bAhw= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys= github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creachadair/cityhash v0.1.0 h1:rzSfISre5xT/EgbVvXBmsUJ58Hsrp8ccYMZVvc5uBmk= github.com/creachadair/cityhash v0.1.0/go.mod h1:bYNmpea/cMZgL9Rjt+ZfDfjaT0hScMO6SSGV1DN5lEM= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXhuXRyumBd6RE= +github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.3 h1:jh22xisGBjrEVnRZ1DVTpBVQm0Xndu8sMl0CWDzSIBI= +github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-errors/errors v1.0.0 h1:2G1gYpeHw4GhLet4Ebp5q9wpnSCAOJNTiJq+I3wJV5I= github.com/go-errors/errors v1.0.0/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -54,49 +92,112 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f h1:XXzyYlFbxK3kWfcmu3Wc+Tv8/QQl/VqwsWuSYF1Rj0s= github.com/google/uuid v1.1.2-0.20190416172445-c2e93f3ae59f/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -104,46 +205,108 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/ory/go-acc v0.2.6 h1:YfI+L9dxI7QCtWn2RbawqO0vXhiThdXu/RgizJBbaq0= +github.com/ory/go-acc v0.2.6/go.mod h1:4Kb/UnPcT8qRAk3IAxta+hvVapdxTLWtrr7bFLlEgpw= +github.com/ory/viper v1.7.5 h1:+xVdq7SU3e1vNaCsk/ixsfxE4zylk1TJUiJrY647jUE= +github.com/ory/viper v1.7.5/go.mod h1:ypOuyJmEUb3oENywQZRgeAMwqgOyDqwboO1tj3DjTaM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 h1:C7kWARE8r64ppRadl40yfNo6pag+G6ocvGU2xZ6yNes= go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -155,77 +318,168 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f h1:EeAYAq+xwFDtvQfGzVRYTNZBFZoKRC5olOUHqOOrZOA= go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 h1:64ChN/hjER/taL4YJuA+gpLfIMT+/NFherRZixbxOhg= +golang.org/x/sys v0.0.0-20210326220804-49726bf1d181/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601 h1:9VBRTdmgQxbs6HE0sUnMrSWNePppAJU07NYvX5dIB04= google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a h1:Ob5/580gVHBJZgXnff1cZDbG+xLtMVE5mDRTe+nIsX4= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index 97f503130..15b824d7c 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -263,7 +263,11 @@ func getIO(genNum uint64, deadlineIO time.Duration, conn net.Conn) (buf []byte, // Read in the header of the request first var hdr ioHeader - conn.SetDeadline(time.Now().Add(deadlineIO)) + err = conn.SetDeadline(time.Now().Add(deadlineIO)) + if err != nil { + return + } + err = binary.Read(conn, binary.BigEndian, &hdr) if err != nil { return diff --git a/retryrpc/client.go b/retryrpc/client.go index 568006eeb..5bc3e7c20 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -280,9 +280,20 @@ func (client *Client) readReplies(callingGenNum uint64) { var localCnt int for { + // Capture copy of net.Conn safely - we might be closing/halting + client.Lock() + if client.halting { + client.Unlock() + return + } + nC := client.connection.castToNetConn() + client.Unlock() + if nil == nC { + return + } // Wait reply from server - buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, client.connection.castToNetConn()) + buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, nC) // This must happen before checking error client.Lock() @@ -486,13 +497,13 @@ func (client *Client) sendMyInfo() (err error) { func (client *Client) reDial() (err error) { var entryState = client.connection.state - client.connection.tlsConfig = &tls.Config{ - RootCAs: client.connection.x509CertPool, - } - // Now dial the server if client.connection.useTLS { + client.connection.tlsConfig = &tls.Config{ + RootCAs: client.connection.x509CertPool, + } + d := &net.Dialer{KeepAlive: client.keepAlivePeriod} tlsConn, dialErr := tls.DialWithDialer(d, "tcp", client.connection.hostPortStr, client.connection.tlsConfig) if dialErr != nil { @@ -584,13 +595,13 @@ func (client *Client) readClientID(callingGenNum uint64) (myUniqueID uint64, err func (client *Client) initialDial() (err error) { var entryState = client.connection.state - client.connection.tlsConfig = &tls.Config{ - RootCAs: client.connection.x509CertPool, - } - // Now dial the server if client.connection.useTLS { + client.connection.tlsConfig = &tls.Config{ + RootCAs: client.connection.x509CertPool, + } + d := &net.Dialer{KeepAlive: client.keepAlivePeriod} tlsConn, dialErr := tls.DialWithDialer(d, "tcp", client.connection.hostPortStr, client.connection.tlsConfig) if dialErr != nil { From af032cdf8f2a9ead94f3c5b3600cd933f764a443 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Mon, 29 Mar 2021 16:54:50 -0700 Subject: [PATCH 092/160] Update Dockerfile for TravisCI This time, goveralls is not configured as a submodule of ProxyFS --- go.sum | 1 + test/container/Dockerfile | 69 ++++++++++++++++++++------------------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/go.sum b/go.sum index 9601e4c5a..239a370fc 100644 --- a/go.sum +++ b/go.sum @@ -422,6 +422,7 @@ golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= diff --git a/test/container/Dockerfile b/test/container/Dockerfile index 6296c652a..ccf987988 100644 --- a/test/container/Dockerfile +++ b/test/container/Dockerfile @@ -32,13 +32,13 @@ FROM centos:7.4.1708 # - automake # - libtool -RUN yum-config-manager --disable CentOS-Base -RUN yum-config-manager --disable CentOS-CR -RUN yum-config-manager --disable CentOS-Debuginfo -RUN yum-config-manager --disable CentOS-fasttrack -RUN yum-config-manager --disable CentOS-Media -RUN yum-config-manager --disable CentOS-Sources -RUN yum-config-manager --disable CentOS-Vault +RUN yum-config-manager --disable CentOS-Base \ + yum-config-manager --disable CentOS-CR \ + yum-config-manager --disable CentOS-Debuginfo \ + yum-config-manager --disable CentOS-fasttrack \ + yum-config-manager --disable CentOS-Media \ + yum-config-manager --disable CentOS-Sources \ + yum-config-manager --disable CentOS-Vault RUN rm -rf \ /etc/yum.repos.d/CentOS-Base.repo \ @@ -49,14 +49,14 @@ RUN rm -rf \ /etc/yum.repos.d/CentOS-Sources.repo \ /etc/yum.repos.d/CentOS-Vault.repo -RUN yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/os/x86_64/ -RUN yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/updates/x86_64/ -RUN yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/extras/x86_64/ -RUN yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/centosplus/x86_64/ -RUN yum-config-manager --enable vault.centos.org_centos_7.4.1708_os_x86_64_ -RUN yum-config-manager --enable vault.centos.org_centos_7.4.1708_updates_x86_64_ -RUN yum-config-manager --enable vault.centos.org_centos_7.4.1708_extras_x86_64_ -RUN yum-config-manager --enable vault.centos.org_centos_7.4.1708_centosplus_x86_64_ +RUN yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/os/x86_64/ \ + yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/updates/x86_64/ \ + yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/extras/x86_64/ \ + yum-config-manager --add-repo http://vault.centos.org/centos/7.4.1708/centosplus/x86_64/ \ + yum-config-manager --enable vault.centos.org_centos_7.4.1708_os_x86_64_ \ + yum-config-manager --enable vault.centos.org_centos_7.4.1708_updates_x86_64_ \ + yum-config-manager --enable vault.centos.org_centos_7.4.1708_extras_x86_64_ \ + yum-config-manager --enable vault.centos.org_centos_7.4.1708_centosplus_x86_64_ \ RUN yum clean all RUN rm -rf /var/cache/yum @@ -81,9 +81,6 @@ RUN yum -y install \ gnutls-devel-3.3.26-9.el7 \ libacl-devel-2.2.51-12.el7 \ openldap-devel-2.4.44-5.el7 \ - samba-4.6.2-12.el7_4 \ - samba-client-4.6.2-12.el7_4 \ - cifs-utils-6.2-10.el7 \ gcc-4.8.5-16.el7_4.2 \ make-3.82-23.el7 \ autoconf-2.69-11.el7 \ @@ -98,20 +95,14 @@ RUN yum -y install rh-python36 && \ ln -s /bin/python3.6 /bin/python3 && \ ln -s /opt/rh/rh-python36/root/usr/include /opt/rh/rh-python36/root/include -# Pip install. Trying to do as much as possible at once. -# Here's an explanation of why we need to install each package: -## Setup ProxyFS build environment -# - requests -## Install Python tox -# - tox RUN pip install --upgrade 'pip<21.0' RUN pip install requests tox==3.5.3 # Install Golang ENV GOLANG_TARFILE_NAME=go1.15.5.linux-amd64.tar.gz -RUN cd /tmp && wget -q https://dl.google.com/go/${GOLANG_TARFILE_NAME} -RUN tar -C /usr/local -xf /tmp/${GOLANG_TARFILE_NAME} -RUN rm -rf /tmp/${GOLANG_TARFILE_NAME} +RUN cd /tmp && wget -q https://dl.google.com/go/${GOLANG_TARFILE_NAME} && \ + tar -C /usr/local -xf /tmp/${GOLANG_TARFILE_NAME} && \ + rm -rf /tmp/${GOLANG_TARFILE_NAME} ENV PATH=$PATH:/usr/local/go/bin # Setup ProxyFS build environment @@ -119,12 +110,22 @@ ENV GOPATH=/gopathroot ENV PATH=$PATH:$GOPATH/bin RUN echo "user_allow_other" >> /etc/fuse.conf -RUN git clone https://github.com/openstack/liberasurecode.git -RUN cd /liberasurecode && ./autogen.sh && ./configure && make && sudo make install && sed -i '1s/^/\/usr\/local\/lib\n/' /etc/ld.so.conf && ldconfig - -RUN git clone https://github.com/openstack/pyeclib.git -RUN cd /pyeclib && pip install -e . && pip install -r test-requirements.txt - -RUN go get github.com/ory/go-acc github.com/mattn/goveralls +# Install liberasurecode +RUN git clone https://github.com/openstack/liberasurecode.git && \ + cd /liberasurecode && \ + ./autogen.sh && \ + ./configure && \ + make && \ + sudo make install && \ + sed -i '1s/^/\/usr\/local\/lib\n/' /etc/ld.so.conf && \ + ldconfig + +# Install pyeclib +RUN git clone https://github.com/openstack/pyeclib.git && \ + cd /pyeclib && \ + pip install -e . && \ + pip install -r test-requirements.txt + +RUN go get github.com/mattn/goveralls CMD ["/bin/bash", "/gopathroot/src/github.com/NVIDIA/proxyfs/test/container/launch.sh"] From d22b22b7e2c082214f5d104f96aa610591c6eee3 Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Thu, 1 Apr 2021 12:25:40 -0700 Subject: [PATCH 093/160] Stress test fixes (#602) * Fix race condition where we are not grabbing client lock consistently * problem found by race detector * fix complaints from VS Code * Update name of bucketstat and use appropriate call --- go.mod | 2 -- retryrpc/api.go | 4 ++-- retryrpc/api_internal.go | 10 +--------- retryrpc/client.go | 10 +++++----- retryrpc/client_tracking.go | 5 +---- retryrpc/retryrpc_test.go | 4 ++-- retryrpc/server.go | 4 ++-- 7 files changed, 13 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index 3ab68a413..ffd849a2f 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,7 @@ require ( github.com/NVIDIA/fission v0.0.0-20210206002952-579a9b261fe0 github.com/NVIDIA/sortedmap v0.0.0-20210206003014-af46f3efdbec github.com/ansel1/merry v1.0.1 - github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a // indirect - github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/creachadair/cityhash v0.1.0 github.com/dgraph-io/ristretto v0.0.3 // indirect github.com/go-errors/errors v1.0.0 // indirect diff --git a/retryrpc/api.go b/retryrpc/api.go index 4e696bcbf..3c51dbbdd 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -105,7 +105,7 @@ func (server *Server) Start() (err error) { return } - if 0 != len(server.tlsCertificate.Certificate) { + if len(server.tlsCertificate.Certificate) != 0 { server.tlsListener = tls.NewListener(server.netListener, tlsConfig) } @@ -194,7 +194,7 @@ func (server *Server) Close() { server.halting = true server.Unlock() - if 0 == len(server.tlsCertificate.Certificate) { + if len(server.tlsCertificate.Certificate) == 0 { err := server.netListener.Close() if err != nil { logger.Errorf("server.netListener.Close() returned err: %v", err) diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index 15b824d7c..444527361 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -38,7 +38,7 @@ type requestID uint64 type statsInfo struct { AddCompleted bucketstats.Total // Number added to completed list RmCompleted bucketstats.Total // Number removed from completed list - RPCLenUsec bucketstats.BucketLog2Round // Tracks length of RPCs + TimeOfRPCUsec bucketstats.BucketLog2Round // Tracks amount of time to call an RPC ReplySize bucketstats.BucketLog2Round // Tracks completed RPC reply size longestRPC time.Duration // Time of longest RPC longestRPCMethod string // Method of longest RPC @@ -80,13 +80,6 @@ type connCtx struct { ci *clientInfo // Back pointer to the CI } -// pendingCtx tracks an individual request from a client -type pendingCtx struct { - lock sync.Mutex - buf []byte // Request - cCtx *connCtx // Most recent connection to return results -} - // methodArgs defines the method provided by the RPC server // as well as the request type and reply type arguments type methodArgs struct { @@ -225,7 +218,6 @@ func setupHdrReply(ioreply *ioReply, t MsgType) { ioreply.Hdr.Version = currentRetryVersion ioreply.Hdr.Type = t ioreply.Hdr.Magic = headerMagic - return } func buildSetIDRequest(myUniqueID uint64) (isreq *internalSetIDRequest, err error) { diff --git a/retryrpc/client.go b/retryrpc/client.go index 5bc3e7c20..889e38dce 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -148,7 +148,7 @@ func (client *Client) sendToServer(crID requestID, ctx *reqCtx, queue bool) { // outstandingRequests queue and resend the request. // // Don't queue the request if we are retransmitting.... - if queue == true { + if queue { client.outstandingRequest[crID] = ctx } @@ -191,16 +191,16 @@ func (client *Client) sendToServer(crID requestID, ctx *reqCtx, queue bool) { fmt.Printf("CLIENT: PARTIAL Write! bytesWritten is: %v len(ctx.ioreq.JReq): %v writeErr: %v\n", bytesWritten, len(ctx.ioreq.JReq), writeErr) */ + genNum := ctx.genNum client.Unlock() // Just return - the retransmit code will start another // sendToServer() goroutine - client.retransmit(ctx.genNum) + client.retransmit(genNum) return } client.Unlock() - return } func (client *Client) notifyReply(buf []byte, genNum uint64) { @@ -216,7 +216,6 @@ func (client *Client) notifyReply(buf []byte, genNum uint64) { // TODO - make log message e := fmt.Errorf("notifyReply failed to unmarshal buf: %+v err: %v", string(buf), err) fmt.Printf("%v\n", e) - client.retransmit(genNum) return } @@ -251,8 +250,9 @@ func (client *Client) notifyReply(buf []byte, genNum uint64) { fmt.Printf("%v\n", e) // Assume read garbage on socket - close the socket and reconnect - client.retransmit(genNum) + // Drop client lock since retransmit() will acquire it. client.Unlock() + client.retransmit(genNum) return } diff --git a/retryrpc/client_tracking.go b/retryrpc/client_tracking.go index 02a07bef1..f811887a5 100644 --- a/retryrpc/client_tracking.go +++ b/retryrpc/client_tracking.go @@ -7,10 +7,7 @@ package retryrpc // keep track of clients. func (ci *clientInfo) isEmpty() bool { - if len(ci.completedRequest) == 0 { - return true - } - return false + return len(ci.completedRequest) == 0 } func (ci *clientInfo) completedCnt() int { diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index 5e2b4b37a..cad4a973a 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -320,7 +320,7 @@ func testBtree(t *testing.T, useTLS bool) { type clientStats struct { AddCompleted bucketstats.Total // Number added to completed list RmCompleted bucketstats.Total // Number removed from completed list - RPCLenUsec bucketstats.BucketLog2Round // Average times of RPCs + TimeOfRPCUsec bucketstats.BucketLog2Round // Average times of RPCs LongestRPCMethod string // Method of longest RPC ReplySize bucketstats.BucketLog2Round // Largest RPC reply size completed LargestReplySizeMethod string // Method of largest RPC reply size completed @@ -359,7 +359,7 @@ func testStatsAndBucketstats(t *testing.T) { // Track duration of all RPCs in a graph start := time.Now() time.Sleep(10 * time.Millisecond) - myClient1.RPCLenUsec.Add(uint64(time.Since(start) / time.Microsecond)) + myClient1.TimeOfRPCUsec.Add(uint64(time.Since(start).Microseconds())) myClient1.ReplySize.Add(8192) // Example of pfsagent #2 diff --git a/retryrpc/server.go b/retryrpc/server.go index d4f8666dd..d144f28f5 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -44,7 +44,7 @@ func (server *Server) run() { defer server.goroutineWG.Done() for { - if 0 == len(server.tlsCertificate.Certificate) { + if len(server.tlsCertificate.Certificate) == 0 { conn, err = server.netListener.Accept() } else { conn, err = server.tlsListener.Accept() @@ -154,7 +154,7 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b // the RPC. startRPC := time.Now() ior := server.callRPCAndFormatReply(buf, ci.myUniqueID, &jReq) - ci.stats.RPCLenUsec.Add(uint64(time.Since(startRPC) / time.Microsecond)) + ci.stats.TimeOfRPCUsec.Add(uint64(time.Since(startRPC).Microseconds())) ci.stats.RPCcompleted.Add(1) // We had to drop the lock before calling the RPC since it From 99b8cbfd02a9831a1814940a76b01f01ebcec83f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 17 Mar 2021 11:15:20 -0700 Subject: [PATCH 094/160] Deprecated the JSON RPC Server... no longer will be needed... ...until, perhaps, BiModal support returns... --- imgr/imgr.conf | 1 - imgr/imgrpkg/globals.go | 6 ------ imgr/imgrpkg/impl.go | 10 ---------- imgr/imgrpkg/json-rpc.go | 12 ------------ 4 files changed, 29 deletions(-) delete mode 100644 imgr/imgrpkg/json-rpc.go diff --git a/imgr/imgr.conf b/imgr/imgr.conf index 68583ee02..365574a70 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -4,7 +4,6 @@ [IMGR] PublicIPAddr: 0.0.0.0 PrivateIPAddr: 0.0.0.0 -JSONRPCPort: 12345 RetryRPCPort: 32356 HTTPServerPort: 15346 diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 5d72f8181..f1bc3c8c3 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -19,7 +19,6 @@ import ( type configStruct struct { PrivateIPAddr string PublicIPAddr string - JSONRPCPort uint16 // To be served only on PrivateIPAddr via TCP RetryRPCPort uint16 // To be served only on PublicIPAddr via TLS HTTPServerPort uint16 // To be served only on PrivateIPAddr via TCP @@ -121,10 +120,6 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { if nil != err { logFatal(err) } - globals.config.JSONRPCPort, err = confMap.FetchOptionValueUint16("IMGR", "JSONRPCPort") - if nil != err { - logFatal(err) - } globals.config.RetryRPCPort, err = confMap.FetchOptionValueUint16("IMGR", "RetryRPCPort") if nil != err { logFatal(err) @@ -224,7 +219,6 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { func uninitializeGlobals() (err error) { globals.config.PrivateIPAddr = "" globals.config.PublicIPAddr = "" - globals.config.JSONRPCPort = 0 globals.config.RetryRPCPort = 0 globals.config.HTTPServerPort = 0 diff --git a/imgr/imgrpkg/impl.go b/imgr/imgrpkg/impl.go index a30e1400e..c6aeb7f2d 100644 --- a/imgr/imgrpkg/impl.go +++ b/imgr/imgrpkg/impl.go @@ -23,11 +23,6 @@ func start(confMap conf.ConfMap) (err error) { return } - err = startJSONRPCServer() - if nil != err { - return - } - err = startHTTPServer() if nil != err { return @@ -42,11 +37,6 @@ func stop() (err error) { return } - err = stopJSONRPCServer() - if nil != err { - return - } - err = stopRetryRPCServer() if nil != err { return diff --git a/imgr/imgrpkg/json-rpc.go b/imgr/imgrpkg/json-rpc.go deleted file mode 100644 index 42b821fb7..000000000 --- a/imgr/imgrpkg/json-rpc.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package imgrpkg - -func startJSONRPCServer() (err error) { - return nil // TODO -} - -func stopJSONRPCServer() (err error) { - return nil // TODO -} From 51ae5e377c989d9fa951a04f6b343ae622aa9797 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 17 Mar 2021 15:35:44 -0700 Subject: [PATCH 095/160] Fixed up imgrpkg/http-server.go --- imgr/imgrpkg/http-server.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index ad56be612..8f323a3f5 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -6,6 +6,7 @@ package imgrpkg import ( "context" "encoding/json" + "fmt" "log" "net" "net/http" @@ -58,13 +59,10 @@ func stopHTTPServer() (err error) { func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { switch request.Method { case http.MethodDelete: - globals.httpServerWG.Add(1) - go serveHTTPDelete(responseWriter, request) + serveHTTPDelete(responseWriter, request) case http.MethodGet: - globals.httpServerWG.Add(1) serveHTTPGet(responseWriter, request) case http.MethodPut: - globals.httpServerWG.Add(1) serveHTTPPut(responseWriter, request) default: responseWriter.WriteHeader(http.StatusMethodNotAllowed) @@ -84,8 +82,6 @@ func serveHTTPDelete(responseWriter http.ResponseWriter, request *http.Request) default: responseWriter.WriteHeader(http.StatusNotFound) } - - globals.httpServerWG.Done() } func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, request *http.Request) { @@ -109,12 +105,6 @@ func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request) { default: responseWriter.WriteHeader(http.StatusNotFound) } - - globals.httpServerWG.Done() -} - -func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.WriteHeader(http.StatusNotImplemented) // TODO } func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Request) { @@ -128,6 +118,7 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ logFatalf("json.Marshal(globals.config) failed: %v", err) } + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(confMapJSON))) responseWriter.Header().Set("Content-Type", "application/json") responseWriter.WriteHeader(http.StatusOK) @@ -145,6 +136,7 @@ func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Reque statsAsString = bucketstats.SprintStats(bucketstats.StatFormatParsable1, "*", "*") + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(statsAsString))) responseWriter.Header().Set("Content-Type", "text/plain") responseWriter.WriteHeader(http.StatusOK) @@ -154,6 +146,10 @@ func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Reque } } +func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request) { + responseWriter.WriteHeader(http.StatusNotImplemented) // TODO +} + func serveHTTPPut(responseWriter http.ResponseWriter, request *http.Request) { var ( path string @@ -167,8 +163,6 @@ func serveHTTPPut(responseWriter http.ResponseWriter, request *http.Request) { default: responseWriter.WriteHeader(http.StatusNotFound) } - - globals.httpServerWG.Done() } func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Request) { From 7cc93864295243f11349b53e65cbd1664483a81d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 17 Mar 2021 17:49:03 -0700 Subject: [PATCH 096/160] Finished framework for Provisioning HTTP Request Handler ...Volume Deletion isn't done yet of course - it has to unmount clients after all... --- imgr/imgrpkg/globals.go | 19 +++ imgr/imgrpkg/http-server.go | 145 +++++++++++++++++----- imgr/imgrpkg/impl.go | 4 +- imgr/imgrpkg/inode.go | 22 ---- imgr/imgrpkg/volume.go | 240 ++++++++++++++++++++++++++++++++++++ 5 files changed, 372 insertions(+), 58 deletions(-) delete mode 100644 imgr/imgrpkg/inode.go create mode 100644 imgr/imgrpkg/volume.go diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index f1bc3c8c3..24ee942c2 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -91,11 +91,30 @@ type statsStruct struct { InodeTableCacheMisses bucketstats.Totaler } +type mountStruct struct { + volume *volumeStruct // volume.{R|}Lock() also protects each mountStruct + mountID string + authTokenValid bool + authToken string + lastAuthTime time.Time +} + +type volumeStruct struct { + sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() + name string + storageURL string + mountMap map[string]*mountStruct // key == mountStruct.mountID + numMountAuthTokenValid uint64 + deleting bool +} + type globalsStruct struct { sync.RWMutex config configStruct logFile *os.File // == nil if config.LogFilePath == "" inodeTableCache sortedmap.BPlusTreeCache + volumeMap sortedmap.LLRBTree // key == volumeStruct.name; value == *volumeStruct + mountMap map[string]*mountStruct // key == mountStruct.mountID httpServer *http.Server httpServerWG sync.WaitGroup stats *statsStruct diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 8f323a3f5..6a0d12c9e 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" "log" "net" "net/http" @@ -57,51 +58,77 @@ func stopHTTPServer() (err error) { } func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + requestBody []byte + requestPath string + ) + + requestPath = strings.TrimRight(request.URL.Path, "/") + + requestBody, err = ioutil.ReadAll(request.Body) + if nil == err { + err = request.Body.Close() + if nil != err { + responseWriter.WriteHeader(http.StatusBadRequest) + return + } + } else { + _ = request.Body.Close() + responseWriter.WriteHeader(http.StatusBadRequest) + return + } + switch request.Method { case http.MethodDelete: - serveHTTPDelete(responseWriter, request) + serveHTTPDelete(responseWriter, request, requestPath) case http.MethodGet: - serveHTTPGet(responseWriter, request) + serveHTTPGet(responseWriter, request, requestPath) case http.MethodPut: - serveHTTPPut(responseWriter, request) + serveHTTPPut(responseWriter, request, requestPath, requestBody) default: responseWriter.WriteHeader(http.StatusMethodNotAllowed) } } -func serveHTTPDelete(responseWriter http.ResponseWriter, request *http.Request) { - var ( - path string - ) - - path = strings.TrimRight(request.URL.Path, "/") - +func serveHTTPDelete(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { switch { - case strings.HasPrefix(path, "/volume"): - serveHTTPDeleteOfVolume(responseWriter, request) + case strings.HasPrefix(requestPath, "/volume"): + serveHTTPDeleteOfVolume(responseWriter, request, requestPath) default: responseWriter.WriteHeader(http.StatusNotFound) } } -func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.WriteHeader(http.StatusNotImplemented) // TODO -} - -func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request) { +func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { var ( - path string + err error + pathSplit []string ) - path = strings.TrimRight(request.URL.Path, "/") + pathSplit = strings.Split(requestPath, "/") + + switch len(pathSplit) { + case 3: + err = deleteVolume(pathSplit[2]) + if nil == err { + responseWriter.WriteHeader(http.StatusNoContent) + } else { + responseWriter.WriteHeader(http.StatusNotFound) + } + default: + responseWriter.WriteHeader(http.StatusBadRequest) + } +} +func serveHTTPGet(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { switch { - case "/config" == path: + case "/config" == requestPath: serveHTTPGetOfConfig(responseWriter, request) - case "/stats" == path: + case "/stats" == requestPath: serveHTTPGetOfStats(responseWriter, request) - case strings.HasPrefix(path, "/volume"): - serveHTTPGetOfVolume(responseWriter, request) + case strings.HasPrefix(requestPath, "/volume"): + serveHTTPGetOfVolume(responseWriter, request, requestPath) default: responseWriter.WriteHeader(http.StatusNotFound) } @@ -146,25 +173,75 @@ func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Reque } } -func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.WriteHeader(http.StatusNotImplemented) // TODO -} - -func serveHTTPPut(responseWriter http.ResponseWriter, request *http.Request) { +func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string) { var ( - path string + err error + jsonToReturn []byte + pathSplit []string ) - path = strings.TrimRight(request.URL.Path, "/") + pathSplit = strings.Split(requestPath, "/") + + switch len(pathSplit) { + case 2: + jsonToReturn = getVolumeListAsJSON() + + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(jsonToReturn))) + responseWriter.Header().Set("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + _, err = responseWriter.Write(jsonToReturn) + if nil != err { + logWarnf("responseWriter.Write(jsonToReturn) failed: %v", err) + } + case 3: + jsonToReturn, err = getVolumeAsJSON(pathSplit[2]) + if nil == err { + responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(jsonToReturn))) + responseWriter.Header().Set("Content-Type", "application/json") + responseWriter.WriteHeader(http.StatusOK) + + _, err = responseWriter.Write(jsonToReturn) + if nil != err { + logWarnf("responseWriter.Write(jsonToReturn) failed: %v", err) + } + } else { + responseWriter.WriteHeader(http.StatusNotFound) + } + default: + responseWriter.WriteHeader(http.StatusBadRequest) + } +} + +func serveHTTPPut(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { switch { - case strings.HasPrefix(path, "/volume"): - serveHTTPPutOfVolume(responseWriter, request) + case strings.HasPrefix(requestPath, "/volume"): + serveHTTPPutOfVolume(responseWriter, request, requestPath, requestBody) default: responseWriter.WriteHeader(http.StatusNotFound) } } -func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Request) { - responseWriter.WriteHeader(http.StatusNotImplemented) // TODO +func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { + var ( + err error + pathSplit []string + storageURL string + ) + + pathSplit = strings.Split(requestPath, "/") + + switch len(pathSplit) { + case 3: + storageURL = string(requestBody[:]) + + err = putVolume(pathSplit[2], storageURL) + if nil == err { + responseWriter.WriteHeader(http.StatusCreated) + } else { + responseWriter.WriteHeader(http.StatusConflict) + } + default: + responseWriter.WriteHeader(http.StatusBadRequest) + } } diff --git a/imgr/imgrpkg/impl.go b/imgr/imgrpkg/impl.go index c6aeb7f2d..e44961009 100644 --- a/imgr/imgrpkg/impl.go +++ b/imgr/imgrpkg/impl.go @@ -13,7 +13,7 @@ func start(confMap conf.ConfMap) (err error) { return } - err = startInodeTableManagement() + err = startVolumeManagement() if nil != err { return } @@ -42,7 +42,7 @@ func stop() (err error) { return } - err = stopInodeTableManagement() + err = stopVolumeManagement() if nil != err { return } diff --git a/imgr/imgrpkg/inode.go b/imgr/imgrpkg/inode.go deleted file mode 100644 index b3fdd2689..000000000 --- a/imgr/imgrpkg/inode.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package imgrpkg - -import ( - "github.com/NVIDIA/sortedmap" -) - -func startInodeTableManagement() (err error) { - globals.inodeTableCache = sortedmap.NewBPlusTreeCache(globals.config.InodeTableCacheEvictLowLimit, globals.config.InodeTableCacheEvictLowLimit) - - err = nil - return -} - -func stopInodeTableManagement() (err error) { - globals.inodeTableCache = nil - - err = nil - return -} diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go new file mode 100644 index 000000000..e1f89cb87 --- /dev/null +++ b/imgr/imgrpkg/volume.go @@ -0,0 +1,240 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +import ( + "encoding/json" + "fmt" + + "github.com/NVIDIA/sortedmap" +) + +func startVolumeManagement() (err error) { + globals.inodeTableCache = sortedmap.NewBPlusTreeCache(globals.config.InodeTableCacheEvictLowLimit, globals.config.InodeTableCacheEvictLowLimit) + globals.volumeMap = sortedmap.NewLLRBTree(sortedmap.CompareString, &globals) + globals.mountMap = make(map[string]*mountStruct) + + err = nil + return +} + +func stopVolumeManagement() (err error) { + globals.inodeTableCache = nil + globals.volumeMap = nil + globals.mountMap = nil + + err = nil + return +} + +func (dummy *globalsStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { + var ( + ok bool + ) + + keyAsString, ok = key.(string) + if ok { + err = nil + } else { + err = fmt.Errorf("volumeMap's DumpKey(%v) called for non-string", key) + } + + return +} + +func (dummy *globalsStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { + var ( + ok bool + valueAsVolume *volumeStruct + ) + + valueAsVolume, ok = value.(*volumeStruct) + if ok { + valueAsString = fmt.Sprintf("%s", valueAsVolume.storageURL) + err = nil + } else { + err = fmt.Errorf("volumeMap's DumpValue(%v) called for non-*volumeStruct", value) + } + + return +} + +func deleteVolume(volumeName string) (err error) { + var ( + ok bool + volumeAsStruct *volumeStruct + volumeAsValue sortedmap.Value + ) + + globals.Lock() + + volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) + if nil != err { + logFatal(err) + } + if !ok { + globals.Unlock() + err = fmt.Errorf("volumeName \"%s\" does not exist", volumeName) + return + } + + volumeAsStruct, ok = volumeAsValue.(*volumeStruct) + if !ok { + logFatalf("[IMGR]globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) + } + + // The following is only temporary... + // TODO: Actually gracefully unmount clients, block new mounts, and lazily remove it + + if 0 != len(volumeAsStruct.mountMap) { + logFatalf("[IMGR]No support for deleting actively mounted volume \"%s\"", volumeName) + } + + ok, err = globals.volumeMap.DeleteByKey(volumeAsStruct.name) + if nil != err { + logFatal(err) + } + if !ok { + logFatalf("[IMGR]globals.volumeMap[\"%s\"] suddenly missing", volumeAsStruct.name) + } + + globals.Unlock() + + err = nil + return +} + +type volumeGETStruct struct { + Name string + StorageURL string + HealthyMounts uint64 + ExpiredMounts uint64 +} + +func getVolumeAsJSON(volumeName string) (volume []byte, err error) { + var ( + ok bool + volumeAsStruct *volumeStruct + volumeAsValue sortedmap.Value + volumeToReturn *volumeGETStruct + ) + + globals.RLock() + + volumeAsValue, ok, err = globals.volumeMap.GetByKey(volumeName) + if nil != err { + logFatal(err) + } + if !ok { + globals.RUnlock() + err = fmt.Errorf("volumeName \"%s\" does not exist", volumeName) + return + } + + volumeAsStruct, ok = volumeAsValue.(*volumeStruct) + if !ok { + logFatalf("[IMGR]globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) + } + + volumeToReturn = &volumeGETStruct{ + Name: volumeAsStruct.name, + StorageURL: volumeAsStruct.storageURL, + HealthyMounts: volumeAsStruct.numMountAuthTokenValid, + ExpiredMounts: uint64(len(volumeAsStruct.mountMap)) - volumeAsStruct.numMountAuthTokenValid, + } + + globals.RUnlock() + + volume, err = json.Marshal(volumeToReturn) + if nil != err { + logFatal(err) + } + + err = nil + return +} + +func getVolumeListAsJSON() (volumeList []byte) { + var ( + err error + ok bool + volumeAsStruct *volumeStruct + volumeAsValue sortedmap.Value + volumeListIndex int + volumeListLen int + volumeListToReturn []*volumeGETStruct + ) + + globals.RLock() + + volumeListLen, err = globals.volumeMap.Len() + if nil != err { + logFatal(err) + } + + volumeListToReturn = make([]*volumeGETStruct, volumeListLen) + + for volumeListIndex = 0; volumeListIndex < volumeListLen; volumeListIndex++ { + _, volumeAsValue, ok, err = globals.volumeMap.GetByIndex(volumeListIndex) + if nil != err { + logFatal(err) + } + if !ok { + logFatalf("[IMGR]globals.volumeMap[] len (%d) is wrong", volumeListLen) + } + + volumeAsStruct, ok = volumeAsValue.(*volumeStruct) + if !ok { + logFatalf("[IMGR]globals.volumeMap[%d] was not a *volumeStruct", volumeListIndex) + } + + volumeListToReturn[volumeListIndex] = &volumeGETStruct{ + Name: volumeAsStruct.name, + StorageURL: volumeAsStruct.storageURL, + HealthyMounts: volumeAsStruct.numMountAuthTokenValid, + ExpiredMounts: uint64(len(volumeAsStruct.mountMap)) - volumeAsStruct.numMountAuthTokenValid, + } + } + + globals.RUnlock() + + volumeList, err = json.Marshal(volumeListToReturn) + if nil != err { + logFatal(err) + } + + return +} + +func putVolume(name string, storageURL string) (err error) { + var ( + ok bool + volume *volumeStruct + ) + + globals.Lock() + + volume = &volumeStruct{ + name: name, + storageURL: storageURL, + mountMap: make(map[string]*mountStruct), + numMountAuthTokenValid: 0, + deleting: false, + } + + ok, err = globals.volumeMap.Put(volume.name, volume) + if nil != err { + logFatal(err) + } + + globals.Unlock() + + if ok { + err = nil + } else { + err = fmt.Errorf("volume \"%s\" already exists", name) + } + + return +} From d7de50ee7dc0a434bc59031f82108cd675426c9d Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 18 Mar 2021 12:22:53 -0700 Subject: [PATCH 097/160] New simplified emswift... now called iswift... to only emulate (auth/normal) Swift Proxy ...this is all we need going forward... no more JSON RPC mechanism... no more "bypass" check... --- Makefile | 4 +- iswift/Makefile | 6 + iswift/dummy_test.go | 11 + iswift/iswift.conf | 12 + iswift/iswiftpkg/Makefile | 6 + iswift/iswiftpkg/api.go | 27 + iswift/iswiftpkg/impl.go | 1353 +++++++++++++++++++++++++++++++++ iswift/iswiftpkg/pkg_test.go | 1365 ++++++++++++++++++++++++++++++++++ iswift/main.go | 67 ++ 9 files changed, 2850 insertions(+), 1 deletion(-) create mode 100644 iswift/Makefile create mode 100644 iswift/dummy_test.go create mode 100644 iswift/iswift.conf create mode 100644 iswift/iswiftpkg/Makefile create mode 100644 iswift/iswiftpkg/api.go create mode 100644 iswift/iswiftpkg/impl.go create mode 100644 iswift/iswiftpkg/pkg_test.go create mode 100644 iswift/main.go diff --git a/Makefile b/Makefile index d65c93082..b1b1cbe10 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,8 @@ gopkgsubdirs = \ version \ emswift/emswiftpkg \ icert/icertpkg \ - imgr/imgrpkg + imgr/imgrpkg \ + iswift/iswiftpkg goplugindirs = \ iauth/iauth-swift @@ -48,6 +49,7 @@ gobinsubdirs = \ icert \ imgr \ inodeworkout \ + iswift \ pfs-crash \ pfs-fsck \ pfs-restart-test \ diff --git a/iswift/Makefile b/iswift/Makefile new file mode 100644 index 000000000..c28a068c8 --- /dev/null +++ b/iswift/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/iswift + +include ../GoMakefile diff --git a/iswift/dummy_test.go b/iswift/dummy_test.go new file mode 100644 index 000000000..d42991728 --- /dev/null +++ b/iswift/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/iswift/iswift.conf b/iswift/iswift.conf new file mode 100644 index 000000000..c324932f5 --- /dev/null +++ b/iswift/iswift.conf @@ -0,0 +1,12 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +[ISWIFT] +SwiftProxyIPAddr: 127.0.0.1 +SwiftProxyTCPPort: 8080 + +MaxAccountNameLength: 256 +MaxContainerNameLength: 256 +MaxObjectNameLength: 1024 +AccountListingLimit: 10000 +ContainerListingLimit: 10000 diff --git a/iswift/iswiftpkg/Makefile b/iswift/iswiftpkg/Makefile new file mode 100644 index 000000000..ed6ce8731 --- /dev/null +++ b/iswift/iswiftpkg/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/iswift/iswiftpkg + +include ../../GoMakefile diff --git a/iswift/iswiftpkg/api.go b/iswift/iswiftpkg/api.go new file mode 100644 index 000000000..918047b7c --- /dev/null +++ b/iswift/iswiftpkg/api.go @@ -0,0 +1,27 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iswiftpkg + +import ( + "github.com/NVIDIA/proxyfs/conf" +) + +// Start is called to start serving the NoAuth Swift Proxy Port and, +// optionally, the Auth Swift Proxy Port +// +func Start(confMap conf.ConfMap) (err error) { + err = start(confMap) + return +} + +// Stop is called to stop serving +// +func Stop() (err error) { + err = stop() + return +} + +func ForceReAuth() { + forceReAuth() +} diff --git a/iswift/iswiftpkg/impl.go b/iswift/iswiftpkg/impl.go new file mode 100644 index 000000000..5bf7bc601 --- /dev/null +++ b/iswift/iswiftpkg/impl.go @@ -0,0 +1,1353 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iswiftpkg + +import ( + "fmt" + "io/ioutil" + "math/rand" + "net" + "net/http" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "golang.org/x/sys/unix" + + "github.com/NVIDIA/sortedmap" + + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/utils" +) + +type swiftAccountStruct struct { + name string + headers http.Header + swiftContainerTree sortedmap.LLRBTree // key is swiftContainerStruct.name; value is *swiftContainerStruct +} + +type swiftContainerStruct struct { + name string + swiftAccount *swiftAccountStruct // back-reference to swiftAccountStruct + headers http.Header + swiftObjectTree sortedmap.LLRBTree // key is swiftObjectStruct.name; value is *swiftObjectStruct +} + +type swiftObjectStruct struct { + name string + swiftContainer *swiftContainerStruct // back-reference to swiftContainerStruct + headers http.Header + contents []byte +} + +const ( + startGETInfoMaxRetries = 10 + startGETInfoRetryDelay = 100 * time.Millisecond +) + +const ( + fixedAuthTokenPrefix = "AUTH_tk0123456789abcde012345678" // To be appended with %08x formatted uint32 nonce + fixedUserToAccountPrefix = "AUTH_" // Prefixed to User truncated before colon (":") if necessary +) + +type rangeStruct struct { + startOffset uint64 + stopOffset uint64 +} + +type stringSet map[string]bool + +var headerNameIgnoreSet = stringSet{"Accept": true, "Accept-Encoding": true, "User-Agent": true, "Content-Length": true} + +type configStruct struct { + SwiftProxyIPAddr string + SwiftProxyTCPPort uint16 + + MaxAccountNameLength uint64 + MaxContainerNameLength uint64 + MaxObjectNameLength uint64 + AccountListingLimit uint64 + ContainerListingLimit uint64 +} + +type globalsStruct struct { + sync.Mutex + config configStruct + emulatorHTTPServer *http.Server + emulatorWG sync.WaitGroup + currentAuthNonce uint32 // incremented each call to forceReAuth() + currentAuthToken string // fmt.Sprintf("%s%08x", fixedAuthTokenPrefix, currentAuthNonce) + swiftAccountMap map[string]*swiftAccountStruct // key is swiftAccountStruct.name; value is *swiftAccountStruct +} + +var globals globalsStruct + +func start(confMap conf.ConfMap) (err error) { + err = initializeGlobals(confMap) + if nil != err { + return + } + + err = startEmulator() + if nil != err { + return + } + + return +} + +func stop() (err error) { + err = stopEmulator() + if nil != err { + return + } + + uninitializeGlobals() + + err = nil + return +} + +func forceReAuth() { + globals.Lock() + globals.currentAuthNonce++ // ignore overflow/wrap + globals.currentAuthToken = fmt.Sprintf("%s%08x", fixedAuthTokenPrefix, globals.currentAuthNonce) + globals.Unlock() +} + +func getCurrentAuthToken() (currentAuthToken string) { + globals.Lock() + currentAuthToken = globals.currentAuthToken + globals.Unlock() + + return +} + +func initializeGlobals(confMap conf.ConfMap) (err error) { + globals.config.SwiftProxyIPAddr, err = confMap.FetchOptionValueString("ISWIFT", "SwiftProxyIPAddr") + if nil != err { + return + } + globals.config.SwiftProxyTCPPort, err = confMap.FetchOptionValueUint16("ISWIFT", "SwiftProxyTCPPort") + if nil != err { + return + } + + globals.config.MaxAccountNameLength, err = confMap.FetchOptionValueUint64("ISWIFT", "MaxAccountNameLength") + if nil != err { + return + } + globals.config.MaxContainerNameLength, err = confMap.FetchOptionValueUint64("ISWIFT", "MaxContainerNameLength") + if nil != err { + return + } + globals.config.MaxObjectNameLength, err = confMap.FetchOptionValueUint64("ISWIFT", "MaxObjectNameLength") + if nil != err { + return + } + globals.config.AccountListingLimit, err = confMap.FetchOptionValueUint64("ISWIFT", "AccountListingLimit") + if nil != err { + return + } + globals.config.ContainerListingLimit, err = confMap.FetchOptionValueUint64("ISWIFT", "ContainerListingLimit") + if nil != err { + return + } + + globals.currentAuthNonce = 0 + globals.currentAuthToken = fmt.Sprintf("%s%08x", fixedAuthTokenPrefix, globals.currentAuthNonce) + + globals.swiftAccountMap = make(map[string]*swiftAccountStruct) + + return +} + +func uninitializeGlobals() { + globals.config.SwiftProxyIPAddr = "" + globals.config.SwiftProxyTCPPort = 0 + + globals.config.MaxAccountNameLength = 0 + globals.config.MaxContainerNameLength = 0 + globals.config.MaxObjectNameLength = 0 + globals.config.AccountListingLimit = 0 + globals.config.ContainerListingLimit = 0 + + globals.swiftAccountMap = make(map[string]*swiftAccountStruct) +} + +func startEmulator() (err error) { + var ( + startGETInfoNumRetries int + ) + + globals.emulatorHTTPServer = &http.Server{ + Addr: net.JoinHostPort(globals.config.SwiftProxyIPAddr, fmt.Sprintf("%d", globals.config.SwiftProxyTCPPort)), + Handler: &globals, + } + + globals.emulatorWG.Add(1) + + go func() { + _ = globals.emulatorHTTPServer.ListenAndServe() + globals.emulatorWG.Done() + }() + + startGETInfoNumRetries = 0 + + for { + _, err = http.Get("http://" + globals.emulatorHTTPServer.Addr + "/info") + if nil == err { + break + } + startGETInfoNumRetries++ + if startGETInfoNumRetries > startGETInfoMaxRetries { + _ = stopEmulator() + err = fmt.Errorf("startEmulator() failed to establish that emulator is up") + return + } + time.Sleep(startGETInfoRetryDelay) + } + + err = nil + return +} + +func stopEmulator() (err error) { + err = globals.emulatorHTTPServer.Close() + if nil != err { + return + } + + globals.emulatorWG.Wait() + + err = nil + return +} + +func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) { + var ( + xAuthKey string + xAuthUser string + xAuthUserSplit2OnColon []string + xStorageURL string + ) + + // Handle the GET of/on info & AuthURL cases + + if http.MethodGet == request.Method { + switch request.URL.Path { + case "/info": + _ = request.Body.Close() + doGET(responseWriter, request) + return + case "/auth/v1.0": + _ = request.Body.Close() + xAuthUser = request.Header.Get("X-Auth-User") + xAuthKey = request.Header.Get("X-Auth-Key") + if ("" == xAuthUser) || ("" == xAuthKey) { + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + xAuthUserSplit2OnColon = strings.SplitN(xAuthUser, ":", 2) + xStorageURL = "http://" + globals.emulatorHTTPServer.Addr + "/v1/" + fixedUserToAccountPrefix + xAuthUserSplit2OnColon[0] + responseWriter.Header().Add("X-Auth-Token", getCurrentAuthToken()) + responseWriter.Header().Add("X-Storage-Url", xStorageURL) + return + default: + // Fall through to normal processing + } + } + + // Require X-Auth-Token to match fixedAuthToken + + if getCurrentAuthToken() != request.Header.Get("X-Auth-Token") { + _ = request.Body.Close() + responseWriter.WriteHeader(http.StatusUnauthorized) + return + } + + // Require "version" portion of request.URL.Path to be "v1" + + if !strings.HasPrefix(request.URL.Path, "/v1/") { + _ = request.Body.Close() + responseWriter.WriteHeader(http.StatusNotFound) + return + } + + // Branch off to individual request method handlers + + switch request.Method { + case http.MethodDelete: + doDELETE(responseWriter, request) + case http.MethodGet: + doGET(responseWriter, request) + case http.MethodHead: + doHEAD(responseWriter, request) + case http.MethodPost: + doPOST(responseWriter, request) + case http.MethodPut: + doPUT(responseWriter, request) + default: + responseWriter.WriteHeader(http.StatusMethodNotAllowed) + } +} + +func (dummy *globalsStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { + keyAsString = fmt.Sprintf("%v", key) + err = nil + return +} + +func (dummy *globalsStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { + valueAsString = fmt.Sprintf("%v", value) + err = nil + return +} + +func parsePath(request *http.Request) (infoOnly bool, swiftAccountName string, swiftContainerName string, swiftObjectName string) { + var ( + pathSplit []string + ) + + infoOnly = false + swiftAccountName = "" + swiftContainerName = "" + swiftObjectName = "" + + if "/info" == request.URL.Path { + infoOnly = true + return + } + + if strings.HasPrefix(request.URL.Path, "/v1/") { + pathSplit = strings.SplitN(request.URL.Path[4:], "/", 3) + swiftAccountName = pathSplit[0] + if 1 == len(pathSplit) { + swiftContainerName = "" + swiftObjectName = "" + } else { + swiftContainerName = pathSplit[1] + if 2 == len(pathSplit) { + swiftObjectName = "" + } else { + swiftObjectName = pathSplit[2] + } + } + } + + return +} + +func parseRangeHeader(request *http.Request, objectLen int) (ranges []rangeStruct, err error) { + var ( + off int + rangeHeaderValue string + rangeHeaderValueSuffix string + rangeString string + rangeStringSlice []string + rangesStrings []string + rangesStringsIndex int + startOffset int64 + stopOffset int64 + ) + + rangeHeaderValue = request.Header.Get("Range") + if "" == rangeHeaderValue { + ranges = make([]rangeStruct, 0) + err = nil + return + } + + if !strings.HasPrefix(rangeHeaderValue, "bytes=") { + err = fmt.Errorf("rangeHeaderValue (%v) does not start with expected \"bytes=\"", rangeHeaderValue) + return + } + + rangeHeaderValueSuffix = rangeHeaderValue[len("bytes="):] + + rangesStrings = strings.SplitN(rangeHeaderValueSuffix, ",", 2) + + ranges = make([]rangeStruct, len(rangesStrings)) + + for rangesStringsIndex, rangeString = range rangesStrings { + rangeStringSlice = strings.SplitN(rangeString, "-", 2) + if 2 != len(rangeStringSlice) { + err = fmt.Errorf("rangeHeaderValue (%v) malformed", rangeHeaderValue) + return + } + if "" == rangeStringSlice[0] { + startOffset = int64(-1) + } else { + off, err = strconv.Atoi(rangeStringSlice[0]) + if nil != err { + err = fmt.Errorf("rangeHeaderValue (%v) malformed (strconv.Atoi() failure: %v)", rangeHeaderValue, err) + return + } + startOffset = int64(off) + } + + if "" == rangeStringSlice[1] { + stopOffset = int64(-1) + } else { + off, err = strconv.Atoi(rangeStringSlice[1]) + if nil != err { + err = fmt.Errorf("rangeHeaderValue (%v) malformed (strconv.Atoi() failure: %v)", rangeHeaderValue, err) + return + } + stopOffset = int64(off) + } + + if ((0 > startOffset) && (0 > stopOffset)) || (startOffset > stopOffset) { + err = fmt.Errorf("rangeHeaderValue (%v) malformed", rangeHeaderValue) + return + } + + if startOffset < 0 { + startOffset = int64(objectLen) - stopOffset + if startOffset < 0 { + err = fmt.Errorf("rangeHeaderValue (%v) malformed...computed startOffset negative", rangeHeaderValue) + return + } + stopOffset = int64(objectLen - 1) + } else if stopOffset < 0 { + stopOffset = int64(objectLen - 1) + } else { + if stopOffset > int64(objectLen-1) { + stopOffset = int64(objectLen - 1) + } + } + + ranges[rangesStringsIndex].startOffset = uint64(startOffset) + ranges[rangesStringsIndex].stopOffset = uint64(stopOffset) + } + + err = nil + return +} + +func locateSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, errno syscall.Errno) { + var ( + ok bool + ) + + swiftAccount, ok = globals.swiftAccountMap[swiftAccountName] + if !ok { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func createSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, errno syscall.Errno) { + var ( + ok bool + ) + + _, ok = globals.swiftAccountMap[swiftAccountName] + if ok { + errno = unix.EEXIST + return + } + swiftAccount = &swiftAccountStruct{ + name: swiftAccountName, + headers: make(http.Header), + swiftContainerTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), + } + globals.swiftAccountMap[swiftAccountName] = swiftAccount + errno = 0 + return +} + +func createOrLocateSwiftAccount(swiftAccountName string) (swiftAccount *swiftAccountStruct, wasCreated bool) { + var ( + ok bool + ) + + swiftAccount, ok = globals.swiftAccountMap[swiftAccountName] + if ok { + wasCreated = false + } else { + swiftAccount = &swiftAccountStruct{ + name: swiftAccountName, + headers: make(http.Header), + swiftContainerTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), + } + globals.swiftAccountMap[swiftAccountName] = swiftAccount + wasCreated = true + } + return +} + +func deleteSwiftAccount(swiftAccountName string, force bool) (errno syscall.Errno) { + var ( + err error + ok bool + swiftAccount *swiftAccountStruct + swiftswiftAccountContainerCount int + ) + + swiftAccount, ok = globals.swiftAccountMap[swiftAccountName] + if ok { + if force { + // ok if account contains data... we'll forget it + } else { + swiftswiftAccountContainerCount, err = swiftAccount.swiftContainerTree.Len() + if nil != err { + panic(err) + } + if 0 != swiftswiftAccountContainerCount { + errno = unix.ENOTEMPTY + return + } + } + delete(globals.swiftAccountMap, swiftAccountName) + } else { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func locateSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, errno syscall.Errno) { + var ( + err error + ok bool + swiftContainerAsValue sortedmap.Value + ) + + swiftContainerAsValue, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) + if nil != err { + panic(err) + } + if ok { + swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) + } else { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func createSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, errno syscall.Errno) { + var ( + err error + ok bool + ) + + _, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) + if nil != err { + panic(err) + } + if ok { + errno = unix.EEXIST + return + } else { + swiftContainer = &swiftContainerStruct{ + name: swiftContainerName, + swiftAccount: swiftAccount, + headers: make(http.Header), + swiftObjectTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), + } + _, err = swiftAccount.swiftContainerTree.Put(swiftContainerName, swiftContainer) + if nil != err { + panic(err) + } + } + errno = 0 + return +} + +func createOrLocateSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (swiftContainer *swiftContainerStruct, wasCreated bool) { + var ( + err error + ok bool + swiftContainerAsValue sortedmap.Value + ) + + swiftContainerAsValue, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) + if nil != err { + panic(err) + } + if ok { + swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) + wasCreated = false + } else { + swiftContainer = &swiftContainerStruct{ + name: swiftContainerName, + swiftAccount: swiftAccount, + headers: make(http.Header), + swiftObjectTree: sortedmap.NewLLRBTree(sortedmap.CompareString, &globals), + } + _, err = swiftAccount.swiftContainerTree.Put(swiftContainerName, swiftContainer) + if nil != err { + panic(err) + } + wasCreated = true + } + return +} + +func deleteSwiftContainer(swiftAccount *swiftAccountStruct, swiftContainerName string) (errno syscall.Errno) { + var ( + err error + ok bool + swiftContainer *swiftContainerStruct + swiftContainerAsValue sortedmap.Value + swiftContainerObjectCount int + ) + + swiftContainerAsValue, ok, err = swiftAccount.swiftContainerTree.GetByKey(swiftContainerName) + if nil != err { + panic(err) + } + if ok { + swiftContainer = swiftContainerAsValue.(*swiftContainerStruct) + swiftContainerObjectCount, err = swiftContainer.swiftObjectTree.Len() + if nil != err { + panic(err) + } + if 0 != swiftContainerObjectCount { + errno = unix.ENOTEMPTY + return + } + _, err = swiftAccount.swiftContainerTree.DeleteByKey(swiftContainerName) + if nil != err { + panic(err) + } + } else { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func locateSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, errno syscall.Errno) { + var ( + err error + ok bool + swiftObjectAsValue sortedmap.Value + ) + + swiftObjectAsValue, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) + if nil != err { + panic(err) + } + if ok { + swiftObject = swiftObjectAsValue.(*swiftObjectStruct) + } else { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func createSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, errno syscall.Errno) { + var ( + err error + ok bool + ) + + _, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) + if nil != err { + panic(err) + } + if ok { + errno = unix.EEXIST + return + } else { + swiftObject = &swiftObjectStruct{name: swiftObjectName, swiftContainer: swiftContainer, contents: []byte{}} + _, err = swiftContainer.swiftObjectTree.Put(swiftObjectName, swiftObject) + if nil != err { + panic(err) + } + } + errno = 0 + return +} + +func createOrLocateSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (swiftObject *swiftObjectStruct, wasCreated bool) { + var ( + err error + ok bool + swiftObjectAsValue sortedmap.Value + ) + + swiftObjectAsValue, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) + if nil != err { + panic(err) + } + if ok { + swiftObject = swiftObjectAsValue.(*swiftObjectStruct) + wasCreated = false + } else { + swiftObject = &swiftObjectStruct{name: swiftObjectName, swiftContainer: swiftContainer, contents: []byte{}} + _, err = swiftContainer.swiftObjectTree.Put(swiftObjectName, swiftObject) + if nil != err { + panic(err) + } + wasCreated = true + } + return +} + +func deleteSwiftObject(swiftContainer *swiftContainerStruct, swiftObjectName string) (errno syscall.Errno) { + var ( + err error + ok bool + ) + + _, ok, err = swiftContainer.swiftObjectTree.GetByKey(swiftObjectName) + if nil != err { + panic(err) + } + if ok { + _, err = swiftContainer.swiftObjectTree.DeleteByKey(swiftObjectName) + if nil != err { + panic(err) + } + } else { + errno = unix.ENOENT + return + } + errno = 0 + return +} + +func doDELETE(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + errno syscall.Errno + infoOnly bool + swiftAccount *swiftAccountStruct + swiftAccountName string + swiftContainer *swiftContainerStruct + swiftContainerName string + swiftObjectName string + ) + + globals.Lock() + defer globals.Unlock() + + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) + if infoOnly || ("" == swiftAccountName) { + responseWriter.WriteHeader(http.StatusForbidden) + } else { + if "" == swiftContainerName { + // DELETE SwiftAccount + errno = deleteSwiftAccount(swiftAccountName, false) + switch errno { + case 0: + responseWriter.WriteHeader(http.StatusNoContent) + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + case unix.ENOTEMPTY: + responseWriter.WriteHeader(http.StatusConflict) + default: + err = fmt.Errorf("deleteSwiftAccount(\"%v\", false) returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } else { + // DELETE SwiftContainer or SwiftObject + swiftAccount, errno = locateSwiftAccount(swiftAccountName) + switch errno { + case 0: + if "" == swiftObjectName { + // DELETE SwiftContainer + errno = deleteSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + responseWriter.WriteHeader(http.StatusNoContent) + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + case unix.ENOTEMPTY: + responseWriter.WriteHeader(http.StatusConflict) + default: + err = fmt.Errorf("deleteSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } else { + // DELETE SwiftObject + swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + errno = deleteSwiftObject(swiftContainer, swiftObjectName) + switch errno { + case 0: + responseWriter.WriteHeader(http.StatusNoContent) + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("deleteSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) + panic(err) + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } + } +} + +func doGET(responseWriter http.ResponseWriter, request *http.Request) { + var ( + boundaryString string + containerIndex int + containerIndexLimit int + err error + errno syscall.Errno + found bool + headerName string + headerValue string + headerValueSlice []string + infoOnly bool + marker string + markerSlice []string + objectIndex int + objectIndexLimit int + ok bool + numContainers int + numObjects int + ranges []rangeStruct + rS rangeStruct + swiftAccount *swiftAccountStruct + swiftAccountName string + swiftContainer *swiftContainerStruct + swiftContainerName string + swiftContainerNameAsKey sortedmap.Key + swiftObject *swiftObjectStruct + swiftObjectName string + swiftObjectNameAsKey sortedmap.Key + ) + + globals.Lock() + defer globals.Unlock() + + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) + if infoOnly { + _, _ = responseWriter.Write(utils.StringToByteSlice("{")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"swift\": {")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_account_name_length\": " + strconv.Itoa(int(globals.config.MaxAccountNameLength)) + ",")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_container_name_length\": " + strconv.Itoa(int(globals.config.MaxContainerNameLength)) + ",")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"max_object_name_length\": " + strconv.Itoa(int(globals.config.MaxObjectNameLength)) + ",")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"account_listing_limit\": " + strconv.Itoa(int(globals.config.AccountListingLimit)) + ",")) + _, _ = responseWriter.Write(utils.StringToByteSlice("\"container_listing_limit\": " + strconv.Itoa(int(globals.config.ContainerListingLimit)))) + _, _ = responseWriter.Write(utils.StringToByteSlice("}")) + _, _ = responseWriter.Write(utils.StringToByteSlice("}")) + } else { + if "" == swiftAccountName { + responseWriter.WriteHeader(http.StatusForbidden) + } else { + swiftAccount, errno = locateSwiftAccount(swiftAccountName) + switch errno { + case 0: + if "" == swiftContainerName { + // GET SwiftAccount + for headerName, headerValueSlice = range swiftAccount.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + numContainers, err = swiftAccount.swiftContainerTree.Len() + if nil != err { + panic(err) + } + if 0 == numContainers { + responseWriter.WriteHeader(http.StatusNoContent) + } else { + marker = "" + markerSlice, ok = request.URL.Query()["marker"] + if ok && (0 < len(markerSlice)) { + marker = markerSlice[0] + } + containerIndex, found, err = swiftAccount.swiftContainerTree.BisectRight(marker) + if nil != err { + panic(err) + } + if found { + containerIndex++ + } + if containerIndex < numContainers { + containerIndexLimit = numContainers + if (containerIndexLimit - containerIndex) > int(globals.config.AccountListingLimit) { + containerIndexLimit = containerIndex + int(globals.config.AccountListingLimit) + } + for containerIndex < containerIndexLimit { + swiftContainerNameAsKey, _, _, err = swiftAccount.swiftContainerTree.GetByIndex(containerIndex) + if nil != err { + panic(err) + } + swiftContainerName = swiftContainerNameAsKey.(string) + _, _ = responseWriter.Write(utils.StringToByteSlice(swiftContainerName)) + _, _ = responseWriter.Write([]byte{'\n'}) + containerIndex++ + } + } else { + responseWriter.WriteHeader(http.StatusNoContent) + } + } + } else { + // GET SwiftContainer or SwiftObject + swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + if "" == swiftObjectName { + // GET SwiftContainer + for headerName, headerValueSlice = range swiftContainer.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + numObjects, err = swiftContainer.swiftObjectTree.Len() + if nil != err { + panic(err) + } + if 0 == numObjects { + responseWriter.WriteHeader(http.StatusNoContent) + } else { + marker = "" + markerSlice, ok = request.URL.Query()["marker"] + if ok && (0 < len(markerSlice)) { + marker = markerSlice[0] + } + objectIndex, found, err = swiftContainer.swiftObjectTree.BisectRight(marker) + if nil != err { + panic(err) + } + if found { + objectIndex++ + } + if objectIndex < numObjects { + objectIndexLimit = numObjects + if (objectIndexLimit - objectIndex) > int(globals.config.ContainerListingLimit) { + objectIndexLimit = objectIndex + int(globals.config.ContainerListingLimit) + } + for objectIndex < objectIndexLimit { + swiftObjectNameAsKey, _, _, err = swiftContainer.swiftObjectTree.GetByIndex(objectIndex) + if nil != err { + panic(err) + } + swiftObjectName = swiftObjectNameAsKey.(string) + _, _ = responseWriter.Write(utils.StringToByteSlice(swiftObjectName)) + _, _ = responseWriter.Write([]byte{'\n'}) + objectIndex++ + } + } else { + responseWriter.WriteHeader(http.StatusNoContent) + } + } + } else { + // GET SwiftObject + swiftObject, errno = locateSwiftObject(swiftContainer, swiftObjectName) + switch errno { + case 0: + for headerName, headerValueSlice = range swiftObject.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + ranges, err = parseRangeHeader(request, len(swiftObject.contents)) + if nil == err { + switch len(ranges) { + case 0: + responseWriter.Header().Add("Content-Type", "application/octet-stream") + responseWriter.WriteHeader(http.StatusOK) + _, _ = responseWriter.Write(swiftObject.contents) + case 1: + responseWriter.Header().Add("Content-Type", "application/octet-stream") + responseWriter.Header().Add("Content-Range", fmt.Sprintf("bytes %d-%d/%d", ranges[0].startOffset, ranges[0].stopOffset, len(swiftObject.contents))) + responseWriter.WriteHeader(http.StatusPartialContent) + _, _ = responseWriter.Write(swiftObject.contents[ranges[0].startOffset:(ranges[0].stopOffset + 1)]) + default: + boundaryString = fmt.Sprintf("%016x%016x", rand.Uint64(), rand.Uint64()) + responseWriter.Header().Add("Content-Type", fmt.Sprintf("multipart/byteranges; boundary=%v", boundaryString)) + responseWriter.WriteHeader(http.StatusPartialContent) + for _, rS = range ranges { + _, _ = responseWriter.Write([]byte("--" + boundaryString + "\r\n")) + _, _ = responseWriter.Write([]byte("Content-Type: application/octet-stream\r\n")) + _, _ = responseWriter.Write([]byte(fmt.Sprintf("Content-Range: bytes %d-%d/%d\r\n", rS.startOffset, rS.stopOffset, len(swiftObject.contents)))) + _, _ = responseWriter.Write([]byte("\r\n")) + _, _ = responseWriter.Write(swiftObject.contents[rS.startOffset:(rS.stopOffset + 1)]) + _, _ = responseWriter.Write([]byte("\r\n")) + } + _, _ = responseWriter.Write([]byte("--" + boundaryString + "--")) + } + } else { + responseWriter.WriteHeader(http.StatusBadRequest) + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } + } +} + +func doHEAD(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + errno syscall.Errno + headerName string + headerValue string + headerValueSlice []string + infoOnly bool + swiftAccount *swiftAccountStruct + swiftAccountName string + swiftContainer *swiftContainerStruct + swiftContainerName string + swiftObject *swiftObjectStruct + swiftObjectName string + ) + + globals.Lock() + defer globals.Unlock() + + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) + if infoOnly || ("" == swiftAccountName) { + responseWriter.WriteHeader(http.StatusForbidden) + } else { + swiftAccount, errno = locateSwiftAccount(swiftAccountName) + switch errno { + case 0: + if "" == swiftContainerName { + // HEAD SwiftAccount + for headerName, headerValueSlice = range swiftAccount.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + responseWriter.WriteHeader(http.StatusNoContent) + } else { + // HEAD SwiftContainer or SwiftObject + swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + if "" == swiftObjectName { + // HEAD SwiftContainer + for headerName, headerValueSlice = range swiftContainer.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + responseWriter.WriteHeader(http.StatusNoContent) + } else { + // HEAD SwiftObject + swiftObject, errno = locateSwiftObject(swiftContainer, swiftObjectName) + switch errno { + case 0: + for headerName, headerValueSlice = range swiftObject.headers { + for _, headerValue = range headerValueSlice { + responseWriter.Header().Add(headerName, headerValue) + } + } + responseWriter.Header().Set("Content-Length", strconv.Itoa(len(swiftObject.contents))) + responseWriter.WriteHeader(http.StatusOK) + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } +} + +func doPOST(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + errno syscall.Errno + headerName string + headerValue string + headerValueSlice []string + headerValueSliceLen int + ignoreHeader bool + infoOnly bool + swiftAccount *swiftAccountStruct + swiftAccountName string + swiftContainer *swiftContainerStruct + swiftContainerName string + swiftObject *swiftObjectStruct + swiftObjectName string + ) + + globals.Lock() + defer globals.Unlock() + + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) + if infoOnly || ("" == swiftAccountName) { + responseWriter.WriteHeader(http.StatusForbidden) + } else { + swiftAccount, errno = locateSwiftAccount(swiftAccountName) + switch errno { + case 0: + if "" == swiftContainerName { + // POST SwiftAccount + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftAccount.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftAccount.headers[headerName] = append(swiftAccount.headers[headerName], headerValue) + } + } + if 0 == len(swiftAccount.headers[headerName]) { + delete(swiftAccount.headers, headerName) + } + } + } + } + responseWriter.WriteHeader(http.StatusNoContent) + } else { + // POST SwiftContainer or SwiftObject + swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + if "" == swiftObjectName { + // POST SwiftContainer + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftContainer.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftContainer.headers[headerName] = append(swiftContainer.headers[headerName], headerValue) + } + } + if 0 == len(swiftContainer.headers[headerName]) { + delete(swiftContainer.headers, headerName) + } + } + } + } + responseWriter.WriteHeader(http.StatusNoContent) + } else { + // POST SwiftObject + swiftObject, errno = locateSwiftObject(swiftContainer, swiftObjectName) + switch errno { + case 0: + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftObject.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftObject.headers[headerName] = append(swiftObject.headers[headerName], headerValue) + } + } + if 0 == len(swiftObject.headers[headerName]) { + delete(swiftObject.headers, headerName) + } + } + } + } + responseWriter.WriteHeader(http.StatusNoContent) + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftObject(\"%v\") returned unexpected errno: %v", swiftObjectName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusNotFound) + default: + err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } +} + +func doPUT(responseWriter http.ResponseWriter, request *http.Request) { + var ( + err error + errno syscall.Errno + headerName string + headerValue string + headerValueSlice []string + headerValueSliceLen int + ignoreHeader bool + infoOnly bool + swiftAccount *swiftAccountStruct + swiftAccountName string + swiftContainer *swiftContainerStruct + swiftContainerName string + swiftObject *swiftObjectStruct + swiftObjectName string + wasCreated bool + ) + + globals.Lock() + defer globals.Unlock() + + infoOnly, swiftAccountName, swiftContainerName, swiftObjectName = parsePath(request) + if infoOnly || ("" == swiftAccountName) { + responseWriter.WriteHeader(http.StatusForbidden) + } else { + if "" == swiftContainerName { + // PUT SwiftAccount + swiftAccount, wasCreated = createOrLocateSwiftAccount(swiftAccountName) + if wasCreated { + swiftAccount.headers = make(http.Header) + } + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftAccount.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftAccount.headers[headerName] = append(swiftAccount.headers[headerName], headerValue) + } + } + if 0 == len(swiftAccount.headers[headerName]) { + delete(swiftAccount.headers, headerName) + } + } + } + } + if wasCreated { + responseWriter.WriteHeader(http.StatusCreated) + } else { + responseWriter.WriteHeader(http.StatusAccepted) + } + } else { + // PUT SwiftContainer or SwiftObject + swiftAccount, errno = locateSwiftAccount(swiftAccountName) + switch errno { + case 0: + if "" == swiftObjectName { + // PUT SwiftContainer + swiftContainer, wasCreated = createOrLocateSwiftContainer(swiftAccount, swiftContainerName) + if wasCreated { + swiftContainer.headers = make(http.Header) + } + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftContainer.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftContainer.headers[headerName] = append(swiftContainer.headers[headerName], headerValue) + } + } + if 0 == len(swiftContainer.headers[headerName]) { + delete(swiftContainer.headers, headerName) + } + } + } + } + if wasCreated { + responseWriter.WriteHeader(http.StatusCreated) + } else { + responseWriter.WriteHeader(http.StatusAccepted) + } + } else { + // PUT SwiftObject + swiftContainer, errno = locateSwiftContainer(swiftAccount, swiftContainerName) + switch errno { + case 0: + swiftObject, wasCreated = createOrLocateSwiftObject(swiftContainer, swiftObjectName) + if wasCreated { + swiftObject.headers = make(http.Header) + } + for headerName, headerValueSlice = range request.Header { + _, ignoreHeader = headerNameIgnoreSet[headerName] + if !ignoreHeader { + headerValueSliceLen = len(headerValueSlice) + if 0 < headerValueSliceLen { + swiftObject.headers[headerName] = make([]string, 0, headerValueSliceLen) + for _, headerValue = range headerValueSlice { + if 0 < len(headerValue) { + swiftObject.headers[headerName] = append(swiftObject.headers[headerName], headerValue) + } + } + if 0 == len(swiftObject.headers[headerName]) { + delete(swiftObject.headers, headerName) + } + } + } + } + swiftObject.contents, _ = ioutil.ReadAll(request.Body) + if wasCreated { + responseWriter.WriteHeader(http.StatusCreated) + } else { + responseWriter.WriteHeader(http.StatusCreated) + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusForbidden) + default: + err = fmt.Errorf("locateSwiftContainer(\"%v\") returned unexpected errno: %v", swiftContainerName, errno) + panic(err) + } + } + case unix.ENOENT: + responseWriter.WriteHeader(http.StatusForbidden) + default: + err = fmt.Errorf("locateSwiftAccount(\"%v\") returned unexpected errno: %v", swiftAccountName, errno) + panic(err) + } + } + } +} diff --git a/iswift/iswiftpkg/pkg_test.go b/iswift/iswiftpkg/pkg_test.go new file mode 100644 index 000000000..48b10236a --- /dev/null +++ b/iswift/iswiftpkg/pkg_test.go @@ -0,0 +1,1365 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iswiftpkg + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" + "strings" + "testing" + + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/utils" +) + +func TestEmulator(t *testing.T) { + var ( + cachedCurrentAuthToken string + confMap conf.ConfMap + confStrings = []string{ + "ISWIFT.SwiftProxyIPAddr=127.0.0.1", + "ISWIFT.SwiftProxyTCPPort=9997", + "ISWIFT.MaxAccountNameLength=256", + "ISWIFT.MaxContainerNameLength=256", + "ISWIFT.MaxObjectNameLength=1024", + "ISWIFT.AccountListingLimit=10000", + "ISWIFT.ContainerListingLimit=10000", + } + contentType string + contentTypeMultiPartBoundary string + err error + errChan chan error + expectedBuf []byte + expectedInfo string + expectedStorageURL string + httpClient *http.Client + httpRequest *http.Request + httpResponse *http.Response + mouseHeaderPresent bool + pipeReader *io.PipeReader + pipeWriter *io.PipeWriter + readBuf []byte + urlForAuth string + urlForInfo string + urlPrefix string + ) + + confMap, err = conf.MakeConfMapFromStrings(confStrings) + if nil != err { + t.Fatalf("conf.MakeConfMapFromStrings(confStrings) returned unexpected error: %v", err) + } + + err = Start(confMap) + if nil != err { + t.Fatalf("Start(confMap) returned unexpected error: %v", err) + } + + // Format URLs + + urlForInfo = "http://" + globals.emulatorHTTPServer.Addr + "/info" + urlForAuth = "http://" + globals.emulatorHTTPServer.Addr + "/auth/v1.0" + urlPrefix = "http://" + globals.emulatorHTTPServer.Addr + "/v1/" + + expectedStorageURL = "http://" + globals.emulatorHTTPServer.Addr + "/v1/" + "AUTH_test" + + // Setup http.Client that we will use for all HTTP requests + + httpClient = &http.Client{} + + // Send a GET for "/info" expecting [ISWIFT] data in compact JSON form + + httpRequest, err = http.NewRequest("GET", urlForInfo, nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + expectedInfo = "{\"swift\": {\"max_account_name_length\": 256,\"max_container_name_length\": 256,\"max_object_name_length\": 1024,\"account_listing_limit\": 10000,\"container_listing_limit\": 10000}}" + if int64(len(expectedInfo)) != httpResponse.ContentLength { + t.Fatalf("GET of /info httpResponse.ContentLength unexpected") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if expectedInfo != utils.ByteSliceToString(readBuf) { + t.Fatalf("GET of /info httpResponse.Body contents unexpected") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for "/auth/v1.0" expecting X-Auth-Token & X-Storage-Url + + httpRequest, err = http.NewRequest("GET", urlForAuth, nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-User", "test:tester") + httpRequest.Header.Add("X-Auth-Key", "testing") + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + cachedCurrentAuthToken = getCurrentAuthToken() + if httpResponse.Header.Get("X-Auth-Token") != cachedCurrentAuthToken { + t.Fatalf("Auth response should have header X-Auth-Token: %s", cachedCurrentAuthToken) + } + if httpResponse.Header.Get("X-Storage-Url") != expectedStorageURL { + t.Fatalf("Auth response should have header X-Storage-Url: %s", expectedStorageURL) + } + + // Send a PUT for account "TestAccount" and header Cat: Dog + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Cat", "Dog") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Force prior Auth to expire + + ForceReAuth() + if getCurrentAuthToken() == cachedCurrentAuthToken { + t.Fatalf("getCurrentAuthToken() not expected to return cachedCurrentAuthToken after call to ForceReAuth()") + } + + // Send a HEAD for account "TestAccount" with stale cachedCurrentAuthToken expecting httpResponse.StatusCode == http.StatusUnauthorized + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusUnauthorized != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Rather than re-authorizing, just update our cachedCurrentAuthToken + + cachedCurrentAuthToken = getCurrentAuthToken() + + // Send a HEAD for account "TestAccount" expecting header Cat: Dog + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for account "TestAccount" and header Mouse: Bird + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "Bird") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if httpResponse.Header.Get("Mouse") != "Bird" { + t.Fatalf("TestAccount should have header Mouse: Bird") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for account "TestAccount" deleting header Mouse + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + _, mouseHeaderPresent = httpResponse.Header["Mouse"] + if mouseHeaderPresent { + t.Fatalf("TestAccount should not have header Mouse") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for account "TestAccount" and header Mouse: Bird + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "Bird") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusAccepted != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for account "TestAccount" expecting header Cat: Dog & Mouse: Bird + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if httpResponse.Header.Get("Mouse") != "Bird" { + t.Fatalf("TestAccount should have header Mouse: Bird") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for account "TestAccount" deleting header Mouse + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusAccepted != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for account "TestAccount" expecting header Cat: Dog & no Mouse header + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + _, mouseHeaderPresent = httpResponse.Header["Mouse"] + if mouseHeaderPresent { + t.Fatalf("TestAccount should not have header Mouse") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for container "TestContainer" and header Cat: Dog + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Cat", "Dog") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" expecting "TestContainer\n" and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if int64(len("TestContainer\n")) != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if "TestContainer\n" != utils.ByteSliceToString(readBuf) { + t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" with marker "AAA" expecting "TestContainer\n" and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount?marker=AAA", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if int64(len("TestContainer\n")) != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if "TestContainer\n" != utils.ByteSliceToString(readBuf) { + t.Fatalf("TestAccount should contain only \"TestContainer\\n\" at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" with marker "ZZZ" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount?marker=ZZZ", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for container "TestContainer" expecting header Cat: Dog + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for container "TestContainer" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for container "TestContainer" and header Mouse: Bird + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "Bird") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + if httpResponse.Header.Get("Mouse") != "Bird" { + t.Fatalf("TestContainer should have header Mouse: Bird") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for container "TestContainer" deleting header Mouse + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + _, mouseHeaderPresent = httpResponse.Header["Mouse"] + if mouseHeaderPresent { + t.Fatalf("TestContainer should not have header Mouse") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for container "TestContainer" and header Mouse: Bird + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "Bird") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusAccepted != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for container "TestContainer" expecting header Cat: Dog & Mouse: Bird + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + if httpResponse.Header.Get("Mouse") != "Bird" { + t.Fatalf("TestContainer should have header Mouse: Bird") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for container "TestContainer" deleting header Mouse + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusAccepted != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for container "TestContainer" expecting header Cat: Dog & no Mouse header + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + _, mouseHeaderPresent = httpResponse.Header["Mouse"] + if mouseHeaderPresent { + t.Fatalf("TestContainer should not have header Mouse") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a non-chunked PUT for object "Foo" to contain []byte{0x00, 0x01, 0x02} + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/Foo", bytes.NewReader([]byte{0x00, 0x01, 0x02})) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a chunked PUT for object "Bar"" with 1st chunk being []byte{0xAA, 0xBB} & 2nd chunk being []byte{0xCC, 0xDD, 0xEE} + + pipeReader, pipeWriter = io.Pipe() + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/Bar", pipeReader) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.ContentLength = -1 + httpRequest.Header.Del("Content-Length") + errChan = make(chan error, 1) + go func() { + var ( + nonShadowingErr error + nonShadowingHTTPResponse *http.Response + ) + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + nonShadowingHTTPResponse, nonShadowingErr = httpClient.Do(httpRequest) + if nil == nonShadowingErr { + httpResponse = nonShadowingHTTPResponse + } + errChan <- nonShadowingErr + }() + _, err = pipeWriter.Write([]byte{0xAA, 0xBB}) + if nil != err { + t.Fatalf("pipeWriter.Write() returned unexpected error: %v", err) + } + _, err = pipeWriter.Write([]byte{0xCC, 0xDD, 0xEE}) + if nil != err { + t.Fatalf("pipeWriter.Write() returned unexpected error: %v", err) + } + err = pipeWriter.Close() + if nil != err { + t.Fatalf("pipeWriter.Close() returned unexpected error: %v", err) + } + err = <-errChan + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for container "TestContainer" expecting "Bar\nFoo\n" and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if int64(len("Bar\nFoo\n")) != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if "Bar\nFoo\n" != utils.ByteSliceToString(readBuf) { + t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for container "TestContainer" with marker "AAA" expecting "Bar\nFoo\n" and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer?marker=AAA", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if int64(len("Bar\nFoo\n")) != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if "Bar\nFoo\n" != utils.ByteSliceToString(readBuf) { + t.Fatalf("TestContainer should contain only \"Bar\\nFoo\\n\" at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for container "TestContainer" with marker "ZZZ" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer?marker=ZZZ", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for object "Foo" expecting Content-Length: 3 + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/Foo", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if 3 != httpResponse.ContentLength { + t.Fatalf("httpResponse.ContentLength contained unexpected value: %v", httpResponse.ContentLength) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a full object GET for object "Foo" expecting []byte{0x00, 0x01, 0x02} + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Foo", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if int64(len([]byte{0x00, 0x01, 0x02})) != httpResponse.ContentLength { + t.Fatalf("Foo should contain precisely []byte{0x00, 0x01, 0x02}") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if 0 != bytes.Compare([]byte{0x00, 0x01, 0x02}, readBuf) { + t.Fatalf("Foo should contain precisely []byte{0x00, 0x01, 0x02}") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a range GET of bytes at offset 1 for length 3 for object "Bar" expecting []byte{0xBB, 0xCC, 0xDD} + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Range", "bytes=1-3") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusPartialContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if int64(len([]byte{0xBB, 0xCC, 0xDD})) != httpResponse.ContentLength { + t.Fatalf("Bar's bytes 1-3 should contain precisely []byte{0xBB, 0xCC, 0xDD}") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if 0 != bytes.Compare([]byte{0xBB, 0xCC, 0xDD}, readBuf) { + t.Fatalf("Bar's bytes 1-3 should contain precisely []byte{0xBB, 0xCC, 0xDD}") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a range GET of bytes at offset 0 for length 2 + // and offset 3 for length of 1 for object "Bar" + // expecting two MIME parts: []byte{0xAA, 0xBB} and []byte{0xDD} + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Range", "bytes=0-1,3-3") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusPartialContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + contentType = httpResponse.Header.Get("Content-Type") + contentTypeMultiPartBoundary = strings.TrimPrefix(contentType, "multipart/byteranges; boundary=") + if (len(contentType) == len(contentTypeMultiPartBoundary)) || (0 == len(contentTypeMultiPartBoundary)) { + t.Fatalf("httpReponse.Header[\"Content-Type\"] contained unexpected value: \"%v\"", contentType) + } + expectedBuf = make([]byte, 0, httpResponse.ContentLength) + expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"\r\n")...) + expectedBuf = append(expectedBuf, []byte("Content-Type: application/octet-stream\r\n")...) + expectedBuf = append(expectedBuf, []byte("Content-Range: bytes 0-1/5\r\n")...) + expectedBuf = append(expectedBuf, []byte("\r\n")...) + expectedBuf = append(expectedBuf, []byte{0xAA, 0xBB}...) + expectedBuf = append(expectedBuf, []byte("\r\n")...) + expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"\r\n")...) + expectedBuf = append(expectedBuf, []byte("Content-Type: application/octet-stream\r\n")...) + expectedBuf = append(expectedBuf, []byte("Content-Range: bytes 3-3/5\r\n")...) + expectedBuf = append(expectedBuf, []byte("\r\n")...) + expectedBuf = append(expectedBuf, []byte{0xDD}...) + expectedBuf = append(expectedBuf, []byte("\r\n")...) + expectedBuf = append(expectedBuf, []byte("--"+contentTypeMultiPartBoundary+"--")...) + if int64(len(expectedBuf)) != httpResponse.ContentLength { + t.Fatalf("Unexpected multi-part GET response Content-Length") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if 0 != bytes.Compare(expectedBuf, readBuf) { + t.Fatalf("Unexpected payload of multi-part GET response") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a tail GET of the last 2 bytes for object "Bar" expecting []byte{0xDD, 0xEE} + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer/Bar", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Range", "bytes=-2") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusPartialContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if int64(len([]byte{0xDD, 0xEE})) != httpResponse.ContentLength { + t.Fatalf("Bar's last 2 bytes should contain precisely []byte{0xDD, 0xEE}") + } + readBuf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + t.Fatalf("ioutil.ReadAll() returned unexpected error: %v", err) + } + if 0 != bytes.Compare([]byte{0xDD, 0xEE}, readBuf) { + t.Fatalf("Bar's last 2 bytes should contain precisely []byte{0xDD, 0xEE}") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a PUT for object "ZigZag" and header Cat: Dog + + httpRequest, err = http.NewRequest("PUT", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Cat", "Dog") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusCreated != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for object "ZigZag" expecting header Cat: Dog + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for object "ZigZag" expecting header Cat: Dog + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for object "ZigZag" and header Mouse: Bird + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "Bird") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for object "ZigZag" expecting header Cat: Dog & Mouse: Bird + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if httpResponse.Header.Get("Mouse") != "Bird" { + t.Fatalf("TestAccount should have header Mouse: Bird") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a POST for object "ZigZag" deleting header Mouse + + httpRequest, err = http.NewRequest("POST", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("Mouse", "") + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a HEAD for object "ZigZag" expecting header Cat: Dog & no Mouse header + + httpRequest, err = http.NewRequest("HEAD", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusOK != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + _, mouseHeaderPresent = httpResponse.Header["Mouse"] + if mouseHeaderPresent { + t.Fatalf("TestAccount should not have header Mouse") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a DELETE for object "Foo" + + httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/Foo", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a DELETE for object "Bar" + + httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/Bar", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a DELETE for object "ZigZag" + + httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer/ZigZag", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for container "TestContainer" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestContainer should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestContainer should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a DELETE for container "TestContainer" + + httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount/TestContainer", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a GET for account "TestAccount" expecting Content-Length: 0 and header Cat: Dog + + httpRequest, err = http.NewRequest("GET", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + if httpResponse.Header.Get("Cat") != "Dog" { + t.Fatalf("TestAccount should have header Cat: Dog") + } + if 0 != httpResponse.ContentLength { + t.Fatalf("TestAccount should contain no elements at this point") + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + // Send a DELETE for account "TestAccount" + + httpRequest, err = http.NewRequest("DELETE", urlPrefix+"TestAccount", nil) + if nil != err { + t.Fatalf("http.NewRequest() returned unexpected error: %v", err) + } + httpRequest.Header.Add("X-Auth-Token", cachedCurrentAuthToken) + httpResponse, err = httpClient.Do(httpRequest) + if nil != err { + t.Fatalf("httpClient.Do() returned unexpected error: %v", err) + } + if http.StatusNoContent != httpResponse.StatusCode { + t.Fatalf("httpResponse.StatusCode contained unexpected value: %v", httpResponse.StatusCode) + } + err = httpResponse.Body.Close() + if nil != err { + t.Fatalf("http.Response.Body.Close() returned unexpected error: %v", err) + } + + err = Stop() + if nil != err { + t.Fatalf("Stop() returned unexpected error: %v", err) + } +} diff --git a/iswift/main.go b/iswift/main.go new file mode 100644 index 000000000..f26cb947f --- /dev/null +++ b/iswift/main.go @@ -0,0 +1,67 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "os/signal" + + "golang.org/x/sys/unix" + + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/iswift/iswiftpkg" +) + +func main() { + var ( + confMap conf.ConfMap + err error + signalChan chan os.Signal + ) + + if len(os.Args) < 2 { + fmt.Fprintf(os.Stderr, "no .conf file specified\n") + os.Exit(1) + } + + confMap, err = conf.MakeConfMapFromFile(os.Args[1]) + if nil != err { + fmt.Fprintf(os.Stderr, "failed to load config: %v\n", err) + os.Exit(1) + } + + err = confMap.UpdateFromStrings(os.Args[2:]) + if nil != err { + fmt.Fprintf(os.Stderr, "failed to apply config overrides: %v\n", err) + os.Exit(1) + } + + // Start Swift Emulation + + err = iswiftpkg.Start(confMap) + if nil != err { + fmt.Fprintf(os.Stderr, "iswiftpkg.Start(confMap) failed: %v\n", err) + os.Exit(1) + } + + // Arm signal handler used to indicate termination & wait on it + // + // Note: signal'd chan must be buffered to avoid race with window between + // arming handler and blocking on the chan read + + signalChan = make(chan os.Signal, 1) + + signal.Notify(signalChan, unix.SIGINT, unix.SIGTERM, unix.SIGHUP) + + _ = <-signalChan + + // Stop Swift Emulation + + err = iswiftpkg.Stop() + if nil != err { + fmt.Fprintf(os.Stderr, "iswiftpkg.Stop() failed: %v\n", err) + os.Exit(1) + } +} From d538f09a020a7db6bba6d7585ed80474d71d5293 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 20 Mar 2021 09:29:17 -0700 Subject: [PATCH 098/160] Just added inodeTable to imgrpkg's volumeStruct --- imgr/imgrpkg/globals.go | 1 + imgr/imgrpkg/volume.go | 1 + 2 files changed, 2 insertions(+) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 24ee942c2..75b07d966 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -106,6 +106,7 @@ type volumeStruct struct { mountMap map[string]*mountStruct // key == mountStruct.mountID numMountAuthTokenValid uint64 deleting bool + inodeTable sortedmap.BPlusTree } type globalsStruct struct { diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index e1f89cb87..faddb2d77 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -221,6 +221,7 @@ func putVolume(name string, storageURL string) (err error) { mountMap: make(map[string]*mountStruct), numMountAuthTokenValid: 0, deleting: false, + inodeTable: nil, } ok, err = globals.volumeMap.Put(volume.name, volume) From c0ef142eac6aab73345215b1c81167e35b3e5a85 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 30 Mar 2021 09:58:28 -0700 Subject: [PATCH 099/160] Added RpcLease "local" benchmark (to contrast with "remote" which uses retryrpc) --- jrpcfs/lease_test.go | 100 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 10 deletions(-) diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index bc751a88b..bd05e9405 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -17,15 +17,16 @@ import ( ) const ( - testRpcLeaseDelayAfterSendingRequest = 10 * time.Millisecond - testRpcLeaseDelayBeforeSendingRequest = 10 * time.Millisecond - testRpcLeaseRetryRPCDeadlineIO = "60s" - testRpcLeaseRetryRPCKeepAlivePeriod = "60s" - testRpcLeaseMultiFirstInodeNumber int64 = 1 - testRpcLeaseMultiNumInstances int = 5 - testRpcLeaseSingleInodeNumber int64 = 1 - testRpcLeaseSingleNumInstances int = 101 // Must be >= 4 - testRpcLeaseTimeFormat = "15:04:05.000" + testRpcLeaseDelayAfterSendingRequest = 10 * time.Millisecond + testRpcLeaseDelayBeforeSendingRequest = 10 * time.Millisecond + testRpcLeaseRetryRPCDeadlineIO = "60s" + testRpcLeaseRetryRPCKeepAlivePeriod = "60s" + testRpcLeaseMultiFirstInodeNumber int64 = 1 + testRpcLeaseMultiNumInstances int = 5 + testRpcLeaseSingleInodeNumber int64 = 1 + testRpcLeaseSingleNumInstances int = 101 // Must be >= 4 + testRpcLeaseTimeFormat = "15:04:05.000" + testRpcLeaseLocalClientID uint64 = 0 ) var ( @@ -45,7 +46,86 @@ type testRpcLeaseClientStruct struct { t *testing.T } -func BenchmarkRpcLease(b *testing.B) { +func BenchmarkRpcLeaseLocal(b *testing.B) { + var ( + benchmarkIteration int + err error + jserver *Server + leaseReply *LeaseReply + leaseRequest *LeaseRequest + mountByAccountNameRequest *MountByAccountNameRequest + mountByAccountNameReply *MountByAccountNameReply + unmountReply *Reply + unmountRequest *UnmountRequest + ) + + jserver = NewServer() + + mountByAccountNameRequest = &MountByAccountNameRequest{ + AccountName: testAccountName, + AuthToken: "", + } + mountByAccountNameReply = &MountByAccountNameReply{} + + err = jserver.RpcMountByAccountName(testRpcLeaseLocalClientID, mountByAccountNameRequest, mountByAccountNameReply) + if nil != err { + b.Fatalf("jserver.RpcMountByAccountName(testRpcLeaseLocalClientID, mountByAccountNameRequest, mountByAccountNameReply) failed: %v", err) + } + + b.ResetTimer() + + for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { + leaseRequest = &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeExclusive, + } + leaseReply = &LeaseReply{} + + err = jserver.RpcLease(leaseRequest, leaseReply) + if nil != err { + b.Fatalf("jserver.RpcLease(leaseRequest, leaseReply) failed: %v", err) + } + + if LeaseReplyTypeExclusive != leaseReply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", leaseReply.LeaseReplyType) + } + + leaseRequest = &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeRelease, + } + leaseReply = &LeaseReply{} + + err = jserver.RpcLease(leaseRequest, leaseReply) + if nil != err { + b.Fatalf("jserver.RpcLease(leaseRequest, leaseReply) failed: %v", err) + } + + if LeaseReplyTypeReleased != leaseReply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", leaseReply.LeaseReplyType) + } + } + + b.StopTimer() + + unmountRequest = &UnmountRequest{ + MountID: mountByAccountNameReply.MountID, + } + unmountReply = &Reply{} + + err = jserver.RpcUnmount(unmountRequest, unmountReply) + if nil != err { + b.Fatalf("jserver.RpcUnmount(unmountRequest, unmountReply) failed: %v", err) + } +} + +func BenchmarkRpcLeaseRemote(b *testing.B) { var ( benchmarkIteration int deadlineIO time.Duration From ebda18f8696f92ae64c68ce7e9ac70f405b9a42b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 30 Mar 2021 14:32:32 -0700 Subject: [PATCH 100/160] Added shortcut to retryrpc benchmarks (both TCP and TLS) to jrpcfs RpcLease tests --- jrpcfs/lease_test.go | 478 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 466 insertions(+), 12 deletions(-) diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index bd05e9405..99198486a 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -4,8 +4,13 @@ package jrpcfs import ( + "crypto/tls" + "crypto/x509" + "encoding/binary" "encoding/json" "fmt" + "io" + "net" "os" "runtime" "strings" @@ -17,16 +22,19 @@ import ( ) const ( - testRpcLeaseDelayAfterSendingRequest = 10 * time.Millisecond - testRpcLeaseDelayBeforeSendingRequest = 10 * time.Millisecond - testRpcLeaseRetryRPCDeadlineIO = "60s" - testRpcLeaseRetryRPCKeepAlivePeriod = "60s" - testRpcLeaseMultiFirstInodeNumber int64 = 1 - testRpcLeaseMultiNumInstances int = 5 - testRpcLeaseSingleInodeNumber int64 = 1 - testRpcLeaseSingleNumInstances int = 101 // Must be >= 4 - testRpcLeaseTimeFormat = "15:04:05.000" - testRpcLeaseLocalClientID uint64 = 0 + testRpcLeaseDelayAfterSendingRequest = 10 * time.Millisecond + testRpcLeaseDelayBeforeSendingRequest = 10 * time.Millisecond + testRpcLeaseRetryRPCDeadlineIO = "60s" + testRpcLeaseRetryRPCKeepAlivePeriod = "60s" + testRpcLeaseMultiFirstInodeNumber int64 = 1 + testRpcLeaseMultiNumInstances int = 5 + testRpcLeaseSingleInodeNumber int64 = 1 + testRpcLeaseSingleNumInstances int = 101 // Must be >= 4 + testRpcLeaseTimeFormat = "15:04:05.000" + testRpcLeaseLocalClientID uint64 = 0 + testRpcLeaseShortcutIPAddrPort = "127.0.0.1:24680" + testRpcLeaseShortcutClientConnMaxRetries = 10 + testRpcLeaseShortcutClientConnRetryDelay = "100ms" ) var ( @@ -46,6 +54,452 @@ type testRpcLeaseClientStruct struct { t *testing.T } +type benchmarkRpcShortcutRequestMethodOnlyStruct struct { // preceeded by a uint32 encoding length in LittleEndian form + Method string +} + +type benchmarkRpcShortcutLeaseRequestStruct struct { // preceeded by a uint32 encoding length in LittleEndian form + Method string + Request *LeaseRequest +} + +type benchmarkRpcShortcutMountByAccountNameRequestStruct struct { // preceeded by a uint32 encoding length in LittleEndian form + Method string + Request *MountByAccountNameRequest +} + +type benchmarkRpcShortcutUnmountRequestStruct struct { // preceeded by a uint32 encoding length in LittleEndian form + Method string + Request *UnmountRequest +} + +type benchmarkRpcShortcutLeaseReplyStruct struct { // preceeded by a uint32 encoding length in LittleEndian form + Err error + Reply *LeaseReply +} + +type benchmarkRpcShortcutMountByAccountNameReplyStruct struct { // preceeded by a uint32 encoding length in LittleEndian form + Err error + Reply *MountByAccountNameReply +} + +type benchmarkRpcShortcutUnmountReplyStruct struct { // preceeded by a uint32 encoding length in LittleEndian form + Err error + Reply *Reply +} + +func BenchmarkRpcLeaseShortcutTCP(b *testing.B) { + benchmarkRpcLeaseShortcut(b, false) +} + +func BenchmarkRpcLeaseShortcutTLS(b *testing.B) { + benchmarkRpcLeaseShortcut(b, true) +} + +func benchmarkRpcLeaseShortcut(b *testing.B, useTLS bool) { + var ( + benchmarkIteration int + doneWG sync.WaitGroup + err error + leaseReply *LeaseReply + leaseReplyWrapped *benchmarkRpcShortcutLeaseReplyStruct + leaseRequest *LeaseRequest + leaseRequestWrapped *benchmarkRpcShortcutLeaseRequestStruct + mountByAccountNameReply *MountByAccountNameReply + mountByAccountNameReplyWrapped *benchmarkRpcShortcutMountByAccountNameReplyStruct + mountByAccountNameRequest *MountByAccountNameRequest + mountByAccountNameRequestWrapped *benchmarkRpcShortcutMountByAccountNameRequestStruct + netConn net.Conn + netConnRetries int + netConnRetryDelay time.Duration + ok bool + rootCACertPool *x509.CertPool + tlsConfig *tls.Config + tlsConn *tls.Conn + unmountReply *Reply + unmountReplyWrapped *benchmarkRpcShortcutUnmountReplyStruct + unmountRequest *UnmountRequest + unmountRequestWrapped *benchmarkRpcShortcutUnmountRequestStruct + ) + + doneWG.Add(1) + go benchmarkRpcLeaseShortcutTCPServer(useTLS, &doneWG) + + netConnRetries = 0 + + netConnRetryDelay, err = time.ParseDuration(testRpcLeaseShortcutClientConnRetryDelay) + if nil != err { + b.Fatalf("time.ParseDuration(testRpcLeaseShortcutClientConnRetryDelay=\"%s\") failed: %v", testRpcLeaseShortcutClientConnRetryDelay, err) + } + + if useTLS { + rootCACertPool = x509.NewCertPool() + ok = rootCACertPool.AppendCertsFromPEM(testTLSCerts.caCertPEMBlock) + if !ok { + b.Fatalf("rootCACertPool.AppendCertsFromPEM(testTLSCerts.caCertPEMBlock) returned !ok") + } + tlsConfig = &tls.Config{ + RootCAs: rootCACertPool, + } + } + + for { + if useTLS { + tlsConn, err = tls.Dial("tcp", testRpcLeaseShortcutIPAddrPort, tlsConfig) + if nil == err { + netConn = tlsConn + break + } + } else { + netConn, err = net.Dial("tcp", testRpcLeaseShortcutIPAddrPort) + if nil == err { + break + } + } + + netConnRetries++ + + if netConnRetries > testRpcLeaseShortcutClientConnMaxRetries { + b.Fatalf("netConnRetries exceeded testRpcLeaseShortcutClientConnMaxRetries (%v)", testRpcLeaseShortcutClientConnMaxRetries) + } + + time.Sleep(netConnRetryDelay) + } + + mountByAccountNameRequest = &MountByAccountNameRequest{ + AccountName: testAccountName, + AuthToken: "", + } + mountByAccountNameReply = &MountByAccountNameReply{} + + mountByAccountNameRequestWrapped = &benchmarkRpcShortcutMountByAccountNameRequestStruct{Method: "RpcMountByAccountName", Request: mountByAccountNameRequest} + mountByAccountNameReplyWrapped = &benchmarkRpcShortcutMountByAccountNameReplyStruct{Reply: mountByAccountNameReply} + + benchmarkRpcLeaseShortcutDoRequest(b, netConn, mountByAccountNameRequestWrapped, mountByAccountNameReplyWrapped) + if nil != mountByAccountNameReplyWrapped.Err { + b.Fatalf("benchmarkRpcLeaseShortcutDoRequest(mountByAccountNameRequestWrapped, mountByAccountNameReplyWrapped) failed: %v", mountByAccountNameReplyWrapped.Err) + } + + b.ResetTimer() + + for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { + leaseRequest = &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeExclusive, + } + leaseReply = &LeaseReply{} + + leaseRequestWrapped = &benchmarkRpcShortcutLeaseRequestStruct{Method: "RpcLease", Request: leaseRequest} + leaseReplyWrapped = &benchmarkRpcShortcutLeaseReplyStruct{Reply: leaseReply} + + benchmarkRpcLeaseShortcutDoRequest(b, netConn, leaseRequestWrapped, leaseReplyWrapped) + if nil != leaseReplyWrapped.Err { + b.Fatalf("benchmarkRpcLeaseShortcutDoRequest(leaseRequestWrapped, leaseReplyWrapped) failed: %v", leaseReplyWrapped.Err) + } + + if LeaseReplyTypeExclusive != leaseReplyWrapped.Reply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", leaseReplyWrapped.Reply.LeaseReplyType) + } + + leaseRequest = &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeRelease, + } + leaseReply = &LeaseReply{} + + leaseRequestWrapped = &benchmarkRpcShortcutLeaseRequestStruct{Method: "RpcLease", Request: leaseRequest} + leaseReplyWrapped = &benchmarkRpcShortcutLeaseReplyStruct{Reply: leaseReply} + + benchmarkRpcLeaseShortcutDoRequest(b, netConn, leaseRequestWrapped, leaseReplyWrapped) + if nil != leaseReplyWrapped.Err { + b.Fatalf("benchmarkRpcLeaseShortcutDoRequest(leaseRequestWrapped, leaseReplyWrapped) failed: %v", leaseReplyWrapped.Err) + } + + if LeaseReplyTypeReleased != leaseReplyWrapped.Reply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", leaseReplyWrapped.Reply.LeaseReplyType) + } + } + + b.StopTimer() + + unmountRequest = &UnmountRequest{ + MountID: mountByAccountNameReply.MountID, + } + unmountReply = &Reply{} + + unmountRequestWrapped = &benchmarkRpcShortcutUnmountRequestStruct{Method: "RpcUnmount", Request: unmountRequest} + unmountReplyWrapped = &benchmarkRpcShortcutUnmountReplyStruct{Reply: unmountReply} + + benchmarkRpcLeaseShortcutDoRequest(b, netConn, unmountRequestWrapped, unmountReplyWrapped) + if nil != unmountReplyWrapped.Err { + b.Fatalf("benchmarkRpcLeaseShortcutDoRequest(unmountRequestWrapped, unmountReplyWrapped) failed: %v", unmountReplyWrapped.Err) + } + + err = netConn.Close() + if nil != err { + b.Fatalf("netConn.Close() failed: %v", err) + } + + doneWG.Wait() +} + +func benchmarkRpcLeaseShortcutTCPServer(useTLS bool, doneWG *sync.WaitGroup) { + var ( + benchmarkRpcShortcutRequestMethodOnly *benchmarkRpcShortcutRequestMethodOnlyStruct + err error + jserver *Server + leaseReply *LeaseReply + leaseReplyWrapped *benchmarkRpcShortcutLeaseReplyStruct + leaseRequest *LeaseRequest + leaseRequestWrapped *benchmarkRpcShortcutLeaseRequestStruct + mountByAccountNameReply *MountByAccountNameReply + mountByAccountNameReplyWrapped *benchmarkRpcShortcutMountByAccountNameReplyStruct + mountByAccountNameRequest *MountByAccountNameRequest + mountByAccountNameRequestWrapped *benchmarkRpcShortcutMountByAccountNameRequestStruct + n int + netConn net.Conn + netListener net.Listener + replyBuf []byte + replyLen uint32 + replyLenBuf []byte + requestBuf []byte + requestLen uint32 + requestLenBuf []byte + tlsConfig *tls.Config + unmountReply *Reply + unmountReplyWrapped *benchmarkRpcShortcutUnmountReplyStruct + unmountRequest *UnmountRequest + unmountRequestWrapped *benchmarkRpcShortcutUnmountRequestStruct + ) + + jserver = NewServer() + + netListener, err = net.Listen("tcp", testRpcLeaseShortcutIPAddrPort) + if nil != err { + panic(fmt.Errorf("net.Listen(\"tcp\", testRpcLeaseShortcutIPAddrPort) failed: %v", err)) + } + + if useTLS { + tlsConfig = &tls.Config{ + Certificates: []tls.Certificate{testTLSCerts.endpointTLSCert}, + } + + netListener = tls.NewListener(netListener, tlsConfig) + } + + netConn, err = netListener.Accept() + if nil != err { + panic(fmt.Errorf("netListener.Accept() failed: %v", err)) + } + + for { + requestLenBuf = make([]byte, 4) + + n, err = netConn.Read(requestLenBuf) + if nil != err { + if io.EOF != err { + panic(fmt.Errorf("netConn.Read(requestLenBuf) failed: %v", err)) + } + + err = netConn.Close() + if nil != err { + panic(fmt.Errorf("netConn.Close() failed: %v", err)) + } + + err = netListener.Close() + if nil != err { + panic(fmt.Errorf("netListener.Close() failed: %v", err)) + } + + doneWG.Done() + + return + } + if n != 4 { + panic(fmt.Errorf("netConn.Read(requestLenBuf) returned n == %v (expected 4)", n)) + } + + requestLen = binary.LittleEndian.Uint32(requestLenBuf) + requestBuf = make([]byte, requestLen) + + n, err = netConn.Read(requestBuf) + if nil != err { + panic(fmt.Errorf("netConn.Read(requestBuf) failed: %v", err)) + } + if n != int(requestLen) { + panic(fmt.Errorf("netConn.Read(requestBuf) returned n == %v (expected %v)", n, requestLen)) + } + + benchmarkRpcShortcutRequestMethodOnly = &benchmarkRpcShortcutRequestMethodOnlyStruct{} + + err = json.Unmarshal(requestBuf, benchmarkRpcShortcutRequestMethodOnly) + if nil != err { + panic(fmt.Errorf("json.Unmarshal(requestBuf, benchmarkRpcShortcutRequestMethodOnly) failed: %v", err)) + } + + switch benchmarkRpcShortcutRequestMethodOnly.Method { + case "RpcMountByAccountName": + mountByAccountNameRequestWrapped = &benchmarkRpcShortcutMountByAccountNameRequestStruct{} + + err = json.Unmarshal(requestBuf, mountByAccountNameRequestWrapped) + if nil != err { + panic(fmt.Errorf("json.Unmarshal(requestBuf, mountByAccountNameRequestWrapped) failed: %v", err)) + } + + mountByAccountNameRequest = mountByAccountNameRequestWrapped.Request + mountByAccountNameReply = &MountByAccountNameReply{} + + err = jserver.RpcMountByAccountName(testRpcLeaseLocalClientID, mountByAccountNameRequest, mountByAccountNameReply) + + mountByAccountNameReplyWrapped = &benchmarkRpcShortcutMountByAccountNameReplyStruct{ + Err: err, + Reply: mountByAccountNameReply, + } + + replyBuf, err = json.Marshal(mountByAccountNameReplyWrapped) + if nil != err { + panic(fmt.Errorf("json.Marshal(mountByAccountNameReplyWrapped) failed")) + } + case "RpcLease": + leaseRequestWrapped = &benchmarkRpcShortcutLeaseRequestStruct{} + + err = json.Unmarshal(requestBuf, leaseRequestWrapped) + if nil != err { + panic(fmt.Errorf("json.Unmarshal(requestBuf, leaseRequestWrapped) failed: %v", err)) + } + + leaseRequest = leaseRequestWrapped.Request + leaseReply = &LeaseReply{} + + err = jserver.RpcLease(leaseRequest, leaseReply) + + leaseReplyWrapped = &benchmarkRpcShortcutLeaseReplyStruct{ + Err: err, + Reply: leaseReply, + } + + replyBuf, err = json.Marshal(leaseReplyWrapped) + if nil != err { + panic(fmt.Errorf("json.Marshal(leaseReplyWrapped) failed")) + } + case "RpcUnmount": + unmountRequestWrapped = &benchmarkRpcShortcutUnmountRequestStruct{} + + err = json.Unmarshal(requestBuf, unmountRequestWrapped) + if nil != err { + panic(fmt.Errorf("json.Unmarshal(requestBuf, unmountRequestWrapped) failed: %v", err)) + } + + unmountRequest = unmountRequestWrapped.Request + unmountReply = &Reply{} + + err = jserver.RpcUnmount(unmountRequest, unmountReply) + + unmountReplyWrapped = &benchmarkRpcShortcutUnmountReplyStruct{ + Err: err, + Reply: unmountReply, + } + + replyBuf, err = json.Marshal(unmountReplyWrapped) + if nil != err { + panic(fmt.Errorf("json.Marshal(unmountReplyWrapped) failed")) + } + default: + panic(fmt.Errorf("benchmarkRpcShortcutRequestMethodOnly.Method (\"%s\") not recognized", benchmarkRpcShortcutRequestMethodOnly.Method)) + } + + replyLen = uint32(len(replyBuf)) + replyLenBuf = make([]byte, 4) + binary.LittleEndian.PutUint32(replyLenBuf, replyLen) + + n, err = netConn.Write(replyLenBuf) + if nil != err { + panic(fmt.Errorf("netConn.Write(replyLenBuf) failed: %v", err)) + } + if n != 4 { + panic(fmt.Errorf("netConn.Write(replyLenBuf) returned n == %v (expected 4)", n)) + } + + n, err = netConn.Write(replyBuf) + if nil != err { + panic(fmt.Errorf("netConn.Write(replyBuf) failed: %v", err)) + } + if n != int(replyLen) { + panic(fmt.Errorf("netConn.Write(replyBuf) returned n == %v (expected %v)", n, replyLen)) + } + } +} + +func benchmarkRpcLeaseShortcutDoRequest(b *testing.B, netConn net.Conn, request interface{}, reply interface{}) { + var ( + err error + n int + replyBuf []byte + replyLen uint32 + replyLenBuf []byte + requestBuf []byte + requestLen uint32 + requestLenBuf []byte + ) + + requestBuf, err = json.Marshal(request) + if nil != err { + b.Fatalf("json.Marshal(request) failed: %v", err) + } + + requestLen = uint32(len(requestBuf)) + requestLenBuf = make([]byte, 4) + binary.LittleEndian.PutUint32(requestLenBuf, requestLen) + + n, err = netConn.Write(requestLenBuf) + if nil != err { + b.Fatalf("netConn.Write(requestLenBuf) failed: %v", err) + } + if n != 4 { + b.Fatalf("netConn.Write(requestLenBuf) returned n == %v (expected 4)", n) + } + + n, err = netConn.Write(requestBuf) + if nil != err { + b.Fatalf("netConn.Write(requestBuf) failed: %v", err) + } + if n != int(requestLen) { + b.Fatalf("netConn.Write(requestBuf) returned n == %v (expected %v)", n, requestLen) + } + + replyLenBuf = make([]byte, 4) + + n, err = netConn.Read(replyLenBuf) + if nil != err { + b.Fatalf("netConn.Read(replyLenBuf) failed: %v", err) + } + if n != 4 { + b.Fatalf("netConn.Read(replyLenBuf) returned n == %v (expected 4)", n) + } + + replyLen = binary.LittleEndian.Uint32(replyLenBuf) + replyBuf = make([]byte, replyLen) + + n, err = netConn.Read(replyBuf) + if nil != err { + b.Fatalf("netConn.Read(replyBuf) failed: %v", err) + } + if n != int(replyLen) { + b.Fatalf("netConn.Read(replyBuf) returned n == %v (expected %v)", n, replyLen) + } + + err = json.Unmarshal(replyBuf, reply) + if nil != err { + b.Fatalf("json.Unmarshal(replyBuf, reply) failed: %v", err) + } +} + func BenchmarkRpcLeaseLocal(b *testing.B) { var ( benchmarkIteration int @@ -53,8 +507,8 @@ func BenchmarkRpcLeaseLocal(b *testing.B) { jserver *Server leaseReply *LeaseReply leaseRequest *LeaseRequest - mountByAccountNameRequest *MountByAccountNameRequest mountByAccountNameReply *MountByAccountNameReply + mountByAccountNameRequest *MountByAccountNameRequest unmountReply *Reply unmountRequest *UnmountRequest ) @@ -133,8 +587,8 @@ func BenchmarkRpcLeaseRemote(b *testing.B) { keepAlivePeriod time.Duration leaseReply *LeaseReply leaseRequest *LeaseRequest - mountByAccountNameRequest *MountByAccountNameRequest mountByAccountNameReply *MountByAccountNameReply + mountByAccountNameRequest *MountByAccountNameRequest retryRPCClient *retryrpc.Client retryrpcClientConfig *retryrpc.ClientConfig testRpcLeaseClient *testRpcLeaseClientStruct From 5ba327c4057600e0f4e67f955cb4f92042490807 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 30 Mar 2021 17:21:52 -0700 Subject: [PATCH 101/160] Tedious hard-coded marshalers for RpcLease benchmarks... not yet measured though --- jrpcfs/lease_test.go | 506 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 504 insertions(+), 2 deletions(-) diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index 99198486a..a34b51dfa 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -4,6 +4,7 @@ package jrpcfs import ( + "bytes" "crypto/tls" "crypto/x509" "encoding/binary" @@ -88,6 +89,507 @@ type benchmarkRpcShortcutUnmountReplyStruct struct { // preceeded by a uint32 en Reply *Reply } +// benchmarkRpcLeaseCustomRequest's are laid out as follows: +// +// RequestLength uint32 - [LittleEndian] does not include RequestLength nor RequestType +// RequestType uint32 - [LittleEndian] one of benchmarkRpcLeaseCustomRequestType* +// Request []byte - optimally packed request of the corresponding type + +// benchmarkRpcLeaseCustomReply's are laid out as follows: +// +// ReplyLenth uint32 - does not include ReplyLength +// Reply []byte - optimally packed RPC error string followed by the reply of the corresponding type + +// In each of the marshaled forms, strings consist of: +// +// StringLength uint32 - [LittleEndian] length of UTF-8 string +// StringBuf []byte - UTF-8 string + +const ( + benchmarkRpcLeaseCustomRequestTypeMountByAccountName uint32 = iota + benchmarkRpcLeaseCustomRequestTypeLease + benchmarkRpcLeaseCustomRequestTypeUnmount +) + +func benchmarkRpcLeaseCustomMountByAccountNameRequestMarshal(mountByAccountNameRequest *MountByAccountNameRequest) (mountByAccountNameRequestBuf []byte) { + var ( + accountNameLen = uint32(len(mountByAccountNameRequest.AccountName)) + authTokenLen = uint32(len(mountByAccountNameRequest.AuthToken)) + bytesBuffer *bytes.Buffer + err error + n int + ) + + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+accountNameLen+4+authTokenLen)) + + err = binary.Write(bytesBuffer, binary.LittleEndian, accountNameLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, accountNameLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(mountByAccountNameRequest.AccountName) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameRequest.AccountName) failed: %v", err)) + } + if uint32(n) != accountNameLen { + panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameRequest.AccountName) returned n==%v (expected %v)", n, accountNameLen)) + } + + err = binary.Write(bytesBuffer, binary.LittleEndian, authTokenLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, authTokenLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(mountByAccountNameRequest.AuthToken) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameRequest.AuthToken) failed: %v", err)) + } + if uint32(n) != authTokenLen { + panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameRequest.AuthToken) returned n==%v (expected %v)", n, authTokenLen)) + } + + mountByAccountNameRequestBuf = bytesBuffer.Bytes() + + return +} + +func benchmarkRpcLeaseCustomMountByAccountNameRequestUnmarshal(mountByAccountNameRequestBuf []byte) (mountByAccountNameRequest *MountByAccountNameRequest, unmarshalErr error) { + var ( + accountNameLen uint32 + authTokenLen uint32 + mountByAccountNameRequestBufLen = uint32(len(mountByAccountNameRequestBuf)) + mountByAccountNameRequestBufPos uint32 = 0 + ) + + mountByAccountNameRequest = &MountByAccountNameRequest{} + + if mountByAccountNameRequestBufPos+4 > mountByAccountNameRequestBufLen { + unmarshalErr = fmt.Errorf("couldn't read accountNameLen") + return + } + + accountNameLen = binary.LittleEndian.Uint32(mountByAccountNameRequestBuf[mountByAccountNameRequestBufPos : mountByAccountNameRequestBufPos+4]) + mountByAccountNameRequestBufPos += 4 + + if mountByAccountNameRequestBufPos+accountNameLen > mountByAccountNameRequestBufLen { + unmarshalErr = fmt.Errorf("couldn't read AccountName") + return + } + + mountByAccountNameRequest.AccountName = string(mountByAccountNameRequestBuf[mountByAccountNameRequestBufPos : mountByAccountNameRequestBufPos+accountNameLen]) + mountByAccountNameRequestBufPos += accountNameLen + + if mountByAccountNameRequestBufPos+4 > mountByAccountNameRequestBufLen { + unmarshalErr = fmt.Errorf("couldn't read authTokenLen") + return + } + + authTokenLen = binary.LittleEndian.Uint32(mountByAccountNameRequestBuf[mountByAccountNameRequestBufPos : mountByAccountNameRequestBufPos+4]) + mountByAccountNameRequestBufPos += 4 + + if mountByAccountNameRequestBufPos+authTokenLen > mountByAccountNameRequestBufLen { + unmarshalErr = fmt.Errorf("couldn't read AuthToken") + return + } + + mountByAccountNameRequest.AuthToken = string(mountByAccountNameRequestBuf[mountByAccountNameRequestBufPos : mountByAccountNameRequestBufPos+authTokenLen]) + mountByAccountNameRequestBufPos += authTokenLen + + unmarshalErr = nil + return +} + +func benchmarkRpcLeaseCustomMountByAccountNameReplyMarshal(rpcErr error, mountByAccountNameReply *MountByAccountNameReply) (mountByAccountNameReplyBuf []byte) { + var ( + bytesBuffer *bytes.Buffer + err error + mountIDLen = uint32(len(mountByAccountNameReply.MountID)) + n int + rpcErrString = rpcErr.Error() + rpcErrStringLen = uint32(len(rpcErrString)) + ) + + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+rpcErrStringLen+4+mountIDLen)) + + err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(string(rpcErrString)) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) + } + if uint32(n) != rpcErrStringLen { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) + } + + err = binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(string(mountByAccountNameReply.MountID)) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameReply.MountID) failed: %v", err)) + } + if uint32(n) != mountIDLen { + panic(fmt.Errorf("bytesBuffer.WriteString(mountByAccountNameReply.MountID) returned n==%v (expected %v)", n, mountIDLen)) + } + + mountByAccountNameReplyBuf = bytesBuffer.Bytes() + + return +} + +func benchmarkRpcLeaseCustomMountByAccountNameReplyUnmarshal(mountByAccountNameReplyBuf []byte) (rpcErr error, mountByAccountNameReply *MountByAccountNameReply, unmarshalErr error) { + var ( + mountByAccountNameReplyBufLen = uint32(len(mountByAccountNameReplyBuf)) + mountByAccountNameReplyBufPos uint32 = 0 + mountIDLen uint32 + rpcErrAsString string + rpcErrLen uint32 + ) + + if mountByAccountNameReplyBufPos+4 > mountByAccountNameReplyBufLen { + unmarshalErr = fmt.Errorf("couldn't read rpcErrLen") + return + } + + rpcErrLen = binary.LittleEndian.Uint32(mountByAccountNameReplyBuf[mountByAccountNameReplyBufPos : mountByAccountNameReplyBufPos+4]) + mountByAccountNameReplyBufPos += 4 + + if 0 == rpcErrLen { + rpcErr = nil + } else { + if mountByAccountNameReplyBufPos+rpcErrLen > mountByAccountNameReplyBufLen { + unmarshalErr = fmt.Errorf("couldn't read rpcErr") + return + } + + rpcErrAsString = string(mountByAccountNameReplyBuf[mountByAccountNameReplyBufPos : mountByAccountNameReplyBufPos+rpcErrLen]) + mountByAccountNameReplyBufPos += rpcErrLen + + rpcErr = fmt.Errorf("%s", rpcErrAsString) + } + + mountByAccountNameReply = &MountByAccountNameReply{} + + if mountByAccountNameReplyBufPos+4 > mountByAccountNameReplyBufLen { + unmarshalErr = fmt.Errorf("couldn't read mountIDLen") + return + } + + mountIDLen = binary.LittleEndian.Uint32(mountByAccountNameReplyBuf[mountByAccountNameReplyBufPos : mountByAccountNameReplyBufPos+4]) + mountByAccountNameReplyBufPos += 4 + + if mountByAccountNameReplyBufPos+mountIDLen > mountByAccountNameReplyBufLen { + unmarshalErr = fmt.Errorf("couldn't read MountID") + return + } + + mountByAccountNameReply.MountID = MountIDAsString(string(mountByAccountNameReplyBuf[mountByAccountNameReplyBufPos : mountByAccountNameReplyBufPos+mountIDLen])) + mountByAccountNameReplyBufPos += mountIDLen + + unmarshalErr = nil + return +} + +func benchmarkRpcLeaseCustomLeaseRequestMarshal(leaseRequest *LeaseRequest) (leaseRequestBuf []byte) { + var ( + bytesBuffer *bytes.Buffer + err error + mountIDLen = uint32(len(leaseRequest.InodeHandle.MountID)) + n int + ) + + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+mountIDLen+8+4)) + + err = binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(string(leaseRequest.InodeHandle.MountID)) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(leaseRequest.InodeHandle.MountID) failed: %v", err)) + } + if uint32(n) != mountIDLen { + panic(fmt.Errorf("bytesBuffer.WriteString(leaseRequest.InodeHandle.MountID) returned n==%v (expected %v)", n, mountIDLen)) + } + + err = binary.Write(bytesBuffer, binary.LittleEndian, uint64(leaseRequest.InodeHandle.InodeNumber)) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, uint64(leaseRequest.InodeHandle.InodeNumber)) failed: %v", err)) + } + + err = binary.Write(bytesBuffer, binary.LittleEndian, leaseRequest.LeaseRequestType) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, leaseRequest.LeaseRequestType) failed: %v", err)) + } + + leaseRequestBuf = bytesBuffer.Bytes() + + return +} + +func benchmarkRpcLeaseCustomLeaseRequestUnmarshal(leaseRequestBuf []byte) (leaseRequest *LeaseRequest, unmarshalErr error) { + var ( + leaseRequestBufLen = uint32(len(leaseRequestBuf)) + leaseRequestBufPos uint32 = 0 + mountIDLen uint32 + ) + + leaseRequest = &LeaseRequest{} + + if leaseRequestBufPos+4 > leaseRequestBufLen { + unmarshalErr = fmt.Errorf("couldn't read mountIDLen") + return + } + + mountIDLen = binary.LittleEndian.Uint32(leaseRequestBuf[leaseRequestBufPos : leaseRequestBufPos+4]) + leaseRequestBufPos += 4 + + if leaseRequestBufPos+mountIDLen > leaseRequestBufLen { + unmarshalErr = fmt.Errorf("couldn't read MountID") + return + } + + leaseRequest.InodeHandle.MountID = MountIDAsString(string(leaseRequestBuf[leaseRequestBufPos : leaseRequestBufPos+mountIDLen])) + leaseRequestBufPos += mountIDLen + + if leaseRequestBufPos+8 > leaseRequestBufLen { + unmarshalErr = fmt.Errorf("couldn't read InodeNumber") + return + } + + leaseRequest.InodeHandle.InodeNumber = int64(binary.LittleEndian.Uint64(leaseRequestBuf[leaseRequestBufPos : leaseRequestBufPos+8])) + leaseRequestBufPos += 8 + + if leaseRequestBufPos+4 > leaseRequestBufLen { + unmarshalErr = fmt.Errorf("couldn't read LeaseRequestType") + return + } + + leaseRequest.LeaseRequestType = LeaseRequestType(binary.LittleEndian.Uint32(leaseRequestBuf[leaseRequestBufPos : leaseRequestBufPos+4])) + leaseRequestBufPos += 4 + + unmarshalErr = nil + return +} + +func benchmarkRpcLeaseCustomLeaseReplyMarshal(rpcErr error, leaseReply *LeaseReply) (leaseReplyBuf []byte) { + var ( + bytesBuffer *bytes.Buffer + err error + n int + rpcErrString = rpcErr.Error() + rpcErrStringLen = uint32(len(rpcErrString)) + ) + + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+4)) + + err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(string(rpcErrString)) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) + } + if uint32(n) != rpcErrStringLen { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) + } + + err = binary.Write(bytesBuffer, binary.LittleEndian, leaseReply.LeaseReplyType) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, leaseReply.LeaseReplyType) failed: %v", err)) + } + + leaseReplyBuf = bytesBuffer.Bytes() + + return +} + +func benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf []byte) (rpcErr error, leaseReply *LeaseReply, unmarshalErr error) { + var ( + leaseReplyBufLen = uint32(len(leaseReplyBuf)) + leaseReplyBufPos uint32 = 0 + rpcErrAsString string + rpcErrLen uint32 + ) + + if leaseReplyBufPos+4 > leaseReplyBufLen { + unmarshalErr = fmt.Errorf("couldn't read rpcErrLen") + return + } + + rpcErrLen = binary.LittleEndian.Uint32(leaseReplyBuf[leaseReplyBufPos : leaseReplyBufPos+4]) + leaseReplyBufPos += 4 + + if 0 == rpcErrLen { + rpcErr = nil + } else { + if leaseReplyBufPos+rpcErrLen > leaseReplyBufLen { + unmarshalErr = fmt.Errorf("couldn't read rpcErr") + return + } + + rpcErrAsString = string(leaseReplyBuf[leaseReplyBufPos : leaseReplyBufPos+rpcErrLen]) + leaseReplyBufPos += rpcErrLen + + rpcErr = fmt.Errorf("%s", rpcErrAsString) + } + + leaseReply = &LeaseReply{} + + if leaseReplyBufPos+4 > leaseReplyBufLen { + unmarshalErr = fmt.Errorf("couldn't read LeaseReplyType") + return + } + + leaseReply.LeaseReplyType = LeaseReplyType(binary.LittleEndian.Uint32(leaseReplyBuf[leaseReplyBufPos : leaseReplyBufPos+4])) + leaseReplyBufPos += 4 + + unmarshalErr = nil + return +} + +func benchmarkRpcLeaseCustomUnmountRequestMarshal(unmountRequest *UnmountRequest) (unmountRequestBuf []byte) { + var ( + bytesBuffer *bytes.Buffer + err error + mountIDLen = uint32(len(unmountRequest.MountID)) + n int + ) + + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+mountIDLen)) + + err = binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(string(unmountRequest.MountID)) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(unmountRequest.MountID) failed: %v", err)) + } + if uint32(n) != mountIDLen { + panic(fmt.Errorf("bytesBuffer.WriteString(unmountRequest.MountID) returned n==%v (expected %v)", n, mountIDLen)) + } + + unmountRequestBuf = bytesBuffer.Bytes() + + return +} + +func benchmarkRpcLeaseCustomUnmountRequestUnmarshal(unmountRequestBuf []byte) (unmountRequest *UnmountRequest, unmarshalErr error) { + var ( + unmountRequestBufLen = uint32(len(unmountRequestBuf)) + unmountRequestBufPos uint32 = 0 + mountIDLen uint32 + ) + + unmountRequest = &UnmountRequest{} + + if unmountRequestBufPos+4 > unmountRequestBufLen { + unmarshalErr = fmt.Errorf("couldn't read mountIDLen") + return + } + + mountIDLen = binary.LittleEndian.Uint32(unmountRequestBuf[unmountRequestBufPos : unmountRequestBufPos+4]) + unmountRequestBufPos += 4 + + if unmountRequestBufPos+mountIDLen > unmountRequestBufLen { + unmarshalErr = fmt.Errorf("couldn't read MountID") + return + } + + unmountRequest.MountID = MountIDAsString(string(unmountRequestBuf[unmountRequestBufPos : unmountRequestBufPos+mountIDLen])) + unmountRequestBufPos += mountIDLen + + unmarshalErr = nil + return +} + +func benchmarkRpcLeaseCustomUnmountReplyMarshal(rpcErr error, unmountReply *Reply) (unmountReplyBuf []byte) { + var ( + bytesBuffer *bytes.Buffer + err error + n int + rpcErrString = rpcErr.Error() + rpcErrStringLen = uint32(len(rpcErrString)) + ) + + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4)) + + err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(string(rpcErrString)) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) + } + if uint32(n) != rpcErrStringLen { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) + } + + unmountReplyBuf = bytesBuffer.Bytes() + + return +} + +func benchmarkRpcLeaseCustomUnmountReplyUnmarshal(unmountReplyBuf []byte) (rpcErr error, unmountReply *Reply, unmarshalErr error) { + var ( + unmountReplyBufLen = uint32(len(unmountReplyBuf)) + unmountReplyBufPos uint32 = 0 + rpcErrAsString string + rpcErrLen uint32 + ) + + if unmountReplyBufPos+4 > unmountReplyBufLen { + unmarshalErr = fmt.Errorf("couldn't read rpcErrLen") + return + } + + rpcErrLen = binary.LittleEndian.Uint32(unmountReplyBuf[unmountReplyBufPos : unmountReplyBufPos+4]) + unmountReplyBufPos += 4 + + if 0 == rpcErrLen { + rpcErr = nil + } else { + if unmountReplyBufPos+rpcErrLen > unmountReplyBufLen { + unmarshalErr = fmt.Errorf("couldn't read rpcErr") + return + } + + rpcErrAsString = string(unmountReplyBuf[unmountReplyBufPos : unmountReplyBufPos+rpcErrLen]) + unmountReplyBufPos += rpcErrLen + + rpcErr = fmt.Errorf("%s", rpcErrAsString) + } + + unmountReply = &Reply{} + + unmarshalErr = nil + return +} + +func BenchmarkRpcLeaseCustomTCP(b *testing.B) { + benchmarkRpcLeaseCustom(b, false) +} + +func BenchmarkRpcLeaseCustomTLS(b *testing.B) { + benchmarkRpcLeaseCustom(b, true) +} + +func benchmarkRpcLeaseCustom(b *testing.B, useTLS bool) { + // TODO +} + func BenchmarkRpcLeaseShortcutTCP(b *testing.B) { benchmarkRpcLeaseShortcut(b, false) } @@ -123,7 +625,7 @@ func benchmarkRpcLeaseShortcut(b *testing.B, useTLS bool) { ) doneWG.Add(1) - go benchmarkRpcLeaseShortcutTCPServer(useTLS, &doneWG) + go benchmarkRpcLeaseShortcutServer(useTLS, &doneWG) netConnRetries = 0 @@ -249,7 +751,7 @@ func benchmarkRpcLeaseShortcut(b *testing.B, useTLS bool) { doneWG.Wait() } -func benchmarkRpcLeaseShortcutTCPServer(useTLS bool, doneWG *sync.WaitGroup) { +func benchmarkRpcLeaseShortcutServer(useTLS bool, doneWG *sync.WaitGroup) { var ( benchmarkRpcShortcutRequestMethodOnly *benchmarkRpcShortcutRequestMethodOnlyStruct err error From 4d9a083417a167180f98a64b5465a1c036b973c0 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 30 Mar 2021 18:28:22 -0700 Subject: [PATCH 102/160] Finished "custom" marshaling in jrpcfs/lease_test.go --- jrpcfs/lease_test.go | 476 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 436 insertions(+), 40 deletions(-) diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index a34b51dfa..c9bfe1a39 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -205,23 +205,35 @@ func benchmarkRpcLeaseCustomMountByAccountNameReplyMarshal(rpcErr error, mountBy err error mountIDLen = uint32(len(mountByAccountNameReply.MountID)) n int - rpcErrString = rpcErr.Error() - rpcErrStringLen = uint32(len(rpcErrString)) + rpcErrString string + rpcErrStringLen uint32 ) - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+rpcErrStringLen+4+mountIDLen)) + if nil == rpcErr { + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+4+mountIDLen)) - err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) - } + err = binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) failed: %v", err)) + } + } else { + rpcErrString = rpcErr.Error() + rpcErrStringLen = uint32(len(rpcErrString)) - n, err = bytesBuffer.WriteString(string(rpcErrString)) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) - } - if uint32(n) != rpcErrStringLen { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+rpcErrStringLen+4+mountIDLen)) + + err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(string(rpcErrString)) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) + } + if uint32(n) != rpcErrStringLen { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) + } } err = binary.Write(bytesBuffer, binary.LittleEndian, mountIDLen) @@ -383,23 +395,35 @@ func benchmarkRpcLeaseCustomLeaseReplyMarshal(rpcErr error, leaseReply *LeaseRep bytesBuffer *bytes.Buffer err error n int - rpcErrString = rpcErr.Error() - rpcErrStringLen = uint32(len(rpcErrString)) + rpcErrString string + rpcErrStringLen uint32 ) - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+4)) + if nil == rpcErr { + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+4)) - err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) - } + err = binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) failed: %v", err)) + } + } else { + rpcErrString = rpcErr.Error() + rpcErrStringLen = uint32(len(rpcErrString)) - n, err = bytesBuffer.WriteString(string(rpcErrString)) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) - } - if uint32(n) != rpcErrStringLen { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+rpcErrStringLen+4)) + + err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(string(rpcErrString)) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) + } + if uint32(n) != rpcErrStringLen { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) + } } err = binary.Write(bytesBuffer, binary.LittleEndian, leaseReply.LeaseReplyType) @@ -518,23 +542,35 @@ func benchmarkRpcLeaseCustomUnmountReplyMarshal(rpcErr error, unmountReply *Repl bytesBuffer *bytes.Buffer err error n int - rpcErrString = rpcErr.Error() - rpcErrStringLen = uint32(len(rpcErrString)) + rpcErrString string + rpcErrStringLen uint32 ) - bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4)) + if nil == rpcErr { + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4)) - err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) - if nil != err { - panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) - } + err = binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, uint32(0)) failed: %v", err)) + } + } else { + rpcErrString = rpcErr.Error() + rpcErrStringLen = uint32(len(rpcErrString)) - n, err = bytesBuffer.WriteString(string(rpcErrString)) - if nil != err { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) - } - if uint32(n) != rpcErrStringLen { - panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) + bytesBuffer = bytes.NewBuffer(make([]byte, 0, 4+rpcErrStringLen)) + + err = binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) + if nil != err { + panic(fmt.Errorf("binary.Write(bytesBuffer, binary.LittleEndian, rpcErrStringLen) failed: %v", err)) + } + + n, err = bytesBuffer.WriteString(string(rpcErrString)) + if nil != err { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) failed: %v", err)) + } + if uint32(n) != rpcErrStringLen { + panic(fmt.Errorf("bytesBuffer.WriteString(rpcErrString) returned n==%v (expected %v)", n, rpcErrStringLen)) + } } unmountReplyBuf = bytesBuffer.Bytes() @@ -587,7 +623,367 @@ func BenchmarkRpcLeaseCustomTLS(b *testing.B) { } func benchmarkRpcLeaseCustom(b *testing.B, useTLS bool) { - // TODO + var ( + benchmarkIteration int + doneWG sync.WaitGroup + err error + leaseReply *LeaseReply + leaseReplyBuf []byte + leaseRequest *LeaseRequest + leaseRequestBuf []byte + mountByAccountNameReply *MountByAccountNameReply + mountByAccountNameReplyBuf []byte + mountByAccountNameRequest *MountByAccountNameRequest + mountByAccountNameRequestBuf []byte + netConn net.Conn + netConnRetries int + netConnRetryDelay time.Duration + ok bool + rootCACertPool *x509.CertPool + rpcErr error + tlsConfig *tls.Config + tlsConn *tls.Conn + unmarshalErr error + unmountReplyBuf []byte + unmountRequest *UnmountRequest + unmountRequestBuf []byte + ) + + doneWG.Add(1) + go benchmarkRpcLeaseCustomServer(useTLS, &doneWG) + + netConnRetries = 0 + + netConnRetryDelay, err = time.ParseDuration(testRpcLeaseShortcutClientConnRetryDelay) + if nil != err { + b.Fatalf("time.ParseDuration(testRpcLeaseShortcutClientConnRetryDelay=\"%s\") failed: %v", testRpcLeaseShortcutClientConnRetryDelay, err) + } + + if useTLS { + rootCACertPool = x509.NewCertPool() + ok = rootCACertPool.AppendCertsFromPEM(testTLSCerts.caCertPEMBlock) + if !ok { + b.Fatalf("rootCACertPool.AppendCertsFromPEM(testTLSCerts.caCertPEMBlock) returned !ok") + } + tlsConfig = &tls.Config{ + RootCAs: rootCACertPool, + } + } + + for { + if useTLS { + tlsConn, err = tls.Dial("tcp", testRpcLeaseShortcutIPAddrPort, tlsConfig) + if nil == err { + netConn = tlsConn + break + } + } else { + netConn, err = net.Dial("tcp", testRpcLeaseShortcutIPAddrPort) + if nil == err { + break + } + } + + netConnRetries++ + + if netConnRetries > testRpcLeaseShortcutClientConnMaxRetries { + b.Fatalf("netConnRetries exceeded testRpcLeaseShortcutClientConnMaxRetries (%v)", testRpcLeaseShortcutClientConnMaxRetries) + } + + time.Sleep(netConnRetryDelay) + } + + mountByAccountNameRequest = &MountByAccountNameRequest{ + AccountName: testAccountName, + AuthToken: "", + } + + mountByAccountNameRequestBuf = benchmarkRpcLeaseCustomMountByAccountNameRequestMarshal(mountByAccountNameRequest) + + mountByAccountNameReplyBuf = benchmarkRpcLeaseCustomDoRequest(b, netConn, benchmarkRpcLeaseCustomRequestTypeMountByAccountName, mountByAccountNameRequestBuf) + + rpcErr, mountByAccountNameReply, unmarshalErr = benchmarkRpcLeaseCustomMountByAccountNameReplyUnmarshal(mountByAccountNameReplyBuf) + if nil != unmarshalErr { + b.Fatalf("benchmarkRpcLeaseCustomMountByAccountNameReplyUnmarshal(mountByAccountNameReplyBuf) returned unmarshalErr: %v", unmarshalErr) + } + if nil != rpcErr { + b.Fatalf("benchmarkRpcLeaseCustomMountByAccountNameReplyUnmarshal(mountByAccountNameReplyBuf) returned rpcErr: %v", rpcErr) + } + + b.ResetTimer() + + for benchmarkIteration = 0; benchmarkIteration < b.N; benchmarkIteration++ { + leaseRequest = &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeExclusive, + } + + leaseRequestBuf = benchmarkRpcLeaseCustomLeaseRequestMarshal(leaseRequest) + + leaseReplyBuf = benchmarkRpcLeaseCustomDoRequest(b, netConn, benchmarkRpcLeaseCustomRequestTypeLease, leaseRequestBuf) + + rpcErr, leaseReply, unmarshalErr = benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) + if nil != unmarshalErr { + b.Fatalf("benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) returned unmarshalErr: %v", unmarshalErr) + } + if nil != rpcErr { + b.Fatalf("benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) returned rpcErr: %v", rpcErr) + } + + if LeaseReplyTypeExclusive != leaseReply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseRequestTypeExclusive", leaseReply.LeaseReplyType) + } + + leaseRequest = &LeaseRequest{ + InodeHandle: InodeHandle{ + MountID: mountByAccountNameReply.MountID, + InodeNumber: testRpcLeaseSingleInodeNumber, + }, + LeaseRequestType: LeaseRequestTypeRelease, + } + + leaseRequestBuf = benchmarkRpcLeaseCustomLeaseRequestMarshal(leaseRequest) + + leaseReplyBuf = benchmarkRpcLeaseCustomDoRequest(b, netConn, benchmarkRpcLeaseCustomRequestTypeLease, leaseRequestBuf) + + rpcErr, leaseReply, unmarshalErr = benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) + if nil != unmarshalErr { + b.Fatalf("benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) returned unmarshalErr: %v", unmarshalErr) + } + if nil != rpcErr { + b.Fatalf("benchmarkRpcLeaseCustomLeaseReplyUnmarshal(leaseReplyBuf) returned rpcErr: %v", rpcErr) + } + + if LeaseReplyTypeReleased != leaseReply.LeaseReplyType { + b.Fatalf("RpcLease() returned LeaseReplyType %v... expected LeaseReplyTypeReleased", leaseReply.LeaseReplyType) + } + } + + b.StopTimer() + + unmountRequest = &UnmountRequest{ + MountID: mountByAccountNameReply.MountID, + } + + unmountRequestBuf = benchmarkRpcLeaseCustomUnmountRequestMarshal(unmountRequest) + + unmountReplyBuf = benchmarkRpcLeaseCustomDoRequest(b, netConn, benchmarkRpcLeaseCustomRequestTypeUnmount, unmountRequestBuf) + + rpcErr, _, unmarshalErr = benchmarkRpcLeaseCustomUnmountReplyUnmarshal(unmountReplyBuf) + if nil != unmarshalErr { + b.Fatalf("benchmarkRpcLeaseCustomUnmountReplyUnmarshal(unmountReplyBuf) returned unmarshalErr: %v", unmarshalErr) + } + if nil != rpcErr { + b.Fatalf("benchmarkRpcLeaseCustomUnmountReplyUnmarshal(unmountReplyBuf) returned rpcErr: %v", rpcErr) + } + + err = netConn.Close() + if nil != err { + b.Fatalf("netConn.Close() failed: %v", err) + } + + doneWG.Wait() +} + +func benchmarkRpcLeaseCustomServer(useTLS bool, doneWG *sync.WaitGroup) { + var ( + err error + jserver *Server + leaseReply *LeaseReply + leaseRequest *LeaseRequest + mountByAccountNameReply *MountByAccountNameReply + mountByAccountNameRequest *MountByAccountNameRequest + n int + netConn net.Conn + netListener net.Listener + replyBuf []byte + replyLen uint32 + replyLenBuf []byte + requestBuf []byte + requestLen uint32 + requestLenAndRequestTypeBuf []byte + requestType uint32 + tlsConfig *tls.Config + unmountReply *Reply + unmountRequest *UnmountRequest + ) + + jserver = NewServer() + + netListener, err = net.Listen("tcp", testRpcLeaseShortcutIPAddrPort) + if nil != err { + panic(fmt.Errorf("net.Listen(\"tcp\", testRpcLeaseShortcutIPAddrPort) failed: %v", err)) + } + + if useTLS { + tlsConfig = &tls.Config{ + Certificates: []tls.Certificate{testTLSCerts.endpointTLSCert}, + } + + netListener = tls.NewListener(netListener, tlsConfig) + } + + netConn, err = netListener.Accept() + if nil != err { + panic(fmt.Errorf("netListener.Accept() failed: %v", err)) + } + + for { + requestLenAndRequestTypeBuf = make([]byte, 8) + + n, err = netConn.Read(requestLenAndRequestTypeBuf) + if nil != err { + if io.EOF != err { + panic(fmt.Errorf("netConn.Read(requestLenAndRequestTypeBuf) failed: %v", err)) + } + + err = netConn.Close() + if nil != err { + panic(fmt.Errorf("netConn.Close() failed: %v", err)) + } + + err = netListener.Close() + if nil != err { + panic(fmt.Errorf("netListener.Close() failed: %v", err)) + } + + doneWG.Done() + + return + } + if n != 8 { + panic(fmt.Errorf("netConn.Read(requestLenAndRequestTypeBuf) returned n == %v (expected 8)", n)) + } + + requestLen = binary.LittleEndian.Uint32(requestLenAndRequestTypeBuf[:4]) + requestType = binary.LittleEndian.Uint32(requestLenAndRequestTypeBuf[4:]) + + requestBuf = make([]byte, requestLen) + + n, err = netConn.Read(requestBuf) + if nil != err { + panic(fmt.Errorf("netConn.Read(requestBuf) failed: %v", err)) + } + if n != int(requestLen) { + panic(fmt.Errorf("netConn.Read(requestBuf) returned n == %v (expected %v)", n, requestLen)) + } + + switch requestType { + case benchmarkRpcLeaseCustomRequestTypeMountByAccountName: + mountByAccountNameRequest, err = benchmarkRpcLeaseCustomMountByAccountNameRequestUnmarshal(requestBuf) + if nil != err { + panic(fmt.Errorf("benchmarkRpcLeaseCustomMountByAccountNameRequestUnmarshal(requestBuf) failed: %v", err)) + } + + mountByAccountNameReply = &MountByAccountNameReply{} + + err = jserver.RpcMountByAccountName(testRpcLeaseLocalClientID, mountByAccountNameRequest, mountByAccountNameReply) + + replyBuf = benchmarkRpcLeaseCustomMountByAccountNameReplyMarshal(err, mountByAccountNameReply) + case benchmarkRpcLeaseCustomRequestTypeLease: + leaseRequest, err = benchmarkRpcLeaseCustomLeaseRequestUnmarshal(requestBuf) + if nil != err { + panic(fmt.Errorf("benchmarkRpcLeaseCustomLeaseRequestUnmarshal(requestBuf) failed: %v", err)) + } + + leaseReply = &LeaseReply{} + + err = jserver.RpcLease(leaseRequest, leaseReply) + + replyBuf = benchmarkRpcLeaseCustomLeaseReplyMarshal(err, leaseReply) + case benchmarkRpcLeaseCustomRequestTypeUnmount: + unmountRequest, err = benchmarkRpcLeaseCustomUnmountRequestUnmarshal(requestBuf) + if nil != err { + panic(fmt.Errorf("benchmarkRpcLeaseCustomUnmountRequestUnmarshal(requestBuf) failed: %v", err)) + } + + unmountReply = &Reply{} + + err = jserver.RpcUnmount(unmountRequest, unmountReply) + + replyBuf = benchmarkRpcLeaseCustomUnmountReplyMarshal(err, unmountReply) + default: + panic(fmt.Errorf("requestType (%v) not recognized", requestType)) + } + + replyLen = uint32(len(replyBuf)) + replyLenBuf = make([]byte, 4) + binary.LittleEndian.PutUint32(replyLenBuf, replyLen) + + n, err = netConn.Write(replyLenBuf) + if nil != err { + panic(fmt.Errorf("netConn.Write(replyLenBuf) failed: %v", err)) + } + if n != 4 { + panic(fmt.Errorf("netConn.Write(replyLenBuf) returned n == %v (expected 4)", n)) + } + + n, err = netConn.Write(replyBuf) + if nil != err { + panic(fmt.Errorf("netConn.Write(replyBuf) failed: %v", err)) + } + if n != int(replyLen) { + panic(fmt.Errorf("netConn.Write(replyBuf) returned n == %v (expected %v)", n, replyLen)) + } + } +} + +func benchmarkRpcLeaseCustomDoRequest(b *testing.B, netConn net.Conn, requestType uint32, requestBuf []byte) (replyBuf []byte) { + var ( + err error + n int + replyLen uint32 + replyLenBuf []byte + requestLen uint32 = uint32(len(requestBuf)) + requestLenAndRequestTypeBuf []byte + ) + + requestLenAndRequestTypeBuf = make([]byte, 8) + + binary.LittleEndian.PutUint32(requestLenAndRequestTypeBuf[:4], requestLen) + binary.LittleEndian.PutUint32(requestLenAndRequestTypeBuf[4:], requestType) + + n, err = netConn.Write(requestLenAndRequestTypeBuf) + if nil != err { + b.Fatalf("netConn.Write(requestLenAndRequestTypeBuf) failed: %v", err) + } + if n != 8 { + b.Fatalf("netConn.Write(requestLenAndRequestTypeBuf) returned n == %v (expected 8)", n) + } + + n, err = netConn.Write(requestBuf) + if nil != err { + b.Fatalf("netConn.Write(requestBuf) failed: %v", err) + } + if n != int(requestLen) { + b.Fatalf("netConn.Write(requestBuf) returned n == %v (expected %v)", n, requestLen) + } + + replyLenBuf = make([]byte, 4) + + n, err = netConn.Read(replyLenBuf) + if nil != err { + b.Fatalf("netConn.Read(replyLenBuf) failed: %v", err) + } + if n != 4 { + b.Fatalf("netConn.Read(replyLenBuf) returned n == %v (expected 4)", n) + } + + replyLen = binary.LittleEndian.Uint32(replyLenBuf) + replyBuf = make([]byte, replyLen) + + n, err = netConn.Read(replyBuf) + if nil != err { + b.Fatalf("netConn.Read(replyBuf) failed: %v", err) + } + if n != int(replyLen) { + b.Fatalf("netConn.Read(replyBuf) returned n == %v (expected %v)", n, replyLen) + } + + return } func BenchmarkRpcLeaseShortcutTCP(b *testing.B) { From e6332889024e349f6454d5efdc35efc33b61ee58 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 1 Apr 2021 15:15:31 -0700 Subject: [PATCH 103/160] Added TLS Cert support in imgrpkg --- imgr/imgr.conf | 11 ++++--- imgr/imgrpkg/globals.go | 62 +++++++++++++++++++++++++++++++++++++++ imgr/imgrpkg/retry-rpc.go | 8 +++++ 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/imgr/imgr.conf b/imgr/imgr.conf index 365574a70..31288e9d5 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 [IMGR] -PublicIPAddr: 0.0.0.0 -PrivateIPAddr: 0.0.0.0 +PublicIPAddr: 172.28.128.2 +PrivateIPAddr: 172.28.128.2 RetryRPCPort: 32356 HTTPServerPort: 15346 @@ -12,6 +12,9 @@ RetryRPCAckTrim: 100ms RetryRPCDeadlineIO: 60s RetryRPCKeepAlivePeriod: 60s +RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, +RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used + MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 @@ -25,6 +28,6 @@ SwiftConnectionPoolSize: 128 InodeTableCacheEvictLowLimit: 10000 InodeTableCacheEvictHighLimit: 10010 -LogFilePath: # imgr.log -LogToConsole: true # false +LogFilePath: # imgr.log +LogToConsole: true # false TraceEnabled: false diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 75b07d966..a5d23e5cf 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -5,6 +5,7 @@ package imgrpkg import ( "container/list" + "fmt" "net/http" "os" "sync" @@ -14,6 +15,7 @@ import ( "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/utils" ) type configStruct struct { @@ -27,6 +29,9 @@ type configStruct struct { RetryRPCDeadlineIO time.Duration RetryRPCKeepAlivePeriod time.Duration + RetryRPCCertFilePath string + RetryRPCKeyFilePath string + MinLeaseDuration time.Duration LeaseInterruptInterval time.Duration LeaseInterruptLimit uint32 @@ -124,6 +129,10 @@ type globalsStruct struct { var globals globalsStruct func initializeGlobals(confMap conf.ConfMap) (err error) { + var ( + configJSONified string + ) + // Default logging related globals globals.config.LogFilePath = "" @@ -170,6 +179,52 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { logFatal(err) } + err = confMap.VerifyOptionIsMissing("IMGR", "RetryRPCCertFilePath") + if nil == err { + err = confMap.VerifyOptionIsMissing("IMGR", "RetryRPCKeyFilePath") + if nil == err { + globals.config.RetryRPCCertFilePath = "" + globals.config.RetryRPCKeyFilePath = "" + } else { + err = confMap.VerifyOptionValueIsEmpty("IMGR", "RetryRPCKeyFilePath") + if nil == err { + globals.config.RetryRPCCertFilePath = "" + globals.config.RetryRPCKeyFilePath = "" + } else { + err = fmt.Errorf("[IMGR]RetryRPCCertFilePath is missing but [IMGR]RetryRPCKeyFilePath is present and non-empty") + logFatal(err) + } + } + } else { + err = confMap.VerifyOptionValueIsEmpty("IMGR", "RetryRPCCertFilePath") + if nil == err { + err = confMap.VerifyOptionIsMissing("IMGR", "RetryRPCKeyFilePath") + if nil == err { + globals.config.RetryRPCCertFilePath = "" + globals.config.RetryRPCKeyFilePath = "" + } else { + err = confMap.VerifyOptionValueIsEmpty("IMGR", "RetryRPCKeyFilePath") + if nil == err { + globals.config.RetryRPCCertFilePath = "" + globals.config.RetryRPCKeyFilePath = "" + } else { + err = fmt.Errorf("[IMGR]RetryRPCCertFilePath is present and empty but [IMGR]RetryRPCKeyFilePath is present and non-empty") + logFatal(err) + } + } + } else { + globals.config.RetryRPCCertFilePath, err = confMap.FetchOptionValueString("IMGR", "RetryRPCCertFilePath") + if nil != err { + logFatal(err) + } else { + globals.config.RetryRPCKeyFilePath, err = confMap.FetchOptionValueString("IMGR", "RetryRPCKeyFilePath") + if nil != err { + logFatal(err) + } + } + } + } + globals.config.MinLeaseDuration, err = confMap.FetchOptionValueDuration("IMGR", "MinLeaseDuration") if nil != err { logFatal(err) @@ -228,6 +283,10 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { logFatal(err) } + configJSONified = utils.JSONify(globals.config, true) + + logInfof("globals.config:\n%s", configJSONified) + globals.stats = &statsStruct{} bucketstats.Register("IMGR", "", globals.stats) @@ -247,6 +306,9 @@ func uninitializeGlobals() (err error) { globals.config.RetryRPCDeadlineIO = time.Duration(0) globals.config.RetryRPCKeepAlivePeriod = time.Duration(0) + globals.config.RetryRPCCertFilePath = "" + globals.config.RetryRPCKeyFilePath = "" + globals.config.MinLeaseDuration = time.Duration(0) globals.config.LeaseInterruptInterval = time.Duration(0) globals.config.LeaseInterruptLimit = 0 diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index a8e44ce77..598738760 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -3,10 +3,18 @@ package imgrpkg +type retryRPCServerStruct struct{} + +var retryRPCServer *retryRPCServerStruct + func startRetryRPCServer() (err error) { + retryRPCServer = &retryRPCServerStruct{} + return nil // TODO } func stopRetryRPCServer() (err error) { + retryRPCServer = nil + return nil // TODO } From f7bbfffe16044400e2f41f117d7f8745b46cb657 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 1 Apr 2021 16:11:52 -0700 Subject: [PATCH 104/160] Added retryrpc RPCs to imgrpkg (just stubs for now...) --- imgr/imgr.conf | 2 + imgr/imgrpkg/globals.go | 9 ++ imgr/imgrpkg/retry-rpc.go | 188 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+) diff --git a/imgr/imgr.conf b/imgr/imgr.conf index 31288e9d5..6bdad24cf 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -15,6 +15,8 @@ RetryRPCKeepAlivePeriod: 60s RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used +FetchNonceRangeTpReturn: 100 + MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index a5d23e5cf..ccee37059 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -32,6 +32,8 @@ type configStruct struct { RetryRPCCertFilePath string RetryRPCKeyFilePath string + FetchNonceRangeTpReturn uint64 + MinLeaseDuration time.Duration LeaseInterruptInterval time.Duration LeaseInterruptLimit uint32 @@ -225,6 +227,11 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { } } + globals.config.FetchNonceRangeTpReturn, err = confMap.FetchOptionValueUint64("IMGR", "FetchNonceRangeTpReturn") + if nil != err { + logFatal(err) + } + globals.config.MinLeaseDuration, err = confMap.FetchOptionValueDuration("IMGR", "MinLeaseDuration") if nil != err { logFatal(err) @@ -309,6 +316,8 @@ func uninitializeGlobals() (err error) { globals.config.RetryRPCCertFilePath = "" globals.config.RetryRPCKeyFilePath = "" + globals.config.FetchNonceRangeTpReturn = 0 + globals.config.MinLeaseDuration = time.Duration(0) globals.config.LeaseInterruptInterval = time.Duration(0) globals.config.LeaseInterruptLimit = 0 diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 598738760..4e83e844b 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -18,3 +18,191 @@ func stopRetryRPCServer() (err error) { return nil // TODO } + +// MountRequestStruct is the request object for RpcMount +// +type MountRequestStruct struct { + VolumeName string + AuthToken string +} + +// MountReplyStruct is the reply object for RpcMount +// +type MountReplyStruct struct { + MountID string +} + +// RpcMount performs a mount of the specified Volume and returns a MountID to be used +// in all subsequent RPCs to reference this Volume by this Client +// +func (dummy *retryRPCServerStruct) RpcMount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { + return nil // TODO +} + +// RenewMountRequestStruct is the request object for RpcRenewMount +// +type RenewMountRequestStruct struct { + MountID string + AuthToken string +} + +// RenewMountReplyStruct is the reply object for RpcRenewMount +// +type RenewMountReplyStruct struct{} + +// RpcRenewMount updates the AuthToken for the specified MountID +// +func (dummy *retryRPCServerStruct) RpcRenewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { + return nil // TODO +} + +// UnmountRequestStruct is the request object for RpcUnmount +// +type UnmountRequestStruct struct { + MountID string +} + +// UnmountReplyStruct is the reply object for RpcUnmount +// +type UnmountReplyStruct struct{} + +// RpcUnmount requests that the given MountID be released (and implicitly releases +// any Leases held by the MountID) +// +func (dummy *retryRPCServerStruct) RpcUnmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { + return nil // TODO +} + +// FetchNonceRangeRequestStruct is the request object for RpcFetchNonceRange +// +type FetchNonceRangeRequestStruct struct { + MountID string +} + +// FetchNonceRangeReplyStruct is the reply object for RpcFetchNonceRange +// +type FetchNonceRangeReplyStruct struct { + NextNonce uint64 + NumNoncesFetched uint64 +} + +// RpcFetchNonceRange requests a range of uint64 nonce values (i.e. values that will +// never be reused) +// +func (dummy *retryRPCServerStruct) RpcFetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { + return nil // TODO +} + +// GetInodeTableEntryRequestStruct is the request object for RpcGetInodeTableEntry +// +type GetInodeTableEntryRequestStruct struct { + MountID string + InodeNumber uint64 +} + +// GetInodeTableEntryReplyStruct is the reply object for RpcGetInodeTableEntry +// +type GetInodeTableEntryReplyStruct struct { + // TODO +} + +// RpcGetInodeTableEntry requests the Inode information for the specified Inode +// (which must have an active Shared or Exclusive Lease granted to the MountID) +// +func (dummy *retryRPCServerStruct) RpcGetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { + return nil // TODO +} + +// PutInodeTableEntriesRequestStruct is the request object for RpcPutInodeTableEntries +// +type PutInodeTableEntriesRequestStruct struct { + MountID string + // TODO +} + +// PutInodeTableEntriesReplyStruct is the reply object for RpcPutInodeTableEntries +// +type PutInodeTableEntriesReplyStruct struct{} + +// RpcPutInodeTableEntries requests an atomic update of the listed Inodes (which must +// each have an active Exclusive Lease granted to the MountID) +// +func (dummy *retryRPCServerStruct) RpcPutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { + return nil // TODO +} + +// LeaseRequestType specifies the requested lease operation +// +type LeaseRequestType uint32 + +const ( + LeaseRequestTypeShared LeaseRequestType = iota // Currently unleased, requesting SharedLease + LeaseRequestTypePromote // Currently SharedLease held, requesting promoting to ExclusiveLease + LeaseRequestTypeExclusive // Currently unleased, requesting ExclusiveLease + LeaseRequestTypeDemote // Currently ExclusiveLease held, requesting demotion to SharedLease + LeaseRequestTypeRelease // Currently SharedLease or ExclusiveLease held, releasing it +) + +// LeaseRequest is the request object for RpcLease +// +type LeaseRequest struct { + MountID string + InodeNumber uint64 + LeaseRequestType // One of LeaseRequestType* +} + +// LeaseReplyType specifies the acknowledgement that the requested lease operation +// has been completed or denied (e.g. when a Promotion request cannot be satisfied +// and the client will soon be receiving a LeaseInterruptTypeRelease) +// +type LeaseReplyType uint32 + +const ( + LeaseReplyTypeDenied LeaseReplyType = iota // Request denied (e.g. Promotion deadlock avoidance) + LeaseReplyTypeShared // SharedLease granted + LeaseReplyTypePromoted // SharedLease promoted to ExclusiveLease + LeaseReplyTypeExclusive // ExclusiveLease granted + LeaseReplyTypeDemoted // ExclusiveLease demoted to SharedLease + LeaseReplyTypeReleased // SharedLease or ExclusiveLease released +) + +// LeaseReply is the reply object for RpcLease +// +type LeaseReply struct { + LeaseReplyType // One of LeaseReplyType* +} + +// RpcLease is a blocking Lease Request +func (dummy *retryRPCServerStruct) RpcLease(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { + return nil // TODO +} + +// RPCInterruptType specifies the action (unmount, demotion, or release) requested by ProxyFS +// of the client in an RpcInterrupt "upcall" to indicate that a lease or leases must be demoted +// or released +// +type RPCInterruptType uint32 + +const ( + // RPCInterruptTypeUnmount indicates all Leases should be released (after performing necessary + // state saving RPCs) and the client should unmount + // + RPCInterruptTypeUnmount RPCInterruptType = iota + + // RPCInterruptTypeDemote indicates the specified LeaseHandle should (at least) be demoted + // from Exclusive to Shared (after performing necessary state saving RPCs) + // + RPCInterruptTypeDemote + + // RPCInterruptTypeRelease indicates the specified LeaseHandle should be released (after + // performing state saving RPCs and invalidating such cached state) + // + RPCInterruptTypeRelease +) + +// RPCInterrupt is the "upcall" mechanism used by ProxyFS to interrupt the client +// +type RPCInterrupt struct { + RPCInterruptType // One of RPCInterruptType* + InodeNumber uint64 // if RPCInterruptType == RPCInterruptTypeUnmount, InodeNumber == 0 (ignored) +} From c2eff1d9b27d83d5f25e2f5faa3177441b6b612e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 2 Apr 2021 15:10:15 -0700 Subject: [PATCH 105/160] Cleaned up the imgrpkg RPC (make it more consistent and add Flush() RPC) --- imgr/imgrpkg/retry-rpc.go | 79 ++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 4e83e844b..6452b6dfd 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -19,115 +19,132 @@ func stopRetryRPCServer() (err error) { return nil // TODO } -// MountRequestStruct is the request object for RpcMount +// MountRequestStruct is the request object for Mount // type MountRequestStruct struct { VolumeName string AuthToken string } -// MountReplyStruct is the reply object for RpcMount +// MountReplyStruct is the reply object for Mount // type MountReplyStruct struct { MountID string } -// RpcMount performs a mount of the specified Volume and returns a MountID to be used +// Mount performs a mount of the specified Volume and returns a MountID to be used // in all subsequent RPCs to reference this Volume by this Client // -func (dummy *retryRPCServerStruct) RpcMount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { +func (dummy *retryRPCServerStruct) Mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { return nil // TODO } -// RenewMountRequestStruct is the request object for RpcRenewMount +// RenewMountRequestStruct is the request object for RenewMount // type RenewMountRequestStruct struct { MountID string AuthToken string } -// RenewMountReplyStruct is the reply object for RpcRenewMount +// RenewMountReplyStruct is the reply object for RenewMount // type RenewMountReplyStruct struct{} -// RpcRenewMount updates the AuthToken for the specified MountID +// RenewMount updates the AuthToken for the specified MountID // -func (dummy *retryRPCServerStruct) RpcRenewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { +func (dummy *retryRPCServerStruct) RenewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { return nil // TODO } -// UnmountRequestStruct is the request object for RpcUnmount +// UnmountRequestStruct is the request object for Unmount // type UnmountRequestStruct struct { MountID string } -// UnmountReplyStruct is the reply object for RpcUnmount +// UnmountReplyStruct is the reply object for Unmount // type UnmountReplyStruct struct{} -// RpcUnmount requests that the given MountID be released (and implicitly releases +// Unmount requests that the given MountID be released (and implicitly releases // any Leases held by the MountID) // -func (dummy *retryRPCServerStruct) RpcUnmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { +func (dummy *retryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { return nil // TODO } -// FetchNonceRangeRequestStruct is the request object for RpcFetchNonceRange +// FetchNonceRangeRequestStruct is the request object for FetchNonceRange // type FetchNonceRangeRequestStruct struct { MountID string } -// FetchNonceRangeReplyStruct is the reply object for RpcFetchNonceRange +// FetchNonceRangeReplyStruct is the reply object for FetchNonceRange // type FetchNonceRangeReplyStruct struct { NextNonce uint64 NumNoncesFetched uint64 } -// RpcFetchNonceRange requests a range of uint64 nonce values (i.e. values that will +// FetchNonceRange requests a range of uint64 nonce values (i.e. values that will // never be reused) // -func (dummy *retryRPCServerStruct) RpcFetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { +func (dummy *retryRPCServerStruct) FetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { return nil // TODO } -// GetInodeTableEntryRequestStruct is the request object for RpcGetInodeTableEntry +// GetInodeTableEntryRequestStruct is the request object for GetInodeTableEntry // type GetInodeTableEntryRequestStruct struct { MountID string InodeNumber uint64 } -// GetInodeTableEntryReplyStruct is the reply object for RpcGetInodeTableEntry +// GetInodeTableEntryReplyStruct is the reply object for GetInodeTableEntry // type GetInodeTableEntryReplyStruct struct { // TODO } -// RpcGetInodeTableEntry requests the Inode information for the specified Inode +// GetInodeTableEntry requests the Inode information for the specified Inode // (which must have an active Shared or Exclusive Lease granted to the MountID) // -func (dummy *retryRPCServerStruct) RpcGetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { +func (dummy *retryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { return nil // TODO } -// PutInodeTableEntriesRequestStruct is the request object for RpcPutInodeTableEntries +// PutInodeTableEntriesRequestStruct is the request object for PutInodeTableEntries // type PutInodeTableEntriesRequestStruct struct { MountID string // TODO } -// PutInodeTableEntriesReplyStruct is the reply object for RpcPutInodeTableEntries +// PutInodeTableEntriesReplyStruct is the reply object for PutInodeTableEntries // type PutInodeTableEntriesReplyStruct struct{} -// RpcPutInodeTableEntries requests an atomic update of the listed Inodes (which must +// PutInodeTableEntries requests an atomic update of the listed Inodes (which must // each have an active Exclusive Lease granted to the MountID) // -func (dummy *retryRPCServerStruct) RpcPutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { +func (dummy *retryRPCServerStruct) PutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { + return nil // TODO +} + +// FlushRequestStruct is the request object for Flush +// +type FlushRequestStruct struct { + MountID string + // TODO +} + +// FlushReplyStruct is the reply object for Flush +// +type FlushReplyStruct struct{} + +// Flush that the results of prior PutInodeTableEntries requests be persisted +// +func (dummy *retryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err error) { return nil // TODO } @@ -143,9 +160,9 @@ const ( LeaseRequestTypeRelease // Currently SharedLease or ExclusiveLease held, releasing it ) -// LeaseRequest is the request object for RpcLease +// LeaseRequestStruct is the request object for Lease // -type LeaseRequest struct { +type LeaseRequestStruct struct { MountID string InodeNumber uint64 LeaseRequestType // One of LeaseRequestType* @@ -166,19 +183,19 @@ const ( LeaseReplyTypeReleased // SharedLease or ExclusiveLease released ) -// LeaseReply is the reply object for RpcLease +// LeaseReplyStruct is the reply object for Lease // -type LeaseReply struct { +type LeaseReplyStruct struct { LeaseReplyType // One of LeaseReplyType* } -// RpcLease is a blocking Lease Request -func (dummy *retryRPCServerStruct) RpcLease(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { +// Lease is a blocking Lease Request +func (dummy *retryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err error) { return nil // TODO } // RPCInterruptType specifies the action (unmount, demotion, or release) requested by ProxyFS -// of the client in an RpcInterrupt "upcall" to indicate that a lease or leases must be demoted +// of the client in an RPCInterrupt "upcall" to indicate that a lease or leases must be demoted // or released // type RPCInterruptType uint32 From e86ac3608a0d470eb07a8b0ef9a671c7d5fb441f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 2 Apr 2021 15:23:16 -0700 Subject: [PATCH 106/160] Make imgrpkg `godoc` be complete --- imgr/imgrpkg/retry-rpc.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 6452b6dfd..2428c620e 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -3,12 +3,12 @@ package imgrpkg -type retryRPCServerStruct struct{} +type RetryRPCServerStruct struct{} -var retryRPCServer *retryRPCServerStruct +var retryRPCServer *RetryRPCServerStruct func startRetryRPCServer() (err error) { - retryRPCServer = &retryRPCServerStruct{} + retryRPCServer = &RetryRPCServerStruct{} return nil // TODO } @@ -35,7 +35,7 @@ type MountReplyStruct struct { // Mount performs a mount of the specified Volume and returns a MountID to be used // in all subsequent RPCs to reference this Volume by this Client // -func (dummy *retryRPCServerStruct) Mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { +func (dummy *RetryRPCServerStruct) Mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { return nil // TODO } @@ -52,7 +52,7 @@ type RenewMountReplyStruct struct{} // RenewMount updates the AuthToken for the specified MountID // -func (dummy *retryRPCServerStruct) RenewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { +func (dummy *RetryRPCServerStruct) RenewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { return nil // TODO } @@ -69,7 +69,7 @@ type UnmountReplyStruct struct{} // Unmount requests that the given MountID be released (and implicitly releases // any Leases held by the MountID) // -func (dummy *retryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { +func (dummy *RetryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { return nil // TODO } @@ -89,7 +89,7 @@ type FetchNonceRangeReplyStruct struct { // FetchNonceRange requests a range of uint64 nonce values (i.e. values that will // never be reused) // -func (dummy *retryRPCServerStruct) FetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { +func (dummy *RetryRPCServerStruct) FetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { return nil // TODO } @@ -109,7 +109,7 @@ type GetInodeTableEntryReplyStruct struct { // GetInodeTableEntry requests the Inode information for the specified Inode // (which must have an active Shared or Exclusive Lease granted to the MountID) // -func (dummy *retryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { +func (dummy *RetryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { return nil // TODO } @@ -127,7 +127,7 @@ type PutInodeTableEntriesReplyStruct struct{} // PutInodeTableEntries requests an atomic update of the listed Inodes (which must // each have an active Exclusive Lease granted to the MountID) // -func (dummy *retryRPCServerStruct) PutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { +func (dummy *RetryRPCServerStruct) PutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { return nil // TODO } @@ -144,7 +144,7 @@ type FlushReplyStruct struct{} // Flush that the results of prior PutInodeTableEntries requests be persisted // -func (dummy *retryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err error) { +func (dummy *RetryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err error) { return nil // TODO } @@ -190,7 +190,7 @@ type LeaseReplyStruct struct { } // Lease is a blocking Lease Request -func (dummy *retryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err error) { +func (dummy *RetryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err error) { return nil // TODO } From 5f59830e5dac9303e09382e9cd29e3e67993c102 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 5 Apr 2021 14:52:50 -0700 Subject: [PATCH 107/160] Made imgr/imgrpkg/api.go cleanly contain requests... Also added missing delete & openCount RPCs to imgr/imgrpkg RPCs --- imgr/imgrpkg/api.go | 266 ++++++++++++++++++++++++++++++++++++++ imgr/imgrpkg/globals.go | 44 ++++--- imgr/imgrpkg/retry-rpc.go | 201 ++-------------------------- imgr/imgrpkg/volume.go | 22 ++-- 4 files changed, 312 insertions(+), 221 deletions(-) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index ba66f6a0e..ea2a66c5a 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -39,3 +39,269 @@ func LogWarnf(format string, args ...interface{}) { func LogInfof(format string, args ...interface{}) { logInfof(format, args...) } + +type RetryRPCServerStruct struct{} + +var retryRPCServer *RetryRPCServerStruct + +func startRetryRPCServer() (err error) { + retryRPCServer = &RetryRPCServerStruct{} + + return nil // TODO +} + +func stopRetryRPCServer() (err error) { + retryRPCServer = nil + + return nil // TODO +} + +// MountRequestStruct is the request object for Mount +// +type MountRequestStruct struct { + VolumeName string + AuthToken string +} + +// MountReplyStruct is the reply object for Mount +// +type MountReplyStruct struct { + MountID string +} + +// Mount performs a mount of the specified Volume and returns a MountID to be used +// in all subsequent RPCs to reference this Volume by this Client. +// +func (dummy *RetryRPCServerStruct) Mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { + return mount(retryRPCClientID, mountRequest, mountReply) +} + +// RenewMountRequestStruct is the request object for RenewMount +// +type RenewMountRequestStruct struct { + MountID string + AuthToken string +} + +// RenewMountReplyStruct is the reply object for RenewMount +// +type RenewMountReplyStruct struct{} + +// RenewMount updates the AuthToken for the specified MountID +// +func (dummy *RetryRPCServerStruct) RenewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { + return renewMount(renewMountRequest, renewMountReply) +} + +// UnmountRequestStruct is the request object for Unmount +// +type UnmountRequestStruct struct { + MountID string +} + +// UnmountReplyStruct is the reply object for Unmount +// +type UnmountReplyStruct struct{} + +// Unmount requests that the given MountID be released (and implicitly releases +// any Leases held by the MountID). +// +func (dummy *RetryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { + return unmount(unmountRequest, unmountReply) +} + +// FetchNonceRangeRequestStruct is the request object for FetchNonceRange +// +type FetchNonceRangeRequestStruct struct { + MountID string +} + +// FetchNonceRangeReplyStruct is the reply object for FetchNonceRange +// +type FetchNonceRangeReplyStruct struct { + NextNonce uint64 + NumNoncesFetched uint64 +} + +// FetchNonceRange requests a range of uint64 nonce values (i.e. values that will +// never be reused). +// +func (dummy *RetryRPCServerStruct) FetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { + return fetchNonceRange(fetchNonceRangeRequest, fetchNonceRangeReply) +} + +// GetInodeTableEntryRequestStruct is the request object for GetInodeTableEntry +// +type GetInodeTableEntryRequestStruct struct { + MountID string + InodeNumber uint64 +} + +// GetInodeTableEntryReplyStruct is the reply object for GetInodeTableEntry +// +type GetInodeTableEntryReplyStruct struct { + // TODO +} + +// GetInodeTableEntry requests the Inode information for the specified Inode +// (which must have an active Shared or Exclusive Lease granted to the MountID). +// +func (dummy *RetryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { + return getInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryReply) +} + +// PutInodeTableEntriesRequestStruct is the request object for PutInodeTableEntries +// +type PutInodeTableEntriesRequestStruct struct { + MountID string + // TODO +} + +// PutInodeTableEntriesReplyStruct is the reply object for PutInodeTableEntries +// +type PutInodeTableEntriesReplyStruct struct{} + +// PutInodeTableEntries requests an atomic update of the listed Inodes (which must +// each have an active Exclusive Lease granted to the MountID). +// +func (dummy *RetryRPCServerStruct) PutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { + return putInodeTableEntries(putInodeTableEntriesRequest, putInodeTableEntriesReply) +} + +// DeleteInodeTableEntryRequestStruct is the request object for DeleteInodeTableEntry +// +type DeleteInodeTableEntryRequestStruct struct { + MountID string + InodeNumber uint64 +} + +// DeleteInodeTableEntryReplyStruct is the reply object for DeleteInodeTableEntry +// +type DeleteInodeTableEntryReplyStruct struct { + // TODO +} + +// DeleteInodeTableEntry requests the specified Inode information be deleted. +// An active Exclusive Lease must be granted to the MountID. Note that +// unless/until the OpenCount for the Inode drops to zero, the Inode will +// still exist. +// +func (dummy *RetryRPCServerStruct) DeleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryReply *DeleteInodeTableEntryReplyStruct) (err error) { + return deleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryReply) +} + +// AdjustInodeTableEntryOpenCountRequestStruct is the request object for AdjustInodeTableEntryOpenCount +// +type AdjustInodeTableEntryOpenCountRequestStruct struct { + MountID string + InodeNumber uint64 +} + +// AdjustInodeTableEntryOpenCountReplyStruct is the reply object for AdjustInodeTableEntryOpenCount +// +type AdjustInodeTableEntryOpenCountReplyStruct struct { + // TODO +} + +// AdjustInodeTableEntryOpenCount requests the specified Inode's OpenCount be +// adjusted. A (Shared or Exclusive) Lease must be granted to the MountID. If +// the adjustment results in an OpenCount of zero and the Inode has been marked +// for deletion by a prior call to DeleteInodeTableEntry, the Inode will be +// deleted. +// +func (dummy *RetryRPCServerStruct) AdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountReply *AdjustInodeTableEntryOpenCountReplyStruct) (err error) { + return adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountReply) +} + +// FlushRequestStruct is the request object for Flush +// +type FlushRequestStruct struct { + MountID string + // TODO +} + +// FlushReplyStruct is the reply object for Flush +// +type FlushReplyStruct struct{} + +// Flush that the results of prior PutInodeTableEntries requests be persisted +// +func (dummy *RetryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err error) { + return flush(flushRequest, flushReply) +} + +// LeaseRequestType specifies the requested lease operation +// +type LeaseRequestType uint32 + +const ( + LeaseRequestTypeShared LeaseRequestType = iota // Currently unleased, requesting SharedLease + LeaseRequestTypePromote // Currently SharedLease held, requesting promoting to ExclusiveLease + LeaseRequestTypeExclusive // Currently unleased, requesting ExclusiveLease + LeaseRequestTypeDemote // Currently ExclusiveLease held, requesting demotion to SharedLease + LeaseRequestTypeRelease // Currently SharedLease or ExclusiveLease held, releasing it +) + +// LeaseRequestStruct is the request object for Lease +// +type LeaseRequestStruct struct { + MountID string + InodeNumber uint64 + LeaseRequestType // One of LeaseRequestType* +} + +// LeaseReplyType specifies the acknowledgement that the requested lease operation +// has been completed or denied (e.g. when a Promotion request cannot be satisfied +// and the client will soon be receiving a LeaseInterruptTypeRelease). +// +type LeaseReplyType uint32 + +const ( + LeaseReplyTypeDenied LeaseReplyType = iota // Request denied (e.g. Promotion deadlock avoidance) + LeaseReplyTypeShared // SharedLease granted + LeaseReplyTypePromoted // SharedLease promoted to ExclusiveLease + LeaseReplyTypeExclusive // ExclusiveLease granted + LeaseReplyTypeDemoted // ExclusiveLease demoted to SharedLease + LeaseReplyTypeReleased // SharedLease or ExclusiveLease released +) + +// LeaseReplyStruct is the reply object for Lease +// +type LeaseReplyStruct struct { + LeaseReplyType // One of LeaseReplyType* +} + +// Lease is a blocking Lease Request +func (dummy *RetryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err error) { + return lease(leaseRequest, leaseReply) +} + +// RPCInterruptType specifies the action (unmount, demotion, or release) requested by ProxyFS +// of the client in an RPCInterrupt "upcall" to indicate that a lease or leases must be demoted +// or released. +// +type RPCInterruptType uint32 + +const ( + // RPCInterruptTypeUnmount indicates all Leases should be released (after performing necessary + // state saving RPCs) and the client should unmount + // + RPCInterruptTypeUnmount RPCInterruptType = iota + + // RPCInterruptTypeDemote indicates the specified LeaseHandle should (at least) be demoted + // from Exclusive to Shared (after performing necessary state saving RPCs) + // + RPCInterruptTypeDemote + + // RPCInterruptTypeRelease indicates the specified LeaseHandle should be released (after + // performing state saving RPCs and invalidating such cached state) + // + RPCInterruptTypeRelease +) + +// RPCInterrupt is the "upcall" mechanism used by ProxyFS to interrupt the client +// +type RPCInterrupt struct { + RPCInterruptType // One of RPCInterruptType* + InodeNumber uint64 // if RPCInterruptType == RPCInterruptTypeUnmount, InodeNumber == 0 (ignored) +} diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index ccee37059..bc3efd1d3 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -99,33 +99,37 @@ type statsStruct struct { } type mountStruct struct { + // reentrancy covered by volumeStruct's sync.RWMutex volume *volumeStruct // volume.{R|}Lock() also protects each mountStruct - mountID string - authTokenValid bool - authToken string - lastAuthTime time.Time + mountID string // + authTokenValid bool // if true, mount is "healthy"; if false, mount is "expired" + authToken string // + lastAuthTime time.Time // used to periodically check TTL of authToken + listElement *list.Element // LRU element on either volumeStruct.{healthy|expired}MountList } type volumeStruct struct { - sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() - name string - storageURL string - mountMap map[string]*mountStruct // key == mountStruct.mountID - numMountAuthTokenValid uint64 - deleting bool - inodeTable sortedmap.BPlusTree + sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() + name string // + storageURL string // + mountMap map[string]*mountStruct // key == mountStruct.mountID + healthyMountList *list.List // LRU of mountStruct's with .authTokenValid == true + expiredMountList *list.List // LRU of mountStruct's with .authTokenValid == false + // TODO uint64 // + deleting bool // + inodeTable sortedmap.BPlusTree // } type globalsStruct struct { - sync.RWMutex - config configStruct - logFile *os.File // == nil if config.LogFilePath == "" - inodeTableCache sortedmap.BPlusTreeCache - volumeMap sortedmap.LLRBTree // key == volumeStruct.name; value == *volumeStruct - mountMap map[string]*mountStruct // key == mountStruct.mountID - httpServer *http.Server - httpServerWG sync.WaitGroup - stats *statsStruct + sync.RWMutex // + config configStruct // + logFile *os.File // == nil if config.LogFilePath == "" + inodeTableCache sortedmap.BPlusTreeCache // + volumeMap sortedmap.LLRBTree // key == volumeStruct.name; value == *volumeStruct + mountMap map[string]*mountStruct // key == mountStruct.mountID + httpServer *http.Server // + httpServerWG sync.WaitGroup // + stats *statsStruct // } var globals globalsStruct diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 2428c620e..b1f767679 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -3,223 +3,42 @@ package imgrpkg -type RetryRPCServerStruct struct{} - -var retryRPCServer *RetryRPCServerStruct - -func startRetryRPCServer() (err error) { - retryRPCServer = &RetryRPCServerStruct{} - +func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { return nil // TODO } -func stopRetryRPCServer() (err error) { - retryRPCServer = nil - +func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { return nil // TODO } -// MountRequestStruct is the request object for Mount -// -type MountRequestStruct struct { - VolumeName string - AuthToken string -} - -// MountReplyStruct is the reply object for Mount -// -type MountReplyStruct struct { - MountID string -} - -// Mount performs a mount of the specified Volume and returns a MountID to be used -// in all subsequent RPCs to reference this Volume by this Client -// -func (dummy *RetryRPCServerStruct) Mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { +func unmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { return nil // TODO } -// RenewMountRequestStruct is the request object for RenewMount -// -type RenewMountRequestStruct struct { - MountID string - AuthToken string -} - -// RenewMountReplyStruct is the reply object for RenewMount -// -type RenewMountReplyStruct struct{} - -// RenewMount updates the AuthToken for the specified MountID -// -func (dummy *RetryRPCServerStruct) RenewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { +func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { return nil // TODO } -// UnmountRequestStruct is the request object for Unmount -// -type UnmountRequestStruct struct { - MountID string -} - -// UnmountReplyStruct is the reply object for Unmount -// -type UnmountReplyStruct struct{} - -// Unmount requests that the given MountID be released (and implicitly releases -// any Leases held by the MountID) -// -func (dummy *RetryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { +func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { return nil // TODO } -// FetchNonceRangeRequestStruct is the request object for FetchNonceRange -// -type FetchNonceRangeRequestStruct struct { - MountID string -} - -// FetchNonceRangeReplyStruct is the reply object for FetchNonceRange -// -type FetchNonceRangeReplyStruct struct { - NextNonce uint64 - NumNoncesFetched uint64 -} - -// FetchNonceRange requests a range of uint64 nonce values (i.e. values that will -// never be reused) -// -func (dummy *RetryRPCServerStruct) FetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { +func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { return nil // TODO } -// GetInodeTableEntryRequestStruct is the request object for GetInodeTableEntry -// -type GetInodeTableEntryRequestStruct struct { - MountID string - InodeNumber uint64 -} - -// GetInodeTableEntryReplyStruct is the reply object for GetInodeTableEntry -// -type GetInodeTableEntryReplyStruct struct { - // TODO -} - -// GetInodeTableEntry requests the Inode information for the specified Inode -// (which must have an active Shared or Exclusive Lease granted to the MountID) -// -func (dummy *RetryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { +func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryReply *DeleteInodeTableEntryReplyStruct) (err error) { return nil // TODO } -// PutInodeTableEntriesRequestStruct is the request object for PutInodeTableEntries -// -type PutInodeTableEntriesRequestStruct struct { - MountID string - // TODO -} - -// PutInodeTableEntriesReplyStruct is the reply object for PutInodeTableEntries -// -type PutInodeTableEntriesReplyStruct struct{} - -// PutInodeTableEntries requests an atomic update of the listed Inodes (which must -// each have an active Exclusive Lease granted to the MountID) -// -func (dummy *RetryRPCServerStruct) PutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { +func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountReply *AdjustInodeTableEntryOpenCountReplyStruct) (err error) { return nil // TODO } -// FlushRequestStruct is the request object for Flush -// -type FlushRequestStruct struct { - MountID string - // TODO -} - -// FlushReplyStruct is the reply object for Flush -// -type FlushReplyStruct struct{} - -// Flush that the results of prior PutInodeTableEntries requests be persisted -// -func (dummy *RetryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err error) { +func flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err error) { return nil // TODO } -// LeaseRequestType specifies the requested lease operation -// -type LeaseRequestType uint32 - -const ( - LeaseRequestTypeShared LeaseRequestType = iota // Currently unleased, requesting SharedLease - LeaseRequestTypePromote // Currently SharedLease held, requesting promoting to ExclusiveLease - LeaseRequestTypeExclusive // Currently unleased, requesting ExclusiveLease - LeaseRequestTypeDemote // Currently ExclusiveLease held, requesting demotion to SharedLease - LeaseRequestTypeRelease // Currently SharedLease or ExclusiveLease held, releasing it -) - -// LeaseRequestStruct is the request object for Lease -// -type LeaseRequestStruct struct { - MountID string - InodeNumber uint64 - LeaseRequestType // One of LeaseRequestType* -} - -// LeaseReplyType specifies the acknowledgement that the requested lease operation -// has been completed or denied (e.g. when a Promotion request cannot be satisfied -// and the client will soon be receiving a LeaseInterruptTypeRelease) -// -type LeaseReplyType uint32 - -const ( - LeaseReplyTypeDenied LeaseReplyType = iota // Request denied (e.g. Promotion deadlock avoidance) - LeaseReplyTypeShared // SharedLease granted - LeaseReplyTypePromoted // SharedLease promoted to ExclusiveLease - LeaseReplyTypeExclusive // ExclusiveLease granted - LeaseReplyTypeDemoted // ExclusiveLease demoted to SharedLease - LeaseReplyTypeReleased // SharedLease or ExclusiveLease released -) - -// LeaseReplyStruct is the reply object for Lease -// -type LeaseReplyStruct struct { - LeaseReplyType // One of LeaseReplyType* -} - -// Lease is a blocking Lease Request -func (dummy *RetryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err error) { +func lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err error) { return nil // TODO } - -// RPCInterruptType specifies the action (unmount, demotion, or release) requested by ProxyFS -// of the client in an RPCInterrupt "upcall" to indicate that a lease or leases must be demoted -// or released -// -type RPCInterruptType uint32 - -const ( - // RPCInterruptTypeUnmount indicates all Leases should be released (after performing necessary - // state saving RPCs) and the client should unmount - // - RPCInterruptTypeUnmount RPCInterruptType = iota - - // RPCInterruptTypeDemote indicates the specified LeaseHandle should (at least) be demoted - // from Exclusive to Shared (after performing necessary state saving RPCs) - // - RPCInterruptTypeDemote - - // RPCInterruptTypeRelease indicates the specified LeaseHandle should be released (after - // performing state saving RPCs and invalidating such cached state) - // - RPCInterruptTypeRelease -) - -// RPCInterrupt is the "upcall" mechanism used by ProxyFS to interrupt the client -// -type RPCInterrupt struct { - RPCInterruptType // One of RPCInterruptType* - InodeNumber uint64 // if RPCInterruptType == RPCInterruptTypeUnmount, InodeNumber == 0 (ignored) -} diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index faddb2d77..6ebc412b9 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -4,6 +4,7 @@ package imgrpkg import ( + "container/list" "encoding/json" "fmt" @@ -140,8 +141,8 @@ func getVolumeAsJSON(volumeName string) (volume []byte, err error) { volumeToReturn = &volumeGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, - HealthyMounts: volumeAsStruct.numMountAuthTokenValid, - ExpiredMounts: uint64(len(volumeAsStruct.mountMap)) - volumeAsStruct.numMountAuthTokenValid, + HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), + ExpiredMounts: uint64(volumeAsStruct.expiredMountList.Len()), } globals.RUnlock() @@ -192,8 +193,8 @@ func getVolumeListAsJSON() (volumeList []byte) { volumeListToReturn[volumeListIndex] = &volumeGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, - HealthyMounts: volumeAsStruct.numMountAuthTokenValid, - ExpiredMounts: uint64(len(volumeAsStruct.mountMap)) - volumeAsStruct.numMountAuthTokenValid, + HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), + ExpiredMounts: uint64(volumeAsStruct.expiredMountList.Len()), } } @@ -216,12 +217,13 @@ func putVolume(name string, storageURL string) (err error) { globals.Lock() volume = &volumeStruct{ - name: name, - storageURL: storageURL, - mountMap: make(map[string]*mountStruct), - numMountAuthTokenValid: 0, - deleting: false, - inodeTable: nil, + name: name, + storageURL: storageURL, + mountMap: make(map[string]*mountStruct), + healthyMountList: list.New(), + expiredMountList: list.New(), + deleting: false, + inodeTable: nil, } ok, err = globals.volumeMap.Put(volume.name, volume) From 492aaa3525b7e0646b08a9602c3321cc54b0cb70 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 5 Apr 2021 16:07:47 -0700 Subject: [PATCH 108/160] Added imgr/imgrpkg package document --- imgr/imgrpkg/api.go | 136 +++++++++++++++++++++++++++++++++----------- 1 file changed, 103 insertions(+), 33 deletions(-) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index ea2a66c5a..987f637aa 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -1,40 +1,109 @@ // Copyright (c) 2015-2021, NVIDIA CORPORATION. // SPDX-License-Identifier: Apache-2.0 +// Package imgrpkg implements the server side Inode Management for ProxyFS volumes. +// The package supports to communications protocols. The balance of the package +// documentation will describe the Go-callable API. Note that func's listed under +// type RetryRPCServerStruct are the RPCs issued by the client side via package +// retryrpc connections (and, thus, not intended to be called directly). +// +// To configure an imgrpkg instance, Start() is called passing, as the sole +// argument, a package conf ConfMap. Here is a sample .conf file: +// +// [IMGR] +// PublicIPAddr: 172.28.128.2 +// PrivateIPAddr: 172.28.128.2 +// RetryRPCPort: 32356 +// HTTPServerPort: 15346 +// +// RetryRPCTTLCompleted: 10m +// RetryRPCAckTrim: 100ms +// RetryRPCDeadlineIO: 60s +// RetryRPCKeepAlivePeriod: 60s +// +// RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, +// RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used +// +// FetchNonceRangeTpReturn: 100 +// +// MinLeaseDuration: 250ms +// LeaseInterruptInterval: 250ms +// LeaseInterruptLimit: 20 +// +// SwiftRetryDelay: 1s +// SwiftRetryExpBackoff: 1.5 +// SwiftRetryLimit: 11 +// +// SwiftConnectionPoolSize: 128 +// +// InodeTableCacheEvictLowLimit: 10000 +// InodeTableCacheEvictHighLimit: 10010 +// +// LogFilePath: # imgr.log +// LogToConsole: true # false +// TraceEnabled: false +// +// Most of the config keys are required and must have values. One exception +// is LogFilePath that will default to "" and, hence, cause logging to not +// go to a file. This might typically be used when LogToConsole is set to true. +// +// The RetryRPC{Cert|Key}FilePath keys are optional and, if provided may be +// empty. In such cases, the retryrpc package will be configured to use TCP. +// If, however, they are present and provide a path or paths to valid Cert|Key +// files, the retryrpc package will be configured to use TLS. In any event, +// the RPCs will be available via :. +// +// In addition to the package retryrpc-exposed RPCs, the package also includes +// an embedded HTTP Server (at URL http://:) +// responses to the following. +// +// DELETE /volume/ +// - deletes the specified +// GET /config +// - returns a JSON document of the supplied ConfMap passed to Start() +// GET /stats +// - returns a package bucketstats dump +// GET /volume +// - returns a JSON document describing details of the specified +// GET /volume/ +// - returns a JSON document describing details of all volumes +// PUT /volume/ +// - content should be a StorageURL (including the Container) +// package imgrpkg import ( "github.com/NVIDIA/proxyfs/conf" ) -// Start is called to start serving +// Start is called to start serving. // func Start(confMap conf.ConfMap) (err error) { err = start(confMap) return } -// Stop is called to stop serving +// Stop is called to stop serving. // func Stop() (err error) { err = stop() return } -// Signal is called to interrupt the server for performing operations such as log rotation +// Signal is called to interrupt the server for performing operations such as log rotation. // func Signal() (err error) { err = signal() return } -// LogWarnf is a wrapper around the internal logWarnf() func called by imgr/main.go::main() +// LogWarnf is a wrapper around the internal logWarnf() func called by imgr/main.go::main(). // func LogWarnf(format string, args ...interface{}) { logWarnf(format, args...) } -// LogInfof is a wrapper around the internal logInfof() func called by imgr/main.go::main() +// LogInfof is a wrapper around the internal logInfof() func called by imgr/main.go::main(). // func LogInfof(format string, args ...interface{}) { logInfof(format, args...) @@ -56,14 +125,14 @@ func stopRetryRPCServer() (err error) { return nil // TODO } -// MountRequestStruct is the request object for Mount +// MountRequestStruct is the request object for Mount. // type MountRequestStruct struct { VolumeName string AuthToken string } -// MountReplyStruct is the reply object for Mount +// MountReplyStruct is the reply object for Mount. // type MountReplyStruct struct { MountID string @@ -76,30 +145,30 @@ func (dummy *RetryRPCServerStruct) Mount(retryRPCClientID uint64, mountRequest * return mount(retryRPCClientID, mountRequest, mountReply) } -// RenewMountRequestStruct is the request object for RenewMount +// RenewMountRequestStruct is the request object for RenewMount. // type RenewMountRequestStruct struct { MountID string AuthToken string } -// RenewMountReplyStruct is the reply object for RenewMount +// RenewMountReplyStruct is the reply object for RenewMount. // type RenewMountReplyStruct struct{} -// RenewMount updates the AuthToken for the specified MountID +// RenewMount updates the AuthToken for the specified MountID. // func (dummy *RetryRPCServerStruct) RenewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { return renewMount(renewMountRequest, renewMountReply) } -// UnmountRequestStruct is the request object for Unmount +// UnmountRequestStruct is the request object for Unmount. // type UnmountRequestStruct struct { MountID string } -// UnmountReplyStruct is the reply object for Unmount +// UnmountReplyStruct is the reply object for Unmount. // type UnmountReplyStruct struct{} @@ -110,13 +179,13 @@ func (dummy *RetryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, return unmount(unmountRequest, unmountReply) } -// FetchNonceRangeRequestStruct is the request object for FetchNonceRange +// FetchNonceRangeRequestStruct is the request object for FetchNonceRange. // type FetchNonceRangeRequestStruct struct { MountID string } -// FetchNonceRangeReplyStruct is the reply object for FetchNonceRange +// FetchNonceRangeReplyStruct is the reply object for FetchNonceRange. // type FetchNonceRangeReplyStruct struct { NextNonce uint64 @@ -130,14 +199,14 @@ func (dummy *RetryRPCServerStruct) FetchNonceRange(fetchNonceRangeRequest *Fetch return fetchNonceRange(fetchNonceRangeRequest, fetchNonceRangeReply) } -// GetInodeTableEntryRequestStruct is the request object for GetInodeTableEntry +// GetInodeTableEntryRequestStruct is the request object for GetInodeTableEntry. // type GetInodeTableEntryRequestStruct struct { MountID string InodeNumber uint64 } -// GetInodeTableEntryReplyStruct is the reply object for GetInodeTableEntry +// GetInodeTableEntryReplyStruct is the reply object for GetInodeTableEntry. // type GetInodeTableEntryReplyStruct struct { // TODO @@ -150,14 +219,14 @@ func (dummy *RetryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest return getInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryReply) } -// PutInodeTableEntriesRequestStruct is the request object for PutInodeTableEntries +// PutInodeTableEntriesRequestStruct is the request object for PutInodeTableEntries. // type PutInodeTableEntriesRequestStruct struct { MountID string // TODO } -// PutInodeTableEntriesReplyStruct is the reply object for PutInodeTableEntries +// PutInodeTableEntriesReplyStruct is the reply object for PutInodeTableEntries. // type PutInodeTableEntriesReplyStruct struct{} @@ -168,14 +237,14 @@ func (dummy *RetryRPCServerStruct) PutInodeTableEntries(putInodeTableEntriesRequ return putInodeTableEntries(putInodeTableEntriesRequest, putInodeTableEntriesReply) } -// DeleteInodeTableEntryRequestStruct is the request object for DeleteInodeTableEntry +// DeleteInodeTableEntryRequestStruct is the request object for DeleteInodeTableEntry. // type DeleteInodeTableEntryRequestStruct struct { MountID string InodeNumber uint64 } -// DeleteInodeTableEntryReplyStruct is the reply object for DeleteInodeTableEntry +// DeleteInodeTableEntryReplyStruct is the reply object for DeleteInodeTableEntry. // type DeleteInodeTableEntryReplyStruct struct { // TODO @@ -190,14 +259,14 @@ func (dummy *RetryRPCServerStruct) DeleteInodeTableEntry(deleteInodeTableEntryRe return deleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryReply) } -// AdjustInodeTableEntryOpenCountRequestStruct is the request object for AdjustInodeTableEntryOpenCount +// AdjustInodeTableEntryOpenCountRequestStruct is the request object for AdjustInodeTableEntryOpenCount. // type AdjustInodeTableEntryOpenCountRequestStruct struct { MountID string InodeNumber uint64 } -// AdjustInodeTableEntryOpenCountReplyStruct is the reply object for AdjustInodeTableEntryOpenCount +// AdjustInodeTableEntryOpenCountReplyStruct is the reply object for AdjustInodeTableEntryOpenCount. // type AdjustInodeTableEntryOpenCountReplyStruct struct { // TODO @@ -213,24 +282,24 @@ func (dummy *RetryRPCServerStruct) AdjustInodeTableEntryOpenCount(adjustInodeTab return adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountReply) } -// FlushRequestStruct is the request object for Flush +// FlushRequestStruct is the request object for Flush. // type FlushRequestStruct struct { MountID string // TODO } -// FlushReplyStruct is the reply object for Flush +// FlushReplyStruct is the reply object for Flush. // type FlushReplyStruct struct{} -// Flush that the results of prior PutInodeTableEntries requests be persisted +// Flush that the results of prior PutInodeTableEntries requests be persisted. // func (dummy *RetryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err error) { return flush(flushRequest, flushReply) } -// LeaseRequestType specifies the requested lease operation +// LeaseRequestType specifies the requested lease operation. // type LeaseRequestType uint32 @@ -242,7 +311,7 @@ const ( LeaseRequestTypeRelease // Currently SharedLease or ExclusiveLease held, releasing it ) -// LeaseRequestStruct is the request object for Lease +// LeaseRequestStruct is the request object for Lease. // type LeaseRequestStruct struct { MountID string @@ -265,13 +334,14 @@ const ( LeaseReplyTypeReleased // SharedLease or ExclusiveLease released ) -// LeaseReplyStruct is the reply object for Lease +// LeaseReplyStruct is the reply object for Lease. // type LeaseReplyStruct struct { LeaseReplyType // One of LeaseReplyType* } -// Lease is a blocking Lease Request +// Lease is a blocking Lease Request. +// func (dummy *RetryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err error) { return lease(leaseRequest, leaseReply) } @@ -284,22 +354,22 @@ type RPCInterruptType uint32 const ( // RPCInterruptTypeUnmount indicates all Leases should be released (after performing necessary - // state saving RPCs) and the client should unmount + // state saving RPCs) and the client should unmount. // RPCInterruptTypeUnmount RPCInterruptType = iota // RPCInterruptTypeDemote indicates the specified LeaseHandle should (at least) be demoted - // from Exclusive to Shared (after performing necessary state saving RPCs) + // from Exclusive to Shared (after performing necessary state saving RPCs). // RPCInterruptTypeDemote // RPCInterruptTypeRelease indicates the specified LeaseHandle should be released (after - // performing state saving RPCs and invalidating such cached state) + // performing state saving RPCs and invalidating such cached state). // RPCInterruptTypeRelease ) -// RPCInterrupt is the "upcall" mechanism used by ProxyFS to interrupt the client +// RPCInterrupt is the "upcall" mechanism used by ProxyFS to interrupt the client. // type RPCInterrupt struct { RPCInterruptType // One of RPCInterruptType* From 822fff7c4e8e1661e938525cda38a0b3df6484fe Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 5 Apr 2021 17:09:42 -0700 Subject: [PATCH 109/160] Fixed icert cli to accept -ttl as uint64 days --- .gitignore | 1 + icert/.gitignore | 1 - icert/main.go | 13 ++++++++----- 3 files changed, 9 insertions(+), 6 deletions(-) delete mode 100644 icert/.gitignore diff --git a/.gitignore b/.gitignore index 21ca5065f..bfa535ff3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .DS_Store .vscode *.swp +*.pem # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o diff --git a/icert/.gitignore b/icert/.gitignore deleted file mode 100644 index cfaad7611..000000000 --- a/icert/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.pem diff --git a/icert/main.go b/icert/main.go index bc27f2b7f..898876e01 100644 --- a/icert/main.go +++ b/icert/main.go @@ -43,7 +43,7 @@ func main() { streetAddressFlag stringSlice postalCodeFlag stringSlice - ttlFlag = flag.Duration("ttl", time.Duration(0), "generated Certificate's time to live") + ttlDaysFlag = flag.Uint64("ttl", uint64(0), "generated Certificate's time to live in days") dnsNamesFlag stringSlice ipAddressesFlag stringSlice @@ -58,6 +58,7 @@ func main() { generateKeyAlgorithm string ipAddresses []net.IP subject pkix.Name + ttl time.Duration ) flag.Var(&organizationFlag, "organization", "generated Certificate's Subject.Organization") @@ -85,7 +86,7 @@ func main() { fmt.Printf(" streetAddressFlag: %v\n", streetAddressFlag) fmt.Printf(" postalCodeFlag: %v\n", postalCodeFlag) fmt.Println() - fmt.Printf(" ttlFlag: %v\n", *ttlFlag) + fmt.Printf(" ttlDaysFlag: %v\n", *ttlDaysFlag) fmt.Println() fmt.Printf(" dnsNamesFlag: %v\n", dnsNamesFlag) fmt.Printf(" ipAddressesFlag: %v\n", ipAddressesFlag) @@ -111,11 +112,13 @@ func main() { os.Exit(1) } - if time.Duration(0) == *ttlFlag { + if uint64(0) == *ttlDaysFlag { fmt.Printf("A non-zero -ttl must be specified\n") os.Exit(1) } + ttl = time.Duration(time.Duration(*ttlDaysFlag*24) * time.Hour) + if ("" == *caCertPemFilePathFlag) || ("" == *caKeyPemFilePathFlag) { fmt.Printf("Both -caCert and -caKey must be specified\n") os.Exit(1) @@ -151,7 +154,7 @@ func main() { } if *caFlag { - _, _, err = icertpkg.GenCACert(generateKeyAlgorithm, subject, *ttlFlag, *caCertPemFilePathFlag, *caKeyPemFilePathFlag) + _, _, err = icertpkg.GenCACert(generateKeyAlgorithm, subject, ttl, *caCertPemFilePathFlag, *caKeyPemFilePathFlag) if nil != err { fmt.Printf("icertpkg.GenCACert() failed: %v\n", err) os.Exit(1) @@ -167,7 +170,7 @@ func main() { ipAddresses = append(ipAddresses, net.ParseIP(ipAddress)) } - _, _, err = icertpkg.GenEndpointCert(generateKeyAlgorithm, subject, dnsNamesFlag, ipAddresses, *ttlFlag, *caCertPemFilePathFlag, *caKeyPemFilePathFlag, *endpointCertPemFilePathFlag, *endpointKeyPemFilePathFlag) + _, _, err = icertpkg.GenEndpointCert(generateKeyAlgorithm, subject, dnsNamesFlag, ipAddresses, ttl, *caCertPemFilePathFlag, *caKeyPemFilePathFlag, *endpointCertPemFilePathFlag, *endpointKeyPemFilePathFlag) if nil != err { fmt.Printf("icertpkg.GenEndpointCert() failed: %v\n", err) os.Exit(1) From 0e820beef1918200f132c24f5d752f7d6be690f0 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 5 Apr 2021 17:42:30 -0700 Subject: [PATCH 110/160] Trying to launch imgr's retryrpc listener... hangs on Close() A few UNDOs in imgr should be removed... --- imgr/imgrpkg/api.go | 14 +--------- imgr/imgrpkg/globals.go | 7 ++--- imgr/imgrpkg/impl.go | 4 +++ imgr/imgrpkg/retry-rpc.go | 57 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 16 deletions(-) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 987f637aa..0dc5e1b97 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -58,7 +58,7 @@ // responses to the following. // // DELETE /volume/ -// - deletes the specified +// - removes the specified from being served // GET /config // - returns a JSON document of the supplied ConfMap passed to Start() // GET /stats @@ -113,18 +113,6 @@ type RetryRPCServerStruct struct{} var retryRPCServer *RetryRPCServerStruct -func startRetryRPCServer() (err error) { - retryRPCServer = &RetryRPCServerStruct{} - - return nil // TODO -} - -func stopRetryRPCServer() (err error) { - retryRPCServer = nil - - return nil // TODO -} - // MountRequestStruct is the request object for Mount. // type MountRequestStruct struct { diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index bc3efd1d3..363daab3e 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -15,6 +15,7 @@ import ( "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/retryrpc" "github.com/NVIDIA/proxyfs/utils" ) @@ -115,9 +116,8 @@ type volumeStruct struct { mountMap map[string]*mountStruct // key == mountStruct.mountID healthyMountList *list.List // LRU of mountStruct's with .authTokenValid == true expiredMountList *list.List // LRU of mountStruct's with .authTokenValid == false - // TODO uint64 // - deleting bool // - inodeTable sortedmap.BPlusTree // + deleting bool // + inodeTable sortedmap.BPlusTree // } type globalsStruct struct { @@ -127,6 +127,7 @@ type globalsStruct struct { inodeTableCache sortedmap.BPlusTreeCache // volumeMap sortedmap.LLRBTree // key == volumeStruct.name; value == *volumeStruct mountMap map[string]*mountStruct // key == mountStruct.mountID + retryrpcServer *retryrpc.Server // httpServer *http.Server // httpServerWG sync.WaitGroup // stats *statsStruct // diff --git a/imgr/imgrpkg/impl.go b/imgr/imgrpkg/impl.go index e44961009..2679a1b55 100644 --- a/imgr/imgrpkg/impl.go +++ b/imgr/imgrpkg/impl.go @@ -4,6 +4,8 @@ package imgrpkg import ( + "fmt" + "github.com/NVIDIA/proxyfs/conf" ) @@ -37,7 +39,9 @@ func stop() (err error) { return } + fmt.Println("UNDO: about to call stopRetryRPCServer()") err = stopRetryRPCServer() + fmt.Println("UNDO: stopRetryRPCServer() returned err:", err) if nil != err { return } diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index b1f767679..db6d99732 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -3,6 +3,63 @@ package imgrpkg +import ( + "crypto/tls" + "fmt" + + "github.com/NVIDIA/proxyfs/retryrpc" +) + +func startRetryRPCServer() (err error) { + var ( + retryrpcServerConfig *retryrpc.ServerConfig + tlsCertificate tls.Certificate + ) + + if "" == globals.config.RetryRPCCertFilePath { // && "" == globals.config.RetryRPCKeyFilePath + tlsCertificate = tls.Certificate{} + } else { // ("" != globals.config.RetryRPCCertFilePath) && ("" != globals.config.RetryRPCKeyFilePath) + tlsCertificate, err = tls.LoadX509KeyPair(globals.config.RetryRPCCertFilePath, globals.config.RetryRPCKeyFilePath) + if nil != err { + return + } + } + + retryrpcServerConfig = &retryrpc.ServerConfig{ + LongTrim: globals.config.RetryRPCTTLCompleted, + ShortTrim: globals.config.RetryRPCAckTrim, + IPAddr: globals.config.PrivateIPAddr, + Port: int(globals.config.RetryRPCPort), + DeadlineIO: globals.config.RetryRPCDeadlineIO, + KeepAlivePeriod: globals.config.RetryRPCKeepAlivePeriod, + TLSCertificate: tlsCertificate, + } + + globals.retryrpcServer = retryrpc.NewServer(retryrpcServerConfig) + + retryRPCServer = &RetryRPCServerStruct{} + + err = globals.retryrpcServer.Register(retryRPCServer) + if nil != err { + return + } + + err = globals.retryrpcServer.Start() + + return nil // err set by globals.retryrpcServer.Start() is sufficient +} + +func stopRetryRPCServer() (err error) { + fmt.Println("UNDO: stopRetryRPCServer() about to call globals.retryrpcServer.Close()") + globals.retryrpcServer.Close() + fmt.Println("UNDO: stopRetryRPCServer() back from call toglobals.retryrpcServer.Close()") + + retryRPCServer = nil + + err = nil + return nil +} + func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { return nil // TODO } From fc11f505ea20e1882e34854818aa46ac2ca68b05 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 5 Apr 2021 20:17:33 -0700 Subject: [PATCH 111/160] Fixed missing Run() call in imgr/imgrpkg/retry-rpc.go::startRetryRPCServer() --- imgr/imgrpkg/impl.go | 4 ---- imgr/imgrpkg/retry-rpc.go | 13 ++++++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/imgr/imgrpkg/impl.go b/imgr/imgrpkg/impl.go index 2679a1b55..e44961009 100644 --- a/imgr/imgrpkg/impl.go +++ b/imgr/imgrpkg/impl.go @@ -4,8 +4,6 @@ package imgrpkg import ( - "fmt" - "github.com/NVIDIA/proxyfs/conf" ) @@ -39,9 +37,7 @@ func stop() (err error) { return } - fmt.Println("UNDO: about to call stopRetryRPCServer()") err = stopRetryRPCServer() - fmt.Println("UNDO: stopRetryRPCServer() returned err:", err) if nil != err { return } diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index db6d99732..e4c3c8f86 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -5,7 +5,6 @@ package imgrpkg import ( "crypto/tls" - "fmt" "github.com/NVIDIA/proxyfs/retryrpc" ) @@ -28,7 +27,7 @@ func startRetryRPCServer() (err error) { retryrpcServerConfig = &retryrpc.ServerConfig{ LongTrim: globals.config.RetryRPCTTLCompleted, ShortTrim: globals.config.RetryRPCAckTrim, - IPAddr: globals.config.PrivateIPAddr, + IPAddr: globals.config.PublicIPAddr, Port: int(globals.config.RetryRPCPort), DeadlineIO: globals.config.RetryRPCDeadlineIO, KeepAlivePeriod: globals.config.RetryRPCKeepAlivePeriod, @@ -45,14 +44,18 @@ func startRetryRPCServer() (err error) { } err = globals.retryrpcServer.Start() + if nil != err { + return + } - return nil // err set by globals.retryrpcServer.Start() is sufficient + globals.retryrpcServer.Run() + + err = nil + return } func stopRetryRPCServer() (err error) { - fmt.Println("UNDO: stopRetryRPCServer() about to call globals.retryrpcServer.Close()") globals.retryrpcServer.Close() - fmt.Println("UNDO: stopRetryRPCServer() back from call toglobals.retryrpcServer.Close()") retryRPCServer = nil From 630782a0785958a0499e5278c0fb946480231c0b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 6 Apr 2021 11:07:05 -0700 Subject: [PATCH 112/160] Added bucketstats calls for each of the Web & RPC APIs (as well as LeaseRequestType) to imgrpkg --- imgr/imgrpkg/globals.go | 29 ++++---- imgr/imgrpkg/http-server.go | 39 +++++++++++ imgr/imgrpkg/retry-rpc.go | 131 ++++++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 12 deletions(-) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 363daab3e..72d23fc01 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -70,7 +70,23 @@ type chunkedPutContextStruct struct { } type statsStruct struct { - MountUsecs bucketstats.BucketLog2Round + DeleteVolumeUsecs bucketstats.BucketLog2Round // DELETE /volume/ + GetConfigUsecs bucketstats.BucketLog2Round // GET /config + GetStatsUsecs bucketstats.BucketLog2Round // GET /stats + GetVolumeListUsecs bucketstats.BucketLog2Round // GET /volume + GetVolumeUsecs bucketstats.BucketLog2Round // GET /volume/ + PutVolumeUsecs bucketstats.BucketLog2Round // PUT /volume/ + + AdjustInodeTableEntryOpenCountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).AdjustInodeTableEntryOpenCount() + DeleteInodeTableEntryUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).DeleteInodeTableEntry() + FetchNonceRangeUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).FetchNonceRange() + FlushUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).Flush() + GetInodeTableEntryUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).GetInodeTableEntry() + LeaseUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).Lease() + MountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).Mount() + PutInodeTableEntriesUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).PutInodeTableEntries() + RenewMountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).RenewMount() + UnmountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).Unmount() SharedLeaseRequestUsecs bucketstats.BucketLog2Round PromoteLeaseRequestUsecs bucketstats.BucketLog2Round @@ -78,21 +94,10 @@ type statsStruct struct { DemoteLeaseRequestUsecs bucketstats.BucketLog2Round ReleaseLeaseRequestUsecs bucketstats.BucketLog2Round - GetInodeTableEntryUsecs bucketstats.BucketLog2Round - PutInodeTableEntriesUsecs bucketstats.BucketLog2Round - FetchNonceRangeUsecs bucketstats.BucketLog2Round - RenewMountUsecs bucketstats.BucketLog2Round - UnmountUsecs bucketstats.BucketLog2Round - UnmountInterrupts bucketstats.Totaler DemoteLeaseInterrupts bucketstats.Totaler RevokeLeaseInterrupts bucketstats.Totaler - GetVolumeListUsecs bucketstats.BucketLog2Round - GetVolumeUsecs bucketstats.BucketLog2Round - PutVolumeUsecs bucketstats.BucketLog2Round - DeleteVolumeUsecs bucketstats.BucketLog2Round - VolumeCheckpointUsecs bucketstats.BucketLog2Round InodeTableCacheHits bucketstats.Totaler diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 6a0d12c9e..1a9f58ef9 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -13,6 +13,7 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/NVIDIA/proxyfs/bucketstats" ) @@ -104,12 +105,19 @@ func serveHTTPDeleteOfVolume(responseWriter http.ResponseWriter, request *http.R var ( err error pathSplit []string + startTime time.Time ) + startTime = time.Now() + pathSplit = strings.Split(requestPath, "/") switch len(pathSplit) { case 3: + defer func() { + globals.stats.DeleteVolumeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + err = deleteVolume(pathSplit[2]) if nil == err { responseWriter.WriteHeader(http.StatusNoContent) @@ -138,8 +146,14 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ var ( confMapJSON []byte err error + startTime time.Time ) + startTime = time.Now() + defer func() { + globals.stats.GetConfigUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + confMapJSON, err = json.Marshal(globals.config) if nil != err { logFatalf("json.Marshal(globals.config) failed: %v", err) @@ -158,9 +172,15 @@ func serveHTTPGetOfConfig(responseWriter http.ResponseWriter, request *http.Requ func serveHTTPGetOfStats(responseWriter http.ResponseWriter, request *http.Request) { var ( err error + startTime time.Time statsAsString string ) + startTime = time.Now() + defer func() { + globals.stats.GetStatsUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + statsAsString = bucketstats.SprintStats(bucketstats.StatFormatParsable1, "*", "*") responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(statsAsString))) @@ -178,12 +198,20 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ err error jsonToReturn []byte pathSplit []string + startTime time.Time ) + startTime = time.Now() + pathSplit = strings.Split(requestPath, "/") switch len(pathSplit) { case 2: + startTime = time.Now() + defer func() { + globals.stats.GetVolumeListUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + jsonToReturn = getVolumeListAsJSON() responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(jsonToReturn))) @@ -195,6 +223,10 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ logWarnf("responseWriter.Write(jsonToReturn) failed: %v", err) } case 3: + defer func() { + globals.stats.GetVolumeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + jsonToReturn, err = getVolumeAsJSON(pathSplit[2]) if nil == err { responseWriter.Header().Set("Content-Length", fmt.Sprintf("%d", len(jsonToReturn))) @@ -226,13 +258,20 @@ func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Requ var ( err error pathSplit []string + startTime time.Time storageURL string ) + startTime = time.Now() + pathSplit = strings.Split(requestPath, "/") switch len(pathSplit) { case 3: + defer func() { + globals.stats.PutVolumeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + storageURL = string(requestBody[:]) err = putVolume(pathSplit[2], storageURL) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index e4c3c8f86..233ca0f7d 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -5,7 +5,10 @@ package imgrpkg import ( "crypto/tls" + "fmt" + "time" + "github.com/NVIDIA/proxyfs/blunder" "github.com/NVIDIA/proxyfs/retryrpc" ) @@ -64,41 +67,169 @@ func stopRetryRPCServer() (err error) { } func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { + var ( + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.MountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + return nil // TODO } func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { + var ( + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.RenewMountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + return nil // TODO } func unmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { + var ( + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.UnmountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + return nil // TODO } func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { + var ( + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.FetchNonceRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + return nil // TODO } func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { + var ( + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.GetInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + return nil // TODO } func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { + var ( + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.PutInodeTableEntriesUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + return nil // TODO } func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryReply *DeleteInodeTableEntryReplyStruct) (err error) { + var ( + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.DeleteInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + return nil // TODO } func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountReply *AdjustInodeTableEntryOpenCountReplyStruct) (err error) { + var ( + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.AdjustInodeTableEntryOpenCountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + return nil // TODO } func flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err error) { + var ( + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.FlushUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + return nil // TODO } func lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err error) { + var ( + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.LeaseUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + switch leaseRequest.LeaseRequestType { + case LeaseRequestTypeShared: + defer func() { + globals.stats.SharedLeaseRequestUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + case LeaseRequestTypePromote: + defer func() { + globals.stats.PromoteLeaseRequestUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + case LeaseRequestTypeExclusive: + defer func() { + globals.stats.ExclusiveLeaseRequestUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + case LeaseRequestTypeDemote: + defer func() { + globals.stats.DemoteLeaseRequestUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + case LeaseRequestTypeRelease: + defer func() { + globals.stats.ReleaseLeaseRequestUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + default: + leaseReply.LeaseReplyType = LeaseReplyTypeDenied + err = fmt.Errorf("LeaseRequestType %v not supported", leaseRequest.LeaseRequestType) + err = blunder.AddError(err, blunder.BadLeaseRequest) + return + } + return nil // TODO } From 007b3a6c18f8c779face115aacd901b25364ae9c Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Wed, 7 Apr 2021 15:34:51 -0700 Subject: [PATCH 113/160] Fix race condition and add bucketstats (#604) * Use bucketstats to narrow down areas eating time * Cleanup bucketstats names and code closer to original. * Don't hold client lock while unmarshaling * sendToServer() must be called in a goroutine. This is because it is called during initial Send() without locking as well as during retransmit() when the client is locked. --- retryrpc/api_internal.go | 15 +++++++++------ retryrpc/client.go | 35 ++++++++++++++++++++++------------- retryrpc/retryrpc_test.go | 2 +- retryrpc/server.go | 18 ++++++++++++------ 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index 444527361..a3577aaf3 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -36,9 +36,12 @@ type requestID uint64 // Useful stats for the clientInfo instance type statsInfo struct { - AddCompleted bucketstats.Total // Number added to completed list - RmCompleted bucketstats.Total // Number removed from completed list - TimeOfRPCUsec bucketstats.BucketLog2Round // Tracks amount of time to call an RPC + TrimAddCompleted bucketstats.Total // Number added to completed list + TrimRmCompleted bucketstats.Total // Number removed from completed list + CallWrapRPCUsec bucketstats.BucketLog2Round // Tracks time to unmarshal request before actual RPC + PreRPCUnmarshalUsec bucketstats.BucketLog2Round // Tracks time to unmarshal request before actual RPC + RPCOnlyUsec bucketstats.BucketLog2Round // Tracks RPC only with no marshaling + PostRPCMarshalUsec bucketstats.BucketLog2Round // Tracks time between RPC returns and marshal the response ReplySize bucketstats.BucketLog2Round // Tracks completed RPC reply size longestRPC time.Duration // Time of longest RPC longestRPCMethod string // Method of longest RPC @@ -163,7 +166,7 @@ type replyCtx struct { // reqCtx exists on the client and tracks a request passed to Send() type reqCtx struct { - ioreq ioRequest // Wrapped request passed to Send() + ioreq *ioRequest // Wrapped request passed to Send() rpcReply interface{} answer chan replyCtx genNum uint64 // Generation number of socket when request sent @@ -198,9 +201,9 @@ type svrResponse struct { Result interface{} `json:"result"` } -func buildIoRequest(jReq jsonRequest) (ioreq *ioRequest, err error) { +func buildIoRequest(jReq *jsonRequest) (ioreq *ioRequest, err error) { ioreq = &ioRequest{} - ioreq.JReq, err = json.Marshal(jReq) + ioreq.JReq, err = json.Marshal(*jReq) if err != nil { return nil, err } diff --git a/retryrpc/client.go b/retryrpc/client.go index 889e38dce..b2843519b 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -31,10 +31,11 @@ const ( // Useful stats for the client side type clientSideStatsInfo struct { - RetransmitsStarted bucketstats.Total // Number of retransmits attempted - SendCalled bucketstats.Total // Number of times Send called - ReplyCalled bucketstats.Total // Number of times receive Reply to RPC - UpcallCalled bucketstats.Total // Number of times received an Upcall + RetransmitsStarted bucketstats.Total // Number of retransmits attempted + SendCalled bucketstats.Total // Number of times Send called + ReplyCalled bucketstats.Total // Number of times receive Reply to RPC + UpcallCalled bucketstats.Total // Number of times received an Upcall + TimeSendRPCUsec bucketstats.BucketLog2Round // Tracks time when Send() called and returned } // TODO - what if RPC was completed on Server1 and before response, @@ -89,6 +90,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter } } } + timeSent := time.Now() // Put request data into structure to be be marshaled into JSON jreq := jsonRequest{Method: method, HighestReplySeen: client.highestConsecutive} @@ -107,7 +109,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter client.Unlock() // Setup ioreq to write structure on socket to server - ioreq, err := buildIoRequest(jreq) + ioreq, err := buildIoRequest(&jreq) if err != nil { e := fmt.Errorf("Client buildIoRequest returned err: %v", err) logger.PanicfWithError(e, "") @@ -115,14 +117,19 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter } // Create context to wait result and to handle retransmits - ctx := &reqCtx{ioreq: *ioreq, rpcReply: rpcReply} + ctx := &reqCtx{ioreq: ioreq, rpcReply: &rpcReply} ctx.answer = make(chan replyCtx) + // Send request to server. + // + // We use a goroutine to be consistent with the way sendToServer() is called + // in the retransmit() case. client.goroutineWG.Add(1) go client.sendToServer(crID, ctx, true) // Now wait for response answer := <-ctx.answer + client.stats.TimeSendRPCUsec.Add(uint64(time.Duration(time.Since(timeSent).Microseconds()))) return answer.err } @@ -133,11 +140,10 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter // At this point, the client will retry the request until either it // completes OR the client is shutdown. func (client *Client) sendToServer(crID requestID, ctx *reqCtx, queue bool) { - defer client.goroutineWG.Done() // Now send the request to the server. - // We need to grab the mutex here to serialize writes on socket + // We need to GRAB THE MUTEX HERE TO SERIALIZE WRITES on socket client.Lock() // Keep track of requests we are sending so we can resend them later @@ -242,6 +248,9 @@ func (client *Client) notifyReply(buf []byte, genNum uint64) { return } + // Carefully drop lock while we are unmarshaling... + client.Unlock() + // Unmarshal the buf into the original reply structure m := svrResponse{Result: ctx.rpcReply} unmarshalErr := json.Unmarshal(buf, &m) @@ -251,12 +260,13 @@ func (client *Client) notifyReply(buf []byte, genNum uint64) { // Assume read garbage on socket - close the socket and reconnect // Drop client lock since retransmit() will acquire it. - client.Unlock() client.retransmit(genNum) return } + client.Lock() delete(client.outstandingRequest, crID) + client.Unlock() // Give reply to blocked send() - most developers test for nil err so // only set it if there is an error @@ -264,7 +274,6 @@ func (client *Client) notifyReply(buf []byte, genNum uint64) { if jReply.ErrStr != "" { r.err = fmt.Errorf("%v", jReply.ErrStr) } - client.Unlock() ctx.answer <- r // Fork off a goroutine to update highestConsecutiveNum @@ -288,14 +297,15 @@ func (client *Client) readReplies(callingGenNum uint64) { } nC := client.connection.castToNetConn() client.Unlock() - if nil == nC { + + if nC == nil { return } // Wait reply from server buf, msgType, getErr := getIO(callingGenNum, client.deadlineIO, nC) - // This must happen before checking error + // Since we reacquired lock - check if now halting client.Lock() if client.halting { client.Unlock() @@ -498,7 +508,6 @@ func (client *Client) reDial() (err error) { var entryState = client.connection.state // Now dial the server - if client.connection.useTLS { client.connection.tlsConfig = &tls.Config{ RootCAs: client.connection.x509CertPool, diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index cad4a973a..6e6b364c0 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -96,7 +96,7 @@ func TestTCPRetryRPC(t *testing.T) { } func TestTLSRetryRPC(t *testing.T) { - testTLSCertsAllocate(t) + testTLSCertsAllocate(t) // Must be first - initializes certificate testRegister(t, true) testServer(t, true) testBtree(t, true) diff --git a/retryrpc/server.go b/retryrpc/server.go index d144f28f5..3dc96785d 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -153,8 +153,8 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b // be unmarshaled again to retrieve the parameters specific to // the RPC. startRPC := time.Now() - ior := server.callRPCAndFormatReply(buf, ci.myUniqueID, &jReq) - ci.stats.TimeOfRPCUsec.Add(uint64(time.Since(startRPC).Microseconds())) + ior := server.callRPCAndFormatReply(buf, ci.myUniqueID, &jReq, &ci.stats) + ci.stats.CallWrapRPCUsec.Add(uint64(time.Since(startRPC).Microseconds())) ci.stats.RPCcompleted.Add(1) // We had to drop the lock before calling the RPC since it @@ -169,7 +169,7 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b // Update completed queue ce := &completedEntry{reply: ior} ci.completedRequest[rID] = ce - ci.stats.AddCompleted.Add(1) + ci.stats.TrimAddCompleted.Add(1) setupHdrReply(ce.reply, RPC) localIOR = *ce.reply sz := uint64(len(ior.JResult)) @@ -356,7 +356,7 @@ func (server *Server) serviceClient(ci *clientInfo, cCtx *connCtx) { } // callRPCAndMarshal calls the RPC and returns results to requestor -func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *jsonRequest) (ior *ioReply) { +func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *jsonRequest, si *statsInfo) (ior *ioReply) { var ( err error returnValues []reflect.Value @@ -372,6 +372,7 @@ func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *j ma := server.svrMap[jReq.Method] if ma != nil { + startUnmarshalRPC := time.Now() // Another unmarshal of buf to find the parameters specific to // this RPC typOfReq = ma.request.Elem() @@ -390,14 +391,17 @@ func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *j // Create the reply structure typOfReply := ma.reply.Elem() myReply := reflect.New(typOfReply) + si.PreRPCUnmarshalUsec.Add(uint64(time.Since(startUnmarshalRPC).Microseconds())) // Call the method function := ma.methodPtr.Func + startRealRPC := time.Now() if ma.passClientID { returnValues = function.Call([]reflect.Value{server.receiver, cid, req, myReply}) } else { returnValues = function.Call([]reflect.Value{server.receiver, req, myReply}) } + si.RPCOnlyUsec.Add(uint64(time.Since(startRealRPC).Microseconds())) // The return value for the method is an error. errInter := returnValues[0].Interface() @@ -417,11 +421,13 @@ func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *j jReply.ErrStr = fmt.Sprintf("errno: %d", unix.ENOENT) } + startReturnRPC := time.Now() // Convert response into JSON for return trip reply.JResult, err = json.Marshal(jReply) if err != nil { logger.PanicfWithError(err, "Unable to marshal jReply: %+v", jReply) } + si.PostRPCMarshalUsec.Add(uint64(time.Duration(time.Since(startReturnRPC).Microseconds()))) return reply } @@ -530,7 +536,7 @@ func (server *Server) trimAClientBasedACK(uniqueID uint64, ci *clientInfo) (numI if ok { ci.completedRequestLRU.Remove(v.lruElem) delete(ci.completedRequest, h) - ci.stats.RmCompleted.Add(1) + ci.stats.TrimRmCompleted.Add(1) numItems++ } } @@ -552,7 +558,7 @@ func (server *Server) trimTLLBased(ci *clientInfo, t time.Time) (numItems int) { eTime := e.Value.(completedLRUEntry).timeCompleted.Add(server.completedLongTTL) if eTime.Before(t) { delete(ci.completedRequest, e.Value.(completedLRUEntry).requestID) - ci.stats.RmCompleted.Add(1) + ci.stats.TrimRmCompleted.Add(1) eTmp := e e = e.Next() From 1f21051e1ba69e045d8f2b49dedc9055af98b583 Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Mon, 12 Apr 2021 13:00:13 -0700 Subject: [PATCH 114/160] Cleanup bucketstats used (#605) --- jrpcfs/lease_test.go | 3 +++ retryrpc/api_internal.go | 12 +++++------- retryrpc/client.go | 26 ++++++++++++++++---------- retryrpc/server.go | 7 ------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/jrpcfs/lease_test.go b/jrpcfs/lease_test.go index c9bfe1a39..83552bd2d 100644 --- a/jrpcfs/lease_test.go +++ b/jrpcfs/lease_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/retryrpc" ) @@ -1570,6 +1571,8 @@ func BenchmarkRpcLeaseRemote(b *testing.B) { b.StopTimer() + fmt.Printf("RetryRPC Bucketstats: %v\n", bucketstats.SprintStats(bucketstats.StatFormatParsable1, "proxyfs.retryrpc", "*")) + unmountRequest = &UnmountRequest{ MountID: mountByAccountNameReply.MountID, } diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index a3577aaf3..cd9395a2b 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -39,9 +39,6 @@ type statsInfo struct { TrimAddCompleted bucketstats.Total // Number added to completed list TrimRmCompleted bucketstats.Total // Number removed from completed list CallWrapRPCUsec bucketstats.BucketLog2Round // Tracks time to unmarshal request before actual RPC - PreRPCUnmarshalUsec bucketstats.BucketLog2Round // Tracks time to unmarshal request before actual RPC - RPCOnlyUsec bucketstats.BucketLog2Round // Tracks RPC only with no marshaling - PostRPCMarshalUsec bucketstats.BucketLog2Round // Tracks time between RPC returns and marshal the response ReplySize bucketstats.BucketLog2Round // Tracks completed RPC reply size longestRPC time.Duration // Time of longest RPC longestRPCMethod string // Method of longest RPC @@ -166,10 +163,11 @@ type replyCtx struct { // reqCtx exists on the client and tracks a request passed to Send() type reqCtx struct { - ioreq *ioRequest // Wrapped request passed to Send() - rpcReply interface{} - answer chan replyCtx - genNum uint64 // Generation number of socket when request sent + ioreq *ioRequest // Wrapped request passed to Send() + rpcReply interface{} + answer chan replyCtx + genNum uint64 // Generation number of socket when request sent + startTime time.Time // Time Send() called sendToServer() } // jsonRequest is used to marshal an RPC request in/out of JSON diff --git a/retryrpc/client.go b/retryrpc/client.go index b2843519b..cb5e5f062 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -31,11 +31,13 @@ const ( // Useful stats for the client side type clientSideStatsInfo struct { - RetransmitsStarted bucketstats.Total // Number of retransmits attempted - SendCalled bucketstats.Total // Number of times Send called - ReplyCalled bucketstats.Total // Number of times receive Reply to RPC - UpcallCalled bucketstats.Total // Number of times received an Upcall - TimeSendRPCUsec bucketstats.BucketLog2Round // Tracks time when Send() called and returned + RetransmitsStarted bucketstats.Total // Number of retransmits attempted + SendCalled bucketstats.Total // Number of times Send called + ReplyCalled bucketstats.Total // Number of times receive Reply to RPC + UpcallCalled bucketstats.Total // Number of times received an Upcall + SeesIOAndSignalsChannel bucketstats.BucketLog2Round // Tracks time after returnReply() -> getIO and reply channel + TimeSendRPCUsec bucketstats.BucketLog2Round // Tracks time when Send() called and returned + SendToServer bucketstats.BucketLog2Round // Tracks time takes to send message to server } // TODO - what if RPC was completed on Server1 and before response, @@ -90,7 +92,6 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter } } } - timeSent := time.Now() // Put request data into structure to be be marshaled into JSON jreq := jsonRequest{Method: method, HighestReplySeen: client.highestConsecutive} @@ -117,7 +118,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter } // Create context to wait result and to handle retransmits - ctx := &reqCtx{ioreq: ioreq, rpcReply: &rpcReply} + ctx := &reqCtx{ioreq: ioreq, rpcReply: &rpcReply, startTime: time.Now()} ctx.answer = make(chan replyCtx) // Send request to server. @@ -129,7 +130,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter // Now wait for response answer := <-ctx.answer - client.stats.TimeSendRPCUsec.Add(uint64(time.Duration(time.Since(timeSent).Microseconds()))) + client.stats.TimeSendRPCUsec.Add(uint64(time.Duration(time.Since(ctx.startTime).Microseconds()))) return answer.err } @@ -142,6 +143,8 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter func (client *Client) sendToServer(crID requestID, ctx *reqCtx, queue bool) { defer client.goroutineWG.Done() + startTime := time.Now() + // Now send the request to the server. // We need to GRAB THE MUTEX HERE TO SERIALIZE WRITES on socket client.Lock() @@ -207,9 +210,10 @@ func (client *Client) sendToServer(crID requestID, ctx *reqCtx, queue bool) { } client.Unlock() + client.stats.SendToServer.Add(uint64(time.Duration(time.Since(startTime).Microseconds()))) } -func (client *Client) notifyReply(buf []byte, genNum uint64) { +func (client *Client) notifyReply(buf []byte, genNum uint64, recvResponse time.Time) { defer client.goroutineWG.Done() // Unmarshal once to get the header fields @@ -275,6 +279,7 @@ func (client *Client) notifyReply(buf []byte, genNum uint64) { r.err = fmt.Errorf("%v", jReply.ErrStr) } ctx.answer <- r + client.stats.SeesIOAndSignalsChannel.Add(uint64(time.Duration(time.Since(recvResponse).Microseconds()))) // Fork off a goroutine to update highestConsecutiveNum go client.updateHighestConsecutiveNum(crID) @@ -330,6 +335,7 @@ func (client *Client) readReplies(callingGenNum uint64) { client.retransmit(callingGenNum) return } + recvResponse := time.Now() // Figure out what type of message it is switch msgType { @@ -338,7 +344,7 @@ func (client *Client) readReplies(callingGenNum uint64) { // and sending the reply to blocked Send() so that this routine // can read the next response. client.goroutineWG.Add(1) - go client.notifyReply(buf, callingGenNum) + go client.notifyReply(buf, callingGenNum, recvResponse) client.stats.ReplyCalled.Add(1) case Upcall: diff --git a/retryrpc/server.go b/retryrpc/server.go index 3dc96785d..f1db1d60e 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -372,7 +372,6 @@ func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *j ma := server.svrMap[jReq.Method] if ma != nil { - startUnmarshalRPC := time.Now() // Another unmarshal of buf to find the parameters specific to // this RPC typOfReq = ma.request.Elem() @@ -391,17 +390,14 @@ func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *j // Create the reply structure typOfReply := ma.reply.Elem() myReply := reflect.New(typOfReply) - si.PreRPCUnmarshalUsec.Add(uint64(time.Since(startUnmarshalRPC).Microseconds())) // Call the method function := ma.methodPtr.Func - startRealRPC := time.Now() if ma.passClientID { returnValues = function.Call([]reflect.Value{server.receiver, cid, req, myReply}) } else { returnValues = function.Call([]reflect.Value{server.receiver, req, myReply}) } - si.RPCOnlyUsec.Add(uint64(time.Since(startRealRPC).Microseconds())) // The return value for the method is an error. errInter := returnValues[0].Interface() @@ -421,13 +417,11 @@ func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *j jReply.ErrStr = fmt.Sprintf("errno: %d", unix.ENOENT) } - startReturnRPC := time.Now() // Convert response into JSON for return trip reply.JResult, err = json.Marshal(jReply) if err != nil { logger.PanicfWithError(err, "Unable to marshal jReply: %+v", jReply) } - si.PostRPCMarshalUsec.Add(uint64(time.Duration(time.Since(startReturnRPC).Microseconds()))) return reply } @@ -457,7 +451,6 @@ func (server *Server) returnResults(ior *ioReply, cCtx *connCtx) { if e != nil { logger.Infof("returnResults() returned err: %v cnt: %v length of JResult: %v", e, cnt, len(ior.JResult)) } - cCtx.Unlock() } From 9539ba6ddce035e29f71b4af581016795571b02a Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 7 Apr 2021 10:25:47 -0700 Subject: [PATCH 115/160] iauth (and swift plug-in) updated to use a normal StorageURL (i.e. /v1/) with specified Container appended --- iauth/README.md | 5 +++-- iauth/iauth-swift/README.md | 22 ++++++++-------------- iauth/iauth-swift/iauth-swift.go | 12 +++++++----- iauth/iauth-swift/plugin_test.go | 3 ++- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/iauth/README.md b/iauth/README.md index e9730e191..bb30d9c78 100644 --- a/iauth/README.md +++ b/iauth/README.md @@ -24,5 +24,6 @@ solution. The only requirements are that: * Credentials to be authorized are provided in a single string (possibly a JSON document) * The plug-in returns both a Swift AuthToken and StorageURL (or an error) * The StorageURL has been properly modified as necessary to ensure - * the proper transport is used (i.e. either "http" or "https") - * the version portion of the path has been set to "proxyfs" (rather than "v1") + * the proper transport (scheme) is used (i.e. either "http" or "https") + * the specified `Account`, if necessary, has been substituted + * the specified `Container` has been appended diff --git a/iauth/iauth-swift/README.md b/iauth/iauth-swift/README.md index 5b76d603b..b20c20fcd 100644 --- a/iauth/iauth-swift/README.md +++ b/iauth/iauth-swift/README.md @@ -10,10 +10,11 @@ is required to be a UTF-8 encoded JSON Document: ``` { - "AuthURL" : "/auth/v1.0>", - "AuthUser" : "", - "AuthKey" : "", - "Account" : "" + "AuthURL" : "/auth/v1.0>", + "AuthUser" : "", + "AuthKey" : "", + "Account" : "", + "Container" : " } ``` @@ -30,19 +31,12 @@ standard Swift Authentication operation: port numbers (i.e. `80` for `http` and `443` for `https`) are not assumed (i.e. port numbers are specified in the URL). -* The path portion of the Storage URL returned by the Swift Proxy will start - with `v1` to indicate the version of the OpenStack Swift protocol being used. - No `v2` has ever been defined, so this is a constant in normal requests. - To indicate the client, in the case of GETs and PUTs, has specified the - `physical` path (i.e. to LogSegments of a FileInode) rather than the - `virtual` path (i.e. the full path of the file in the file system), this - protocol version will be changed to `proxyfs`. This updated protocol verion - will also be used with the `PROXYFS` method used to pass a `mount` request, - via JSON RPC, over to the ProxyFS process serving the volume to be mounted. - * The final element of the path portion of the Storage URL returned by the Swift Proxy will typically be the Account associated with the specified AuthUser (e.g. AuthUser `test` typically has a corresponding Account named `AUTH_test`). The volume being accessed may, however be stored in a different Account than this. As such, the account element of the path will be replaced with the `Account` as requested. + +* The specified Container must be appended to the Storage URL delineated from + the perhaps updated Account portion by a slash ("/"). diff --git a/iauth/iauth-swift/iauth-swift.go b/iauth/iauth-swift/iauth-swift.go index f92bedf6a..e856d44bb 100644 --- a/iauth/iauth-swift/iauth-swift.go +++ b/iauth/iauth-swift/iauth-swift.go @@ -13,10 +13,11 @@ import ( ) type authInStruct struct { - AuthURL string - AuthUser string - AuthKey string - Account string + AuthURL string + AuthUser string + AuthKey string + Account string + Container string } // PerformAuth accepts a JSON string, performs the OpenStack Swift Authorization @@ -76,9 +77,10 @@ func PerformAuth(authInJSON string) (authToken string, storageURL string, err er storageURLSplit = strings.Split(storageURL, "/") - storageURLSplit[3] = "proxyfs" storageURLSplit[4] = authIn.Account + storageURLSplit = append(storageURLSplit, authIn.Container) + storageURL = strings.Join(storageURLSplit, "/") err = nil diff --git a/iauth/iauth-swift/plugin_test.go b/iauth/iauth-swift/plugin_test.go index 020c2b9b4..c218faf4c 100644 --- a/iauth/iauth-swift/plugin_test.go +++ b/iauth/iauth-swift/plugin_test.go @@ -46,7 +46,8 @@ func TestSwiftAuth(t *testing.T) { " \"AuthURL\" : \"http://127.0.0.1:9997/auth/v1.0\"," + " \"AuthUser\" : \"test:tester\"," + " \"AuthKey\" : \"testing\"," + - " \"Account\" : \"AUTH_test\"" + + " \"Account\" : \"AUTH_test\"," + + " \"Container\" : \"con\"" + "}" authToken, storageURL, err = PerformAuth(authInJSON) From da82193221371f632f1749b2610a26dc70168f0c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 7 Apr 2021 18:22:29 -0700 Subject: [PATCH 116/160] Flushed out on-disk format in package ilayer Note: marshaling code not yet added --- Makefile | 1 + ilayout/Makefile | 6 ++ ilayout/dummy_test.go | 11 +++ ilayout/on-disk.go | 220 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 238 insertions(+) create mode 100644 ilayout/Makefile create mode 100644 ilayout/dummy_test.go create mode 100644 ilayout/on-disk.go diff --git a/Makefile b/Makefile index b1b1cbe10..9f265da5f 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ gopkgsubdirs = \ headhunter \ httpserver \ iauth \ + ilayout \ inode \ jrpcfs \ liveness \ diff --git a/ilayout/Makefile b/ilayout/Makefile new file mode 100644 index 000000000..1c8f51c9b --- /dev/null +++ b/ilayout/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/ilayout + +include ../GoMakefile diff --git a/ilayout/dummy_test.go b/ilayout/dummy_test.go new file mode 100644 index 000000000..a529b67ce --- /dev/null +++ b/ilayout/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package ilayout + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/ilayout/on-disk.go b/ilayout/on-disk.go new file mode 100644 index 000000000..2e12a9409 --- /dev/null +++ b/ilayout/on-disk.go @@ -0,0 +1,220 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package ilayout + +import "time" + +// CheckpointHeaderVersionV* specifies the format of the CheckPointHeader. +// The CheckPointHeaderVersion must always be fetched by scanning the entire +// CheckPointHeader using a %016X format specifier. This value will then be +// used to interpret the remaining characters of the CheckPointHeader string. +// +const ( + CheckPointHeaderVersionV1 uint64 = 1 +) + +// CheckPointHeaderV1Struct specifies the format of the CheckPointHeader +// as of V1. +// +// The contents of the struct are serialized as space separated fields formatted +// via %016X numbers. +// +type CheckPointHeaderV1Struct struct { + Version uint64 // == CheckPointHeaderVersionV1 + SuperBlockObjectNumber uint64 // Identifies the Object containing the SuperBlock at the end + SuperBlockLength uint64 // Total length of the SuperBlock found at the end of the Object indicated by SuperBlockObjectNumber + ReservedToNonce uint64 // Ensures all numbers requiring uniqueness (e.g. Object numbers, Inode numbers) are never reused +} + +// SuperBlockVersionV* specifies the format of all preceeding bytes +// referenced by CheckPointHeaderV1Struct.SuperBlock{ObjectNumber|Length}. +// +// The value is stored in BigEndian (binary uint64) format. +// +const ( + SuperBlockVersionV1 uint64 = 1 +) + +// InodeTableLayoutEntryV1Struct specifies the layout of the InodeTable B+Tree in Objects. +// Since any modification of the Volume will result in a fresh SuperBlockV1Struct +// being written to a new Object, when BytesReferenced drops to zero, the Object +// may be deleted. +// +// The struct is serialized as a sequence of BigEndian (binary uint64) formatted fields. +// +type InodeTableLayoutEntryV1Struct struct { + ObjectNumber uint64 // Identifies the Object containing the page(s) of the InodeTable B+Tree + ObjectSize uint64 // Number of bytes written to the Object + BytesReferenced uint64 // Number of bytes currently referenced in the Object +} + +// SuperBlockStruct specifies the format of the SuperBlock found at +// the CheckPointHeaderV1Struct.SuperBlockLength trailing bytes of +// the Object indicated by CheckPointHeaderV1Struct.SuperBlockObjectNumber. +// Note that the ...Length includes the SuperBlockVersionV1 value appended +// to it. +// +// The InodeTable is a B+Tree where the Key is the uint64 InodeNumber. The Value +// is +// +// The struct is serialized as a sequence of BigEndian (binary uint64) formatted fields. +// The InodeTableLayout slice is serialized by a preceeding BigEndian (binary uint64) +// count of the number of InodeTableLayoutEntryV1Struct's followed by the serialization +// of each one. +// +// Note that the CheckPointHeaderV1Struct.SuperBlockLength also includes the bytes for +// holding SuperBlockVersionV1 that is appended. +// +type SuperBlockV1Struct struct { + InodeTableRootObjectNumber uint64 // Identifies the Object containing the root of the InodeTable + InodeTableRootObjectOffset uint64 // Starting offset in the Object of the root of the InodeTable + InodeTableRootObjectLength uint64 // Number of bytes in the Object of the root of the InodeTable + InodeTableLayout []InodeTableLayoutEntryV1Struct // Describes the data and space occupied by the the InodeTable +} + +// InodeTableEntryVersionV* specifies the format of all preceeding bytes +// in an InodeTable entry's Value InodeTableEntryStruct. +// +// The value is stored in BigEndian (binary uint64) format. +// +const ( + InodeTableEntryVersionV1 uint64 = 1 +) + +// InodeTableEntryValueV1Struct specifies the format of the bytes in the InodeTable entry's +// Value preceeding InodeTableEntryVersionV1. +// +// The struct is serialized as a sequence of BigEndian (binary uint64) formatted fields. +// +// Note that there is no InodeTableEntryKeyV1Struct as it is simply a uint64 InodeNumber. +// +type InodeTableEntryValueV1Struct struct { + InodeHeadObjectNumber uint64 // Identifies the Object containing InodeHeadV*Struct + InodeHeadLength uint64 // Total length of the InodeHead found at the end of the Object indicated by InodeHeadObjectNumber +} + +// InodeHeadVersionV* specifies the format of all preceeding bytes referenced +// by InodeTableEntryValueV1Struct.InodeHead{ObjectNumber|Length}. +// +// The value is stored in BigEndian (binary uint64) format. +// +const ( + InodeHeadVersionV1 uint64 = 1 +) + +// InodeType* specifies the type of Inode. +// +const ( + InodeTypeDir uint8 = 0 + InodeTypeFile uint8 = 1 + InodeTypeSymlink uint8 = 2 +) + +// InodeLinkTableEntryStruct specifies the layout of an InodeHeadV1Struct.LinkTable's entry. +// +// The struct's uint64 field is serialized in BigEndian (binary uint64) format followed by +// the struct's string field serialized as a BigEndian (binary uint64) length followed by +// the bytes of the string. +// +type InodeLinkTableEntryStruct struct { + ParentDirInodeNumber uint64 + ParentDirEntryName string +} + +// InodeModeMask provides a bound on the acceptable values of an Inode's Mode field's +// protection bits (i.e. rwx bits for each of user, group, and other). +// +// The value is stored in BigEndian (binary uint16) format. +// +const ( + InodeModeMask uint16 = 0o777 +) + +// InodeStreamTableEntryStruct specifies the layout of an InodeHeadV1Struct.StreamTable's entry. +// +// The struct is serialized be treating both fields as an array of bytes preceeded +// by a BigEndian (binary uint64) length. +// +type InodeStreamTableEntryStruct struct { + Name string + Value []byte +} + +// InodeHeadLayoutEntryV1Struct is utilized in both DirInode's and FileInode's. For DirInode's, +// it specifies the layout of the Directory B+Tree in Objects. For FileInode's, it specifies +// the layout of the ExtentMap B+Tree as well as the File's contents in Objects. Since any +// modification of the Inode will result in a fresh InodeHeadStruct being written to a new +// Object, when BytesReferenced drops to zero, the Object may be deleted. +// +// The struct is serialized as a sequence of BigEndian (binary uint64) formatted fields. +// +type InodeHeadLayoutEntryV1Struct struct { + ObjectNumber uint64 // For DirInode's: + // Identifies the Object containing the page(s) of the Directory B+Tree + // For FileInode's: + // Identifies the Object containing the page(s) of the ExtentMap B+Tree + // as well as the bytes of the File's contents + ObjectSize uint64 // Number of bytes written to the Object + BytesReferenced uint64 // Number of bytes currently referenced in the Object +} + +// InodeHeadV1Struct specifies the layout of an Inode. +// +// The struct is serializes as a sequence of fields. +// For uint* fields, BigEndian format is used. +// For table fields, a uint64 length in BigEndian format is followed by the serialization specified in +// the table entry struct. +// For time.Time fields, a string (i.e. uint64 length followed by that number of bytes) is used. +// The string is itself time.RFC3339 formatted +// +// Note that the InodeTableEntryValueV1Struct.InodeHeadLength also includes the bytes for +// holding InodeHeadVersionV1 that is appended. +// +type InodeHeadV1Struct struct { + InodeNumber uint64 + InodeType uint8 // One of InodeType* + LinkTable []InodeLinkTableEntryStruct // List of Directory Entry references to this Inode + Size uint64 + CreationTime time.Time + ModificationTime time.Time + AccessTimeTime time.Time + AttrChangeTimeTime time.Time + Mode uint16 // Must be <= InodeModeMask (Note: does not include InodeType encoding) + UserID uint64 + GroupID uint64 + StreamTable []InodeStreamTableEntryStruct // List of Alternate Data Streams for this Inode + PayloadObjectNumber uint64 // For Dir & File Inodes, identifies the Object containing the root of the Directory or ExtentMap B+Tree + PayloadObjectOffset uint64 // For Dir & File Inodes, starting offset in the Object of the root of the Directory or ExtentMap B+Tree + PayloadObjectLength uint64 // For Dir & File Inodes, number of bytes in the Object of the root of the Directory or ExtentMap B+Tree + SymlinkTarget string // For Symlink Inodes, the target of the link + Layout []InodeHeadLayoutEntryV1Struct // Describes the data and space occupied by the the InodeTable +} + +// DirectoryEntryValueV1Struct specifies the format, for an Inode of type InodeTypeDir, +// of the bytes in a .Payload-identified B+Tree's Value. +// +// The struct is serialized as a sequence of uint* fields in BigEndian (binary uint64) format. +// +// Note that there is no DirectoryEntryKeyV1Struct as it is simply a string serialized +// by a uint64 length in BigEndian format followed by the bytes of the string. +// +type DirectoryEntryValueV1Struct struct { + InodeNumber uint64 + InodeType uint8 +} + +// ExtentMapEntryValueV1Struct specifies the format, for an Inode of type InodeTypeFile, +// of the bytes in a .Payload-identified B+Tree's Value. +// +// The struct is serialized as a sequence of uint64 fields in BigEndian (binary uint64) format. +// +// Note that there is no ExtentMapEntryKeyV1Struct as it is simply a uint64 serialized +// in BigEndian format. +// +type ExtentMapEntryValueV1Struct struct { + FileOffset uint64 // Offset from the start of the File + Length uint64 // Length of this extent (both in the File and in the Object) + ObjectNumber uint64 // Identifies the Object containing this extent's data + ObjectOffset uint64 // Starting offset in the Object of this extent's data +} From 12577f30bd9ad60c1fd9e486894cc720ed6fa34b Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 7 Apr 2021 19:36:32 -0700 Subject: [PATCH 117/160] Added ilayout API & API Test boilerplates --- ilayout/{on-disk.go => api.go} | 80 +++++++++++++++++++++++++++++++--- ilayout/api_test.go | 23 ++++++++++ ilayout/dummy_test.go | 11 ----- ilayout/impl.go | 56 ++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 16 deletions(-) rename ilayout/{on-disk.go => api.go} (76%) create mode 100644 ilayout/api_test.go delete mode 100644 ilayout/dummy_test.go create mode 100644 ilayout/impl.go diff --git a/ilayout/on-disk.go b/ilayout/api.go similarity index 76% rename from ilayout/on-disk.go rename to ilayout/api.go index 2e12a9409..3fcca83aa 100644 --- a/ilayout/on-disk.go +++ b/ilayout/api.go @@ -3,7 +3,9 @@ package ilayout -import "time" +import ( + "time" +) // CheckpointHeaderVersionV* specifies the format of the CheckPointHeader. // The CheckPointHeaderVersion must always be fetched by scanning the entire @@ -14,6 +16,13 @@ const ( CheckPointHeaderVersionV1 uint64 = 1 ) +// UnmarshalCheckPointHeaderVersion extracts checkPointHeaderVersion from checkpointHeaderString. +// +func UnmarshalCheckPointHeaderVersion(checkpointHeaderString string) (checkPointHeaderVersion uint64, err error) { + checkPointHeaderVersion, err = unmarshalCheckPointHeaderVersion(checkpointHeaderString) + return +} + // CheckPointHeaderV1Struct specifies the format of the CheckPointHeader // as of V1. // @@ -27,6 +36,16 @@ type CheckPointHeaderV1Struct struct { ReservedToNonce uint64 // Ensures all numbers requiring uniqueness (e.g. Object numbers, Inode numbers) are never reused } +func (checkPointHeaderV1 *CheckPointHeaderV1Struct) MarshalCheckPointHeaderV1() (checkPointHeaderString string, err error) { + checkPointHeaderString, err = checkPointHeaderV1.marshalCheckPointHeaderV1() + return +} + +func UnmarshalCheckPointHeaderV1(checkPointHeaderString string) (checkPointHeaderV1 *CheckPointHeaderV1Struct, err error) { + checkPointHeaderV1, err = UnmarshalCheckPointHeaderV1(checkPointHeaderString) + return +} + // SuperBlockVersionV* specifies the format of all preceeding bytes // referenced by CheckPointHeaderV1Struct.SuperBlock{ObjectNumber|Length}. // @@ -36,6 +55,13 @@ const ( SuperBlockVersionV1 uint64 = 1 ) +// UnmarshalSuperBlockVersion extracts superBlockVersion from superBlockBuf. +// +func UnmarshalSuperBlockVersion(superBlockBuf []byte) (superBlockVersion uint64, err error) { + superBlockVersion, err = unmarshalSuperBlockVersion(superBlockBuf) + return +} + // InodeTableLayoutEntryV1Struct specifies the layout of the InodeTable B+Tree in Objects. // Since any modification of the Volume will result in a fresh SuperBlockV1Struct // being written to a new Object, when BytesReferenced drops to zero, the Object @@ -73,17 +99,34 @@ type SuperBlockV1Struct struct { InodeTableLayout []InodeTableLayoutEntryV1Struct // Describes the data and space occupied by the the InodeTable } -// InodeTableEntryVersionV* specifies the format of all preceeding bytes +func (superBlockV1 *SuperBlockV1Struct) MarshalSuperBlockV1() (superBlockV1Buf []byte, err error) { + superBlockV1Buf, err = superBlockV1.marshalSuperBlockV1() + return +} + +func UnmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1Struct, err error) { + superBlockV1, err = unmarshalSuperBlockV1(superBlockV1Buf) + return +} + +// InodeTableEntryValueVersionV* specifies the format of all preceeding bytes // in an InodeTable entry's Value InodeTableEntryStruct. // // The value is stored in BigEndian (binary uint64) format. // const ( - InodeTableEntryVersionV1 uint64 = 1 + InodeTableEntryValueVersionV1 uint64 = 1 ) +// UnmarshalInodeTableEntryValueVersion extracts inodeTableEntryValueVersion from inodeTableEntryValueBuf. +// +func UnmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf []byte) (inodeTableEntryValueVersion uint64, err error) { + inodeTableEntryValueVersion, err = UnmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf) + return +} + // InodeTableEntryValueV1Struct specifies the format of the bytes in the InodeTable entry's -// Value preceeding InodeTableEntryVersionV1. +// Value preceeding InodeTableEntryValueVersionV1. // // The struct is serialized as a sequence of BigEndian (binary uint64) formatted fields. // @@ -94,6 +137,16 @@ type InodeTableEntryValueV1Struct struct { InodeHeadLength uint64 // Total length of the InodeHead found at the end of the Object indicated by InodeHeadObjectNumber } +func (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct) MarshalInodeTableEntryValueV1() (inodeTableEntryValueV1Buf []byte, err error) { + inodeTableEntryValueV1Buf, err = inodeTableEntryValueV1.marshalInodeTableEntryValueV1() + return +} + +func UnmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct, err error) { + inodeTableEntryValueV1, err = unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf) + return +} + // InodeHeadVersionV* specifies the format of all preceeding bytes referenced // by InodeTableEntryValueV1Struct.InodeHead{ObjectNumber|Length}. // @@ -103,6 +156,13 @@ const ( InodeHeadVersionV1 uint64 = 1 ) +// UnmarshalInodeHeadVersion extracts inodeHeadVersion from inodeHeadBuf. +// +func UnmarshalInodeHeadVersion(inodeHeadBuf []byte) (inodeHeadVersion uint64, err error) { + inodeHeadVersion, err = unmarshalInodeHeadVersion(inodeHeadBuf) + return +} + // InodeType* specifies the type of Inode. // const ( @@ -166,7 +226,7 @@ type InodeHeadLayoutEntryV1Struct struct { // For table fields, a uint64 length in BigEndian format is followed by the serialization specified in // the table entry struct. // For time.Time fields, a string (i.e. uint64 length followed by that number of bytes) is used. -// The string is itself time.RFC3339 formatted +// The string is itself time.RFC3339 formatted. // // Note that the InodeTableEntryValueV1Struct.InodeHeadLength also includes the bytes for // holding InodeHeadVersionV1 that is appended. @@ -191,6 +251,16 @@ type InodeHeadV1Struct struct { Layout []InodeHeadLayoutEntryV1Struct // Describes the data and space occupied by the the InodeTable } +func (inodeHeadV1 *InodeHeadV1Struct) MarshalInodeHeadV1() (inodeHeadV1Buf []byte, err error) { + inodeHeadV1Buf, err = inodeHeadV1.marshalInodeHeadV1() + return +} + +func UnmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct, err error) { + inodeHeadV1, err = unmarshalInodeHeadV1(inodeHeadV1Buf) + return +} + // DirectoryEntryValueV1Struct specifies the format, for an Inode of type InodeTypeDir, // of the bytes in a .Payload-identified B+Tree's Value. // diff --git a/ilayout/api_test.go b/ilayout/api_test.go new file mode 100644 index 000000000..e7eff3de7 --- /dev/null +++ b/ilayout/api_test.go @@ -0,0 +1,23 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package ilayout + +import ( + "testing" +) + +func TestAPI(t *testing.T) { + t.Logf("TODO: test MarshalCheckPointHeaderV1()") + t.Logf("TODO: test UnmarshalCheckPointHeaderVersion()") + t.Logf("TODO: test UnmarshalCheckPointHeaderV1()") + t.Logf("TODO: test MarshalSuperBlockV1()") + t.Logf("TODO: test UnmarshalSuperBlockVersion()") + t.Logf("TODO: test UnmarshalSuperBlockV1()") + t.Logf("TODO: test MarshalInodeTableEntryValueV1()") + t.Logf("TODO: test UnmarshalInodeTableEntryValueVersion()") + t.Logf("TODO: test UnmarshalInodeTableEntryValueV1()") + t.Logf("TODO: test MarshalInodeHeadV1()") + t.Logf("TODO: test UnmarshalInodeHeadVersion()") + t.Logf("TODO: test UnmarshalInodeHeadV1()") +} diff --git a/ilayout/dummy_test.go b/ilayout/dummy_test.go deleted file mode 100644 index a529b67ce..000000000 --- a/ilayout/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package ilayout - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/ilayout/impl.go b/ilayout/impl.go new file mode 100644 index 000000000..57098af8c --- /dev/null +++ b/ilayout/impl.go @@ -0,0 +1,56 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package ilayout + +import ( + "fmt" +) + +func unmarshalCheckPointHeaderVersion(checkpointHeaderString string) (checkPointHeaderVersion uint64, err error) { + return 0, fmt.Errorf("TODO") +} + +func (checkPointHeaderV1 *CheckPointHeaderV1Struct) marshalCheckPointHeaderV1() (checkPointHeaderString string, err error) { + return "", fmt.Errorf("TODO") +} + +func unmarshalCheckPointHeaderV1(checkPointHeaderString string) (checkPointHeaderV1 *CheckPointHeaderV1Struct, err error) { + return nil, fmt.Errorf("TODO") +} + +func unmarshalSuperBlockVersion(superBlockBuf []byte) (superBlockVersion uint64, err error) { + return 0, fmt.Errorf("TODO") +} + +func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf []byte, err error) { + return nil, fmt.Errorf("TODO") +} + +func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1Struct, err error) { + return nil, fmt.Errorf("TODO") +} + +func unmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf []byte) (inodeTableEntryValueVersion uint64, err error) { + return 0, fmt.Errorf("TODO") +} + +func (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct) marshalInodeTableEntryValueV1() (inodeTableEntryValueV1Buf []byte, err error) { + return nil, fmt.Errorf("TODO") +} + +func unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct, err error) { + return nil, fmt.Errorf("TODO") +} + +func unmarshalInodeHeadVersion(inodeHeadBuf []byte) (inodeHeadVersion uint64, err error) { + return 0, fmt.Errorf("TODO") +} + +func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byte, err error) { + return nil, fmt.Errorf("TODO") +} + +func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct, err error) { + return nil, fmt.Errorf("TODO") +} From bcdf06760e00dc769734602322033df83f4f7a47 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 8 Apr 2021 15:48:03 -0700 Subject: [PATCH 118/160] Implement & test ilayout/api.go marshaling for all but InodeHeadV1Struct Also switched to LittleEndian --- ilayout/api.go | 60 +++---- ilayout/api_test.go | 129 +++++++++++++- ilayout/impl.go | 398 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 534 insertions(+), 53 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index 3fcca83aa..79b3af595 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -36,20 +36,20 @@ type CheckPointHeaderV1Struct struct { ReservedToNonce uint64 // Ensures all numbers requiring uniqueness (e.g. Object numbers, Inode numbers) are never reused } -func (checkPointHeaderV1 *CheckPointHeaderV1Struct) MarshalCheckPointHeaderV1() (checkPointHeaderString string, err error) { - checkPointHeaderString, err = checkPointHeaderV1.marshalCheckPointHeaderV1() +func (checkPointHeaderV1 *CheckPointHeaderV1Struct) MarshalCheckPointHeaderV1() (checkPointHeaderV1String string, err error) { + checkPointHeaderV1String, err = checkPointHeaderV1.marshalCheckPointHeaderV1() return } -func UnmarshalCheckPointHeaderV1(checkPointHeaderString string) (checkPointHeaderV1 *CheckPointHeaderV1Struct, err error) { - checkPointHeaderV1, err = UnmarshalCheckPointHeaderV1(checkPointHeaderString) +func UnmarshalCheckPointHeaderV1(checkPointHeaderV1String string) (checkPointHeaderV1 *CheckPointHeaderV1Struct, err error) { + checkPointHeaderV1, err = unmarshalCheckPointHeaderV1(checkPointHeaderV1String) return } // SuperBlockVersionV* specifies the format of all preceeding bytes // referenced by CheckPointHeaderV1Struct.SuperBlock{ObjectNumber|Length}. // -// The value is stored in BigEndian (binary uint64) format. +// The value is stored in LittleEndian format. // const ( SuperBlockVersionV1 uint64 = 1 @@ -67,7 +67,7 @@ func UnmarshalSuperBlockVersion(superBlockBuf []byte) (superBlockVersion uint64, // being written to a new Object, when BytesReferenced drops to zero, the Object // may be deleted. // -// The struct is serialized as a sequence of BigEndian (binary uint64) formatted fields. +// The struct is serialized as a sequence of LittleEndian formatted fields. // type InodeTableLayoutEntryV1Struct struct { ObjectNumber uint64 // Identifies the Object containing the page(s) of the InodeTable B+Tree @@ -82,12 +82,12 @@ type InodeTableLayoutEntryV1Struct struct { // to it. // // The InodeTable is a B+Tree where the Key is the uint64 InodeNumber. The Value -// is +// is a InodeTableEntryValueV1Struct. // -// The struct is serialized as a sequence of BigEndian (binary uint64) formatted fields. -// The InodeTableLayout slice is serialized by a preceeding BigEndian (binary uint64) -// count of the number of InodeTableLayoutEntryV1Struct's followed by the serialization -// of each one. +// The struct is serialized as a sequence of LittleEndian formatted fields. +// The InodeTableLayout slice is serialized by a preceeding LittleEndian +// count of the number of InodeTableLayoutEntryV1Struct's followed by the +// serialization of each one. // // Note that the CheckPointHeaderV1Struct.SuperBlockLength also includes the bytes for // holding SuperBlockVersionV1 that is appended. @@ -109,10 +109,10 @@ func UnmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St return } -// InodeTableEntryValueVersionV* specifies the format of all preceeding bytes +// InodeTableEntryValueVersionV* specifies the format of all following bytes // in an InodeTable entry's Value InodeTableEntryStruct. // -// The value is stored in BigEndian (binary uint64) format. +// The value is stored in LittleEndian format. // const ( InodeTableEntryValueVersionV1 uint64 = 1 @@ -121,14 +121,14 @@ const ( // UnmarshalInodeTableEntryValueVersion extracts inodeTableEntryValueVersion from inodeTableEntryValueBuf. // func UnmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf []byte) (inodeTableEntryValueVersion uint64, err error) { - inodeTableEntryValueVersion, err = UnmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf) + inodeTableEntryValueVersion, err = unmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf) return } // InodeTableEntryValueV1Struct specifies the format of the bytes in the InodeTable entry's -// Value preceeding InodeTableEntryValueVersionV1. +// Value following InodeTableEntryValueVersionV1. // -// The struct is serialized as a sequence of BigEndian (binary uint64) formatted fields. +// The struct is serialized as a sequence of LittleEndian formatted fields. // // Note that there is no InodeTableEntryKeyV1Struct as it is simply a uint64 InodeNumber. // @@ -150,7 +150,7 @@ func UnmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTab // InodeHeadVersionV* specifies the format of all preceeding bytes referenced // by InodeTableEntryValueV1Struct.InodeHead{ObjectNumber|Length}. // -// The value is stored in BigEndian (binary uint64) format. +// The value is stored in LittleEndian format. // const ( InodeHeadVersionV1 uint64 = 1 @@ -173,8 +173,8 @@ const ( // InodeLinkTableEntryStruct specifies the layout of an InodeHeadV1Struct.LinkTable's entry. // -// The struct's uint64 field is serialized in BigEndian (binary uint64) format followed by -// the struct's string field serialized as a BigEndian (binary uint64) length followed by +// The struct's uint64 field is serialized in LittleEndian format followed by +// the struct's string field serialized as a LittleEndian length followed by // the bytes of the string. // type InodeLinkTableEntryStruct struct { @@ -185,7 +185,7 @@ type InodeLinkTableEntryStruct struct { // InodeModeMask provides a bound on the acceptable values of an Inode's Mode field's // protection bits (i.e. rwx bits for each of user, group, and other). // -// The value is stored in BigEndian (binary uint16) format. +// The value is stored in LittleEndian format. // const ( InodeModeMask uint16 = 0o777 @@ -194,7 +194,7 @@ const ( // InodeStreamTableEntryStruct specifies the layout of an InodeHeadV1Struct.StreamTable's entry. // // The struct is serialized be treating both fields as an array of bytes preceeded -// by a BigEndian (binary uint64) length. +// by a LittleEndian length. // type InodeStreamTableEntryStruct struct { Name string @@ -207,7 +207,7 @@ type InodeStreamTableEntryStruct struct { // modification of the Inode will result in a fresh InodeHeadStruct being written to a new // Object, when BytesReferenced drops to zero, the Object may be deleted. // -// The struct is serialized as a sequence of BigEndian (binary uint64) formatted fields. +// The struct is serialized as a sequence of LittleEndian formatted fields. // type InodeHeadLayoutEntryV1Struct struct { ObjectNumber uint64 // For DirInode's: @@ -222,9 +222,9 @@ type InodeHeadLayoutEntryV1Struct struct { // InodeHeadV1Struct specifies the layout of an Inode. // // The struct is serializes as a sequence of fields. -// For uint* fields, BigEndian format is used. -// For table fields, a uint64 length in BigEndian format is followed by the serialization specified in -// the table entry struct. +// For uint* fields, LittleEndian format is used. +// For table fields, a uint64 length in LittleEndian format is followed by the serialization +// specified in the table entry struct. // For time.Time fields, a string (i.e. uint64 length followed by that number of bytes) is used. // The string is itself time.RFC3339 formatted. // @@ -235,7 +235,7 @@ type InodeHeadV1Struct struct { InodeNumber uint64 InodeType uint8 // One of InodeType* LinkTable []InodeLinkTableEntryStruct // List of Directory Entry references to this Inode - Size uint64 + Size uint64 // Only applicable to File Inodes CreationTime time.Time ModificationTime time.Time AccessTimeTime time.Time @@ -264,10 +264,10 @@ func UnmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct // DirectoryEntryValueV1Struct specifies the format, for an Inode of type InodeTypeDir, // of the bytes in a .Payload-identified B+Tree's Value. // -// The struct is serialized as a sequence of uint* fields in BigEndian (binary uint64) format. +// The struct is serialized as a sequence of uint* fields in LittleEndian format. // // Note that there is no DirectoryEntryKeyV1Struct as it is simply a string serialized -// by a uint64 length in BigEndian format followed by the bytes of the string. +// by a uint64 length in LittleEndian format followed by the bytes of the string. // type DirectoryEntryValueV1Struct struct { InodeNumber uint64 @@ -277,10 +277,10 @@ type DirectoryEntryValueV1Struct struct { // ExtentMapEntryValueV1Struct specifies the format, for an Inode of type InodeTypeFile, // of the bytes in a .Payload-identified B+Tree's Value. // -// The struct is serialized as a sequence of uint64 fields in BigEndian (binary uint64) format. +// The struct is serialized as a sequence of uint64 fields in LittleEndian format. // // Note that there is no ExtentMapEntryKeyV1Struct as it is simply a uint64 serialized -// in BigEndian format. +// in LittleEndian format. // type ExtentMapEntryValueV1Struct struct { FileOffset uint64 // Offset from the start of the File diff --git a/ilayout/api_test.go b/ilayout/api_test.go index e7eff3de7..bbb6bae66 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -8,15 +8,126 @@ import ( ) func TestAPI(t *testing.T) { - t.Logf("TODO: test MarshalCheckPointHeaderV1()") - t.Logf("TODO: test UnmarshalCheckPointHeaderVersion()") - t.Logf("TODO: test UnmarshalCheckPointHeaderV1()") - t.Logf("TODO: test MarshalSuperBlockV1()") - t.Logf("TODO: test UnmarshalSuperBlockVersion()") - t.Logf("TODO: test UnmarshalSuperBlockV1()") - t.Logf("TODO: test MarshalInodeTableEntryValueV1()") - t.Logf("TODO: test UnmarshalInodeTableEntryValueVersion()") - t.Logf("TODO: test UnmarshalInodeTableEntryValueV1()") + var ( + err error + + testCheckPointHeaderV1 = &CheckPointHeaderV1Struct{ + Version: CheckPointHeaderVersionV1, + SuperBlockObjectNumber: 2, + SuperBlockLength: 3, + ReservedToNonce: 4, + } + + marshaledCheckPointHeaderV1 string + unmarshaledCheckPointHeaderVersion uint64 + unmarshaledCheckPointHeaderV1 *CheckPointHeaderV1Struct + + testSuperBlockV1 = &SuperBlockV1Struct{ + InodeTableRootObjectNumber: 1, + InodeTableRootObjectOffset: 2, + InodeTableRootObjectLength: 3, + InodeTableLayout: []InodeTableLayoutEntryV1Struct{ + { + ObjectNumber: 4, + ObjectSize: 5, + BytesReferenced: 6, + }, + { + ObjectNumber: 7, + ObjectSize: 8, + BytesReferenced: 9, + }, + }, + } + + marshaledSuperBlockV1 []byte + unmarshaledSuperBlockVersion uint64 + unmarshaledSuperBlockV1 *SuperBlockV1Struct + + inodeTableLayoutIndex int + + testInodeTableEntryValueV1 = &InodeTableEntryValueV1Struct{ + InodeHeadObjectNumber: 2, + InodeHeadLength: 3, + } + + marshaledInodeTableEntryValueV1 []byte + unmarshaledInodeTableEntryValueVersion uint64 + unmarshaledInodeTableEntryValueV1 *InodeTableEntryValueV1Struct + ) + + marshaledCheckPointHeaderV1, err = testCheckPointHeaderV1.MarshalCheckPointHeaderV1() + if nil != err { + t.Fatal(err) + } + + unmarshaledCheckPointHeaderVersion, err = UnmarshalCheckPointHeaderVersion(marshaledCheckPointHeaderV1) + if nil != err { + t.Fatal(err) + } + if CheckPointHeaderVersionV1 != unmarshaledCheckPointHeaderVersion { + t.Fatalf("Bad unmarshaledCheckPointHeaderVersion (%016X) - expected CheckPointHeaderVersionV1 (%016X)", unmarshaledCheckPointHeaderVersion, CheckPointHeaderVersionV1) + } + + unmarshaledCheckPointHeaderV1, err = UnmarshalCheckPointHeaderV1(marshaledCheckPointHeaderV1) + if nil != err { + t.Fatal(err) + } + if *testCheckPointHeaderV1 != *unmarshaledCheckPointHeaderV1 { + t.Fatalf("Bad unmarshaledCheckPointHeaderV1 (%#v) - expected testCheckPointHeaderV1 (%#v)", unmarshaledCheckPointHeaderV1, testCheckPointHeaderV1) + } + + marshaledSuperBlockV1, err = testSuperBlockV1.MarshalSuperBlockV1() + if nil != err { + t.Fatal(err) + } + + unmarshaledSuperBlockVersion, err = UnmarshalSuperBlockVersion(marshaledSuperBlockV1) + if nil != err { + t.Fatal(err) + } + if SuperBlockVersionV1 != unmarshaledSuperBlockVersion { + t.Fatalf("Bad unmarshaledSuperBlockVersion (%016X) - expected SuperBlockVersionV1 (%016X)", unmarshaledSuperBlockVersion, SuperBlockVersionV1) + } + + unmarshaledSuperBlockV1, err = UnmarshalSuperBlockV1(marshaledSuperBlockV1) + if nil != err { + t.Fatal(err) + } + if (testSuperBlockV1.InodeTableRootObjectNumber != unmarshaledSuperBlockV1.InodeTableRootObjectNumber) || + (testSuperBlockV1.InodeTableRootObjectOffset != unmarshaledSuperBlockV1.InodeTableRootObjectOffset) || + (testSuperBlockV1.InodeTableRootObjectLength != unmarshaledSuperBlockV1.InodeTableRootObjectLength) || + (len(testSuperBlockV1.InodeTableLayout) != len(unmarshaledSuperBlockV1.InodeTableLayout)) { + t.Fatalf("Bad unmarshaledSuperBlockV1 (%#v) - expected testSuperBlockV1 (%#v) [Case 1]", unmarshaledSuperBlockV1, testSuperBlockV1) + } + for inodeTableLayoutIndex = range testSuperBlockV1.InodeTableLayout { + if testSuperBlockV1.InodeTableLayout[inodeTableLayoutIndex] != unmarshaledSuperBlockV1.InodeTableLayout[inodeTableLayoutIndex] { + t.Fatalf("Bad unmarshaledSuperBlockV1 (%#v) - expected testSuperBlockV1 (%#v) [Case 2]", unmarshaledSuperBlockV1, testSuperBlockV1) + } + } + + marshaledInodeTableEntryValueV1, err = testInodeTableEntryValueV1.MarshalInodeTableEntryValueV1() + if nil != err { + t.Fatal(err) + } + + unmarshaledInodeTableEntryValueVersion, err = UnmarshalInodeTableEntryValueVersion(marshaledInodeTableEntryValueV1) + if nil != err { + t.Fatal(err) + } + if InodeTableEntryValueVersionV1 != unmarshaledInodeTableEntryValueVersion { + t.Fatalf("Bad unmarshaledSuperBlockVersion (%016X) - expected SuperBlockVersionV1 (%016X)", unmarshaledSuperBlockVersion, SuperBlockVersionV1) + } + + unmarshaledInodeTableEntryValueV1, err = UnmarshalInodeTableEntryValueV1(marshaledInodeTableEntryValueV1) + if nil != err { + t.Fatal(err) + } + if (testInodeTableEntryValueV1.InodeHeadObjectNumber != unmarshaledInodeTableEntryValueV1.InodeHeadObjectNumber) || + (testInodeTableEntryValueV1.InodeHeadLength != unmarshaledInodeTableEntryValueV1.InodeHeadLength) { + t.Fatalf("Bad unmarshaledInodeTableEntryValueV1 (%#v) - expected testInodeTableEntryValueV1 (%#v)", unmarshaledInodeTableEntryValueV1, testInodeTableEntryValueV1) + } + t.Logf("TODO: test MarshalInodeHeadV1()") t.Logf("TODO: test UnmarshalInodeHeadVersion()") t.Logf("TODO: test UnmarshalInodeHeadV1()") diff --git a/ilayout/impl.go b/ilayout/impl.go index 57098af8c..afd91c303 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -8,49 +8,419 @@ import ( ) func unmarshalCheckPointHeaderVersion(checkpointHeaderString string) (checkPointHeaderVersion uint64, err error) { - return 0, fmt.Errorf("TODO") + _, err = fmt.Sscanf(checkpointHeaderString, "%016X", &checkPointHeaderVersion) + + return } -func (checkPointHeaderV1 *CheckPointHeaderV1Struct) marshalCheckPointHeaderV1() (checkPointHeaderString string, err error) { - return "", fmt.Errorf("TODO") +func (checkPointHeaderV1 *CheckPointHeaderV1Struct) marshalCheckPointHeaderV1() (checkPointHeaderV1String string, err error) { + checkPointHeaderV1String = fmt.Sprintf("%016X %016X %016X %016X", checkPointHeaderV1.Version, checkPointHeaderV1.SuperBlockObjectNumber, checkPointHeaderV1.SuperBlockLength, checkPointHeaderV1.ReservedToNonce) + + err = nil + return } -func unmarshalCheckPointHeaderV1(checkPointHeaderString string) (checkPointHeaderV1 *CheckPointHeaderV1Struct, err error) { - return nil, fmt.Errorf("TODO") +func unmarshalCheckPointHeaderV1(checkPointHeaderV1String string) (checkPointHeaderV1 *CheckPointHeaderV1Struct, err error) { + checkPointHeaderV1 = &CheckPointHeaderV1Struct{} + + _, err = fmt.Sscanf(checkPointHeaderV1String, "%016X %016X %016X %016X", &checkPointHeaderV1.Version, &checkPointHeaderV1.SuperBlockObjectNumber, &checkPointHeaderV1.SuperBlockLength, &checkPointHeaderV1.ReservedToNonce) + if (nil == err) && (CheckPointHeaderVersionV1 != checkPointHeaderV1.Version) { + err = fmt.Errorf("Version mismatch... found %016X... expected %016X", checkPointHeaderV1.Version, CheckPointHeaderVersionV1) + } + + return } func unmarshalSuperBlockVersion(superBlockBuf []byte) (superBlockVersion uint64, err error) { - return 0, fmt.Errorf("TODO") + var ( + superBlockBufLen int + ) + + superBlockBufLen = len(superBlockBuf) + + if superBlockBufLen < 8 { + err = fmt.Errorf("Insufficient space in superBlockBuf for superBlockVersion") + return + } + + superBlockVersion, _, err = leGetUint64FromBuf(superBlockBuf, superBlockBufLen-8) + + return } func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf []byte, err error) { - return nil, fmt.Errorf("TODO") + var ( + curPos int + inodeTableLayoutIndex int + ) + + superBlockV1Buf = make([]byte, 8+8+8+(len(superBlockV1.InodeTableLayout)*(8+8+8))+8) + + curPos = 0 + + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableRootObjectNumber) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableRootObjectOffset) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableRootObjectLength) + if nil != err { + return + } + + for inodeTableLayoutIndex = 0; inodeTableLayoutIndex < len(superBlockV1.InodeTableLayout); inodeTableLayoutIndex++ { + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectNumber) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectSize) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableLayout[inodeTableLayoutIndex].BytesReferenced) + if nil != err { + return + } + } + + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, SuperBlockVersionV1) + if nil != err { + return + } + + err = nil + return } func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1Struct, err error) { - return nil, fmt.Errorf("TODO") + var ( + curPos int + inodeTableLayoutEntryV1 InodeTableLayoutEntryV1Struct + superBlockVersion uint64 + ) + + superBlockV1 = &SuperBlockV1Struct{ + InodeTableLayout: make([]InodeTableLayoutEntryV1Struct, 0), + } + + curPos = 0 + + superBlockV1.InodeTableRootObjectNumber, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + + superBlockV1.InodeTableRootObjectOffset, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + + superBlockV1.InodeTableRootObjectLength, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + + for (len(superBlockV1Buf) - curPos) > 8 { + inodeTableLayoutEntryV1.ObjectNumber, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + + inodeTableLayoutEntryV1.ObjectSize, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + + inodeTableLayoutEntryV1.BytesReferenced, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + + superBlockV1.InodeTableLayout = append(superBlockV1.InodeTableLayout, inodeTableLayoutEntryV1) + } + + superBlockVersion, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + if superBlockVersion != SuperBlockVersionV1 { + err = fmt.Errorf("Incorrect Version for superBlockV1Buf") + return + } + + if curPos != len(superBlockV1Buf) { + err = fmt.Errorf("Incorrect size for superBlockV1Buf") + return + } + + err = nil + return } func unmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf []byte) (inodeTableEntryValueVersion uint64, err error) { - return 0, fmt.Errorf("TODO") + inodeTableEntryValueVersion, _, err = leGetUint64FromBuf(inodeTableEntryValueBuf, 0) + + return } func (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct) marshalInodeTableEntryValueV1() (inodeTableEntryValueV1Buf []byte, err error) { - return nil, fmt.Errorf("TODO") + var ( + curPos int + ) + + inodeTableEntryValueV1Buf = make([]byte, 8+8+8) + + curPos = 0 + + curPos, err = lePutUint64ToBuf(inodeTableEntryValueV1Buf, curPos, InodeTableEntryValueVersionV1) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeTableEntryValueV1Buf, curPos, inodeTableEntryValueV1.InodeHeadObjectNumber) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeTableEntryValueV1Buf, curPos, inodeTableEntryValueV1.InodeHeadLength) + if nil != err { + return + } + + err = nil + return } func unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct, err error) { - return nil, fmt.Errorf("TODO") + var ( + curPos int + inodeTableEntryValueVersion uint64 + ) + + curPos = 0 + + inodeTableEntryValueVersion, curPos, err = leGetUint64FromBuf(inodeTableEntryValueV1Buf, curPos) + if nil != err { + return + } + if InodeTableEntryValueVersionV1 != inodeTableEntryValueVersion { + err = fmt.Errorf("Incorrect Version for inodeTableEntryValueV1Buf") + return + } + + inodeTableEntryValueV1 = &InodeTableEntryValueV1Struct{} + + inodeTableEntryValueV1.InodeHeadObjectNumber, curPos, err = leGetUint64FromBuf(inodeTableEntryValueV1Buf, curPos) + if nil != err { + return + } + + inodeTableEntryValueV1.InodeHeadLength, curPos, err = leGetUint64FromBuf(inodeTableEntryValueV1Buf, curPos) + if nil != err { + return + } + + if curPos != len(inodeTableEntryValueV1Buf) { + err = fmt.Errorf("Incorrect size for inodeTableEntryValueV1Buf") + return + } + + err = nil + return } func unmarshalInodeHeadVersion(inodeHeadBuf []byte) (inodeHeadVersion uint64, err error) { - return 0, fmt.Errorf("TODO") + var ( + inodeHeadBufLen int + ) + + inodeHeadBufLen = len(inodeHeadBuf) + + if inodeHeadBufLen < 8 { + err = fmt.Errorf("Insufficient space in inodeHeadBuf for inodeHeadVersion") + return + } + + inodeHeadVersion, _, err = leGetUint64FromBuf(inodeHeadBuf, inodeHeadBufLen-8) + + return } func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byte, err error) { - return nil, fmt.Errorf("TODO") + return nil, fmt.Errorf("TODO - marshalInodeHeadV1()") } func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct, err error) { - return nil, fmt.Errorf("TODO") + return nil, fmt.Errorf("TODO - unmarshalInodeHeadV1()") +} + +func leGetUint8FromBuf(buf []byte, curPos int) (u8 uint8, nextPos int, err error) { + nextPos = curPos + 1 + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for uint8") + return + } + + u8 = buf[curPos] + + err = nil + return +} + +func lePutUint8ToBuf(buf []byte, curPos int, u8 uint8) (nextPos int, err error) { + nextPos = curPos + 1 + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for uint8") + return + } + + buf[curPos] = u8 + + err = nil + return +} + +func leGetUint16FromBuf(buf []byte, curPos int) (u16 uint16, nextPos int, err error) { + nextPos = curPos + 2 + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for uint16") + return + } + + u16 = uint16(buf[curPos]) + (uint16(buf[curPos+1]) << 8) + err = nil + return +} + +func lePutUint16ToBuf(buf []byte, curPos int, u16 uint16) (nextPos int, err error) { + nextPos = curPos + 2 + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for uint16") + return + } + + buf[curPos] = uint8(u16 & 0xFF) + buf[curPos+1] = uint8(u16 >> 8) + + err = nil + return +} + +func leGetUint32FromBuf(buf []byte, curPos int) (u32 uint32, nextPos int, err error) { + nextPos = curPos + 4 + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for uint32") + return + } + + u32 = uint32(buf[curPos]) + (uint32(buf[curPos+1]) << 8) + (uint32(buf[curPos+2]) << 16) + (uint32(buf[curPos+3]) << 24) + + err = nil + return +} + +func lePutUint32ToBuf(buf []byte, curPos int, u32 uint32) (nextPos int, err error) { + nextPos = curPos + 4 + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for uint32") + return + } + + buf[curPos] = uint8(u32 & 0xFF) + buf[curPos+1] = uint8((u32 >> 8) & 0xFF) + buf[curPos+2] = uint8((u32 >> 16) & 0xFF) + buf[curPos+3] = uint8(u32 >> 24) + + err = nil + return +} + +func leGetUint64FromBuf(buf []byte, curPos int) (u64 uint64, nextPos int, err error) { + nextPos = curPos + 8 + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for uint64") + return + } + + u64 = uint64(buf[curPos]) + (uint64(buf[curPos+1]) << 8) + (uint64(buf[curPos+2]) << 16) + (uint64(buf[curPos+3]) << 24) + (uint64(buf[curPos+2]) << 32) + (uint64(buf[curPos+3]) << 40) + (uint64(buf[curPos+2]) << 48) + (uint64(buf[curPos+3]) << 56) + err = nil + + return +} + +func lePutUint64ToBuf(buf []byte, curPos int, u64 uint64) (nextPos int, err error) { + nextPos = curPos + 8 + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for uint64") + return + } + + buf[curPos] = uint8(u64 & 0xFF) + buf[curPos+1] = uint8((u64 >> 8) & 0xFF) + buf[curPos+2] = uint8((u64 >> 16) & 0xFF) + buf[curPos+1] = uint8((u64 >> 24) & 0xFF) + buf[curPos+2] = uint8((u64 >> 32) & 0xFF) + buf[curPos+1] = uint8((u64 >> 40) & 0xFF) + buf[curPos+2] = uint8((u64 >> 48) & 0xFF) + buf[curPos+3] = uint8(u64 >> 56) + + err = nil + return +} + +func leGetStringFromBuf(buf []byte, curPos int) (s string, nextPos int, err error) { + var ( + sLen uint64 + ) + + sLen, nextPos, err = leGetUint64FromBuf(buf, curPos) + if nil != err { + return + } + + if sLen > (uint64(len(buf)) - uint64(nextPos)) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for string of reported length") + return + } + + s = string(buf[nextPos : nextPos+int(sLen)]) + nextPos += int(sLen) + + err = nil + return +} + +func lePutStringToBuf(buf []byte, curPos int, s string) (nextPos int, err error) { + nextPos, err = lePutUint64ToBuf(buf, curPos, uint64(len(s))) + if nil != err { + return + } + + curPos = nextPos + nextPos += len(s) + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for string") + return + } + + copy(buf[curPos:nextPos], []byte(s)) + + err = nil + return } From 6f1b25bf4026aad215c13bd409477364b21588ac Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 8 Apr 2021 17:24:05 -0700 Subject: [PATCH 119/160] Implement & test ilayout/api.go marshaling for all but InodeHeadV1Struct...but not unmarshaling --- ilayout/api.go | 4 +- ilayout/api_test.go | 79 ++++++++++++- ilayout/impl.go | 282 ++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 333 insertions(+), 32 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index 79b3af595..7cecf4943 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -238,8 +238,8 @@ type InodeHeadV1Struct struct { Size uint64 // Only applicable to File Inodes CreationTime time.Time ModificationTime time.Time - AccessTimeTime time.Time - AttrChangeTimeTime time.Time + AccessTime time.Time + AttrChangeTime time.Time Mode uint16 // Must be <= InodeModeMask (Note: does not include InodeType encoding) UserID uint64 GroupID uint64 diff --git a/ilayout/api_test.go b/ilayout/api_test.go index bbb6bae66..1479a2e5a 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -5,6 +5,7 @@ package ilayout import ( "testing" + "time" ) func TestAPI(t *testing.T) { @@ -54,6 +55,61 @@ func TestAPI(t *testing.T) { marshaledInodeTableEntryValueV1 []byte unmarshaledInodeTableEntryValueVersion uint64 unmarshaledInodeTableEntryValueV1 *InodeTableEntryValueV1Struct + + testStartTime = time.Now() + + testInodeHeadV1 = &InodeHeadV1Struct{ + InodeNumber: 1, + InodeType: InodeTypeSymlink, + LinkTable: []InodeLinkTableEntryStruct{ + { + ParentDirInodeNumber: 101, + ParentDirEntryName: "101", + }, + { + ParentDirInodeNumber: 102, + ParentDirEntryName: "102", + }, + }, + Size: 3, + CreationTime: testStartTime.AddDate(0, 0, -4), + ModificationTime: testStartTime.AddDate(0, 0, -3), + AccessTime: testStartTime.AddDate(0, 0, -1), + AttrChangeTime: testStartTime.AddDate(0, 0, -2), + Mode: 0o456, + UserID: 7, + GroupID: 8, + StreamTable: []InodeStreamTableEntryStruct{ + { + Name: "201", + Value: []byte{2, 0, 1}, + }, + { + Name: "202", + Value: []byte{2, 0, 2}, + }, + }, + PayloadObjectNumber: 11, + PayloadObjectOffset: 12, + PayloadObjectLength: 13, + SymlinkTarget: "sym-target", + Layout: []InodeHeadLayoutEntryV1Struct{ + { + ObjectNumber: 311, + ObjectSize: 312, + BytesReferenced: 313, + }, + { + ObjectNumber: 321, + ObjectSize: 322, + BytesReferenced: 323, + }, + }, + } + + marshaledInodeHeadV1 []byte + unmarshaledInodeHeadVersion uint64 + // unmarshaledInodeHeadV1 *InodeHeadV1Struct ) marshaledCheckPointHeaderV1, err = testCheckPointHeaderV1.MarshalCheckPointHeaderV1() @@ -74,7 +130,7 @@ func TestAPI(t *testing.T) { t.Fatal(err) } if *testCheckPointHeaderV1 != *unmarshaledCheckPointHeaderV1 { - t.Fatalf("Bad unmarshaledCheckPointHeaderV1 (%#v) - expected testCheckPointHeaderV1 (%#v)", unmarshaledCheckPointHeaderV1, testCheckPointHeaderV1) + t.Fatalf("Bad unmarshaledCheckPointHeaderV1 (%+v) - expected testCheckPointHeaderV1 (%+v)", unmarshaledCheckPointHeaderV1, testCheckPointHeaderV1) } marshaledSuperBlockV1, err = testSuperBlockV1.MarshalSuperBlockV1() @@ -98,11 +154,11 @@ func TestAPI(t *testing.T) { (testSuperBlockV1.InodeTableRootObjectOffset != unmarshaledSuperBlockV1.InodeTableRootObjectOffset) || (testSuperBlockV1.InodeTableRootObjectLength != unmarshaledSuperBlockV1.InodeTableRootObjectLength) || (len(testSuperBlockV1.InodeTableLayout) != len(unmarshaledSuperBlockV1.InodeTableLayout)) { - t.Fatalf("Bad unmarshaledSuperBlockV1 (%#v) - expected testSuperBlockV1 (%#v) [Case 1]", unmarshaledSuperBlockV1, testSuperBlockV1) + t.Fatalf("Bad unmarshaledSuperBlockV1 (%+v) - expected testSuperBlockV1 (%+v) [Case 1]", unmarshaledSuperBlockV1, testSuperBlockV1) } for inodeTableLayoutIndex = range testSuperBlockV1.InodeTableLayout { if testSuperBlockV1.InodeTableLayout[inodeTableLayoutIndex] != unmarshaledSuperBlockV1.InodeTableLayout[inodeTableLayoutIndex] { - t.Fatalf("Bad unmarshaledSuperBlockV1 (%#v) - expected testSuperBlockV1 (%#v) [Case 2]", unmarshaledSuperBlockV1, testSuperBlockV1) + t.Fatalf("Bad unmarshaledSuperBlockV1 (%+v) - expected testSuperBlockV1 (%+v) [Case 2]", unmarshaledSuperBlockV1, testSuperBlockV1) } } @@ -125,10 +181,21 @@ func TestAPI(t *testing.T) { } if (testInodeTableEntryValueV1.InodeHeadObjectNumber != unmarshaledInodeTableEntryValueV1.InodeHeadObjectNumber) || (testInodeTableEntryValueV1.InodeHeadLength != unmarshaledInodeTableEntryValueV1.InodeHeadLength) { - t.Fatalf("Bad unmarshaledInodeTableEntryValueV1 (%#v) - expected testInodeTableEntryValueV1 (%#v)", unmarshaledInodeTableEntryValueV1, testInodeTableEntryValueV1) + t.Fatalf("Bad unmarshaledInodeTableEntryValueV1 (%+v) - expected testInodeTableEntryValueV1 (%+v)", unmarshaledInodeTableEntryValueV1, testInodeTableEntryValueV1) + } + + marshaledInodeHeadV1, err = testInodeHeadV1.MarshalInodeHeadV1() + if nil != err { + t.Fatal(err) + } + + unmarshaledInodeHeadVersion, err = UnmarshalInodeHeadVersion(marshaledInodeHeadV1) + if nil != err { + t.Fatal(err) + } + if InodeHeadVersionV1 != unmarshaledInodeHeadVersion { + t.Fatalf("Bad unmarshaledInodeHeadVersion (%016X) - expected InodeHeadVersionV1 (%016X)", unmarshaledInodeHeadVersion, InodeHeadVersionV1) } - t.Logf("TODO: test MarshalInodeHeadV1()") - t.Logf("TODO: test UnmarshalInodeHeadVersion()") t.Logf("TODO: test UnmarshalInodeHeadV1()") } diff --git a/ilayout/impl.go b/ilayout/impl.go index afd91c303..d3df52ded 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -5,6 +5,7 @@ package ilayout import ( "fmt" + "time" ) func unmarshalCheckPointHeaderVersion(checkpointHeaderString string) (checkPointHeaderVersion uint64, err error) { @@ -54,7 +55,7 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ inodeTableLayoutIndex int ) - superBlockV1Buf = make([]byte, 8+8+8+(len(superBlockV1.InodeTableLayout)*(8+8+8))+8) + superBlockV1Buf = make([]byte, 8+8+8+8+(len(superBlockV1.InodeTableLayout)*(8+8+8))+8) curPos = 0 @@ -73,6 +74,11 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ return } + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, uint64(len(superBlockV1.InodeTableLayout))) + if nil != err { + return + } + for inodeTableLayoutIndex = 0; inodeTableLayoutIndex < len(superBlockV1.InodeTableLayout); inodeTableLayoutIndex++ { curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectNumber) if nil != err { @@ -101,14 +107,13 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1Struct, err error) { var ( - curPos int - inodeTableLayoutEntryV1 InodeTableLayoutEntryV1Struct - superBlockVersion uint64 + curPos int + inodeTableLayoutIndex uint64 + inodeTableLayoutLen uint64 + superBlockVersion uint64 ) - superBlockV1 = &SuperBlockV1Struct{ - InodeTableLayout: make([]InodeTableLayoutEntryV1Struct, 0), - } + superBlockV1 = &SuperBlockV1Struct{} curPos = 0 @@ -127,23 +132,28 @@ func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St return } - for (len(superBlockV1Buf) - curPos) > 8 { - inodeTableLayoutEntryV1.ObjectNumber, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + inodeTableLayoutLen, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + + superBlockV1.InodeTableLayout = make([]InodeTableLayoutEntryV1Struct, inodeTableLayoutLen) + + for inodeTableLayoutIndex = 0; inodeTableLayoutIndex < inodeTableLayoutLen; inodeTableLayoutIndex++ { + superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectNumber, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - inodeTableLayoutEntryV1.ObjectSize, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectSize, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - inodeTableLayoutEntryV1.BytesReferenced, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeTableLayout[inodeTableLayoutIndex].BytesReferenced, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - - superBlockV1.InodeTableLayout = append(superBlockV1.InodeTableLayout, inodeTableLayoutEntryV1) } superBlockVersion, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) @@ -254,7 +264,188 @@ func unmarshalInodeHeadVersion(inodeHeadBuf []byte) (inodeHeadVersion uint64, er } func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byte, err error) { - return nil, fmt.Errorf("TODO - marshalInodeHeadV1()") + var ( + accessTimeAsRFC3339String string + attrChangeTimeAsRFC3339String string + creationTimeAsRFC3339String string + curPos int + inodeHeadV1BufLen int + layoutIndex int + linkTableIndex int + modificationTimeAsRFC3339String string + streamTableIndex int + ) + + creationTimeAsRFC3339String = inodeHeadV1.CreationTime.Format(time.RFC3339) + modificationTimeAsRFC3339String = inodeHeadV1.ModificationTime.Format(time.RFC3339) + accessTimeAsRFC3339String = inodeHeadV1.AccessTime.Format(time.RFC3339) + attrChangeTimeAsRFC3339String = inodeHeadV1.AttrChangeTime.Format(time.RFC3339) + + inodeHeadV1BufLen = 8 + 1 + + inodeHeadV1BufLen += 8 + + for linkTableIndex = 0; linkTableIndex < len(inodeHeadV1.LinkTable); linkTableIndex++ { + inodeHeadV1BufLen += 8 + 8 + len(inodeHeadV1.LinkTable[linkTableIndex].ParentDirEntryName) + } + + inodeHeadV1BufLen += 8 + + inodeHeadV1BufLen += (8 + len(creationTimeAsRFC3339String)) + (8 + len(modificationTimeAsRFC3339String)) + (8 + len(accessTimeAsRFC3339String)) + (8 + len(attrChangeTimeAsRFC3339String)) + + inodeHeadV1BufLen += 2 + 8 + 8 + + inodeHeadV1BufLen += 8 + + for streamTableIndex = 0; streamTableIndex < len(inodeHeadV1.StreamTable); streamTableIndex++ { + inodeHeadV1BufLen += 8 + len(inodeHeadV1.StreamTable[streamTableIndex].Name) + 8 + len(inodeHeadV1.StreamTable[streamTableIndex].Value) + } + + inodeHeadV1BufLen += 8 + 8 + 8 + + inodeHeadV1BufLen += 8 + len(inodeHeadV1.SymlinkTarget) + + inodeHeadV1BufLen += 8 + (len(inodeHeadV1.Layout) * (8 + 8 + 8)) + + inodeHeadV1BufLen += 8 + + inodeHeadV1Buf = make([]byte, inodeHeadV1BufLen) + + curPos = 0 + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.InodeNumber) + if nil != err { + return + } + + curPos, err = lePutUint8ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.InodeType) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, uint64(len(inodeHeadV1.LinkTable))) + if nil != err { + return + } + + for linkTableIndex = 0; linkTableIndex < len(inodeHeadV1.LinkTable); linkTableIndex++ { + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.LinkTable[linkTableIndex].ParentDirInodeNumber) + if nil != err { + return + } + + curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.LinkTable[linkTableIndex].ParentDirEntryName) + if nil != err { + return + } + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Size) + if nil != err { + return + } + + curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, creationTimeAsRFC3339String) + if nil != err { + return + } + + curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, modificationTimeAsRFC3339String) + if nil != err { + return + } + + curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, accessTimeAsRFC3339String) + if nil != err { + return + } + + curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, attrChangeTimeAsRFC3339String) + if nil != err { + return + } + + curPos, err = lePutUint16ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Mode) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.UserID) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.GroupID) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, uint64(len(inodeHeadV1.StreamTable))) + if nil != err { + return + } + + for streamTableIndex = 0; streamTableIndex < len(inodeHeadV1.StreamTable); streamTableIndex++ { + curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.StreamTable[streamTableIndex].Name) + if nil != err { + return + } + + curPos, err = lePutByteSliceToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.StreamTable[streamTableIndex].Value) + if nil != err { + return + } + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.PayloadObjectNumber) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.PayloadObjectOffset) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.PayloadObjectLength) + if nil != err { + return + } + + curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.SymlinkTarget) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, uint64(len(inodeHeadV1.Layout))) + if nil != err { + return + } + + for layoutIndex = 0; layoutIndex < len(inodeHeadV1.Layout); layoutIndex++ { + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Layout[layoutIndex].ObjectNumber) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Layout[layoutIndex].ObjectSize) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Layout[layoutIndex].BytesReferenced) + if nil != err { + return + } + } + + curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, InodeHeadVersionV1) + if nil != err { + return + } + + err = nil + return } func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct, err error) { @@ -383,43 +574,86 @@ func lePutUint64ToBuf(buf []byte, curPos int, u64 uint64) (nextPos int, err erro return } -func leGetStringFromBuf(buf []byte, curPos int) (s string, nextPos int, err error) { +func leGetStringFromBuf(buf []byte, curPos int) (str string, nextPos int, err error) { var ( - sLen uint64 + strLen uint64 ) - sLen, nextPos, err = leGetUint64FromBuf(buf, curPos) + strLen, nextPos, err = leGetUint64FromBuf(buf, curPos) if nil != err { return } - if sLen > (uint64(len(buf)) - uint64(nextPos)) { + if strLen > (uint64(len(buf)) - uint64(nextPos)) { err = fmt.Errorf("Insufficient space in buf[curPos:] for string of reported length") return } - s = string(buf[nextPos : nextPos+int(sLen)]) - nextPos += int(sLen) + str = string(buf[nextPos : nextPos+int(strLen)]) + nextPos += int(strLen) err = nil return } -func lePutStringToBuf(buf []byte, curPos int, s string) (nextPos int, err error) { - nextPos, err = lePutUint64ToBuf(buf, curPos, uint64(len(s))) +func lePutStringToBuf(buf []byte, curPos int, str string) (nextPos int, err error) { + nextPos, err = lePutUint64ToBuf(buf, curPos, uint64(len(str))) if nil != err { return } curPos = nextPos - nextPos += len(s) + nextPos += len(str) if nextPos > len(buf) { err = fmt.Errorf("Insufficient space in buf[curPos:] for string") return } - copy(buf[curPos:nextPos], []byte(s)) + copy(buf[curPos:nextPos], []byte(str)) + + err = nil + return +} + +func leGetByteSliceFromBuf(buf []byte, curPos int) (byteSlice []byte, nextPos int, err error) { + var ( + byteSliceLen uint64 + ) + + byteSliceLen, nextPos, err = leGetUint64FromBuf(buf, curPos) + if nil != err { + return + } + + if byteSliceLen > (uint64(len(buf)) - uint64(nextPos)) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for []byte of reported length") + return + } + + byteSlice = make([]byte, byteSliceLen) + copy(byteSlice, buf[nextPos:nextPos+int(byteSliceLen)]) + nextPos += int(byteSliceLen) + + err = nil + return +} + +func lePutByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { + nextPos, err = lePutUint64ToBuf(buf, curPos, uint64(len(byteSlice))) + if nil != err { + return + } + + curPos = nextPos + nextPos += len(byteSlice) + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for []byte") + return + } + + copy(buf[curPos:nextPos], byteSlice) err = nil return From c021100e0116307e567a6ef22d7ca7b156579f7e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 8 Apr 2021 17:52:16 -0700 Subject: [PATCH 120/160] Partially implemented ilayout/api.go unmarshaling for InodeHeadV1Struct...but it has problems --- ilayout/api_test.go | 9 ++- ilayout/impl.go | 188 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 194 insertions(+), 3 deletions(-) diff --git a/ilayout/api_test.go b/ilayout/api_test.go index 1479a2e5a..e575b0f3f 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -109,7 +109,7 @@ func TestAPI(t *testing.T) { marshaledInodeHeadV1 []byte unmarshaledInodeHeadVersion uint64 - // unmarshaledInodeHeadV1 *InodeHeadV1Struct + unmarshaledInodeHeadV1 *InodeHeadV1Struct ) marshaledCheckPointHeaderV1, err = testCheckPointHeaderV1.MarshalCheckPointHeaderV1() @@ -197,5 +197,10 @@ func TestAPI(t *testing.T) { t.Fatalf("Bad unmarshaledInodeHeadVersion (%016X) - expected InodeHeadVersionV1 (%016X)", unmarshaledInodeHeadVersion, InodeHeadVersionV1) } - t.Logf("TODO: test UnmarshalInodeHeadV1()") + unmarshaledInodeHeadV1, err = UnmarshalInodeHeadV1(marshaledInodeHeadV1) + if nil != err { + t.Fatal(err) + } + + t.Logf("TODO: finish testing UnmarshalInodeHeadV1(); %+v", unmarshaledInodeHeadV1) } diff --git a/ilayout/impl.go b/ilayout/impl.go index d3df52ded..67ffda9c6 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -449,7 +449,193 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt } func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct, err error) { - return nil, fmt.Errorf("TODO - unmarshalInodeHeadV1()") + var ( + accessTimeAsRFC3339String string + attrChangeTimeAsRFC3339String string + creationTimeAsRFC3339String string + curPos int + inodeHeadVersion uint64 + layoutIndex uint64 + layoutLen uint64 + linkTableIndex uint64 + linkTableLen uint64 + modificationTimeAsRFC3339String string + streamTableIndex uint64 + streamTableLen uint64 + ) + + inodeHeadV1 = &InodeHeadV1Struct{} + + curPos = 0 + + inodeHeadV1.InodeNumber, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.InodeType, curPos, err = leGetUint8FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + linkTableLen, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.LinkTable = make([]InodeLinkTableEntryStruct, linkTableLen) + + for linkTableIndex = 0; linkTableIndex < linkTableLen; linkTableIndex++ { + inodeHeadV1.LinkTable[linkTableIndex].ParentDirInodeNumber, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.LinkTable[linkTableIndex].ParentDirEntryName, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + } + + inodeHeadV1.Size, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + creationTimeAsRFC3339String, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.CreationTime, err = time.Parse(time.RFC3339, creationTimeAsRFC3339String) + if nil != err { + return + } + + modificationTimeAsRFC3339String, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.ModificationTime, err = time.Parse(time.RFC3339, modificationTimeAsRFC3339String) + if nil != err { + return + } + + accessTimeAsRFC3339String, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.AccessTime, err = time.Parse(time.RFC3339, accessTimeAsRFC3339String) + if nil != err { + return + } + + attrChangeTimeAsRFC3339String, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.AttrChangeTime, err = time.Parse(time.RFC3339, attrChangeTimeAsRFC3339String) + if nil != err { + return + } + + inodeHeadV1.Mode, curPos, err = leGetUint16FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.UserID, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.GroupID, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + streamTableLen, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.StreamTable = make([]InodeStreamTableEntryStruct, streamTableLen) + + for streamTableIndex = 0; streamTableIndex < streamTableLen; streamTableIndex++ { + inodeHeadV1.StreamTable[streamTableIndex].Name, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.StreamTable[streamTableIndex].Value, curPos, err = leGetByteSliceFromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + } + + inodeHeadV1.PayloadObjectNumber, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.PayloadObjectOffset, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.PayloadObjectLength, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.SymlinkTarget, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + layoutLen, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.Layout = make([]InodeHeadLayoutEntryV1Struct, layoutLen) + + for layoutIndex = 0; layoutIndex < layoutLen; layoutIndex++ { + inodeHeadV1.Layout[layoutIndex].ObjectNumber, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.Layout[layoutIndex].ObjectSize, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + + inodeHeadV1.Layout[layoutIndex].BytesReferenced, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + } + + inodeHeadVersion, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + if nil != err { + return + } + if inodeHeadVersion != InodeHeadVersionV1 { + err = fmt.Errorf("Incorrect Version for inodeHeadV1Buf") + return + } + + if curPos != len(inodeHeadV1Buf) { + err = fmt.Errorf("Incorrect size for inodeHeadV1Buf") + return + } + + err = nil + return } func leGetUint8FromBuf(buf []byte, curPos int) (u8 uint8, nextPos int, err error) { From fd5a975f69e48c9d173735604a4e6af3eea424d6 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Thu, 8 Apr 2021 19:51:17 -0700 Subject: [PATCH 121/160] Completed implementation & test of ilayout/api.go unmarshaling for InodeHeadV1Struct Also added statfs() data to SuperBlockV1Struct --- ilayout/api.go | 3 +++ ilayout/api_test.go | 59 ++++++++++++++++++++++++++++++++++++++------- ilayout/impl.go | 42 +++++++++++++++++++++++++++----- 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index 7cecf4943..c193ae319 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -97,6 +97,9 @@ type SuperBlockV1Struct struct { InodeTableRootObjectOffset uint64 // Starting offset in the Object of the root of the InodeTable InodeTableRootObjectLength uint64 // Number of bytes in the Object of the root of the InodeTable InodeTableLayout []InodeTableLayoutEntryV1Struct // Describes the data and space occupied by the the InodeTable + InodeObjectCount uint64 // Number of Objects holding Inodes + InodeObjectSize uint64 // Sum of sizes of all Objects holding Inodes + InodeBytesReferenced uint64 // Sum of bytes referenced in all Objects holding Inodes } func (superBlockV1 *SuperBlockV1Struct) MarshalSuperBlockV1() (superBlockV1Buf []byte, err error) { diff --git a/ilayout/api_test.go b/ilayout/api_test.go index e575b0f3f..4525afa70 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -4,6 +4,7 @@ package ilayout import ( + "bytes" "testing" "time" ) @@ -29,16 +30,19 @@ func TestAPI(t *testing.T) { InodeTableRootObjectLength: 3, InodeTableLayout: []InodeTableLayoutEntryV1Struct{ { - ObjectNumber: 4, - ObjectSize: 5, - BytesReferenced: 6, + ObjectNumber: 11, + ObjectSize: 12, + BytesReferenced: 13, }, { - ObjectNumber: 7, - ObjectSize: 8, - BytesReferenced: 9, + ObjectNumber: 21, + ObjectSize: 22, + BytesReferenced: 23, }, }, + InodeObjectCount: 4, + InodeObjectSize: 5, + InodeBytesReferenced: 6, } marshaledSuperBlockV1 []byte @@ -56,7 +60,7 @@ func TestAPI(t *testing.T) { unmarshaledInodeTableEntryValueVersion uint64 unmarshaledInodeTableEntryValueV1 *InodeTableEntryValueV1Struct - testStartTime = time.Now() + testStartTime = time.Now().Truncate(time.Second) testInodeHeadV1 = &InodeHeadV1Struct{ InodeNumber: 1, @@ -110,6 +114,10 @@ func TestAPI(t *testing.T) { marshaledInodeHeadV1 []byte unmarshaledInodeHeadVersion uint64 unmarshaledInodeHeadV1 *InodeHeadV1Struct + + linkTableIndex int + streamTableIndex int + layoutIndex int ) marshaledCheckPointHeaderV1, err = testCheckPointHeaderV1.MarshalCheckPointHeaderV1() @@ -201,6 +209,39 @@ func TestAPI(t *testing.T) { if nil != err { t.Fatal(err) } - - t.Logf("TODO: finish testing UnmarshalInodeHeadV1(); %+v", unmarshaledInodeHeadV1) + if (testInodeHeadV1.InodeNumber != unmarshaledInodeHeadV1.InodeNumber) || + (testInodeHeadV1.InodeType != unmarshaledInodeHeadV1.InodeType) || + (len(testInodeHeadV1.LinkTable) != len(unmarshaledInodeHeadV1.LinkTable)) || + (testInodeHeadV1.Size != unmarshaledInodeHeadV1.Size) || + (testInodeHeadV1.CreationTime != unmarshaledInodeHeadV1.CreationTime) || + (testInodeHeadV1.ModificationTime != unmarshaledInodeHeadV1.ModificationTime) || + (testInodeHeadV1.AccessTime != unmarshaledInodeHeadV1.AccessTime) || + (testInodeHeadV1.AttrChangeTime != unmarshaledInodeHeadV1.AttrChangeTime) || + (testInodeHeadV1.Mode != unmarshaledInodeHeadV1.Mode) || + (testInodeHeadV1.UserID != unmarshaledInodeHeadV1.UserID) || + (testInodeHeadV1.GroupID != unmarshaledInodeHeadV1.GroupID) || + (len(testInodeHeadV1.StreamTable) != len(unmarshaledInodeHeadV1.StreamTable)) || + (testInodeHeadV1.PayloadObjectNumber != unmarshaledInodeHeadV1.PayloadObjectNumber) || + (testInodeHeadV1.PayloadObjectOffset != unmarshaledInodeHeadV1.PayloadObjectOffset) || + (testInodeHeadV1.PayloadObjectLength != unmarshaledInodeHeadV1.PayloadObjectLength) || + (testInodeHeadV1.SymlinkTarget != unmarshaledInodeHeadV1.SymlinkTarget) || + (len(testInodeHeadV1.Layout) != len(unmarshaledInodeHeadV1.Layout)) { + t.Fatalf("Bad unmarshaledInodeHeadV1 (%+v) - expected testInodeHeadV1 (%+v) [Case 1]", unmarshaledInodeHeadV1, testInodeHeadV1) + } + for linkTableIndex = range testInodeHeadV1.LinkTable { + if testInodeHeadV1.LinkTable[linkTableIndex] != unmarshaledInodeHeadV1.LinkTable[linkTableIndex] { + t.Fatalf("Bad unmarshaledInodeHeadV1 (%+v) - expected testInodeHeadV1 (%+v) [Case 2]", unmarshaledInodeHeadV1, testInodeHeadV1) + } + } + for streamTableIndex = range testInodeHeadV1.StreamTable { + if (testInodeHeadV1.StreamTable[streamTableIndex].Name != unmarshaledInodeHeadV1.StreamTable[streamTableIndex].Name) || + (0 != bytes.Compare(testInodeHeadV1.StreamTable[streamTableIndex].Value, unmarshaledInodeHeadV1.StreamTable[streamTableIndex].Value)) { + t.Fatalf("Bad unmarshaledInodeHeadV1 (%+v) - expected testInodeHeadV1 (%+v) [Case 3]", unmarshaledInodeHeadV1, testInodeHeadV1) + } + } + for layoutIndex = range testInodeHeadV1.Layout { + if testInodeHeadV1.Layout[layoutIndex] != unmarshaledInodeHeadV1.Layout[layoutIndex] { + t.Fatalf("Bad unmarshaledInodeHeadV1 (%+v) - expected testInodeHeadV1 (%+v) [Case 4]", unmarshaledInodeHeadV1, testInodeHeadV1) + } + } } diff --git a/ilayout/impl.go b/ilayout/impl.go index 67ffda9c6..9555b61e8 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -55,7 +55,7 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ inodeTableLayoutIndex int ) - superBlockV1Buf = make([]byte, 8+8+8+8+(len(superBlockV1.InodeTableLayout)*(8+8+8))+8) + superBlockV1Buf = make([]byte, 8+8+8+8+(len(superBlockV1.InodeTableLayout)*(8+8+8))+8+8+8+8) curPos = 0 @@ -96,6 +96,21 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ } } + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeObjectCount) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeObjectSize) + if nil != err { + return + } + + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeBytesReferenced) + if nil != err { + return + } + curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, SuperBlockVersionV1) if nil != err { return @@ -156,6 +171,21 @@ func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St } } + superBlockV1.InodeObjectCount, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + + superBlockV1.InodeObjectSize, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + + superBlockV1.InodeBytesReferenced, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + if nil != err { + return + } + superBlockVersion, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return @@ -750,11 +780,11 @@ func lePutUint64ToBuf(buf []byte, curPos int, u64 uint64) (nextPos int, err erro buf[curPos] = uint8(u64 & 0xFF) buf[curPos+1] = uint8((u64 >> 8) & 0xFF) buf[curPos+2] = uint8((u64 >> 16) & 0xFF) - buf[curPos+1] = uint8((u64 >> 24) & 0xFF) - buf[curPos+2] = uint8((u64 >> 32) & 0xFF) - buf[curPos+1] = uint8((u64 >> 40) & 0xFF) - buf[curPos+2] = uint8((u64 >> 48) & 0xFF) - buf[curPos+3] = uint8(u64 >> 56) + buf[curPos+3] = uint8((u64 >> 24) & 0xFF) + buf[curPos+4] = uint8((u64 >> 32) & 0xFF) + buf[curPos+5] = uint8((u64 >> 40) & 0xFF) + buf[curPos+6] = uint8((u64 >> 48) & 0xFF) + buf[curPos+7] = uint8(u64 >> 56) err = nil return From b4a6d1c1db1e85a8e3d0bc40d5323b7f430da654 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 9 Apr 2021 12:42:02 -0700 Subject: [PATCH 122/160] Added framework for POST to format a volume in imgrpkg imgr/imgrpkg/volume.go::postVolume() is the TODO to complete this support --- imgr/imgr.conf | 48 ++++++++++--------- imgr/imgrpkg/api.go | 94 +++++++++++++++++++++++++------------ imgr/imgrpkg/globals.go | 16 +++++-- imgr/imgrpkg/http-server.go | 70 ++++++++++++++++++++++++--- imgr/imgrpkg/volume.go | 5 ++ 5 files changed, 171 insertions(+), 62 deletions(-) diff --git a/imgr/imgr.conf b/imgr/imgr.conf index 6bdad24cf..7034dae56 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -2,34 +2,36 @@ # SPDX-License-Identifier: Apache-2.0 [IMGR] -PublicIPAddr: 172.28.128.2 -PrivateIPAddr: 172.28.128.2 -RetryRPCPort: 32356 -HTTPServerPort: 15346 +PublicIPAddr: 172.28.128.2 +PrivateIPAddr: 172.28.128.2 +RetryRPCPort: 32356 +HTTPServerPort: 15346 -RetryRPCTTLCompleted: 10m -RetryRPCAckTrim: 100ms -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s +RetryRPCTTLCompleted: 10m +RetryRPCAckTrim: 100ms +RetryRPCDeadlineIO: 60s +RetryRPCKeepAlivePeriod: 60s -RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, -RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used +RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, +RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used -FetchNonceRangeTpReturn: 100 +FetchNonceRangeToReturn: 100 -MinLeaseDuration: 250ms -LeaseInterruptInterval: 250ms -LeaseInterruptLimit: 20 +MinLeaseDuration: 250ms +LeaseInterruptInterval: 250ms +LeaseInterruptLimit: 20 -SwiftRetryDelay: 1s -SwiftRetryExpBackoff: 1.5 -SwiftRetryLimit: 11 +SwiftRetryDelay: 1s +SwiftRetryExpBackoff: 1.5 +SwiftRetryLimit: 11 -SwiftConnectionPoolSize: 128 +SwiftConnectionPoolSize: 128 -InodeTableCacheEvictLowLimit: 10000 -InodeTableCacheEvictHighLimit: 10010 +InodeTableCacheEvictLowLimit: 10000 +InodeTableCacheEvictHighLimit: 10010 -LogFilePath: # imgr.log -LogToConsole: true # false -TraceEnabled: false +InodeTableMaxInodesPerBPlusTreePage: 2048 + +LogFilePath: # imgr.log +LogToConsole: true # false +TraceEnabled: false diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 0dc5e1b97..c4589c006 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -11,37 +11,39 @@ // argument, a package conf ConfMap. Here is a sample .conf file: // // [IMGR] -// PublicIPAddr: 172.28.128.2 -// PrivateIPAddr: 172.28.128.2 -// RetryRPCPort: 32356 -// HTTPServerPort: 15346 +// PublicIPAddr: 172.28.128.2 +// PrivateIPAddr: 172.28.128.2 +// RetryRPCPort: 32356 +// HTTPServerPort: 15346 // -// RetryRPCTTLCompleted: 10m -// RetryRPCAckTrim: 100ms -// RetryRPCDeadlineIO: 60s -// RetryRPCKeepAlivePeriod: 60s +// RetryRPCTTLCompleted: 10m +// RetryRPCAckTrim: 100ms +// RetryRPCDeadlineIO: 60s +// RetryRPCKeepAlivePeriod: 60s // -// RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, -// RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used +// RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, +// RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used // -// FetchNonceRangeTpReturn: 100 +// FetchNonceRangeToReturn: 100 // -// MinLeaseDuration: 250ms -// LeaseInterruptInterval: 250ms -// LeaseInterruptLimit: 20 +// MinLeaseDuration: 250ms +// LeaseInterruptInterval: 250ms +// LeaseInterruptLimit: 20 // -// SwiftRetryDelay: 1s -// SwiftRetryExpBackoff: 1.5 -// SwiftRetryLimit: 11 +// SwiftRetryDelay: 1s +// SwiftRetryExpBackoff: 1.5 +// SwiftRetryLimit: 11 // -// SwiftConnectionPoolSize: 128 +// SwiftConnectionPoolSize: 128 // -// InodeTableCacheEvictLowLimit: 10000 -// InodeTableCacheEvictHighLimit: 10010 +// InodeTableCacheEvictLowLimit: 10000 +// InodeTableCacheEvictHighLimit: 10010 // -// LogFilePath: # imgr.log -// LogToConsole: true # false -// TraceEnabled: false +// InodeTableMaxInodesPerBPlusTreePage: 2048 +// +// LogFilePath: # imgr.log +// LogToConsole: true # false +// TraceEnabled: false // // Most of the config keys are required and must have values. One exception // is LogFilePath that will default to "" and, hence, cause logging to not @@ -55,20 +57,52 @@ // // In addition to the package retryrpc-exposed RPCs, the package also includes // an embedded HTTP Server (at URL http://:) -// responses to the following. +// responses to the following: // // DELETE /volume/ -// - removes the specified from being served +// +// This will cause the specified to no longer be served. Note that +// this does not actually affect the contents of the associated Container. +// // GET /config -// - returns a JSON document of the supplied ConfMap passed to Start() +// +// This will return a JSON document that matches the conf.ConfMap used to +// launch this package. +// // GET /stats -// - returns a package bucketstats dump +// +// This will return a raw bucketstats dump. +// // GET /volume -// - returns a JSON document describing details of the specified +// +// This will return a JSON document containing an array of volumes currently +// being served with details about each. +// // GET /volume/ -// - returns a JSON document describing details of all volumes +// +// This will return a JSON document containing only the specified +// details (assuming it is currently being served). +// +// POST /volume/ +// Content-Type: application/json +// +// { +// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con" +// } +// +// This will cause the specified to be served. The StorageURL +// specified in the JSON document content identifies the Container to first +// format and then serve. +// // PUT /volume/ -// - content should be a StorageURL (including the Container) +// Content-Type: application/json +// +// { +// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con" +// } +// +// This will cause the specified to be served. The StorageURL +// specified in the JSON document content identifies the Container to serve. // package imgrpkg diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 72d23fc01..11da87b5e 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -33,7 +33,7 @@ type configStruct struct { RetryRPCCertFilePath string RetryRPCKeyFilePath string - FetchNonceRangeTpReturn uint64 + FetchNonceRangeToReturn uint64 MinLeaseDuration time.Duration LeaseInterruptInterval time.Duration @@ -48,6 +48,8 @@ type configStruct struct { InodeTableCacheEvictLowLimit uint64 InodeTableCacheEvictHighLimit uint64 + InodeTableMaxInodesPerBPlusTreePage uint64 + LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled LogToConsole bool TraceEnabled bool @@ -75,6 +77,7 @@ type statsStruct struct { GetStatsUsecs bucketstats.BucketLog2Round // GET /stats GetVolumeListUsecs bucketstats.BucketLog2Round // GET /volume GetVolumeUsecs bucketstats.BucketLog2Round // GET /volume/ + PostVolumeUsecs bucketstats.BucketLog2Round // POST /volume/ PutVolumeUsecs bucketstats.BucketLog2Round // PUT /volume/ AdjustInodeTableEntryOpenCountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).AdjustInodeTableEntryOpenCount() @@ -237,7 +240,7 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { } } - globals.config.FetchNonceRangeTpReturn, err = confMap.FetchOptionValueUint64("IMGR", "FetchNonceRangeTpReturn") + globals.config.FetchNonceRangeToReturn, err = confMap.FetchOptionValueUint64("IMGR", "FetchNonceRangeToReturn") if nil != err { logFatal(err) } @@ -282,6 +285,11 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { logFatal(err) } + globals.config.InodeTableMaxInodesPerBPlusTreePage, err = confMap.FetchOptionValueUint64("IMGR", "InodeTableMaxInodesPerBPlusTreePage") + if nil != err { + logFatal(err) + } + globals.config.LogFilePath, err = confMap.FetchOptionValueString("IMGR", "LogFilePath") if nil != err { err = confMap.VerifyOptionValueIsEmpty("IMGR", "LogFilePath") @@ -326,7 +334,7 @@ func uninitializeGlobals() (err error) { globals.config.RetryRPCCertFilePath = "" globals.config.RetryRPCKeyFilePath = "" - globals.config.FetchNonceRangeTpReturn = 0 + globals.config.FetchNonceRangeToReturn = 0 globals.config.MinLeaseDuration = time.Duration(0) globals.config.LeaseInterruptInterval = time.Duration(0) @@ -341,6 +349,8 @@ func uninitializeGlobals() (err error) { globals.config.InodeTableCacheEvictLowLimit = 0 globals.config.InodeTableCacheEvictHighLimit = 0 + globals.config.InodeTableMaxInodesPerBPlusTreePage = 0 + globals.config.LogFilePath = "" globals.config.LogToConsole = false globals.config.TraceEnabled = false diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 1a9f58ef9..67cbb19f4 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -85,6 +85,8 @@ func (dummy *globalsStruct) ServeHTTP(responseWriter http.ResponseWriter, reques serveHTTPDelete(responseWriter, request, requestPath) case http.MethodGet: serveHTTPGet(responseWriter, request, requestPath) + case http.MethodPost: + serveHTTPPost(responseWriter, request, requestPath, requestBody) case http.MethodPut: serveHTTPPut(responseWriter, request, requestPath, requestBody) default: @@ -245,6 +247,54 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ } } +func serveHTTPPost(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { + switch { + case strings.HasPrefix(requestPath, "/volume"): + serveHTTPPostOfVolume(responseWriter, request, requestPath, requestBody) + default: + responseWriter.WriteHeader(http.StatusNotFound) + } +} + +type serveHTTPPostOfVolumeRequestBodyAsJSONStruct struct { + StorageURL string +} + +func serveHTTPPostOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { + var ( + err error + pathSplit []string + requestBodyAsJSON serveHTTPPostOfVolumeRequestBodyAsJSONStruct + startTime time.Time + ) + + startTime = time.Now() + + pathSplit = strings.Split(requestPath, "/") + + switch len(pathSplit) { + case 3: + defer func() { + globals.stats.PostVolumeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + err = json.Unmarshal(requestBody, &requestBodyAsJSON) + if nil != err { + responseWriter.WriteHeader(http.StatusBadRequest) + return + } + + err = postVolume(pathSplit[2], requestBodyAsJSON.StorageURL) + if nil == err { + responseWriter.WriteHeader(http.StatusCreated) + } else { + responseWriter.WriteHeader(http.StatusConflict) + } + default: + responseWriter.WriteHeader(http.StatusBadRequest) + } +} + func serveHTTPPut(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { switch { case strings.HasPrefix(requestPath, "/volume"): @@ -254,12 +304,16 @@ func serveHTTPPut(responseWriter http.ResponseWriter, request *http.Request, req } } +type serveHTTPPutOfVolumeRequestBodyAsJSONStruct struct { + StorageURL string +} + func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { var ( - err error - pathSplit []string - startTime time.Time - storageURL string + err error + pathSplit []string + requestBodyAsJSON serveHTTPPutOfVolumeRequestBodyAsJSONStruct + startTime time.Time ) startTime = time.Now() @@ -272,9 +326,13 @@ func serveHTTPPutOfVolume(responseWriter http.ResponseWriter, request *http.Requ globals.stats.PutVolumeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - storageURL = string(requestBody[:]) + err = json.Unmarshal(requestBody, &requestBodyAsJSON) + if nil != err { + responseWriter.WriteHeader(http.StatusBadRequest) + return + } - err = putVolume(pathSplit[2], storageURL) + err = putVolume(pathSplit[2], requestBodyAsJSON.StorageURL) if nil == err { responseWriter.WriteHeader(http.StatusCreated) } else { diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 6ebc412b9..2427dca8f 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -208,6 +208,11 @@ func getVolumeListAsJSON() (volumeList []byte) { return } +func postVolume(name string, storageURL string) (err error) { + err = fmt.Errorf("TODO") + return +} + func putVolume(name string, storageURL string) (err error) { var ( ok bool From 1f15c1f81e0c997c831c2a90442cb69cf7b1eec9 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 9 Apr 2021 19:58:06 -0700 Subject: [PATCH 123/160] Lots of work to complete imgr/imgrpkg/volume.go::postVolume()... Also added a bunch of text in the Overview of Package ilayout --- ilayout/api.go | 232 ++++++++++++++++++++- ilayout/api_test.go | 50 ++++- ilayout/impl.go | 389 ++++++++++++++++++++++-------------- imgr/imgrpkg/api.go | 3 +- imgr/imgrpkg/http-server.go | 3 +- imgr/imgrpkg/volume.go | 233 ++++++++++++++++++++- 6 files changed, 753 insertions(+), 157 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index c193ae319..81eb980b1 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -1,6 +1,59 @@ // Copyright (c) 2015-2021, NVIDIA CORPORATION. // SPDX-License-Identifier: Apache-2.0 +// Package ilayout specifies the "on disk" format of the file system metadata. +// +// The file system heavily leverages package sortedmap to provide a pageable +// B+Tree metadata system capable of very large inode counts and sizes. Several +// data structures are also "versioned" to enable gradual modification. +// +// The structure of the file system assumes an underlying object store system +// (e.g. an S3 Bucket or OpenStack Swift Container) that may or may not support +// consistency. As such, no Object (or any other numerically identified entity +// in the file system) is ever written twice. This means that the file system +// operates in a write journaled mode. +// +// At the top level, the file system periodically checkpoints. This enables the +// file system to be complemented by the extreme scale of such Object Store +// systems despite such often suffering long latencies and coarse granularity +// of writes. The CheckPointHeader is the changing indication of the most +// recent checkpoint performed. +// +// From the latest checkpoint, the file system's SuperBlock is located. This +// SuperBlock is responsible for tracking the location, in the Object Store, +// of the Inodes (identified by InodeNumber's). This is the first use case +// for the pageable B+Tree mechanism. +// +// The file system's Inodes are stored in unique Objects. The "tail" of the +// most recent Object written for the Inode contains the Inode's state +// (e.g. Mode, UserID, CreationTime, etc...). The Inode's "type" will be +// either a Directory, a normal File, or a Symbolic Link. +// +// For a Directory Inodes, large directory entry counts call for a second +// use case for the pageable B+Tree mechanism. Each entry represents a +// mapping from a "basename" to another Inode in the file system. As an +// optimization, this mapping also records the InodeType of the refernced +// Inode. This Directory B+Tree is stored in the same sequence of Objects +// as that holding the Directory Inode's state. +// +// For normal File Inodes, large amounts of data must be managed. File data +// is represented as a sequence of "extents" that may be individually modest +// in size but may be very large in number. This requirement presents the +// third use case for the pageable B+Tree mechanism. This time, the entries +// in the ExtentMap B+Tree are mappings from File "offsets" and "extent lengths" +// to elsewhere in the Object Store objects where the actual File data is held. +// Both the ExtentMap B+Tree and the File data extents are stored in the same +// sequence of Objects that hold the File Inode's state. +// +// For Symbolic Link Inodes, there is no need for anything more than the +// Inode's "state". Here, the "state" adds in the SymLinkTarget. +// +// In addition to structures and constants laying out the file system's "on-disk" +// format, several marshaling func's are provided to convert between this +// "on disk" format and an "in memory" equivalent. These func's are both high +// level (e.g. "superblock") and low level (e.g. uint64) to assist in managing +// the "on disk" representation of the file system. +// package ilayout import ( @@ -36,11 +89,15 @@ type CheckPointHeaderV1Struct struct { ReservedToNonce uint64 // Ensures all numbers requiring uniqueness (e.g. Object numbers, Inode numbers) are never reused } +// MarshalCheckPointHeaderV1 encodes checkPointHeaderV1 to checkpointHeaderString. +// func (checkPointHeaderV1 *CheckPointHeaderV1Struct) MarshalCheckPointHeaderV1() (checkPointHeaderV1String string, err error) { checkPointHeaderV1String, err = checkPointHeaderV1.marshalCheckPointHeaderV1() return } +// UnmarshalCheckPointHeaderV1 decodes checkPointHeaderV1 from checkpointHeaderString. +// func UnmarshalCheckPointHeaderV1(checkPointHeaderV1String string) (checkPointHeaderV1 *CheckPointHeaderV1Struct, err error) { checkPointHeaderV1, err = unmarshalCheckPointHeaderV1(checkPointHeaderV1String) return @@ -102,11 +159,15 @@ type SuperBlockV1Struct struct { InodeBytesReferenced uint64 // Sum of bytes referenced in all Objects holding Inodes } +// MarshalSuperBlockV1 encodes superBlockV1 to superBlockV1Buf. +// func (superBlockV1 *SuperBlockV1Struct) MarshalSuperBlockV1() (superBlockV1Buf []byte, err error) { superBlockV1Buf, err = superBlockV1.marshalSuperBlockV1() return } +// UnmarshalSuperBlockV1 decodes superBlockV1 from superBlockV1Buf. +// func UnmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1Struct, err error) { superBlockV1, err = unmarshalSuperBlockV1(superBlockV1Buf) return @@ -140,11 +201,15 @@ type InodeTableEntryValueV1Struct struct { InodeHeadLength uint64 // Total length of the InodeHead found at the end of the Object indicated by InodeHeadObjectNumber } +// MarshalInodeTableEntryValueV1 encodes inodeTableEntryValueV1 to inodeTableEntryValueV1Buf. +// func (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct) MarshalInodeTableEntryValueV1() (inodeTableEntryValueV1Buf []byte, err error) { inodeTableEntryValueV1Buf, err = inodeTableEntryValueV1.marshalInodeTableEntryValueV1() return } +// UnmarshalInodeTableEntryValueV1 decodes inodeTableEntryValueV1 from inodeTableEntryValueV1Buf. +// func UnmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct, err error) { inodeTableEntryValueV1, err = unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf) return @@ -166,12 +231,18 @@ func UnmarshalInodeHeadVersion(inodeHeadBuf []byte) (inodeHeadVersion uint64, er return } +// RootDirInodeNumber is the InodeNumber for the directory at the root of the file system. +// +const ( + RootDirInodeNumber uint64 = 1 +) + // InodeType* specifies the type of Inode. // const ( InodeTypeDir uint8 = 0 InodeTypeFile uint8 = 1 - InodeTypeSymlink uint8 = 2 + InodeTypeSymLink uint8 = 2 ) // InodeLinkTableEntryStruct specifies the layout of an InodeHeadV1Struct.LinkTable's entry. @@ -228,8 +299,7 @@ type InodeHeadLayoutEntryV1Struct struct { // For uint* fields, LittleEndian format is used. // For table fields, a uint64 length in LittleEndian format is followed by the serialization // specified in the table entry struct. -// For time.Time fields, a string (i.e. uint64 length followed by that number of bytes) is used. -// The string is itself time.RFC3339 formatted. +// For time.Time fields, a uint64 in LittleEndian is used to hold the UnixNano() equivalent. // // Note that the InodeTableEntryValueV1Struct.InodeHeadLength also includes the bytes for // holding InodeHeadVersionV1 that is appended. @@ -250,15 +320,19 @@ type InodeHeadV1Struct struct { PayloadObjectNumber uint64 // For Dir & File Inodes, identifies the Object containing the root of the Directory or ExtentMap B+Tree PayloadObjectOffset uint64 // For Dir & File Inodes, starting offset in the Object of the root of the Directory or ExtentMap B+Tree PayloadObjectLength uint64 // For Dir & File Inodes, number of bytes in the Object of the root of the Directory or ExtentMap B+Tree - SymlinkTarget string // For Symlink Inodes, the target of the link + SymLinkTarget string // For SymLink Inodes, the target of the link Layout []InodeHeadLayoutEntryV1Struct // Describes the data and space occupied by the the InodeTable } +// MarshalInodeHeadV1 encodes inodeHeadV1 to inodeHeadV1Buf. +// func (inodeHeadV1 *InodeHeadV1Struct) MarshalInodeHeadV1() (inodeHeadV1Buf []byte, err error) { inodeHeadV1Buf, err = inodeHeadV1.marshalInodeHeadV1() return } +// UnmarshalInodeHeadV1 decodes inodeHeadV1 from inodeHeadV1Buf. +// func UnmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct, err error) { inodeHeadV1, err = unmarshalInodeHeadV1(inodeHeadV1Buf) return @@ -277,6 +351,20 @@ type DirectoryEntryValueV1Struct struct { InodeType uint8 } +// MarshalDirectoryEntryValueV1 encodes directoryEntryValueV1 to directoryEntryValueV1Buf. +// +func (directoryEntryValueV1 *DirectoryEntryValueV1Struct) MarshalDirectoryEntryValueV1() (directoryEntryValueV1Buf []byte, err error) { + directoryEntryValueV1Buf, err = directoryEntryValueV1.marshalDirectoryEntryValueV1() + return +} + +// UnmarshalDirectoryEntryValueV1 decodes directoryEntryValueV1 from directoryEntryValueV1Buf. +// +func UnmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf []byte) (directoryEntryValueV1 *DirectoryEntryValueV1Struct, err error) { + directoryEntryValueV1, err = unmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf) + return +} + // ExtentMapEntryValueV1Struct specifies the format, for an Inode of type InodeTypeFile, // of the bytes in a .Payload-identified B+Tree's Value. // @@ -291,3 +379,139 @@ type ExtentMapEntryValueV1Struct struct { ObjectNumber uint64 // Identifies the Object containing this extent's data ObjectOffset uint64 // Starting offset in the Object of this extent's data } + +// MarshalExtentMapEntryValueV1 encodes directoryEntryValueV1 to directoryEntryValueV1Buf. +// +func (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct) MarshalExtentMapEntryValueV1() (extentMapEntryValueV1Buf []byte, err error) { + extentMapEntryValueV1Buf, err = extentMapEntryValueV1.marshalExtentMapEntryValueV1() + return +} + +// UnmarshalExtentMapEntryValueV1 decodes directoryEntryValueV1 from directoryEntryValueV1Buf. +// +func UnmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf []byte) (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct, err error) { + extentMapEntryValueV1, err = unmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf) + return +} + +// GetLEUint8FromBuf fetches a uint8 from buf starting at curPos. +// +// The returned nextPos indicates where the next field (if any) should be read from. +// +func GetLEUint8FromBuf(buf []byte, curPos int) (u8 uint8, nextPos int, err error) { + u8, nextPos, err = getLEUint8FromBuf(buf, curPos) + return +} + +// PutLEUint8ToBuf writes a uint8 to buf starting at curPos. +// +// The returned nextPost indicates where the next field (if any) should be written. +// +func PutLEUint8ToBuf(buf []byte, curPos int, u8 uint8) (nextPos int, err error) { + nextPos, err = putLEUint8ToBuf(buf, curPos, u8) + return +} + +// GetLEUint16FromBuf fetches a uint16 from buf starting at curPos. +// +// The uint16 is assumed to have been written in LittleEndian byte order. +// The returned nextPos indicates where the next field (if any) should be read from. +// +func GetLEUint16FromBuf(buf []byte, curPos int) (u16 uint16, nextPos int, err error) { + u16, nextPos, err = getLEUint16FromBuf(buf, curPos) + return +} + +// PutLEUint16ToBuf writes a uint16 to buf starting at curPos. +// +// The uint16 is written in LittleEndian byte order. +// The returned nextPost indicates where the next field (if any) should be written. +// +func PutLEUint16ToBuf(buf []byte, curPos int, u16 uint16) (nextPos int, err error) { + nextPos, err = putLEUint16ToBuf(buf, curPos, u16) + return +} + +// GetLEUint32FromBuf fetches a uint32 from buf starting at curPos. +// +// The uint32 is assumed to have been written in LittleEndian byte order. +// The returned nextPos indicates where the next field (if any) should be read from. +// +func GetLEUint32FromBuf(buf []byte, curPos int) (u32 uint32, nextPos int, err error) { + u32, nextPos, err = getLEUint32FromBuf(buf, curPos) + return +} + +// PutLEUint32ToBuf writes a uint32 to buf starting at curPos. +// +// The uint32 is written in LittleEndian byte order. +// The returned nextPost indicates where the next field (if any) should be written. +// +func PutLEUint32ToBuf(buf []byte, curPos int, u32 uint32) (nextPos int, err error) { + nextPos, err = putLEUint32ToBuf(buf, curPos, u32) + return +} + +// GetLEUint64FromBuf fetches a uint64 from buf starting at curPos. +// +// The uint64 is assumed to have been written in LittleEndian byte order. +// The returned nextPos indicates where the next field (if any) should be read from. +// +func GetLEUint64FromBuf(buf []byte, curPos int) (u64 uint64, nextPos int, err error) { + u64, nextPos, err = getLEUint64FromBuf(buf, curPos) + return +} + +// PutLEUint64ToBuf writes a uint64 to buf starting at curPos. +// +// The uint64 is written in LittleEndian byte order. +// The returned nextPost indicates where the next field (if any) should be written. +// +func PutLEUint64ToBuf(buf []byte, curPos int, u64 uint64) (nextPos int, err error) { + nextPos, err = putLEUint64ToBuf(buf, curPos, u64) + return +} + +// GetLEStringFromBuf fetches a string from buf starting at curPos. +// +// The string is assumed to have been written as a LittleEndian byte order uint64 +// length followed by the bytes that make up the string. The returned nextPos +// indicates where the next field (if any) should be read from. +// +func GetLEStringFromBuf(buf []byte, curPos int) (str string, nextPos int, err error) { + str, nextPos, err = getLEStringFromBuf(buf, curPos) + return +} + +// PutLEStringToBuf writes a string to buf starting at curPos. +// +// The string is written as a LittleEndian byte order uint64 length followed by the +// bytes that make up the string. The returned nextPost indicates where the next field +// (if any) should be written. +// +func PutLEStringToBuf(buf []byte, curPos int, str string) (nextPos int, err error) { + nextPos, err = putLEStringToBuf(buf, curPos, str) + return +} + +// GetLEByteSliceFromBuf fetches a []byte from buf starting at curPos. +// +// The []byte is assumed to have been written as a LittleEndian byte order uint64 +// length followed by the bytes that make up the []byte. The returned nextPos +// indicates where the next field (if any) should be read from. +// +func GetLEByteSliceFromBuf(buf []byte, curPos int) (byteSlice []byte, nextPos int, err error) { + byteSlice, nextPos, err = getLEByteSliceFromBuf(buf, curPos) + return +} + +// PutLEByteSliceToBuf writes a []byte to buf starting at curPos. +// +// The []byte is written aas a LittleEndian byte order uint64 length followed by the +// bytes that make up the []byte. The returned nextPost indicates where the next field +// (if any) should be written. +// +func PutLEByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { + nextPos, err = putLEByteSliceToBuf(buf, curPos, byteSlice) + return +} diff --git a/ilayout/api_test.go b/ilayout/api_test.go index 4525afa70..efd61e520 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -64,7 +64,7 @@ func TestAPI(t *testing.T) { testInodeHeadV1 = &InodeHeadV1Struct{ InodeNumber: 1, - InodeType: InodeTypeSymlink, + InodeType: InodeTypeSymLink, LinkTable: []InodeLinkTableEntryStruct{ { ParentDirInodeNumber: 101, @@ -96,7 +96,7 @@ func TestAPI(t *testing.T) { PayloadObjectNumber: 11, PayloadObjectOffset: 12, PayloadObjectLength: 13, - SymlinkTarget: "sym-target", + SymLinkTarget: "sym-link-target", Layout: []InodeHeadLayoutEntryV1Struct{ { ObjectNumber: 311, @@ -118,6 +118,24 @@ func TestAPI(t *testing.T) { linkTableIndex int streamTableIndex int layoutIndex int + + testDirectoryEntryValueV1 = &DirectoryEntryValueV1Struct{ + InodeNumber: 1, + InodeType: 2, + } + + marshaledDirectoryEntryValueV1 []byte + unmarshaledDirectoryEntryValueV1 *DirectoryEntryValueV1Struct + + testExtentMapEntryValueV1 = &ExtentMapEntryValueV1Struct{ + FileOffset: 1, + Length: 2, + ObjectNumber: 3, + ObjectOffset: 4, + } + + marshaledExtentMapEntryValueV1 []byte + unmarshaledExtentMapEntryValueV1 *ExtentMapEntryValueV1Struct ) marshaledCheckPointHeaderV1, err = testCheckPointHeaderV1.MarshalCheckPointHeaderV1() @@ -224,7 +242,7 @@ func TestAPI(t *testing.T) { (testInodeHeadV1.PayloadObjectNumber != unmarshaledInodeHeadV1.PayloadObjectNumber) || (testInodeHeadV1.PayloadObjectOffset != unmarshaledInodeHeadV1.PayloadObjectOffset) || (testInodeHeadV1.PayloadObjectLength != unmarshaledInodeHeadV1.PayloadObjectLength) || - (testInodeHeadV1.SymlinkTarget != unmarshaledInodeHeadV1.SymlinkTarget) || + (testInodeHeadV1.SymLinkTarget != unmarshaledInodeHeadV1.SymLinkTarget) || (len(testInodeHeadV1.Layout) != len(unmarshaledInodeHeadV1.Layout)) { t.Fatalf("Bad unmarshaledInodeHeadV1 (%+v) - expected testInodeHeadV1 (%+v) [Case 1]", unmarshaledInodeHeadV1, testInodeHeadV1) } @@ -244,4 +262,30 @@ func TestAPI(t *testing.T) { t.Fatalf("Bad unmarshaledInodeHeadV1 (%+v) - expected testInodeHeadV1 (%+v) [Case 4]", unmarshaledInodeHeadV1, testInodeHeadV1) } } + + marshaledDirectoryEntryValueV1, err = testDirectoryEntryValueV1.MarshalDirectoryEntryValueV1() + if nil != err { + t.Fatal(err) + } + + unmarshaledDirectoryEntryValueV1, err = UnmarshalDirectoryEntryValueV1(marshaledDirectoryEntryValueV1) + if nil != err { + t.Fatal(err) + } + if *testDirectoryEntryValueV1 != *unmarshaledDirectoryEntryValueV1 { + t.Fatalf("Bad unmarshaledDirectoryEntryValueV1 (%+v) - expected testDirectoryEntryValueV1 (%+v)", unmarshaledDirectoryEntryValueV1, testDirectoryEntryValueV1) + } + + marshaledExtentMapEntryValueV1, err = testExtentMapEntryValueV1.MarshalExtentMapEntryValueV1() + if nil != err { + t.Fatal(err) + } + + unmarshaledExtentMapEntryValueV1, err = UnmarshalExtentMapEntryValueV1(marshaledExtentMapEntryValueV1) + if nil != err { + t.Fatal(err) + } + if *testExtentMapEntryValueV1 != *unmarshaledExtentMapEntryValueV1 { + t.Fatalf("Bad unmarshaledExtentMapEntryValueV1 (%+v) - expected testExtentMapEntryValueV1 (%+v)", unmarshaledExtentMapEntryValueV1, testExtentMapEntryValueV1) + } } diff --git a/ilayout/impl.go b/ilayout/impl.go index 9555b61e8..70d7184b9 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -44,7 +44,7 @@ func unmarshalSuperBlockVersion(superBlockBuf []byte) (superBlockVersion uint64, return } - superBlockVersion, _, err = leGetUint64FromBuf(superBlockBuf, superBlockBufLen-8) + superBlockVersion, _, err = getLEUint64FromBuf(superBlockBuf, superBlockBufLen-8) return } @@ -59,59 +59,59 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ curPos = 0 - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableRootObjectNumber) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableRootObjectNumber) if nil != err { return } - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableRootObjectOffset) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableRootObjectOffset) if nil != err { return } - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableRootObjectLength) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableRootObjectLength) if nil != err { return } - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, uint64(len(superBlockV1.InodeTableLayout))) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, uint64(len(superBlockV1.InodeTableLayout))) if nil != err { return } for inodeTableLayoutIndex = 0; inodeTableLayoutIndex < len(superBlockV1.InodeTableLayout); inodeTableLayoutIndex++ { - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectNumber) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectNumber) if nil != err { return } - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectSize) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectSize) if nil != err { return } - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableLayout[inodeTableLayoutIndex].BytesReferenced) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeTableLayout[inodeTableLayoutIndex].BytesReferenced) if nil != err { return } } - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeObjectCount) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeObjectCount) if nil != err { return } - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeObjectSize) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeObjectSize) if nil != err { return } - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeBytesReferenced) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, superBlockV1.InodeBytesReferenced) if nil != err { return } - curPos, err = lePutUint64ToBuf(superBlockV1Buf, curPos, SuperBlockVersionV1) + curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, SuperBlockVersionV1) if nil != err { return } @@ -132,22 +132,22 @@ func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St curPos = 0 - superBlockV1.InodeTableRootObjectNumber, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeTableRootObjectNumber, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - superBlockV1.InodeTableRootObjectOffset, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeTableRootObjectOffset, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - superBlockV1.InodeTableRootObjectLength, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeTableRootObjectLength, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - inodeTableLayoutLen, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + inodeTableLayoutLen, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } @@ -155,38 +155,38 @@ func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St superBlockV1.InodeTableLayout = make([]InodeTableLayoutEntryV1Struct, inodeTableLayoutLen) for inodeTableLayoutIndex = 0; inodeTableLayoutIndex < inodeTableLayoutLen; inodeTableLayoutIndex++ { - superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectNumber, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectNumber, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectSize, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeTableLayout[inodeTableLayoutIndex].ObjectSize, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - superBlockV1.InodeTableLayout[inodeTableLayoutIndex].BytesReferenced, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeTableLayout[inodeTableLayoutIndex].BytesReferenced, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } } - superBlockV1.InodeObjectCount, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeObjectCount, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - superBlockV1.InodeObjectSize, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeObjectSize, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - superBlockV1.InodeBytesReferenced, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockV1.InodeBytesReferenced, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } - superBlockVersion, curPos, err = leGetUint64FromBuf(superBlockV1Buf, curPos) + superBlockVersion, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) if nil != err { return } @@ -205,7 +205,7 @@ func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St } func unmarshalInodeTableEntryValueVersion(inodeTableEntryValueBuf []byte) (inodeTableEntryValueVersion uint64, err error) { - inodeTableEntryValueVersion, _, err = leGetUint64FromBuf(inodeTableEntryValueBuf, 0) + inodeTableEntryValueVersion, _, err = getLEUint64FromBuf(inodeTableEntryValueBuf, 0) return } @@ -219,17 +219,17 @@ func (inodeTableEntryValueV1 *InodeTableEntryValueV1Struct) marshalInodeTableEnt curPos = 0 - curPos, err = lePutUint64ToBuf(inodeTableEntryValueV1Buf, curPos, InodeTableEntryValueVersionV1) + curPos, err = putLEUint64ToBuf(inodeTableEntryValueV1Buf, curPos, InodeTableEntryValueVersionV1) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeTableEntryValueV1Buf, curPos, inodeTableEntryValueV1.InodeHeadObjectNumber) + curPos, err = putLEUint64ToBuf(inodeTableEntryValueV1Buf, curPos, inodeTableEntryValueV1.InodeHeadObjectNumber) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeTableEntryValueV1Buf, curPos, inodeTableEntryValueV1.InodeHeadLength) + curPos, err = putLEUint64ToBuf(inodeTableEntryValueV1Buf, curPos, inodeTableEntryValueV1.InodeHeadLength) if nil != err { return } @@ -246,7 +246,7 @@ func unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTab curPos = 0 - inodeTableEntryValueVersion, curPos, err = leGetUint64FromBuf(inodeTableEntryValueV1Buf, curPos) + inodeTableEntryValueVersion, curPos, err = getLEUint64FromBuf(inodeTableEntryValueV1Buf, curPos) if nil != err { return } @@ -257,12 +257,12 @@ func unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTab inodeTableEntryValueV1 = &InodeTableEntryValueV1Struct{} - inodeTableEntryValueV1.InodeHeadObjectNumber, curPos, err = leGetUint64FromBuf(inodeTableEntryValueV1Buf, curPos) + inodeTableEntryValueV1.InodeHeadObjectNumber, curPos, err = getLEUint64FromBuf(inodeTableEntryValueV1Buf, curPos) if nil != err { return } - inodeTableEntryValueV1.InodeHeadLength, curPos, err = leGetUint64FromBuf(inodeTableEntryValueV1Buf, curPos) + inodeTableEntryValueV1.InodeHeadLength, curPos, err = getLEUint64FromBuf(inodeTableEntryValueV1Buf, curPos) if nil != err { return } @@ -288,29 +288,20 @@ func unmarshalInodeHeadVersion(inodeHeadBuf []byte) (inodeHeadVersion uint64, er return } - inodeHeadVersion, _, err = leGetUint64FromBuf(inodeHeadBuf, inodeHeadBufLen-8) + inodeHeadVersion, _, err = getLEUint64FromBuf(inodeHeadBuf, inodeHeadBufLen-8) return } func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byte, err error) { var ( - accessTimeAsRFC3339String string - attrChangeTimeAsRFC3339String string - creationTimeAsRFC3339String string - curPos int - inodeHeadV1BufLen int - layoutIndex int - linkTableIndex int - modificationTimeAsRFC3339String string - streamTableIndex int + curPos int + inodeHeadV1BufLen int + layoutIndex int + linkTableIndex int + streamTableIndex int ) - creationTimeAsRFC3339String = inodeHeadV1.CreationTime.Format(time.RFC3339) - modificationTimeAsRFC3339String = inodeHeadV1.ModificationTime.Format(time.RFC3339) - accessTimeAsRFC3339String = inodeHeadV1.AccessTime.Format(time.RFC3339) - attrChangeTimeAsRFC3339String = inodeHeadV1.AttrChangeTime.Format(time.RFC3339) - inodeHeadV1BufLen = 8 + 1 inodeHeadV1BufLen += 8 @@ -319,11 +310,7 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt inodeHeadV1BufLen += 8 + 8 + len(inodeHeadV1.LinkTable[linkTableIndex].ParentDirEntryName) } - inodeHeadV1BufLen += 8 - - inodeHeadV1BufLen += (8 + len(creationTimeAsRFC3339String)) + (8 + len(modificationTimeAsRFC3339String)) + (8 + len(accessTimeAsRFC3339String)) + (8 + len(attrChangeTimeAsRFC3339String)) - - inodeHeadV1BufLen += 2 + 8 + 8 + inodeHeadV1BufLen += 8 + 8 + 8 + 8 + 8 + 2 + 8 + 8 inodeHeadV1BufLen += 8 @@ -333,7 +320,7 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt inodeHeadV1BufLen += 8 + 8 + 8 - inodeHeadV1BufLen += 8 + len(inodeHeadV1.SymlinkTarget) + inodeHeadV1BufLen += 8 + len(inodeHeadV1.SymLinkTarget) inodeHeadV1BufLen += 8 + (len(inodeHeadV1.Layout) * (8 + 8 + 8)) @@ -343,133 +330,133 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt curPos = 0 - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.InodeNumber) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.InodeNumber) if nil != err { return } - curPos, err = lePutUint8ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.InodeType) + curPos, err = putLEUint8ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.InodeType) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, uint64(len(inodeHeadV1.LinkTable))) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(len(inodeHeadV1.LinkTable))) if nil != err { return } for linkTableIndex = 0; linkTableIndex < len(inodeHeadV1.LinkTable); linkTableIndex++ { - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.LinkTable[linkTableIndex].ParentDirInodeNumber) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.LinkTable[linkTableIndex].ParentDirInodeNumber) if nil != err { return } - curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.LinkTable[linkTableIndex].ParentDirEntryName) + curPos, err = putLEStringToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.LinkTable[linkTableIndex].ParentDirEntryName) if nil != err { return } } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Size) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Size) if nil != err { return } - curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, creationTimeAsRFC3339String) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(inodeHeadV1.CreationTime.UnixNano())) if nil != err { return } - curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, modificationTimeAsRFC3339String) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(inodeHeadV1.ModificationTime.UnixNano())) if nil != err { return } - curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, accessTimeAsRFC3339String) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(inodeHeadV1.AccessTime.UnixNano())) if nil != err { return } - curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, attrChangeTimeAsRFC3339String) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(inodeHeadV1.AttrChangeTime.UnixNano())) if nil != err { return } - curPos, err = lePutUint16ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Mode) + curPos, err = putLEUint16ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Mode) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.UserID) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.UserID) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.GroupID) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.GroupID) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, uint64(len(inodeHeadV1.StreamTable))) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(len(inodeHeadV1.StreamTable))) if nil != err { return } for streamTableIndex = 0; streamTableIndex < len(inodeHeadV1.StreamTable); streamTableIndex++ { - curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.StreamTable[streamTableIndex].Name) + curPos, err = putLEStringToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.StreamTable[streamTableIndex].Name) if nil != err { return } - curPos, err = lePutByteSliceToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.StreamTable[streamTableIndex].Value) + curPos, err = putLEByteSliceToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.StreamTable[streamTableIndex].Value) if nil != err { return } } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.PayloadObjectNumber) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.PayloadObjectNumber) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.PayloadObjectOffset) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.PayloadObjectOffset) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.PayloadObjectLength) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.PayloadObjectLength) if nil != err { return } - curPos, err = lePutStringToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.SymlinkTarget) + curPos, err = putLEStringToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.SymLinkTarget) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, uint64(len(inodeHeadV1.Layout))) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, uint64(len(inodeHeadV1.Layout))) if nil != err { return } for layoutIndex = 0; layoutIndex < len(inodeHeadV1.Layout); layoutIndex++ { - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Layout[layoutIndex].ObjectNumber) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Layout[layoutIndex].ObjectNumber) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Layout[layoutIndex].ObjectSize) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Layout[layoutIndex].ObjectSize) if nil != err { return } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Layout[layoutIndex].BytesReferenced) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, inodeHeadV1.Layout[layoutIndex].BytesReferenced) if nil != err { return } } - curPos, err = lePutUint64ToBuf(inodeHeadV1Buf, curPos, InodeHeadVersionV1) + curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, InodeHeadVersionV1) if nil != err { return } @@ -480,35 +467,35 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct, err error) { var ( - accessTimeAsRFC3339String string - attrChangeTimeAsRFC3339String string - creationTimeAsRFC3339String string - curPos int - inodeHeadVersion uint64 - layoutIndex uint64 - layoutLen uint64 - linkTableIndex uint64 - linkTableLen uint64 - modificationTimeAsRFC3339String string - streamTableIndex uint64 - streamTableLen uint64 + accessTimeAsUnixTimeInNs uint64 + attrChangeTimeAsUnixTimeInNs uint64 + creationTimeAsUnixTimeInNs uint64 + curPos int + inodeHeadVersion uint64 + layoutIndex uint64 + layoutLen uint64 + linkTableIndex uint64 + linkTableLen uint64 + modificationTimeAsUnixTimeInNs uint64 + streamTableIndex uint64 + streamTableLen uint64 ) inodeHeadV1 = &InodeHeadV1Struct{} curPos = 0 - inodeHeadV1.InodeNumber, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.InodeNumber, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.InodeType, curPos, err = leGetUint8FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.InodeType, curPos, err = getLEUint8FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - linkTableLen, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + linkTableLen, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } @@ -516,78 +503,66 @@ func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct inodeHeadV1.LinkTable = make([]InodeLinkTableEntryStruct, linkTableLen) for linkTableIndex = 0; linkTableIndex < linkTableLen; linkTableIndex++ { - inodeHeadV1.LinkTable[linkTableIndex].ParentDirInodeNumber, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.LinkTable[linkTableIndex].ParentDirInodeNumber, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.LinkTable[linkTableIndex].ParentDirEntryName, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.LinkTable[linkTableIndex].ParentDirEntryName, curPos, err = getLEStringFromBuf(inodeHeadV1Buf, curPos) if nil != err { return } } - inodeHeadV1.Size, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.Size, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - creationTimeAsRFC3339String, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + creationTimeAsUnixTimeInNs, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.CreationTime, err = time.Parse(time.RFC3339, creationTimeAsRFC3339String) - if nil != err { - return - } + inodeHeadV1.CreationTime = time.Unix(0, int64(creationTimeAsUnixTimeInNs)) - modificationTimeAsRFC3339String, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + modificationTimeAsUnixTimeInNs, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.ModificationTime, err = time.Parse(time.RFC3339, modificationTimeAsRFC3339String) - if nil != err { - return - } + inodeHeadV1.ModificationTime = time.Unix(0, int64(modificationTimeAsUnixTimeInNs)) - accessTimeAsRFC3339String, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + accessTimeAsUnixTimeInNs, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.AccessTime, err = time.Parse(time.RFC3339, accessTimeAsRFC3339String) - if nil != err { - return - } + inodeHeadV1.AccessTime = time.Unix(0, int64(accessTimeAsUnixTimeInNs)) - attrChangeTimeAsRFC3339String, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + attrChangeTimeAsUnixTimeInNs, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.AttrChangeTime, err = time.Parse(time.RFC3339, attrChangeTimeAsRFC3339String) - if nil != err { - return - } + inodeHeadV1.AttrChangeTime = time.Unix(0, int64(attrChangeTimeAsUnixTimeInNs)) - inodeHeadV1.Mode, curPos, err = leGetUint16FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.Mode, curPos, err = getLEUint16FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.UserID, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.UserID, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.GroupID, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.GroupID, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - streamTableLen, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + streamTableLen, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } @@ -595,38 +570,38 @@ func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct inodeHeadV1.StreamTable = make([]InodeStreamTableEntryStruct, streamTableLen) for streamTableIndex = 0; streamTableIndex < streamTableLen; streamTableIndex++ { - inodeHeadV1.StreamTable[streamTableIndex].Name, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.StreamTable[streamTableIndex].Name, curPos, err = getLEStringFromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.StreamTable[streamTableIndex].Value, curPos, err = leGetByteSliceFromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.StreamTable[streamTableIndex].Value, curPos, err = getLEByteSliceFromBuf(inodeHeadV1Buf, curPos) if nil != err { return } } - inodeHeadV1.PayloadObjectNumber, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.PayloadObjectNumber, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.PayloadObjectOffset, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.PayloadObjectOffset, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.PayloadObjectLength, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.PayloadObjectLength, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.SymlinkTarget, curPos, err = leGetStringFromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.SymLinkTarget, curPos, err = getLEStringFromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - layoutLen, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + layoutLen, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } @@ -634,23 +609,23 @@ func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct inodeHeadV1.Layout = make([]InodeHeadLayoutEntryV1Struct, layoutLen) for layoutIndex = 0; layoutIndex < layoutLen; layoutIndex++ { - inodeHeadV1.Layout[layoutIndex].ObjectNumber, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.Layout[layoutIndex].ObjectNumber, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.Layout[layoutIndex].ObjectSize, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.Layout[layoutIndex].ObjectSize, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } - inodeHeadV1.Layout[layoutIndex].BytesReferenced, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadV1.Layout[layoutIndex].BytesReferenced, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } } - inodeHeadVersion, curPos, err = leGetUint64FromBuf(inodeHeadV1Buf, curPos) + inodeHeadVersion, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) if nil != err { return } @@ -668,7 +643,129 @@ func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct return } -func leGetUint8FromBuf(buf []byte, curPos int) (u8 uint8, nextPos int, err error) { +func (directoryEntryValueV1 *DirectoryEntryValueV1Struct) marshalDirectoryEntryValueV1() (directoryEntryValueV1Buf []byte, err error) { + var ( + curPos int + ) + + directoryEntryValueV1Buf = make([]byte, 8+1) + + curPos = 0 + + curPos, err = putLEUint64ToBuf(directoryEntryValueV1Buf, curPos, directoryEntryValueV1.InodeNumber) + if nil != err { + return + } + + curPos, err = putLEUint8ToBuf(directoryEntryValueV1Buf, curPos, directoryEntryValueV1.InodeType) + if nil != err { + return + } + + err = nil + return +} + +func unmarshalDirectoryEntryValueV1(directoryEntryValueV1Buf []byte) (directoryEntryValueV1 *DirectoryEntryValueV1Struct, err error) { + var ( + curPos int + ) + + curPos = 0 + + directoryEntryValueV1 = &DirectoryEntryValueV1Struct{} + + directoryEntryValueV1.InodeNumber, curPos, err = getLEUint64FromBuf(directoryEntryValueV1Buf, curPos) + if nil != err { + return + } + + directoryEntryValueV1.InodeType, curPos, err = getLEUint8FromBuf(directoryEntryValueV1Buf, curPos) + if nil != err { + return + } + + if curPos != len(directoryEntryValueV1Buf) { + err = fmt.Errorf("Incorrect size for directoryEntryValueV1Buf") + return + } + + err = nil + return +} + +func (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct) marshalExtentMapEntryValueV1() (extentMapEntryValueV1Buf []byte, err error) { + var ( + curPos int + ) + + extentMapEntryValueV1Buf = make([]byte, 8+8+8+8) + + curPos = 0 + + curPos, err = putLEUint64ToBuf(extentMapEntryValueV1Buf, curPos, extentMapEntryValueV1.FileOffset) + if nil != err { + return + } + + curPos, err = putLEUint64ToBuf(extentMapEntryValueV1Buf, curPos, extentMapEntryValueV1.Length) + if nil != err { + return + } + + curPos, err = putLEUint64ToBuf(extentMapEntryValueV1Buf, curPos, extentMapEntryValueV1.ObjectNumber) + if nil != err { + return + } + + curPos, err = putLEUint64ToBuf(extentMapEntryValueV1Buf, curPos, extentMapEntryValueV1.ObjectOffset) + if nil != err { + return + } + + err = nil + return +} + +func unmarshalExtentMapEntryValueV1(extentMapEntryValueV1Buf []byte) (extentMapEntryValueV1 *ExtentMapEntryValueV1Struct, err error) { + var ( + curPos int + ) + + curPos = 0 + + extentMapEntryValueV1 = &ExtentMapEntryValueV1Struct{} + + extentMapEntryValueV1.FileOffset, curPos, err = getLEUint64FromBuf(extentMapEntryValueV1Buf, curPos) + if nil != err { + return + } + + extentMapEntryValueV1.Length, curPos, err = getLEUint64FromBuf(extentMapEntryValueV1Buf, curPos) + if nil != err { + return + } + + extentMapEntryValueV1.ObjectNumber, curPos, err = getLEUint64FromBuf(extentMapEntryValueV1Buf, curPos) + if nil != err { + return + } + + extentMapEntryValueV1.ObjectOffset, curPos, err = getLEUint64FromBuf(extentMapEntryValueV1Buf, curPos) + if nil != err { + return + } + + if curPos != len(extentMapEntryValueV1Buf) { + err = fmt.Errorf("Incorrect size for extentMapEntryValueV1Buf") + return + } + + err = nil + return +} + +func getLEUint8FromBuf(buf []byte, curPos int) (u8 uint8, nextPos int, err error) { nextPos = curPos + 1 if nextPos > len(buf) { @@ -682,7 +779,7 @@ func leGetUint8FromBuf(buf []byte, curPos int) (u8 uint8, nextPos int, err error return } -func lePutUint8ToBuf(buf []byte, curPos int, u8 uint8) (nextPos int, err error) { +func putLEUint8ToBuf(buf []byte, curPos int, u8 uint8) (nextPos int, err error) { nextPos = curPos + 1 if nextPos > len(buf) { @@ -696,7 +793,7 @@ func lePutUint8ToBuf(buf []byte, curPos int, u8 uint8) (nextPos int, err error) return } -func leGetUint16FromBuf(buf []byte, curPos int) (u16 uint16, nextPos int, err error) { +func getLEUint16FromBuf(buf []byte, curPos int) (u16 uint16, nextPos int, err error) { nextPos = curPos + 2 if nextPos > len(buf) { @@ -709,7 +806,7 @@ func leGetUint16FromBuf(buf []byte, curPos int) (u16 uint16, nextPos int, err er return } -func lePutUint16ToBuf(buf []byte, curPos int, u16 uint16) (nextPos int, err error) { +func putLEUint16ToBuf(buf []byte, curPos int, u16 uint16) (nextPos int, err error) { nextPos = curPos + 2 if nextPos > len(buf) { @@ -724,7 +821,7 @@ func lePutUint16ToBuf(buf []byte, curPos int, u16 uint16) (nextPos int, err erro return } -func leGetUint32FromBuf(buf []byte, curPos int) (u32 uint32, nextPos int, err error) { +func getLEUint32FromBuf(buf []byte, curPos int) (u32 uint32, nextPos int, err error) { nextPos = curPos + 4 if nextPos > len(buf) { @@ -738,7 +835,7 @@ func leGetUint32FromBuf(buf []byte, curPos int) (u32 uint32, nextPos int, err er return } -func lePutUint32ToBuf(buf []byte, curPos int, u32 uint32) (nextPos int, err error) { +func putLEUint32ToBuf(buf []byte, curPos int, u32 uint32) (nextPos int, err error) { nextPos = curPos + 4 if nextPos > len(buf) { @@ -755,7 +852,7 @@ func lePutUint32ToBuf(buf []byte, curPos int, u32 uint32) (nextPos int, err erro return } -func leGetUint64FromBuf(buf []byte, curPos int) (u64 uint64, nextPos int, err error) { +func getLEUint64FromBuf(buf []byte, curPos int) (u64 uint64, nextPos int, err error) { nextPos = curPos + 8 if nextPos > len(buf) { @@ -763,13 +860,13 @@ func leGetUint64FromBuf(buf []byte, curPos int) (u64 uint64, nextPos int, err er return } - u64 = uint64(buf[curPos]) + (uint64(buf[curPos+1]) << 8) + (uint64(buf[curPos+2]) << 16) + (uint64(buf[curPos+3]) << 24) + (uint64(buf[curPos+2]) << 32) + (uint64(buf[curPos+3]) << 40) + (uint64(buf[curPos+2]) << 48) + (uint64(buf[curPos+3]) << 56) + u64 = uint64(buf[curPos]) + (uint64(buf[curPos+1]) << 8) + (uint64(buf[curPos+2]) << 16) + (uint64(buf[curPos+3]) << 24) + (uint64(buf[curPos+4]) << 32) + (uint64(buf[curPos+5]) << 40) + (uint64(buf[curPos+6]) << 48) + (uint64(buf[curPos+7]) << 56) err = nil return } -func lePutUint64ToBuf(buf []byte, curPos int, u64 uint64) (nextPos int, err error) { +func putLEUint64ToBuf(buf []byte, curPos int, u64 uint64) (nextPos int, err error) { nextPos = curPos + 8 if nextPos > len(buf) { @@ -790,12 +887,12 @@ func lePutUint64ToBuf(buf []byte, curPos int, u64 uint64) (nextPos int, err erro return } -func leGetStringFromBuf(buf []byte, curPos int) (str string, nextPos int, err error) { +func getLEStringFromBuf(buf []byte, curPos int) (str string, nextPos int, err error) { var ( strLen uint64 ) - strLen, nextPos, err = leGetUint64FromBuf(buf, curPos) + strLen, nextPos, err = getLEUint64FromBuf(buf, curPos) if nil != err { return } @@ -812,8 +909,8 @@ func leGetStringFromBuf(buf []byte, curPos int) (str string, nextPos int, err er return } -func lePutStringToBuf(buf []byte, curPos int, str string) (nextPos int, err error) { - nextPos, err = lePutUint64ToBuf(buf, curPos, uint64(len(str))) +func putLEStringToBuf(buf []byte, curPos int, str string) (nextPos int, err error) { + nextPos, err = putLEUint64ToBuf(buf, curPos, uint64(len(str))) if nil != err { return } @@ -832,12 +929,12 @@ func lePutStringToBuf(buf []byte, curPos int, str string) (nextPos int, err erro return } -func leGetByteSliceFromBuf(buf []byte, curPos int) (byteSlice []byte, nextPos int, err error) { +func getLEByteSliceFromBuf(buf []byte, curPos int) (byteSlice []byte, nextPos int, err error) { var ( byteSliceLen uint64 ) - byteSliceLen, nextPos, err = leGetUint64FromBuf(buf, curPos) + byteSliceLen, nextPos, err = getLEUint64FromBuf(buf, curPos) if nil != err { return } @@ -855,8 +952,8 @@ func leGetByteSliceFromBuf(buf []byte, curPos int) (byteSlice []byte, nextPos in return } -func lePutByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { - nextPos, err = lePutUint64ToBuf(buf, curPos, uint64(len(byteSlice))) +func putLEByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { + nextPos, err = putLEUint64ToBuf(buf, curPos, uint64(len(byteSlice))) if nil != err { return } diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index c4589c006..2f02080b5 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -87,7 +87,8 @@ // Content-Type: application/json // // { -// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con" +// "StorageURL": "http://172.28.128.2:8080/v1/AUTH_test/con", +// "AuthToken" : "AUTH_tk0123456789abcde0123456789abcdef0" // } // // This will cause the specified to be served. The StorageURL diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 67cbb19f4..dabcb580f 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -258,6 +258,7 @@ func serveHTTPPost(responseWriter http.ResponseWriter, request *http.Request, re type serveHTTPPostOfVolumeRequestBodyAsJSONStruct struct { StorageURL string + AuthToken string } func serveHTTPPostOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { @@ -284,7 +285,7 @@ func serveHTTPPostOfVolume(responseWriter http.ResponseWriter, request *http.Req return } - err = postVolume(pathSplit[2], requestBodyAsJSON.StorageURL) + err = postVolume(pathSplit[2], requestBodyAsJSON.StorageURL, requestBodyAsJSON.AuthToken) if nil == err { responseWriter.WriteHeader(http.StatusCreated) } else { diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 2427dca8f..45c283a8d 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -7,7 +7,10 @@ import ( "container/list" "encoding/json" "fmt" + "time" + "github.com/NVIDIA/proxyfs/ilayout" + "github.com/NVIDIA/proxyfs/utils" "github.com/NVIDIA/sortedmap" ) @@ -208,8 +211,234 @@ func getVolumeListAsJSON() (volumeList []byte) { return } -func postVolume(name string, storageURL string) (err error) { - err = fmt.Errorf("TODO") +type rootDirDirectoryCallbacksStruct struct { + objectNumber uint64 + buf []byte +} + +func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { + err = fmt.Errorf("Not implemented") + return +} + +func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { + err = fmt.Errorf("Not implemented") + return +} + +func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { + err = fmt.Errorf("Not implemented") + return +} + +func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { + objectNumber = rootDirDirectoryCallbacks.objectNumber + objectOffset = uint64(len(rootDirDirectoryCallbacks.buf)) + + rootDirDirectoryCallbacks.buf = append(rootDirDirectoryCallbacks.buf, nodeByteSlice...) + + err = nil + return +} + +func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { + err = nil + return +} + +func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { + var ( + keyAsString string + nextPos int + ok bool + ) + + keyAsString, ok = key.(string) + if !ok { + err = fmt.Errorf("*rootDirDirectoryCallbacksStruct).PackKey(key:%v) called with non-string", key) + return + } + + packedKey = make([]byte, 8+len(keyAsString)) + + nextPos, err = ilayout.PutLEStringToBuf(packedKey, 0, keyAsString) + if nil != err { + return + } + + if len(packedKey) != nextPos { + err = fmt.Errorf("*rootDirDirectoryCallbacksStruct).PackKey(key:%s) logic error", keyAsString) + return + } + + err = nil + return +} + +func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { + err = fmt.Errorf("Not implemented") + return +} + +func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { + var ( + nextPos int + ok bool + valueAsDirectoryEntryValueV1 ilayout.DirectoryEntryValueV1Struct + ) + + valueAsDirectoryEntryValueV1, ok = value.(ilayout.DirectoryEntryValueV1Struct) + if !ok { + err = fmt.Errorf("*rootDirDirectoryCallbacksStruct).PackValue(value:%v) called with non-DirectoryEntryValueV1Struct", value) + return + } + + packedValue = make([]byte, 8+1) + + nextPos, err = ilayout.PutLEUint64ToBuf(packedValue, 0, valueAsDirectoryEntryValueV1.InodeNumber) + if nil != err { + return + } + + nextPos, err = ilayout.PutLEUint8ToBuf(packedValue, nextPos, valueAsDirectoryEntryValueV1.InodeType) + if nil != err { + return + } + + if len(packedValue) != nextPos { + err = fmt.Errorf("*rootDirDirectoryCallbacksStruct).PackKey(key:%#v) logic error", valueAsDirectoryEntryValueV1) + return + } + + err = nil + return +} + +func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { + err = fmt.Errorf("Not implemented") + return +} + +func postVolume(name string, storageURL string, authToken string) (err error) { + var ( + ok bool + reservedToNonce uint64 + rootDirDirectory sortedmap.BPlusTree + rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct + rootDirInodeHeadV1 *ilayout.InodeHeadV1Struct + rootDirInodeObjectLength uint64 + rootDirInodeObjectNumber uint64 + rootDirInodeObjectOffset uint64 + superBlockObjectNumber uint64 + timeNow = time.Now() + ) + + // Reserve some Nonce values + + rootDirInodeObjectNumber = ilayout.RootDirInodeNumber + 1 + superBlockObjectNumber = rootDirInodeObjectNumber + 1 + + reservedToNonce = superBlockObjectNumber + + // Construct RootDirInode + + rootDirDirectoryCallbacks = &rootDirDirectoryCallbacksStruct{ + objectNumber: rootDirInodeObjectNumber, + buf: make([]byte, 0), + } + + rootDirDirectory = sortedmap.NewBPlusTree( + globals.config.InodeTableMaxInodesPerBPlusTreePage, + sortedmap.CompareString, + rootDirDirectoryCallbacks, + nil) + + ok, err = rootDirDirectory.Put( + ".", + ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: ilayout.RootDirInodeNumber, + InodeType: ilayout.InodeTypeDir, + }) + if nil != err { + return + } + if !ok { + err = fmt.Errorf("rootDirDirectory.Put(\".\",) returned !ok") + return + } + + ok, err = rootDirDirectory.Put( + "..", + ilayout.DirectoryEntryValueV1Struct{ + InodeNumber: ilayout.RootDirInodeNumber, + InodeType: ilayout.InodeTypeDir, + }) + if nil != err { + return + } + if !ok { + err = fmt.Errorf("rootDirDirectory.Put(\".\",) returned !ok") + return + } + + _, rootDirInodeObjectOffset, rootDirInodeObjectLength, err = rootDirDirectory.Flush(false) + if nil != err { + return + } + + rootDirInodeHeadV1 = &ilayout.InodeHeadV1Struct{ + InodeNumber: ilayout.RootDirInodeNumber, + InodeType: ilayout.InodeTypeDir, + LinkTable: []ilayout.InodeLinkTableEntryStruct{ + { + ParentDirInodeNumber: ilayout.RootDirInodeNumber, + ParentDirEntryName: ".", + }, + { + ParentDirInodeNumber: ilayout.RootDirInodeNumber, + ParentDirEntryName: "..", + }, + }, + Size: 0, + CreationTime: timeNow, + ModificationTime: timeNow, + AccessTime: timeNow, + AttrChangeTime: timeNow, + Mode: ilayout.InodeModeMask, + UserID: 0, + GroupID: 0, + StreamTable: []ilayout.InodeStreamTableEntryStruct{}, + PayloadObjectNumber: rootDirInodeObjectNumber, + PayloadObjectOffset: rootDirInodeObjectOffset, + PayloadObjectLength: rootDirInodeObjectLength, + SymLinkTarget: "", + Layout: []ilayout.InodeHeadLayoutEntryV1Struct{ + { + ObjectNumber: rootDirInodeObjectNumber, + ObjectSize: uint64(len(rootDirDirectoryCallbacks.buf)), + BytesReferenced: uint64(len(rootDirDirectoryCallbacks.buf)), + }, + }, + } + + fmt.Printf("UNDO: reservedToNonce == %v\n", reservedToNonce) + fmt.Printf("UNDO: rootDirInodeHeadV1:\n%s\n", utils.JSONify(rootDirInodeHeadV1, true)) + + // TODO: + // 1 - Append marshaled rootDirInodeHeadV1 to rootDirDirectoryCallbacks.buf + // 2 - PUT rootDirDirectoryCallbacks.buf contents to / + // 3 - Create InodeTable to include RootDirInode + // 4 - Flush InodeTable to inodeTableCallbacks.buf + // 5 - Craft SuperBlock to reference flushed InodeTable + // 6 - Append marshaled SuperBlock to inodeTableCallbacks.buf + // 7 - PUT inodeTableCallbacks.buf contents to / + // 8 - Craft CheckPointHeader to reference superBlockObjectNumber & reservedToNonce + // 9 - PUT X-Checkpoint-Header to + + // Finally, start serving the volume (and pass thru that err return) + + err = putVolume(name, storageURL) + return } From a212ebdf1e6b1ea7cccc9d860e2b0596c55804b7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sat, 10 Apr 2021 11:31:59 -0700 Subject: [PATCH 124/160] Knocked off two TODOs in imgr/imgrpkg/volume.go::postVolume() Also shortened retry total duration dramatically --- imgr/imgr.conf | 7 +- imgr/imgrpkg/api.go | 7 +- imgr/imgrpkg/globals.go | 13 ++ imgr/imgrpkg/impl.go | 10 + imgr/imgrpkg/swift-client.go | 370 +++++++++++++++++++++++++++++++++++ imgr/imgrpkg/volume.go | 15 +- 6 files changed, 414 insertions(+), 8 deletions(-) create mode 100644 imgr/imgrpkg/swift-client.go diff --git a/imgr/imgr.conf b/imgr/imgr.conf index 7034dae56..4ed05c052 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -21,10 +21,11 @@ MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 -SwiftRetryDelay: 1s -SwiftRetryExpBackoff: 1.5 -SwiftRetryLimit: 11 +SwiftRetryDelay: 100ms +SwiftRetryExpBackoff: 2 +SwiftRetryLimit: 4 +SwiftTimeout: 10m SwiftConnectionPoolSize: 128 InodeTableCacheEvictLowLimit: 10000 diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 2f02080b5..e26c254c4 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -30,10 +30,11 @@ // LeaseInterruptInterval: 250ms // LeaseInterruptLimit: 20 // -// SwiftRetryDelay: 1s -// SwiftRetryExpBackoff: 1.5 -// SwiftRetryLimit: 11 +// SwiftRetryDelay: 100ms +// SwiftRetryExpBackoff: 2 +// SwiftRetryLimit: 4 // +// SwiftTimeout: 10m // SwiftConnectionPoolSize: 128 // // InodeTableCacheEvictLowLimit: 10000 diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 11da87b5e..c09fc569f 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -43,6 +43,7 @@ type configStruct struct { SwiftRetryExpBackoff float64 SwiftRetryLimit uint32 + SwiftTimeout time.Duration SwiftConnectionPoolSize uint32 InodeTableCacheEvictLowLimit uint64 @@ -105,6 +106,12 @@ type statsStruct struct { InodeTableCacheHits bucketstats.Totaler InodeTableCacheMisses bucketstats.Totaler + + SwiftContainerHeaderGetUsecs bucketstats.BucketLog2Round + SwiftContainerHeaderSetUsecs bucketstats.BucketLog2Round + SwiftObjectGetRangeUsecs bucketstats.BucketLog2Round + SwiftObjectGetTailUsecs bucketstats.BucketLog2Round + SwiftObjectPutUsecs bucketstats.BucketLog2Round } type mountStruct struct { @@ -135,6 +142,7 @@ type globalsStruct struct { inodeTableCache sortedmap.BPlusTreeCache // volumeMap sortedmap.LLRBTree // key == volumeStruct.name; value == *volumeStruct mountMap map[string]*mountStruct // key == mountStruct.mountID + httpClient *http.Client // retryrpcServer *retryrpc.Server // httpServer *http.Server // httpServerWG sync.WaitGroup // @@ -271,6 +279,10 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { logFatal(err) } + globals.config.SwiftTimeout, err = confMap.FetchOptionValueDuration("IMGR", "SwiftTimeout") + if nil != err { + logFatal(err) + } globals.config.SwiftConnectionPoolSize, err = confMap.FetchOptionValueUint32("IMGR", "SwiftConnectionPoolSize") if nil != err { logFatal(err) @@ -344,6 +356,7 @@ func uninitializeGlobals() (err error) { globals.config.SwiftRetryExpBackoff = 0.0 globals.config.SwiftRetryLimit = 0 + globals.config.SwiftTimeout = time.Duration(0) globals.config.SwiftConnectionPoolSize = 0 globals.config.InodeTableCacheEvictLowLimit = 0 diff --git a/imgr/imgrpkg/impl.go b/imgr/imgrpkg/impl.go index e44961009..833aa2172 100644 --- a/imgr/imgrpkg/impl.go +++ b/imgr/imgrpkg/impl.go @@ -13,6 +13,11 @@ func start(confMap conf.ConfMap) (err error) { return } + err = startSwiftClient() + if nil != err { + return + } + err = startVolumeManagement() if nil != err { return @@ -47,6 +52,11 @@ func stop() (err error) { return } + err = stopSwiftClient() + if nil != err { + return + } + err = uninitializeGlobals() return diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go new file mode 100644 index 000000000..a3c145db0 --- /dev/null +++ b/imgr/imgrpkg/swift-client.go @@ -0,0 +1,370 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +import ( + "bytes" + "fmt" + "io/ioutil" + "net/http" + "time" +) + +func startSwiftClient() (err error) { + var ( + customTransport *http.Transport + defaultTransport *http.Transport + ok bool + ) + + defaultTransport, ok = http.DefaultTransport.(*http.Transport) + if !ok { + err = fmt.Errorf("http.DefaultTransport.(*http.Transport) returned !ok\n") + return + } + + customTransport = &http.Transport{ // Up-to-date as of Golang 1.11 + Proxy: defaultTransport.Proxy, + DialContext: defaultTransport.DialContext, + Dial: defaultTransport.Dial, + DialTLS: defaultTransport.DialTLS, + TLSClientConfig: defaultTransport.TLSClientConfig, + TLSHandshakeTimeout: globals.config.SwiftTimeout, + DisableKeepAlives: false, + DisableCompression: defaultTransport.DisableCompression, + MaxIdleConns: int(globals.config.SwiftConnectionPoolSize), + MaxIdleConnsPerHost: int(globals.config.SwiftConnectionPoolSize), + MaxConnsPerHost: int(globals.config.SwiftConnectionPoolSize), + IdleConnTimeout: globals.config.SwiftTimeout, + ResponseHeaderTimeout: globals.config.SwiftTimeout, + ExpectContinueTimeout: globals.config.SwiftTimeout, + TLSNextProto: defaultTransport.TLSNextProto, + ProxyConnectHeader: defaultTransport.ProxyConnectHeader, + MaxResponseHeaderBytes: defaultTransport.MaxResponseHeaderBytes, + } + + globals.httpClient = &http.Client{ + Transport: customTransport, + Timeout: globals.config.SwiftTimeout, + } + + err = nil + return +} + +func stopSwiftClient() (err error) { + err = nil + return +} + +func swiftContainerHeaderGet(storageURL string, authToken string, headerName string) (headerValue string, err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.SwiftContainerHeaderGetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + httpRequest, err = http.NewRequest("HEAD", storageURL, nil) + if nil != err { + return + } + + if "" != authToken { + httpRequest.Header["X-Auth-Token"] = []string{authToken} + } + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) + return + } + + _, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + return + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + headerValue = httpResponse.Header.Get(headerName) + + err = nil + return + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return +} + +func swiftContainerHeaderSet(storageURL string, authToken string, headerName string, headerValue string) (err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.SwiftContainerHeaderSetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + httpRequest, err = http.NewRequest("POST", storageURL, nil) + if nil != err { + return + } + + httpRequest.Header[headerName] = []string{headerValue} + + if "" != authToken { + httpRequest.Header["X-Auth-Token"] = []string{authToken} + } + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) + return + } + + _, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + return + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + err = nil + return + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return +} + +func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + objectURL string + rangeHeaderValue string + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.SwiftObjectGetRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) + rangeHeaderValue = fmt.Sprintf("bytes=%d-%d", objectOffset, (objectOffset + objectLength - 1)) + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + httpRequest, err = http.NewRequest("GET", objectURL, nil) + if nil != err { + return + } + + httpRequest.Header["Range"] = []string{rangeHeaderValue} + + if "" != authToken { + httpRequest.Header["X-Auth-Token"] = []string{authToken} + } + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) + return + } + + buf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + return + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + err = nil + return + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return +} + +func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64, objectLength uint64) (buf []byte, err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + objectURL string + rangeHeaderValue string + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.SwiftObjectGetTailUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) + rangeHeaderValue = fmt.Sprintf("bytes=-%d", objectLength) + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + httpRequest, err = http.NewRequest("GET", objectURL, nil) + if nil != err { + return + } + + httpRequest.Header["Range"] = []string{rangeHeaderValue} + + if "" != authToken { + httpRequest.Header["X-Auth-Token"] = []string{authToken} + } + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) + return + } + + buf, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + return + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + err = nil + return + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return +} + +func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, buf []byte) (err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + objectURL string + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.SwiftObjectPutUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + httpRequest, err = http.NewRequest("PUT", objectURL, bytes.NewReader(buf)) + if nil != err { + return + } + + if "" != authToken { + httpRequest.Header["X-Auth-Token"] = []string{authToken} + } + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) + return + } + + _, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + return + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + err = nil + return + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return +} diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 45c283a8d..f13b70cd6 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -326,6 +326,7 @@ func postVolume(name string, storageURL string, authToken string) (err error) { rootDirDirectory sortedmap.BPlusTree rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct rootDirInodeHeadV1 *ilayout.InodeHeadV1Struct + rootDirInodeHeadV1Buf []byte rootDirInodeObjectLength uint64 rootDirInodeObjectNumber uint64 rootDirInodeObjectOffset uint64 @@ -421,12 +422,22 @@ func postVolume(name string, storageURL string, authToken string) (err error) { }, } + rootDirInodeHeadV1Buf, err = rootDirInodeHeadV1.MarshalInodeHeadV1() + if nil != err { + return + } + fmt.Printf("UNDO: reservedToNonce == %v\n", reservedToNonce) fmt.Printf("UNDO: rootDirInodeHeadV1:\n%s\n", utils.JSONify(rootDirInodeHeadV1, true)) + rootDirDirectoryCallbacks.buf = append(rootDirDirectoryCallbacks.buf, rootDirInodeHeadV1Buf...) + + err = swiftObjectPut(storageURL, authToken, rootDirInodeObjectNumber, rootDirDirectoryCallbacks.buf) + if nil != err { + return + } + // TODO: - // 1 - Append marshaled rootDirInodeHeadV1 to rootDirDirectoryCallbacks.buf - // 2 - PUT rootDirDirectoryCallbacks.buf contents to / // 3 - Create InodeTable to include RootDirInode // 4 - Flush InodeTable to inodeTableCallbacks.buf // 5 - Craft SuperBlock to reference flushed InodeTable From b112eb7d1b6209b1295ca966c7c63f1fff67990e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Sun, 11 Apr 2021 15:08:31 -0700 Subject: [PATCH 125/160] Nearly complete imgr SuperBlock filled out - TODO: write Ckpt Hdr --- imgr/imgr.conf | 51 +++--- imgr/imgrpkg/api.go | 51 +++--- imgr/imgrpkg/globals.go | 8 +- imgr/imgrpkg/swift-client.go | 8 +- imgr/imgrpkg/volume.go | 320 ++++++++++++++++++++++++++++++----- 5 files changed, 342 insertions(+), 96 deletions(-) diff --git a/imgr/imgr.conf b/imgr/imgr.conf index 4ed05c052..0c5eee0fa 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -2,37 +2,38 @@ # SPDX-License-Identifier: Apache-2.0 [IMGR] -PublicIPAddr: 172.28.128.2 -PrivateIPAddr: 172.28.128.2 -RetryRPCPort: 32356 -HTTPServerPort: 15346 +PublicIPAddr: 172.28.128.2 +PrivateIPAddr: 172.28.128.2 +RetryRPCPort: 32356 +HTTPServerPort: 15346 -RetryRPCTTLCompleted: 10m -RetryRPCAckTrim: 100ms -RetryRPCDeadlineIO: 60s -RetryRPCKeepAlivePeriod: 60s +RetryRPCTTLCompleted: 10m +RetryRPCAckTrim: 100ms +RetryRPCDeadlineIO: 60s +RetryRPCKeepAlivePeriod: 60s -RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, -RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used +RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, +RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used -FetchNonceRangeToReturn: 100 +FetchNonceRangeToReturn: 100 -MinLeaseDuration: 250ms -LeaseInterruptInterval: 250ms -LeaseInterruptLimit: 20 +MinLeaseDuration: 250ms +LeaseInterruptInterval: 250ms +LeaseInterruptLimit: 20 -SwiftRetryDelay: 100ms -SwiftRetryExpBackoff: 2 -SwiftRetryLimit: 4 +SwiftRetryDelay: 100ms +SwiftRetryExpBackoff: 2 +SwiftRetryLimit: 4 -SwiftTimeout: 10m -SwiftConnectionPoolSize: 128 +SwiftTimeout: 10m +SwiftConnectionPoolSize: 128 -InodeTableCacheEvictLowLimit: 10000 -InodeTableCacheEvictHighLimit: 10010 +InodeTableCacheEvictLowLimit: 10000 +InodeTableCacheEvictHighLimit: 10010 -InodeTableMaxInodesPerBPlusTreePage: 2048 +InodeTableMaxInodesPerBPlusTreePage: 2048 +RootDirMaxDirEntriesPerBPlusTreePage: 1024 -LogFilePath: # imgr.log -LogToConsole: true # false -TraceEnabled: false +LogFilePath: # imgr.log +LogToConsole: true # false +TraceEnabled: false diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index e26c254c4..5d75d8cb2 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -11,40 +11,41 @@ // argument, a package conf ConfMap. Here is a sample .conf file: // // [IMGR] -// PublicIPAddr: 172.28.128.2 -// PrivateIPAddr: 172.28.128.2 -// RetryRPCPort: 32356 -// HTTPServerPort: 15346 +// PublicIPAddr: 172.28.128.2 +// PrivateIPAddr: 172.28.128.2 +// RetryRPCPort: 32356 +// HTTPServerPort: 15346 // -// RetryRPCTTLCompleted: 10m -// RetryRPCAckTrim: 100ms -// RetryRPCDeadlineIO: 60s -// RetryRPCKeepAlivePeriod: 60s +// RetryRPCTTLCompleted: 10m +// RetryRPCAckTrim: 100ms +// RetryRPCDeadlineIO: 60s +// RetryRPCKeepAlivePeriod: 60s // -// RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, -// RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used +// RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, +// RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used // -// FetchNonceRangeToReturn: 100 +// FetchNonceRangeToReturn: 100 // -// MinLeaseDuration: 250ms -// LeaseInterruptInterval: 250ms -// LeaseInterruptLimit: 20 +// MinLeaseDuration: 250ms +// LeaseInterruptInterval: 250ms +// LeaseInterruptLimit: 20 // -// SwiftRetryDelay: 100ms -// SwiftRetryExpBackoff: 2 -// SwiftRetryLimit: 4 +// SwiftRetryDelay: 100ms +// SwiftRetryExpBackoff: 2 +// SwiftRetryLimit: 4 // -// SwiftTimeout: 10m -// SwiftConnectionPoolSize: 128 +// SwiftTimeout: 10m +// SwiftConnectionPoolSize: 128 // -// InodeTableCacheEvictLowLimit: 10000 -// InodeTableCacheEvictHighLimit: 10010 +// InodeTableCacheEvictLowLimit: 10000 +// InodeTableCacheEvictHighLimit: 10010 // -// InodeTableMaxInodesPerBPlusTreePage: 2048 +// InodeTableMaxInodesPerBPlusTreePage: 2048 +// RootDirMaxDirEntriesPerBPlusTreePage: 1024 // -// LogFilePath: # imgr.log -// LogToConsole: true # false -// TraceEnabled: false +// LogFilePath: # imgr.log +// LogToConsole: true # false +// TraceEnabled: false // // Most of the config keys are required and must have values. One exception // is LogFilePath that will default to "" and, hence, cause logging to not diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index c09fc569f..af9d3142e 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -49,7 +49,8 @@ type configStruct struct { InodeTableCacheEvictLowLimit uint64 InodeTableCacheEvictHighLimit uint64 - InodeTableMaxInodesPerBPlusTreePage uint64 + InodeTableMaxInodesPerBPlusTreePage uint64 + RootDirMaxDirEntriesPerBPlusTreePage uint64 LogFilePath string // Unless starting with '/', relative to $CWD; == "" means disabled LogToConsole bool @@ -301,6 +302,10 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { if nil != err { logFatal(err) } + globals.config.RootDirMaxDirEntriesPerBPlusTreePage, err = confMap.FetchOptionValueUint64("IMGR", "RootDirMaxDirEntriesPerBPlusTreePage") + if nil != err { + logFatal(err) + } globals.config.LogFilePath, err = confMap.FetchOptionValueString("IMGR", "LogFilePath") if nil != err { @@ -363,6 +368,7 @@ func uninitializeGlobals() (err error) { globals.config.InodeTableCacheEvictHighLimit = 0 globals.config.InodeTableMaxInodesPerBPlusTreePage = 0 + globals.config.RootDirMaxDirEntriesPerBPlusTreePage = 0 globals.config.LogFilePath = "" globals.config.LogToConsole = false diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index a3c145db0..bbcca420b 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -4,8 +4,8 @@ package imgrpkg import ( - "bytes" "fmt" + "io" "io/ioutil" "net/http" "time" @@ -308,7 +308,7 @@ func swiftObjectGetTail(storageURL string, authToken string, objectNumber uint64 return } -func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, buf []byte) (err error) { +func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, body io.ReadSeeker) (err error) { var ( httpRequest *http.Request httpResponse *http.Response @@ -329,7 +329,9 @@ func swiftObjectPut(storageURL string, authToken string, objectNumber uint64, bu nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - httpRequest, err = http.NewRequest("PUT", objectURL, bytes.NewReader(buf)) + body.Seek(0, io.SeekStart) + + httpRequest, err = http.NewRequest("PUT", objectURL, body) if nil != err { return } diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index f13b70cd6..165469ad8 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -7,6 +7,7 @@ import ( "container/list" "encoding/json" "fmt" + "io" "time" "github.com/NVIDIA/proxyfs/ilayout" @@ -211,42 +212,84 @@ func getVolumeListAsJSON() (volumeList []byte) { return } -type rootDirDirectoryCallbacksStruct struct { +type postVolumeRootDirDirectoryCallbacksStruct struct { + io.ReadSeeker + sortedmap.BPlusTreeCallbacks objectNumber uint64 - buf []byte + body []byte + readPos int64 } -func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) Read(p []byte) (n int, err error) { + n = copy(p, postVolumeRootDirDirectoryCallbacks.body[postVolumeRootDirDirectoryCallbacks.readPos:]) + postVolumeRootDirDirectoryCallbacks.readPos += int64(n) + + if postVolumeRootDirDirectoryCallbacks.readPos == int64(len(postVolumeRootDirDirectoryCallbacks.body)) { + err = io.EOF + } else { + err = nil + } + return +} +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) Seek(offset int64, whence int) (int64, error) { + var ( + newOffset int64 + ) + + switch whence { + case io.SeekStart: + newOffset = offset + case io.SeekCurrent: + newOffset = postVolumeRootDirDirectoryCallbacks.readPos + offset + case io.SeekEnd: + newOffset = postVolumeRootDirDirectoryCallbacks.readPos + offset + default: + return 0, fmt.Errorf("invalid whence (%d)", whence) + } + + if newOffset < 0 { + return 0, fmt.Errorf("resultant offset cannot be negative") + } + if newOffset > int64(len(postVolumeRootDirDirectoryCallbacks.body)) { + return 0, fmt.Errorf("resultant offset cannot be beyond len(body)") + } + + postVolumeRootDirDirectoryCallbacks.readPos = newOffset + + return newOffset, nil +} + +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { err = fmt.Errorf("Not implemented") return } -func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { err = fmt.Errorf("Not implemented") return } -func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { err = fmt.Errorf("Not implemented") return } -func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { - objectNumber = rootDirDirectoryCallbacks.objectNumber - objectOffset = uint64(len(rootDirDirectoryCallbacks.buf)) +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { + objectNumber = postVolumeRootDirDirectoryCallbacks.objectNumber + objectOffset = uint64(len(postVolumeRootDirDirectoryCallbacks.body)) - rootDirDirectoryCallbacks.buf = append(rootDirDirectoryCallbacks.buf, nodeByteSlice...) + postVolumeRootDirDirectoryCallbacks.body = append(postVolumeRootDirDirectoryCallbacks.body, nodeByteSlice...) err = nil return } -func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { err = nil return } -func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { var ( keyAsString string nextPos int @@ -255,7 +298,7 @@ func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PackKey(key so keyAsString, ok = key.(string) if !ok { - err = fmt.Errorf("*rootDirDirectoryCallbacksStruct).PackKey(key:%v) called with non-string", key) + err = fmt.Errorf("(*postVolumeRootDirDirectoryCallbacksStruct).PackKey(key:%v) called with non-string", key) return } @@ -267,7 +310,7 @@ func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PackKey(key so } if len(packedKey) != nextPos { - err = fmt.Errorf("*rootDirDirectoryCallbacksStruct).PackKey(key:%s) logic error", keyAsString) + err = fmt.Errorf("(*postVolumeRootDirDirectoryCallbacksStruct).PackKey(key:%s) logic error", keyAsString) return } @@ -275,12 +318,12 @@ func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PackKey(key so return } -func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { err = fmt.Errorf("Not implemented") return } -func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { var ( nextPos int ok bool @@ -289,7 +332,7 @@ func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PackValue(valu valueAsDirectoryEntryValueV1, ok = value.(ilayout.DirectoryEntryValueV1Struct) if !ok { - err = fmt.Errorf("*rootDirDirectoryCallbacksStruct).PackValue(value:%v) called with non-DirectoryEntryValueV1Struct", value) + err = fmt.Errorf("(*postVolumeRootDirDirectoryCallbacksStruct).PackValue(value:%v) called with non-DirectoryEntryValueV1Struct", value) return } @@ -306,7 +349,7 @@ func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PackValue(valu } if len(packedValue) != nextPos { - err = fmt.Errorf("*rootDirDirectoryCallbacksStruct).PackKey(key:%#v) logic error", valueAsDirectoryEntryValueV1) + err = fmt.Errorf("(*postVolumeRootDirDirectoryCallbacksStruct).PackKey(key:%#v) logic error", valueAsDirectoryEntryValueV1) return } @@ -314,24 +357,163 @@ func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) PackValue(valu return } -func (rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { +func (postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { + err = fmt.Errorf("Not implemented") + return +} + +type postVolumeSuperBlockInodeTableCallbacksStruct struct { + io.ReadSeeker + sortedmap.BPlusTreeCallbacks + objectNumber uint64 + body []byte + readPos int64 +} + +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) Read(p []byte) (n int, err error) { + n = copy(p, postVolumeSuperBlockInodeTableCallbacks.body[postVolumeSuperBlockInodeTableCallbacks.readPos:]) + postVolumeSuperBlockInodeTableCallbacks.readPos += int64(n) + + if postVolumeSuperBlockInodeTableCallbacks.readPos == int64(len(postVolumeSuperBlockInodeTableCallbacks.body)) { + err = io.EOF + } else { + err = nil + } + return +} +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) Seek(offset int64, whence int) (int64, error) { + var ( + newOffset int64 + ) + + switch whence { + case io.SeekStart: + newOffset = offset + case io.SeekCurrent: + newOffset = postVolumeSuperBlockInodeTableCallbacks.readPos + offset + case io.SeekEnd: + newOffset = postVolumeSuperBlockInodeTableCallbacks.readPos + offset + default: + return 0, fmt.Errorf("invalid whence (%d)", whence) + } + + if newOffset < 0 { + return 0, fmt.Errorf("resultant offset cannot be negative") + } + if newOffset > int64(len(postVolumeSuperBlockInodeTableCallbacks.body)) { + return 0, fmt.Errorf("resultant offset cannot be beyond len(body)") + } + + postVolumeSuperBlockInodeTableCallbacks.readPos = newOffset + + return newOffset, nil +} + +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) DumpKey(key sortedmap.Key) (keyAsString string, err error) { + err = fmt.Errorf("Not implemented") + return +} + +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) DumpValue(value sortedmap.Value) (valueAsString string, err error) { + err = fmt.Errorf("Not implemented") + return +} + +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) GetNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (nodeByteSlice []byte, err error) { + err = fmt.Errorf("Not implemented") + return +} + +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) PutNode(nodeByteSlice []byte) (objectNumber uint64, objectOffset uint64, err error) { + objectNumber = postVolumeSuperBlockInodeTableCallbacks.objectNumber + objectOffset = uint64(len(postVolumeSuperBlockInodeTableCallbacks.body)) + + postVolumeSuperBlockInodeTableCallbacks.body = append(postVolumeSuperBlockInodeTableCallbacks.body, nodeByteSlice...) + + err = nil + return +} + +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) DiscardNode(objectNumber uint64, objectOffset uint64, objectLength uint64) (err error) { + err = nil + return +} + +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) PackKey(key sortedmap.Key) (packedKey []byte, err error) { + var ( + keyAsUint64 uint64 + nextPos int + ok bool + ) + + keyAsUint64, ok = key.(uint64) + if !ok { + err = fmt.Errorf("(*postVolumeSuperBlockInodeTableCallbacksStruct).PackKey(key:%v) called with non-uint64", key) + return + } + + packedKey = make([]byte, 8) + + nextPos, err = ilayout.PutLEUint64ToBuf(packedKey, 0, keyAsUint64) + if nil != err { + return + } + + if len(packedKey) != nextPos { + err = fmt.Errorf("(*postVolumeSuperBlockInodeTableCallbacksStruct).PackKey(key:%016X) logic error", keyAsUint64) + return + } + + err = nil + return +} + +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) UnpackKey(payloadData []byte) (key sortedmap.Key, bytesConsumed uint64, err error) { + err = fmt.Errorf("Not implemented") + return +} + +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) PackValue(value sortedmap.Value) (packedValue []byte, err error) { + var ( + ok bool + valueAsInodeTableEntryValueV1 ilayout.InodeTableEntryValueV1Struct + ) + + valueAsInodeTableEntryValueV1, ok = value.(ilayout.InodeTableEntryValueV1Struct) + if !ok { + err = fmt.Errorf("(*postVolumeSuperBlockInodeTableCallbacksStruct).PackValue(value:%v) called with non-InodeTableEntryValueV1Struct", value) + return + } + + packedValue, err = valueAsInodeTableEntryValueV1.MarshalInodeTableEntryValueV1() + + return +} + +func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct) UnpackValue(payloadData []byte) (value sortedmap.Value, bytesConsumed uint64, err error) { err = fmt.Errorf("Not implemented") return } func postVolume(name string, storageURL string, authToken string) (err error) { var ( - ok bool - reservedToNonce uint64 - rootDirDirectory sortedmap.BPlusTree - rootDirDirectoryCallbacks *rootDirDirectoryCallbacksStruct - rootDirInodeHeadV1 *ilayout.InodeHeadV1Struct - rootDirInodeHeadV1Buf []byte - rootDirInodeObjectLength uint64 - rootDirInodeObjectNumber uint64 - rootDirInodeObjectOffset uint64 - superBlockObjectNumber uint64 - timeNow = time.Now() + ok bool + inodeTable sortedmap.BPlusTree + postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct + postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct + reservedToNonce uint64 + rootDirDirectory sortedmap.BPlusTree + rootDirInodeHeadV1 *ilayout.InodeHeadV1Struct + rootDirInodeHeadV1Buf []byte + rootDirInodeObjectLength uint64 + rootDirInodeObjectNumber uint64 + rootDirInodeObjectOffset uint64 + superBlockObjectLength uint64 + superBlockObjectNumber uint64 + superBlockObjectOffset uint64 + superBlockV1 *ilayout.SuperBlockV1Struct + superBlockV1Buf []byte + timeNow = time.Now() ) // Reserve some Nonce values @@ -343,15 +525,16 @@ func postVolume(name string, storageURL string, authToken string) (err error) { // Construct RootDirInode - rootDirDirectoryCallbacks = &rootDirDirectoryCallbacksStruct{ + postVolumeRootDirDirectoryCallbacks = &postVolumeRootDirDirectoryCallbacksStruct{ objectNumber: rootDirInodeObjectNumber, - buf: make([]byte, 0), + body: make([]byte, 0), + readPos: 0, } rootDirDirectory = sortedmap.NewBPlusTree( - globals.config.InodeTableMaxInodesPerBPlusTreePage, + globals.config.RootDirMaxDirEntriesPerBPlusTreePage, sortedmap.CompareString, - rootDirDirectoryCallbacks, + postVolumeRootDirDirectoryCallbacks, nil) ok, err = rootDirDirectory.Put( @@ -416,8 +599,8 @@ func postVolume(name string, storageURL string, authToken string) (err error) { Layout: []ilayout.InodeHeadLayoutEntryV1Struct{ { ObjectNumber: rootDirInodeObjectNumber, - ObjectSize: uint64(len(rootDirDirectoryCallbacks.buf)), - BytesReferenced: uint64(len(rootDirDirectoryCallbacks.buf)), + ObjectSize: uint64(len(postVolumeRootDirDirectoryCallbacks.body)), + BytesReferenced: uint64(len(postVolumeRootDirDirectoryCallbacks.body)), }, }, } @@ -430,19 +613,72 @@ func postVolume(name string, storageURL string, authToken string) (err error) { fmt.Printf("UNDO: reservedToNonce == %v\n", reservedToNonce) fmt.Printf("UNDO: rootDirInodeHeadV1:\n%s\n", utils.JSONify(rootDirInodeHeadV1, true)) - rootDirDirectoryCallbacks.buf = append(rootDirDirectoryCallbacks.buf, rootDirInodeHeadV1Buf...) + postVolumeRootDirDirectoryCallbacks.body = append(postVolumeRootDirDirectoryCallbacks.body, rootDirInodeHeadV1Buf...) + + err = swiftObjectPut(storageURL, authToken, rootDirInodeObjectNumber, postVolumeRootDirDirectoryCallbacks) + if nil != err { + return + } + + // Construct SuperBlock + + postVolumeSuperBlockInodeTableCallbacks = &postVolumeSuperBlockInodeTableCallbacksStruct{ + objectNumber: superBlockObjectNumber, + body: make([]byte, 0), + readPos: 0, + } + + inodeTable = sortedmap.NewBPlusTree( + globals.config.InodeTableMaxInodesPerBPlusTreePage, + sortedmap.CompareUint64, + postVolumeSuperBlockInodeTableCallbacks, + nil) + + ok, err = inodeTable.Put( + ilayout.RootDirInodeNumber, + ilayout.InodeTableEntryValueV1Struct{ + InodeHeadObjectNumber: ilayout.RootDirInodeNumber, + InodeHeadLength: uint64(len(rootDirInodeHeadV1Buf)), // TODO + }) + if nil != err { + return + } + if !ok { + err = fmt.Errorf("inodeTable.Put(RootDirInodeNumber,) returned !ok") + return + } + + _, superBlockObjectOffset, superBlockObjectLength, err = inodeTable.Flush(false) + if nil != err { + return + } + + fmt.Printf("UNDO: superBlockObjectOffset: %v\n", superBlockObjectOffset) + fmt.Printf("UNDO: superBlockObjectLength: %d\n", superBlockObjectLength) + + superBlockV1 = &ilayout.SuperBlockV1Struct{ + InodeTableRootObjectNumber: superBlockObjectNumber, + InodeTableRootObjectOffset: superBlockObjectOffset, + InodeTableRootObjectLength: superBlockObjectLength, + InodeTableLayout: []ilayout.InodeTableLayoutEntryV1Struct{}, // TODO + InodeObjectCount: 1, + InodeObjectSize: uint64(len(postVolumeRootDirDirectoryCallbacks.body)), + InodeBytesReferenced: 0, // TODO + } + + superBlockV1Buf, err = superBlockV1.MarshalSuperBlockV1() + if nil != err { + return + } + + postVolumeSuperBlockInodeTableCallbacks.body = append(postVolumeSuperBlockInodeTableCallbacks.body, superBlockV1Buf...) - err = swiftObjectPut(storageURL, authToken, rootDirInodeObjectNumber, rootDirDirectoryCallbacks.buf) + err = swiftObjectPut(storageURL, authToken, superBlockObjectNumber, postVolumeSuperBlockInodeTableCallbacks) if nil != err { return } // TODO: - // 3 - Create InodeTable to include RootDirInode - // 4 - Flush InodeTable to inodeTableCallbacks.buf - // 5 - Craft SuperBlock to reference flushed InodeTable - // 6 - Append marshaled SuperBlock to inodeTableCallbacks.buf - // 7 - PUT inodeTableCallbacks.buf contents to / // 8 - Craft CheckPointHeader to reference superBlockObjectNumber & reservedToNonce // 9 - PUT X-Checkpoint-Header to From 3fc2bb805f80721b5f57f5fbb2b2d62541d14b5c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 12 Apr 2021 09:28:51 -0700 Subject: [PATCH 126/160] Finished formatting (POST /volume) in imgr --- ilayout/api.go | 7 +++++ imgr/imgrpkg/api.go | 6 ++-- imgr/imgrpkg/http-server.go | 40 +++++++++++---------------- imgr/imgrpkg/volume.go | 55 +++++++++++++++++++++++-------------- 4 files changed, 59 insertions(+), 49 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index 81eb980b1..d4ceaa111 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -60,6 +60,13 @@ import ( "time" ) +// CheckPointHeaderName specifies the name of the CheckPointHeader applied +// to the Container to identify the SuperBlock. +// +const ( + CheckPointHeaderName = "X-Container-Meta-Check-Point" +) + // CheckpointHeaderVersionV* specifies the format of the CheckPointHeader. // The CheckPointHeaderVersion must always be fetched by scanning the entire // CheckPointHeader using a %016X format specifier. This value will then be diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 5d75d8cb2..eb1b5d2a2 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -85,7 +85,7 @@ // This will return a JSON document containing only the specified // details (assuming it is currently being served). // -// POST /volume/ +// POST /volume // Content-Type: application/json // // { @@ -93,9 +93,7 @@ // "AuthToken" : "AUTH_tk0123456789abcde0123456789abcdef0" // } // -// This will cause the specified to be served. The StorageURL -// specified in the JSON document content identifies the Container to first -// format and then serve. +// This will cause the specified StorageURL to be formatted. // // PUT /volume/ // Content-Type: application/json diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index dabcb580f..9e5f455f8 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -249,8 +249,8 @@ func serveHTTPGetOfVolume(responseWriter http.ResponseWriter, request *http.Requ func serveHTTPPost(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { switch { - case strings.HasPrefix(requestPath, "/volume"): - serveHTTPPostOfVolume(responseWriter, request, requestPath, requestBody) + case "/volume" == requestPath: + serveHTTPPostOfVolume(responseWriter, request, requestBody) default: responseWriter.WriteHeader(http.StatusNotFound) } @@ -261,38 +261,30 @@ type serveHTTPPostOfVolumeRequestBodyAsJSONStruct struct { AuthToken string } -func serveHTTPPostOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestPath string, requestBody []byte) { +func serveHTTPPostOfVolume(responseWriter http.ResponseWriter, request *http.Request, requestBody []byte) { var ( err error - pathSplit []string requestBodyAsJSON serveHTTPPostOfVolumeRequestBodyAsJSONStruct startTime time.Time ) startTime = time.Now() - pathSplit = strings.Split(requestPath, "/") - - switch len(pathSplit) { - case 3: - defer func() { - globals.stats.PostVolumeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() - - err = json.Unmarshal(requestBody, &requestBodyAsJSON) - if nil != err { - responseWriter.WriteHeader(http.StatusBadRequest) - return - } + defer func() { + globals.stats.PostVolumeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() - err = postVolume(pathSplit[2], requestBodyAsJSON.StorageURL, requestBodyAsJSON.AuthToken) - if nil == err { - responseWriter.WriteHeader(http.StatusCreated) - } else { - responseWriter.WriteHeader(http.StatusConflict) - } - default: + err = json.Unmarshal(requestBody, &requestBodyAsJSON) + if nil != err { responseWriter.WriteHeader(http.StatusBadRequest) + return + } + + err = postVolume(requestBodyAsJSON.StorageURL, requestBodyAsJSON.AuthToken) + if nil == err { + responseWriter.WriteHeader(http.StatusCreated) + } else { + responseWriter.WriteHeader(http.StatusConflict) } } diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 165469ad8..283b6d90d 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -11,7 +11,6 @@ import ( "time" "github.com/NVIDIA/proxyfs/ilayout" - "github.com/NVIDIA/proxyfs/utils" "github.com/NVIDIA/sortedmap" ) @@ -495,10 +494,12 @@ func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCal return } -func postVolume(name string, storageURL string, authToken string) (err error) { +func postVolume(storageURL string, authToken string) (err error) { var ( - ok bool + checkPointHeaderV1 *ilayout.CheckPointHeaderV1Struct + checkPointHeaderV1String string inodeTable sortedmap.BPlusTree + ok bool postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCallbacksStruct reservedToNonce uint64 @@ -523,7 +524,7 @@ func postVolume(name string, storageURL string, authToken string) (err error) { reservedToNonce = superBlockObjectNumber - // Construct RootDirInode + // Create RootDirInode postVolumeRootDirDirectoryCallbacks = &postVolumeRootDirDirectoryCallbacksStruct{ objectNumber: rootDirInodeObjectNumber, @@ -610,9 +611,6 @@ func postVolume(name string, storageURL string, authToken string) (err error) { return } - fmt.Printf("UNDO: reservedToNonce == %v\n", reservedToNonce) - fmt.Printf("UNDO: rootDirInodeHeadV1:\n%s\n", utils.JSONify(rootDirInodeHeadV1, true)) - postVolumeRootDirDirectoryCallbacks.body = append(postVolumeRootDirDirectoryCallbacks.body, rootDirInodeHeadV1Buf...) err = swiftObjectPut(storageURL, authToken, rootDirInodeObjectNumber, postVolumeRootDirDirectoryCallbacks) @@ -620,7 +618,7 @@ func postVolume(name string, storageURL string, authToken string) (err error) { return } - // Construct SuperBlock + // Create SuperBlock postVolumeSuperBlockInodeTableCallbacks = &postVolumeSuperBlockInodeTableCallbacksStruct{ objectNumber: superBlockObjectNumber, @@ -638,7 +636,7 @@ func postVolume(name string, storageURL string, authToken string) (err error) { ilayout.RootDirInodeNumber, ilayout.InodeTableEntryValueV1Struct{ InodeHeadObjectNumber: ilayout.RootDirInodeNumber, - InodeHeadLength: uint64(len(rootDirInodeHeadV1Buf)), // TODO + InodeHeadLength: uint64(len(rootDirInodeHeadV1Buf)), }) if nil != err { return @@ -653,17 +651,20 @@ func postVolume(name string, storageURL string, authToken string) (err error) { return } - fmt.Printf("UNDO: superBlockObjectOffset: %v\n", superBlockObjectOffset) - fmt.Printf("UNDO: superBlockObjectLength: %d\n", superBlockObjectLength) - superBlockV1 = &ilayout.SuperBlockV1Struct{ InodeTableRootObjectNumber: superBlockObjectNumber, InodeTableRootObjectOffset: superBlockObjectOffset, InodeTableRootObjectLength: superBlockObjectLength, - InodeTableLayout: []ilayout.InodeTableLayoutEntryV1Struct{}, // TODO - InodeObjectCount: 1, - InodeObjectSize: uint64(len(postVolumeRootDirDirectoryCallbacks.body)), - InodeBytesReferenced: 0, // TODO + InodeTableLayout: []ilayout.InodeTableLayoutEntryV1Struct{ + { + ObjectNumber: superBlockObjectNumber, + ObjectSize: uint64(len(postVolumeSuperBlockInodeTableCallbacks.body)), + BytesReferenced: uint64(len(postVolumeSuperBlockInodeTableCallbacks.body)), + }, + }, + InodeObjectCount: 1, + InodeObjectSize: uint64(len(postVolumeRootDirDirectoryCallbacks.body)), + InodeBytesReferenced: uint64(len(postVolumeRootDirDirectoryCallbacks.body)), } superBlockV1Buf, err = superBlockV1.MarshalSuperBlockV1() @@ -678,14 +679,26 @@ func postVolume(name string, storageURL string, authToken string) (err error) { return } - // TODO: - // 8 - Craft CheckPointHeader to reference superBlockObjectNumber & reservedToNonce - // 9 - PUT X-Checkpoint-Header to + // Create CheckPointHeader + + checkPointHeaderV1 = &ilayout.CheckPointHeaderV1Struct{ + Version: ilayout.CheckPointHeaderVersionV1, + SuperBlockObjectNumber: superBlockObjectNumber, + SuperBlockLength: uint64(len(superBlockV1Buf)), + ReservedToNonce: reservedToNonce, + } - // Finally, start serving the volume (and pass thru that err return) + checkPointHeaderV1String, err = checkPointHeaderV1.MarshalCheckPointHeaderV1() + if nil != err { + return + } - err = putVolume(name, storageURL) + err = swiftContainerHeaderSet(storageURL, authToken, ilayout.CheckPointHeaderName, checkPointHeaderV1String) + if nil != err { + return + } + err = nil return } From 1d6b32fcffd710620a4a29d76f55c62c854d8c4e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 12 Apr 2021 18:17:39 -0700 Subject: [PATCH 127/160] Added imgr/imgrpkg/http-server_test.go... fully testing WebAPI of imgr Also added missing Object DELETE method... since imgr will be responsible for deleting Objects --- imgr/imgrpkg/dummy_test.go | 11 -- imgr/imgrpkg/globals.go | 21 ++- imgr/imgrpkg/http-server.go | 19 ++- imgr/imgrpkg/http-server_test.go | 109 ++++++++++++ imgr/imgrpkg/swift-client.go | 61 +++++++ imgr/imgrpkg/utils_test.go | 278 +++++++++++++++++++++++++++++++ imgr/imgrpkg/volume.go | 30 ++-- 7 files changed, 498 insertions(+), 31 deletions(-) delete mode 100644 imgr/imgrpkg/dummy_test.go create mode 100644 imgr/imgrpkg/http-server_test.go create mode 100644 imgr/imgrpkg/utils_test.go diff --git a/imgr/imgrpkg/dummy_test.go b/imgr/imgrpkg/dummy_test.go deleted file mode 100644 index 0b0d310b5..000000000 --- a/imgr/imgrpkg/dummy_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2015-2021, NVIDIA CORPORATION. -// SPDX-License-Identifier: Apache-2.0 - -package imgrpkg - -import ( - "testing" -) - -func TestDummy(t *testing.T) { -} diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index af9d3142e..e75beca3a 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -15,6 +15,7 @@ import ( "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/proxyfs/retryrpc" "github.com/NVIDIA/proxyfs/utils" ) @@ -110,6 +111,7 @@ type statsStruct struct { SwiftContainerHeaderGetUsecs bucketstats.BucketLog2Round SwiftContainerHeaderSetUsecs bucketstats.BucketLog2Round + SwiftObjectDeleteUsecs bucketstats.BucketLog2Round SwiftObjectGetRangeUsecs bucketstats.BucketLog2Round SwiftObjectGetTailUsecs bucketstats.BucketLog2Round SwiftObjectPutUsecs bucketstats.BucketLog2Round @@ -126,14 +128,17 @@ type mountStruct struct { } type volumeStruct struct { - sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() - name string // - storageURL string // - mountMap map[string]*mountStruct // key == mountStruct.mountID - healthyMountList *list.List // LRU of mountStruct's with .authTokenValid == true - expiredMountList *list.List // LRU of mountStruct's with .authTokenValid == false - deleting bool // - inodeTable sortedmap.BPlusTree // + sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() + name string // + storageURL string // + mountMap map[string]*mountStruct // key == mountStruct.mountID + healthyMountList *list.List // LRU of mountStruct's with .authTokenValid == true + expiredMountList *list.List // LRU of mountStruct's with .authTokenValid == false + deleting bool // + checkPointHeader *ilayout.CheckPointHeaderV1Struct // == nil if not currently mounted and/or checkpointing + superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing + inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing + pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber } type globalsStruct struct { diff --git a/imgr/imgrpkg/http-server.go b/imgr/imgrpkg/http-server.go index 9e5f455f8..d0d19e1c7 100644 --- a/imgr/imgrpkg/http-server.go +++ b/imgr/imgrpkg/http-server.go @@ -18,9 +18,15 @@ import ( "github.com/NVIDIA/proxyfs/bucketstats" ) +const ( + startHTTPServerUpCheckDelay = 100 * time.Millisecond + startHTTPServerUpCheckMaxRetries = 10 +) + func startHTTPServer() (err error) { var ( - ipAddrTCPPort string + ipAddrTCPPort string + startHTTPServerUpCheckRetries uint32 ) ipAddrTCPPort = net.JoinHostPort(globals.config.PrivateIPAddr, strconv.Itoa(int(globals.config.HTTPServerPort))) @@ -45,7 +51,16 @@ func startHTTPServer() (err error) { globals.httpServerWG.Done() }() - err = nil + for startHTTPServerUpCheckRetries = 0; startHTTPServerUpCheckRetries < startHTTPServerUpCheckMaxRetries; startHTTPServerUpCheckRetries++ { + _, err = http.Get("http://" + ipAddrTCPPort + "/config") + if nil == err { + return + } + + time.Sleep(startHTTPServerUpCheckDelay) + } + + err = fmt.Errorf("startHTTPServerUpCheckMaxRetries (%v) exceeded", startHTTPServerUpCheckMaxRetries) return } diff --git a/imgr/imgrpkg/http-server_test.go b/imgr/imgrpkg/http-server_test.go new file mode 100644 index 000000000..fb3f98202 --- /dev/null +++ b/imgr/imgrpkg/http-server_test.go @@ -0,0 +1,109 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "github.com/NVIDIA/proxyfs/ilayout" +) + +func TestHTTPServer(t *testing.T) { + var ( + err error + getRequestHeaders http.Header + getResponseHeaders http.Header + postRequestBody string + putRequestBody string + responseBody []byte + responseBodyExpected string + ) + + testSetup(t) + + _, _, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/config", nil, nil) + if nil != err { + t.Fatalf("GET /config failed: %v", err) + } + + _, _, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/stats", nil, nil) + if nil != err { + t.Fatalf("GET /stats failed: %v", err) + } + + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume", nil, nil) + if nil != err { + t.Fatalf("GET /volume [case 1] failed: %v", err) + } + if "[]" != string(responseBody[:]) { + t.Fatalf("GET /volume [case 1] should have returned \"[]\" - it returned \"%s\"", string(responseBody[:])) + } + + postRequestBody = fmt.Sprintf("{\"StorageURL\":\"%s\",\"AuthToken\":\"%s\"}", testGlobals.containerURL, testGlobals.authToken) + + _, _, err = testDoHTTPRequest("POST", testGlobals.httpServerURL+"/volume", nil, strings.NewReader(postRequestBody)) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"POST\", testGlobals.httpServerURL+\"/volume\", nil, strings.NewReader(postRequestBody)) failed: %v", err) + } + + getRequestHeaders = make(http.Header) + + getRequestHeaders["X-Auth-Token"] = []string{testGlobals.authToken} + + getResponseHeaders, responseBody, err = testDoHTTPRequest("GET", testGlobals.containerURL, getRequestHeaders, nil) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL, getRequestHeaders, nil) failed: %v", err) + } + if getResponseHeaders.Get(ilayout.CheckPointHeaderName) != "0000000000000001 0000000000000003 0000000000000058 0000000000000003" { + t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL, getRequestHeaders, nil) returned unexpected %s: \"%s\"", ilayout.CheckPointHeaderName, getResponseHeaders.Get(ilayout.CheckPointHeaderName)) + } + if "0000000000000002\n0000000000000003\n" != string(responseBody[:]) { + t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL, getRequestHeaders, nil) returned unexpected Object List: \"%s\"", string(responseBody[:])) + } + + putRequestBody = fmt.Sprintf("{\"StorageURL\":\"%s\"}", testGlobals.containerURL) + + _, _, err = testDoHTTPRequest("PUT", testGlobals.httpServerURL+"/volume/"+testVolume, nil, strings.NewReader(putRequestBody)) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) failed: %v", err) + } + + responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"HealthyMounts\":0,\"ExpiredMounts\":0}", testVolume, testGlobals.containerURL) + + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) + if nil != err { + t.Fatalf("GET /volume/%s failed: %v", testVolume, err) + } + if string(responseBody[:]) != responseBodyExpected { + t.Fatalf("GET /volume/%s returned unexpected responseBody: \"%s\"", testVolume, responseBody) + } + + responseBodyExpected = "[" + responseBodyExpected + "]" + + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume", nil, nil) + if nil != err { + t.Fatalf("GET /volume [case 2] failed: %v", err) + } + if string(responseBody[:]) != responseBodyExpected { + t.Fatalf("GET /volume [case 2] returned unexpected responseBody: \"%s\"", responseBody) + } + + _, _, err = testDoHTTPRequest("DELETE", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"DELETE\", testGlobals.httpServerURL+\"/volume/\"+testVolume, nil, nil) failed: %v", err) + } + + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume", nil, nil) + if nil != err { + t.Fatalf("GET /volume [case 3] failed: %v", err) + } + if "[]" != string(responseBody[:]) { + t.Fatalf("GET /volume [case 3] should have returned \"[]\" - it returned \"%s\"", string(responseBody[:])) + } + + testTeardown(t) +} diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index bbcca420b..dcd620653 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -178,6 +178,67 @@ func swiftContainerHeaderSet(storageURL string, authToken string, headerName str return } +func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) (err error) { + var ( + httpRequest *http.Request + httpResponse *http.Response + nextSwiftRetryDelay time.Duration + numSwiftRetries uint32 + objectURL string + startTime time.Time + ) + + startTime = time.Now() + + defer func() { + globals.stats.SwiftObjectDeleteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + }() + + objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) + + nextSwiftRetryDelay = globals.config.SwiftRetryDelay + + for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { + httpRequest, err = http.NewRequest("DELETE", objectURL, nil) + if nil != err { + return + } + + if "" != authToken { + httpRequest.Header["X-Auth-Token"] = []string{authToken} + } + + httpResponse, err = globals.httpClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) + return + } + + _, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) + return + } + + if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { + err = nil + return + } + + time.Sleep(nextSwiftRetryDelay) + + nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) + } + + err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") + return +} + func swiftObjectGetRange(storageURL string, authToken string, objectNumber uint64, objectOffset uint64, objectLength uint64) (buf []byte, err error) { var ( httpRequest *http.Request diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go new file mode 100644 index 000000000..ac141f142 --- /dev/null +++ b/imgr/imgrpkg/utils_test.go @@ -0,0 +1,278 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +import ( + "crypto/x509/pkix" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "os" + "testing" + "time" + + "github.com/NVIDIA/proxyfs/conf" + "github.com/NVIDIA/proxyfs/icert/icertpkg" + "github.com/NVIDIA/proxyfs/iswift/iswiftpkg" +) + +const ( + testIPAddr = "127.0.0.1" + testRetryRPCPort = 32357 + testHTTPServerPort = 15347 + testSwiftProxyTCPPort = 24367 + testSwiftAuthUser = "test" + testSwiftAuthKey = "test" + testContainer = "testContainer" + testVolume = "testVolume" +) + +type testGlobalsStruct struct { + tempDir string + caCertFile string + caKeyFile string + endpointCertFile string + endpointKeyFile string + confMap conf.ConfMap + httpServerURL string + authURL string + authToken string + accountURL string + containerURL string +} + +var testGlobals *testGlobalsStruct + +func testSetup(t *testing.T) { + var ( + authRequestHeaders http.Header + authResponseHeaders http.Header + confStrings []string + err error + putAccountRequestHeaders http.Header + putContainerRequestHeaders http.Header + tempDir string + ) + + tempDir, err = ioutil.TempDir("", "imgrpkg_test") + if nil != err { + t.Fatalf("ioutil.TempDir(\"\", \"imgrpkg_test\") failed: %v", err) + } + + testGlobals = &testGlobalsStruct{ + tempDir: tempDir, + caCertFile: tempDir + "/caCertFile", + caKeyFile: tempDir + "/caKeyFile", + endpointCertFile: tempDir + "/endpoingCertFile", + endpointKeyFile: tempDir + "/endpointKeyFile", + httpServerURL: fmt.Sprintf("http://%s:%d", testIPAddr, testHTTPServerPort), + authURL: fmt.Sprintf("http://%s:%d/auth/v1.0", testIPAddr, testSwiftProxyTCPPort), + } + + _, _, err = icertpkg.GenCACert( + icertpkg.GenerateKeyAlgorithmEd25519, + pkix.Name{ + Organization: []string{"Test Organization CA"}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + time.Hour, + testGlobals.caCertFile, + testGlobals.caKeyFile) + if nil != err { + t.Fatalf("icertpkg.GenCACert() failed: %v", err) + } + + _, _, err = icertpkg.GenEndpointCert( + icertpkg.GenerateKeyAlgorithmEd25519, + pkix.Name{ + Organization: []string{"Test Organization Endpoint"}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + []string{}, + []net.IP{net.ParseIP(testIPAddr)}, + time.Hour, + testGlobals.caCertFile, + testGlobals.caKeyFile, + testGlobals.endpointCertFile, + testGlobals.endpointKeyFile) + if nil != err { + t.Fatalf("icertpkg.GenEndpointCert() failed: %v", err) + } + + confStrings = []string{ + "IMGR.PublicIPAddr=" + testIPAddr, + "IMGR.PrivateIPAddr=" + testIPAddr, + "IMGR.RetryRPCPort=" + fmt.Sprintf("%d", testRetryRPCPort), + "IMGR.HTTPServerPort=" + fmt.Sprintf("%d", testHTTPServerPort), + + "IMGR.RetryRPCTTLCompleted=10m", + "IMGR.RetryRPCAckTrim=100ms", + "IMGR.RetryRPCDeadlineIO=60s", + "IMGR.RetryRPCKeepAlivePeriod=60s", + + "IMGR.RetryRPCCertFilePath=" + testGlobals.endpointCertFile, + "IMGR.RetryRPCKeyFilePath=" + testGlobals.endpointKeyFile, + + "IMGR.FetchNonceRangeToReturn=100", + + "IMGR.MinLeaseDuration=250ms", + "IMGR.LeaseInterruptInterval=250ms", + "IMGR.LeaseInterruptLimit=20", + + "IMGR.SwiftRetryDelay=100ms", + "IMGR.SwiftRetryExpBackoff=2", + "IMGR.SwiftRetryLimit=4", + + "IMGR.SwiftTimeout=10m", + "IMGR.SwiftConnectionPoolSize=128", + + "IMGR.InodeTableCacheEvictLowLimit=10000", + "IMGR.InodeTableCacheEvictHighLimit=10010", + + "IMGR.InodeTableMaxInodesPerBPlusTreePage=2048", + "IMGR.RootDirMaxDirEntriesPerBPlusTreePage=1024", + + "IMGR.LogFilePath=", + "IMGR.LogToConsole=true", + "IMGR.TraceEnabled=false", + + "ISWIFT.SwiftProxyIPAddr=" + testIPAddr, + "ISWIFT.SwiftProxyTCPPort=" + fmt.Sprintf("%d", testSwiftProxyTCPPort), + + "ISWIFT.MaxAccountNameLength=256", + "ISWIFT.MaxContainerNameLength=256", + "ISWIFT.MaxObjectNameLength=1024", + "ISWIFT.AccountListingLimit=10000", + "ISWIFT.ContainerListingLimit=10000", + } + + testGlobals.confMap, err = conf.MakeConfMapFromStrings(confStrings) + if nil != err { + t.Fatalf("conf.MakeConfMapFromStrings(confStrings) failed: %v", err) + } + + err = iswiftpkg.Start(testGlobals.confMap) + if nil != err { + t.Fatalf("iswifpkg.Start(testGlobals.confMap) failed: %v", err) + } + + authRequestHeaders = make(http.Header) + + authRequestHeaders["X-Auth-User"] = []string{testSwiftAuthUser} + authRequestHeaders["X-Auth-Key"] = []string{testSwiftAuthKey} + + authResponseHeaders, _, err = testDoHTTPRequest("GET", testGlobals.authURL, authRequestHeaders, nil) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.authURL, authRequestHeaders) failed: %v", err) + } + + testGlobals.authToken = authResponseHeaders.Get("X-Auth-Token") + testGlobals.accountURL = authResponseHeaders.Get("X-Storage-Url") + + testGlobals.containerURL = testGlobals.accountURL + "/" + testContainer + + putAccountRequestHeaders = make(http.Header) + + putAccountRequestHeaders["X-Auth-Token"] = []string{testGlobals.authToken} + + _, _, err = testDoHTTPRequest("PUT", testGlobals.accountURL, putAccountRequestHeaders, nil) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.accountURL, putAccountRequestHeaders) failed: %v", err) + } + + putContainerRequestHeaders = make(http.Header) + + putContainerRequestHeaders["X-Auth-Token"] = []string{testGlobals.authToken} + + _, _, err = testDoHTTPRequest("PUT", testGlobals.containerURL, putContainerRequestHeaders, nil) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.storageURL, putContainerRequestHeaders) failed: %v", err) + } + + err = Start(testGlobals.confMap) + if nil != err { + t.Fatalf("Start(testGlobals.confMap) failed: %v", err) + } +} + +func testTeardown(t *testing.T) { + var ( + err error + ) + + err = Stop() + if nil != err { + t.Fatalf("Stop() failed: %v", err) + } + + err = iswiftpkg.Stop() + if nil != err { + t.Fatalf("iswiftpkg.Stop() failed: %v", err) + } + + err = os.RemoveAll(testGlobals.tempDir) + if nil != err { + t.Fatalf("os.RemoveAll(testGlobals.tempDir) failed: %v", err) + } + + testGlobals = nil +} + +func testDoHTTPRequest(method string, url string, requestHeaders http.Header, requestBody io.Reader) (responseHeaders http.Header, responseBody []byte, err error) { + var ( + headerKey string + headerValues []string + httpRequest *http.Request + httpResponse *http.Response + ) + + httpRequest, err = http.NewRequest(method, url, requestBody) + if nil != err { + err = fmt.Errorf("http.NewRequest(\"%s\", \"%s\", nil) failed: %v", method, url, err) + return + } + + if nil != requestHeaders { + for headerKey, headerValues = range requestHeaders { + httpRequest.Header[headerKey] = headerValues + } + } + + httpResponse, err = http.DefaultClient.Do(httpRequest) + if nil != err { + err = fmt.Errorf("http.Do(httpRequest) failed: %v", err) + return + } + + responseBody, err = ioutil.ReadAll(httpResponse.Body) + if nil != err { + err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v", err) + return + } + err = httpResponse.Body.Close() + if nil != err { + err = fmt.Errorf("httpResponse.Body.Close() failed: %v", err) + return + } + + if (200 > httpResponse.StatusCode) || (299 < httpResponse.StatusCode) { + err = fmt.Errorf("httpResponse.StatusCode unexpected: %s", httpResponse.Status) + return + } + + responseHeaders = httpResponse.Header + + err = nil + return +} diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 283b6d90d..748819efd 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -141,6 +141,9 @@ func getVolumeAsJSON(volumeName string) (volume []byte, err error) { logFatalf("[IMGR]globals.volumeMap[\"%s\"] was not a *volumeStruct", volumeName) } + volumeAsStruct.RLock() + globals.RUnlock() + volumeToReturn = &volumeGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, @@ -148,7 +151,7 @@ func getVolumeAsJSON(volumeName string) (volume []byte, err error) { ExpiredMounts: uint64(volumeAsStruct.expiredMountList.Len()), } - globals.RUnlock() + volumeAsStruct.RUnlock() volume, err = json.Marshal(volumeToReturn) if nil != err { @@ -193,12 +196,16 @@ func getVolumeListAsJSON() (volumeList []byte) { logFatalf("[IMGR]globals.volumeMap[%d] was not a *volumeStruct", volumeListIndex) } + volumeAsStruct.RLock() + volumeListToReturn[volumeListIndex] = &volumeGETStruct{ Name: volumeAsStruct.name, StorageURL: volumeAsStruct.storageURL, HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), ExpiredMounts: uint64(volumeAsStruct.expiredMountList.Len()), } + + volumeAsStruct.RUnlock() } globals.RUnlock() @@ -708,18 +715,21 @@ func putVolume(name string, storageURL string) (err error) { volume *volumeStruct ) - globals.Lock() - volume = &volumeStruct{ - name: name, - storageURL: storageURL, - mountMap: make(map[string]*mountStruct), - healthyMountList: list.New(), - expiredMountList: list.New(), - deleting: false, - inodeTable: nil, + name: name, + storageURL: storageURL, + mountMap: make(map[string]*mountStruct), + healthyMountList: list.New(), + expiredMountList: list.New(), + deleting: false, + checkPointHeader: nil, + superBlock: nil, + inodeTable: nil, + pendingObjectDeleteSet: make(map[uint64]struct{}), } + globals.Lock() + ok, err = globals.volumeMap.Put(volume.name, volume) if nil != err { logFatal(err) From aaa44fced738f38f0803dc238e23fc1bf66f1a5c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 13 Apr 2021 09:05:01 -0700 Subject: [PATCH 128/160] Inserted placeholders for iclient, the replacement for pfsagentd --- Makefile | 2 ++ iclient/Makefile | 6 ++++++ iclient/README.md | 3 +++ iclient/dummy_test.go | 11 +++++++++++ iclient/iclientpkg/Makefile | 6 ++++++ iclient/iclientpkg/api.go | 4 ++++ iclient/iclientpkg/api_test.go | 12 ++++++++++++ iclient/iclientpkg/impl.go | 4 ++++ iclient/main.go | 6 ++++++ 9 files changed, 54 insertions(+) create mode 100644 iclient/Makefile create mode 100644 iclient/README.md create mode 100644 iclient/dummy_test.go create mode 100644 iclient/iclientpkg/Makefile create mode 100644 iclient/iclientpkg/api.go create mode 100644 iclient/iclientpkg/api_test.go create mode 100644 iclient/iclientpkg/impl.go create mode 100644 iclient/main.go diff --git a/Makefile b/Makefile index 9f265da5f..4f11d6f5c 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,7 @@ gopkgsubdirs = \ version \ emswift/emswiftpkg \ icert/icertpkg \ + iclient/iclientpkg \ imgr/imgrpkg \ iswift/iswiftpkg @@ -48,6 +49,7 @@ gobinsubdirs = \ emswift \ fsworkout \ icert \ + iclient \ imgr \ inodeworkout \ iswift \ diff --git a/iclient/Makefile b/iclient/Makefile new file mode 100644 index 000000000..13a1dc7ad --- /dev/null +++ b/iclient/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/iclient + +include ../GoMakefile diff --git a/iclient/README.md b/iclient/README.md new file mode 100644 index 000000000..d42c85e0c --- /dev/null +++ b/iclient/README.md @@ -0,0 +1,3 @@ +# iclient + +A FUSE presentation of a ProxyFS Volume. diff --git a/iclient/dummy_test.go b/iclient/dummy_test.go new file mode 100644 index 000000000..d42991728 --- /dev/null +++ b/iclient/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/iclient/iclientpkg/Makefile b/iclient/iclientpkg/Makefile new file mode 100644 index 000000000..43cd6e749 --- /dev/null +++ b/iclient/iclientpkg/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/iclient/iclientpkg + +include ../../GoMakefile diff --git a/iclient/iclientpkg/api.go b/iclient/iclientpkg/api.go new file mode 100644 index 000000000..a38d3fc2e --- /dev/null +++ b/iclient/iclientpkg/api.go @@ -0,0 +1,4 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg diff --git a/iclient/iclientpkg/api_test.go b/iclient/iclientpkg/api_test.go new file mode 100644 index 000000000..5cb237705 --- /dev/null +++ b/iclient/iclientpkg/api_test.go @@ -0,0 +1,12 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg + +import ( + "testing" +) + +func TestAPI(t *testing.T) { + t.Logf("TODO") +} diff --git a/iclient/iclientpkg/impl.go b/iclient/iclientpkg/impl.go new file mode 100644 index 000000000..a38d3fc2e --- /dev/null +++ b/iclient/iclientpkg/impl.go @@ -0,0 +1,4 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package iclientpkg diff --git a/iclient/main.go b/iclient/main.go new file mode 100644 index 000000000..0f815e8ac --- /dev/null +++ b/iclient/main.go @@ -0,0 +1,6 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +func main() {} From f748240cb585d69dd85662ccd52e09bb2c3c1109 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Mon, 12 Apr 2021 13:27:04 -0500 Subject: [PATCH 129/160] bump swift to ss-release-2.27.0.2 --- pfs_middleware/tox.ini | 2 +- saio/container/Dockerfile | 2 +- saio/vagrant_provision.sh | 2 +- sait/vagrant_provision.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pfs_middleware/tox.ini b/pfs_middleware/tox.ini index 69d951627..d576e6747 100644 --- a/pfs_middleware/tox.ini +++ b/pfs_middleware/tox.ini @@ -7,7 +7,7 @@ usedevelop = True deps = lint: flake8 !lint: -r{toxinidir}/test-requirements.txt - release: git+https://github.com/NVIDIA/swift.git@ss-release-2.26.0.16 + release: git+https://github.com/NVIDIA/swift.git@ss-release-2.27.0.2 minver: http://tarballs.openstack.org/swift/swift-2.9.0.tar.gz master: http://tarballs.openstack.org/swift/swift-master.tar.gz commands = python -m unittest discover diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index d77ac2052..001448946 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -3,7 +3,7 @@ FROM centos:7.4.1708 -ARG SwiftVersion=ss-release-2.26.0.16 +ARG SwiftVersion=ss-release-2.27.0.2 ARG GolangVersion=1.15.5 ARG ProxyFS_Version=stable diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index 91f6fa915..4e745a347 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -228,7 +228,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.26.0.16 +git checkout ss-release-2.27.0.2 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index 021562e2c..6cc0c9342 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -230,7 +230,7 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.26.0.16 +git checkout ss-release-2.27.0.2 pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet From 398ed9912082988f77a4d4845c1e69118c6a46ca Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Tue, 13 Apr 2021 18:40:14 -0700 Subject: [PATCH 130/160] Add per method bucketstats (#608) * Add unit test for per method bucketstats * Add bucketstats tracking per method * Dump server side stats on proxyfsd when a client connection is closed * Dump bucketstats when client.Close() called --- retryrpc/api.go | 8 ++--- retryrpc/api_internal.go | 9 +++++ retryrpc/client.go | 2 +- retryrpc/client_tracking.go | 54 +++++++++++++++++++++++++++++- retryrpc/retryrpc_test.go | 67 +++++++++++++++++++++++++++++++++++++ retryrpc/server.go | 31 +++++++---------- retryrpc/svrmap.go | 28 ++++++++-------- 7 files changed, 161 insertions(+), 38 deletions(-) diff --git a/retryrpc/api.go b/retryrpc/api.go index 3c51dbbdd..4b8d45fd4 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -223,8 +223,7 @@ func (server *Server) Close() { // Cleanup bucketstats so that unit tests can run for _, ci := range server.perClientInfo { ci.Lock() - s10 := strconv.FormatInt(int64(ci.myUniqueID), 10) - bucketstats.UnRegister("proxyfs.retryrpc", s10) + ci.unregsiterMethodStats(server) ci.Unlock() } @@ -396,6 +395,7 @@ func (client *Client) Close() { // Wait for the goroutines to return client.goroutineWG.Wait() - bucketstats.UnRegister("proxyfs.retryrpc", client.GetStatsGroupName()) - + logger.Infof("bucketstats for myUniqueID: '%v' - %s\n", client.myUniqueID, + bucketstats.SprintStats(bucketstats.StatFormatParsable1, bucketStatsPkgName, client.GetStatsGroupName())) + bucketstats.UnRegister(bucketStatsPkgName, client.GetStatsGroupName()) } diff --git a/retryrpc/api_internal.go b/retryrpc/api_internal.go index cd9395a2b..7650a8a97 100644 --- a/retryrpc/api_internal.go +++ b/retryrpc/api_internal.go @@ -30,10 +30,18 @@ const ( const ( currentRetryVersion = 1 + bucketStatsPkgName = "proxyfs.retryrpc" ) type requestID uint64 +// methodStats tracks per method stats +type methodStats struct { + Method string // Name of method + Count bucketstats.Total // Number of times this method called + TimeOfRPCCall bucketstats.BucketLog2Round // Length of time of this method of RPC +} + // Useful stats for the clientInfo instance type statsInfo struct { TrimAddCompleted bucketstats.Total // Number added to completed list @@ -47,6 +55,7 @@ type statsInfo struct { RPCattempted bucketstats.Total // Number of RPCs attempted - may be completed or in process RPCcompleted bucketstats.Total // Number of RPCs which completed - incremented after call returns RPCretried bucketstats.Total // Number of RPCs which were just pulled from completed list + PerMethodStats map[string]*methodStats // Per method bucketstats } // Server side data structure storing per client information diff --git a/retryrpc/client.go b/retryrpc/client.go index cb5e5f062..6400a79f3 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -74,7 +74,7 @@ func (client *Client) send(method string, rpcRequest interface{}, rpcReply inter // Now that we have a connection - we can setup bucketstats if client.connection.state == CONNECTED { - bucketstats.Register("proxyfs.retryrpc", client.GetStatsGroupName(), &client.stats) + bucketstats.Register(bucketStatsPkgName, client.GetStatsGroupName(), &client.stats) } break } diff --git a/retryrpc/client_tracking.go b/retryrpc/client_tracking.go index f811887a5..615a88c1c 100644 --- a/retryrpc/client_tracking.go +++ b/retryrpc/client_tracking.go @@ -3,8 +3,16 @@ package retryrpc +import ( + "container/list" + "strconv" + + "github.com/NVIDIA/proxyfs/bucketstats" + "github.com/NVIDIA/proxyfs/logger" +) + // This file contains functions in the server which -// keep track of clients. +// track clients and initialize clientInfo. func (ci *clientInfo) isEmpty() bool { return len(ci.completedRequest) == 0 @@ -13,3 +21,47 @@ func (ci *clientInfo) isEmpty() bool { func (ci *clientInfo) completedCnt() int { return len(ci.completedRequest) } + +func methodAndName(name string, method string) string { + return name + "-" + method +} + +func initClientInfo(cCtx *connCtx, newUniqueID uint64, server *Server) (ci *clientInfo) { + ci = &clientInfo{cCtx: cCtx, myUniqueID: newUniqueID} + ci.completedRequest = make(map[requestID]*completedEntry) + ci.completedRequestLRU = list.New() + ci.stats.PerMethodStats = make(map[string]*methodStats) + + idAsStr := strconv.FormatInt(int64(newUniqueID), 10) + bucketstats.Register(bucketStatsPkgName, idAsStr, &ci.stats) + + // Register per method stats + for m := range server.svrMap { + ms := &methodStats{Method: m} + ci.stats.PerMethodStats[m] = ms + bucketstats.Register(bucketStatsPkgName, methodAndName(strconv.FormatInt(int64(ci.myUniqueID), 10), m), ms) + } + return +} + +// Bump bucketstats for this method +func (ci *clientInfo) setMethodStats(method string, deltaTime uint64) { + ms := ci.stats.PerMethodStats[method] + ms.Count.Add(1) + ms.TimeOfRPCCall.Add(deltaTime) +} + +// Unregister per method bucketstats for this client +func (ci *clientInfo) unregsiterMethodStats(server *Server) { + idAsStr := strconv.FormatInt(int64(ci.myUniqueID), 10) + + logger.Infof("bucketstats for myUniqueID: '%v' - %s\n", ci.myUniqueID, + bucketstats.SprintStats(bucketstats.StatFormatParsable1, bucketStatsPkgName, "*")) + for m := range server.svrMap { + logger.Infof("bucketstats myUniqueID: '%v' method: '%v'- %s\n", ci.myUniqueID, m, + bucketstats.SprintStats(bucketstats.StatFormatParsable1, bucketStatsPkgName, + methodAndName(strconv.FormatInt(int64(ci.myUniqueID), 10), m))) + bucketstats.UnRegister(bucketStatsPkgName, methodAndName(strconv.FormatInt(int64(ci.myUniqueID), 10), m)) + } + bucketstats.UnRegister(bucketStatsPkgName, idAsStr) +} diff --git a/retryrpc/retryrpc_test.go b/retryrpc/retryrpc_test.go index 6e6b364c0..6a3ed0f46 100644 --- a/retryrpc/retryrpc_test.go +++ b/retryrpc/retryrpc_test.go @@ -6,7 +6,9 @@ package retryrpc import ( "crypto/tls" "crypto/x509/pkix" + "fmt" "net" + "strconv" "testing" "time" @@ -378,3 +380,68 @@ func testStatsAndBucketstats(t *testing.T) { bucketstats.UnRegister("proxyfs.retryrpc", myUniqueClient1) bucketstats.UnRegister("proxyfs.retryrpc", myUniqueClient2) } + +// Unit test to show per method bucketstats +// bucketstats does not automatically register stats in a map, etc. +type methodStatsDemo struct { + Method string // Name of method + Count bucketstats.Total // Number of times this method called + TimeOfRPCCall bucketstats.BucketLog2Round // Length of time of this method of RPC +} + +type methodClientStats struct { + AddCompleted bucketstats.Total // Number added to completed list + PerMethodStats map[string]*methodStatsDemo // Example of per method stats +} + +func mAndN(name string, method string) string { + return name + "-" + method +} + +func TestMethodStatsAndBucketstats(t *testing.T) { + var ( + myClient1 methodClientStats + myUniqueClient1 = "1111111" + method = "method" + ) + + // Register from bucketstats from pfsagent #1 + bucketstats.Register("proxyfs.retryrpc", myUniqueClient1, &myClient1) + + // Register per method stats + myClient1.PerMethodStats = make(map[string]*methodStatsDemo) + for i := 0; i < 10; i++ { + msd := &methodStatsDemo{} + ms := method + strconv.Itoa(i) + myClient1.PerMethodStats[ms] = msd + bucketstats.Register("proxyfs.retryrpc", mAndN(myUniqueClient1, ms), msd) + } + + // Completed list stats + myClient1.AddCompleted.Add(1) + + // Per method stats + start := time.Now() + time.Sleep(10 * time.Millisecond) + for i := 0; i < 10; i++ { + ms := method + strconv.Itoa(i) + msd, _ := myClient1.PerMethodStats[ms] + msd.Count.Add(uint64(i)) + msd.TimeOfRPCCall.Add(uint64(time.Since(start).Microseconds())) + myClient1.PerMethodStats[ms] = msd + } + + // Dump stats + fmt.Printf("pfsagent #1: %s\n", bucketstats.SprintStats(bucketstats.StatFormatParsable1, "proxyfs.retryrpc", myUniqueClient1)) + for i := 0; i < 10; i++ { + ms := method + strconv.Itoa(i) + fmt.Printf("method: %s\n", bucketstats.SprintStats(bucketstats.StatFormatParsable1, "proxyfs.retryrpc", mAndN(myUniqueClient1, ms))) + } + + // Unregister clients from bucketstats + bucketstats.UnRegister("proxyfs.retryrpc", myUniqueClient1) + for i := 0; i < 10; i++ { + ms := method + strconv.Itoa(i) + bucketstats.UnRegister("proxyfs.retryrpc", mAndN(myUniqueClient1, ms)) + } +} diff --git a/retryrpc/server.go b/retryrpc/server.go index f1db1d60e..8ee6894b4 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -11,11 +11,9 @@ import ( "net" "os" "reflect" - "strconv" "sync" "time" - "github.com/NVIDIA/proxyfs/bucketstats" "github.com/NVIDIA/proxyfs/logger" "golang.org/x/sys/unix" ) @@ -153,7 +151,7 @@ func (server *Server) processRequest(ci *clientInfo, myConnCtx *connCtx, buf []b // be unmarshaled again to retrieve the parameters specific to // the RPC. startRPC := time.Now() - ior := server.callRPCAndFormatReply(buf, ci.myUniqueID, &jReq, &ci.stats) + ior := server.callRPCAndFormatReply(buf, ci, &jReq) ci.stats.CallWrapRPCUsec.Add(uint64(time.Since(startRPC).Microseconds())) ci.stats.RPCcompleted.Add(1) @@ -268,21 +266,20 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err ci.Unlock() } } else { + + // First time we have seen this client var ( localIOR ioReply ) - // First time we have seen this client - // Create a unique client ID and return to client server.Lock() server.clientIDNonce++ newUniqueID := server.clientIDNonce // Setup new client data structures - c := &clientInfo{cCtx: cCtx, myUniqueID: newUniqueID} - c.completedRequest = make(map[requestID]*completedEntry) - c.completedRequestLRU = list.New() + c := initClientInfo(cCtx, newUniqueID, server) + server.perClientInfo[newUniqueID] = c server.Unlock() ci = c @@ -290,9 +287,6 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err cCtx.ci = ci cCtx.Unlock() - s10 := strconv.FormatInt(int64(newUniqueID), 10) - bucketstats.Register("proxyfs.retryrpc", s10, &c.stats) - // Build reply and send back to client var e error localIOR.JResult, e = json.Marshal(newUniqueID) @@ -356,7 +350,7 @@ func (server *Server) serviceClient(ci *clientInfo, cCtx *connCtx) { } // callRPCAndMarshal calls the RPC and returns results to requestor -func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *jsonRequest, si *statsInfo) (ior *ioReply) { +func (server *Server) callRPCAndFormatReply(buf []byte, ci *clientInfo, jReq *jsonRequest) (ior *ioReply) { var ( err error returnValues []reflect.Value @@ -365,7 +359,7 @@ func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *j ) // Setup the reply structure with common fields - reply := &ioReply{} + ior = &ioReply{} rid := jReq.RequestID jReply := &jsonReply{MyUniqueID: jReq.MyUniqueID, RequestID: rid} @@ -385,7 +379,7 @@ func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *j return } req := reflect.ValueOf(dummyReq) - cid := reflect.ValueOf(clientID) + cid := reflect.ValueOf(ci.myUniqueID) // Create the reply structure typOfReply := ma.reply.Elem() @@ -393,11 +387,13 @@ func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *j // Call the method function := ma.methodPtr.Func + t := time.Now() if ma.passClientID { returnValues = function.Call([]reflect.Value{server.receiver, cid, req, myReply}) } else { returnValues = function.Call([]reflect.Value{server.receiver, req, myReply}) } + ci.setMethodStats(jReq.Method, uint64(time.Since(t).Microseconds())) // The return value for the method is an error. errInter := returnValues[0].Interface() @@ -418,12 +414,12 @@ func (server *Server) callRPCAndFormatReply(buf []byte, clientID uint64, jReq *j } // Convert response into JSON for return trip - reply.JResult, err = json.Marshal(jReply) + ior.JResult, err = json.Marshal(jReply) if err != nil { logger.PanicfWithError(err, "Unable to marshal jReply: %+v", jReply) } - return reply + return } func (server *Server) returnResults(ior *ioReply, cCtx *connCtx) { @@ -497,8 +493,7 @@ func (server *Server) trimCompleted(t time.Time, long bool) { ci.Lock() ci.cCtx.Lock() if ci.isEmpty() && ci.cCtx.serviceClientExited { - s10 := strconv.FormatInt(int64(ci.myUniqueID), 10) - bucketstats.UnRegister("proxyfs.retryrpc", s10) + ci.unregsiterMethodStats(server) delete(server.perClientInfo, key) logger.Infof("Trim - DELETE inactive clientInfo with ID: %v", ci.myUniqueID) } diff --git a/retryrpc/svrmap.go b/retryrpc/svrmap.go index 810140542..25a06c628 100644 --- a/retryrpc/svrmap.go +++ b/retryrpc/svrmap.go @@ -20,8 +20,8 @@ var ( func (server *Server) buildSvrMap(typ reflect.Type) { for m := 0; m < typ.NumMethod(); m++ { method := typ.Method(m) - mtype := method.Type - mname := method.Name + mType := method.Type + mName := method.Name // Just like net/rpc, we have these requirements on methods: // - must be exported @@ -34,27 +34,27 @@ func (server *Server) buildSvrMap(typ reflect.Type) { // Will have 3 arguments if pass request/reply // Will have 4 arguments if expect clientID and request/reply - if (mtype.NumIn() != 3) && (mtype.NumIn() != 4) { + if (mType.NumIn() != 3) && (mType.NumIn() != 4) { continue } - if mtype.NumOut() != 1 { + if mType.NumOut() != 1 { continue } - returnType := mtype.Out(0) + returnType := mType.Out(0) if returnType != typeOfError { continue } // We save off the request type so we know how to unmarshal the request. // We use the reply type to allocate the reply struct and marshal the response. - if mtype.NumIn() == 3 { - argType := mtype.In(1) + if mType.NumIn() == 3 { + argType := mType.In(1) if !isExportedOrBuiltinType(argType) { continue } - replyType := mtype.In(2) + replyType := mType.In(2) if replyType.Kind() != reflect.Ptr { continue } @@ -64,20 +64,20 @@ func (server *Server) buildSvrMap(typ reflect.Type) { } ma := methodArgs{methodPtr: &method, passClientID: false, request: argType, reply: replyType} - server.svrMap[mname] = &ma - } else if mtype.NumIn() == 4 { - argType := mtype.In(2) + server.svrMap[mName] = &ma + } else if mType.NumIn() == 4 { + argType := mType.In(2) if !isExportedOrBuiltinType(argType) { continue } // Check if first argument is uint64 for clientID - clientIDType := mtype.In(1) + clientIDType := mType.In(1) if clientIDType.Kind() != reflect.Uint64 { continue } - replyType := mtype.In(3) + replyType := mType.In(3) if replyType.Kind() != reflect.Ptr { continue } @@ -87,7 +87,7 @@ func (server *Server) buildSvrMap(typ reflect.Type) { } ma := methodArgs{methodPtr: &method, passClientID: true, request: argType, reply: replyType} - server.svrMap[mname] = &ma + server.svrMap[mName] = &ma } } } From c93d87f37866e6dac5878386aa8049b03db215b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Wed, 14 Apr 2021 16:16:14 -0700 Subject: [PATCH 131/160] Automatically get latest Swift tag in SAIO/SAIT and Tox (#609) --- Makefile | 2 +- pfs_middleware/tox.ini | 2 +- saio/container/Dockerfile | 4 ++-- saio/vagrant_provision.sh | 3 ++- sait/vagrant_provision.sh | 3 ++- 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4f11d6f5c..b532f071a 100644 --- a/Makefile +++ b/Makefile @@ -198,7 +198,7 @@ pre-generate: python-test: cd meta_middleware && tox -e lint - cd pfs_middleware && tox -e py27-release,py27-minver,py36-release,lint + cd pfs_middleware && LATEST_SWIFT_TAG=` git ls-remote --refs --tags https://github.com/NVIDIA/swift.git | cut -d '/' -f 3 | grep ^ss-release- | sort --version-sort | tail -n 1 ` tox -e py27-release,py27-minver,py36-release,lint test: @set -e; \ diff --git a/pfs_middleware/tox.ini b/pfs_middleware/tox.ini index d576e6747..3f3ace192 100644 --- a/pfs_middleware/tox.ini +++ b/pfs_middleware/tox.ini @@ -7,7 +7,7 @@ usedevelop = True deps = lint: flake8 !lint: -r{toxinidir}/test-requirements.txt - release: git+https://github.com/NVIDIA/swift.git@ss-release-2.27.0.2 + release: git+https://github.com/NVIDIA/swift.git@{env:LATEST_SWIFT_TAG} minver: http://tarballs.openstack.org/swift/swift-2.9.0.tar.gz master: http://tarballs.openstack.org/swift/swift-master.tar.gz commands = python -m unittest discover diff --git a/saio/container/Dockerfile b/saio/container/Dockerfile index 001448946..8c406d5c6 100644 --- a/saio/container/Dockerfile +++ b/saio/container/Dockerfile @@ -3,7 +3,7 @@ FROM centos:7.4.1708 -ARG SwiftVersion=ss-release-2.27.0.2 +ARG SwiftVersion ARG GolangVersion=1.15.5 ARG ProxyFS_Version=stable @@ -111,7 +111,7 @@ RUN pip install -r test-requirements.txt WORKDIR /home/swift RUN git clone https://github.com/NVIDIA/swift.git WORKDIR /home/swift/swift -RUN git checkout $SwiftVersion +RUN if [[ -n "$SwiftVersion" ]] ; then git checkout $SwiftVersion ; else git checkout $( git describe --tags --abbrev=0 $(git rev-list --tags --max-count=1) ) ; fi RUN pip install wheel RUN python setup.py bdist_wheel RUN yum remove -y python-greenlet diff --git a/saio/vagrant_provision.sh b/saio/vagrant_provision.sh index 4e745a347..da3bf7525 100644 --- a/saio/vagrant_provision.sh +++ b/saio/vagrant_provision.sh @@ -228,7 +228,8 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.27.0.2 +export SWIFT_TAG="$( git describe --tags --abbrev=0 $(git rev-list --tags --max-count=1) )" +git checkout $SWIFT_TAG pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet diff --git a/sait/vagrant_provision.sh b/sait/vagrant_provision.sh index 6cc0c9342..4816229e6 100644 --- a/sait/vagrant_provision.sh +++ b/sait/vagrant_provision.sh @@ -230,7 +230,8 @@ echo "export ST_KEY=testing" >> ~vagrant/.bash_profile cd ~swift git clone https://github.com/NVIDIA/swift.git cd swift -git checkout ss-release-2.27.0.2 +export SWIFT_TAG="$( git describe --tags --abbrev=0 $(git rev-list --tags --max-count=1) )" +git checkout $SWIFT_TAG pip install wheel python setup.py bdist_wheel yum remove -y python-greenlet From 680b0f0eebebd00c7083f79eb62318611d9c70cd Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 16 Apr 2021 15:27:39 -0700 Subject: [PATCH 132/160] Forced use of 512 Byte BlkSize in PFSAgent... to honor stat()'s clients' insistence on ignoring it --- pfsagentd/README.md | 1 - pfsagentd/fission.go | 8 +++++--- pfsagentd/globals.go | 10 ---------- pfsagentd/pfsagent.conf | 1 - pfsagentd/setup_teardown_test.go | 1 - 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/pfsagentd/README.md b/pfsagentd/README.md index 85852d523..e890b32a2 100644 --- a/pfsagentd/README.md +++ b/pfsagentd/README.md @@ -70,7 +70,6 @@ ReadDirPlusEnabled: false XAttrEnabled: false EntryDuration: 0s AttrDuration: 0s -AttrBlockSize: 65536 ReaddirMaxEntries: 1024 FUSEMaxBackground: 100 FUSECongestionThreshhold: 0 diff --git a/pfsagentd/fission.go b/pfsagentd/fission.go index a3eb1c9d5..adc028d63 100644 --- a/pfsagentd/fission.go +++ b/pfsagentd/fission.go @@ -21,6 +21,8 @@ import ( ) const ( + attrBlockSize = uint32(512) + initOutFlagsMaskReadDirPlusDisabled = uint32(0) | fission.InitFlagsAsyncRead | fission.InitFlagsFileOps | @@ -87,9 +89,9 @@ func convertErrToErrno(err error, defaultErrno syscall.Errno) (errno syscall.Err func fixAttrSizes(attr *fission.Attr) { if syscall.S_IFREG == (attr.Mode & syscall.S_IFMT) { - attr.Blocks = attr.Size + (globals.config.AttrBlockSize - 1) - attr.Blocks /= globals.config.AttrBlockSize - attr.BlkSize = uint32(globals.config.AttrBlockSize) + attr.Blocks = attr.Size + (uint64(attrBlockSize) - 1) + attr.Blocks /= uint64(attrBlockSize) + attr.BlkSize = attrBlockSize } else { attr.Size = 0 attr.Blocks = 0 diff --git a/pfsagentd/globals.go b/pfsagentd/globals.go index 9f5becc87..a73c5ce48 100644 --- a/pfsagentd/globals.go +++ b/pfsagentd/globals.go @@ -9,7 +9,6 @@ import ( "io" "io/ioutil" "log" - "math" "net/http" "os" "os/exec" @@ -67,7 +66,6 @@ type configStruct struct { XAttrEnabled bool EntryDuration time.Duration AttrDuration time.Duration - AttrBlockSize uint64 ReaddirMaxEntries uint64 FUSEMaxBackground uint16 FUSECongestionThreshhold uint16 @@ -616,14 +614,6 @@ func initializeGlobals(confMap conf.ConfMap) { logFatal(err) } - globals.config.AttrBlockSize, err = confMap.FetchOptionValueUint64("Agent", "AttrBlockSize") - if nil != err { - logFatal(err) - } - if (0 == globals.config.AttrBlockSize) || (math.MaxUint32 < globals.config.AttrBlockSize) { - logFatalf("AttrBlockSize must be non-zero and fit in a uint32") - } - globals.config.ReaddirMaxEntries, err = confMap.FetchOptionValueUint64("Agent", "ReaddirMaxEntries") if nil != err { logFatal(err) diff --git a/pfsagentd/pfsagent.conf b/pfsagentd/pfsagent.conf index dcefa6b98..ebf2e1c75 100644 --- a/pfsagentd/pfsagent.conf +++ b/pfsagentd/pfsagent.conf @@ -36,7 +36,6 @@ ReadDirPlusEnabled: false XAttrEnabled: false EntryDuration: 0s AttrDuration: 0s -AttrBlockSize: 65536 ReaddirMaxEntries: 1024 FUSEMaxBackground: 100 FUSECongestionThreshhold: 0 diff --git a/pfsagentd/setup_teardown_test.go b/pfsagentd/setup_teardown_test.go index 3f9f68a16..916e1cbc3 100644 --- a/pfsagentd/setup_teardown_test.go +++ b/pfsagentd/setup_teardown_test.go @@ -120,7 +120,6 @@ func testSetup(t *testing.T) { "Agent.XAttrEnabled=false", "Agent.EntryDuration=10s", "Agent.AttrDuration=10s", - "Agent.AttrBlockSize=65536", "Agent.ReaddirMaxEntries=1024", "Agent.FUSEMaxBackground=100", "Agent.FUSECongestionThreshhold=0", From e54eed484089aa822d536d6f2c1e4b74c5dca3ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Fri, 16 Apr 2021 16:52:36 -0700 Subject: [PATCH 133/160] GitHub actions (#610) Replacing TravisCI with GitHub Actions --- .github/workflows/run-unit-tests.yml | 19 +++++++ .travis.yml | 17 ------ test/container/launch.sh | 6 +- utils/api_test.go | 84 ++++++++++++++-------------- 4 files changed, 64 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/run-unit-tests.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/run-unit-tests.yml b/.github/workflows/run-unit-tests.yml new file mode 100644 index 000000000..0aa4c36d0 --- /dev/null +++ b/.github/workflows/run-unit-tests.yml @@ -0,0 +1,19 @@ +name: Unit Tests +on: [push] +jobs: + Build-and-unit-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + submodules: false + - name: Build, run unit tests, and get coverage + run: docker run -e "COVERALLS_TOKEN=${{ secrets.COVERALLS_TOKEN }}" -e "GIT_BRANCH=${{ github.ref }}" --cap-add SYS_ADMIN --device /dev/fuse -i -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs swiftstack/proxyfs_unit_tests + - name: Slack Notification + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + SLACK_COLOR: ${{ job.status }} + SLACK_MESSAGE: 'Unit tests status: ${{ job.status }}' + if: always() diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0ccebea4b..000000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -services: -- docker - -git: - submodules: false - depth: false - -notifications: - slack: - rooms: - - secure: "qd9CCq5BJXolPXPvqXaeUcz1l1PchWO25efOhEyH0fE3SJfZVfAymUHLXblgd7GDD/lMJCJOngpBBiaDw/D2lBbTxHsZUliSKo5esg7Bxwm3MVNK3J5EKl63TPiLsbxrhAqcmrTf5DJHclJSnrZggv9RePZbZGNx+ByinC0DexZDps86kDK0KSlX2cox3wiuw4kRDyj37J+MY2o3OFk6riMqGBjnY0+ct3N5QN9NBkfWUQZ6T2pZ5X3+v12e6Q013wexfzOuRR/sJc4QXIfG6cJkR7JgE7CGxXVbqK1GGwHiKa1r0MIvOZQ5ggiCuHQq6ryowWLIk+0fu9y62HJ8tpCCNYz538XZAGDPxKQpxBGRtikgHwp3yfKZbQcHOJzFOLmtK9oYVV7WC6q6/dzYLpZLMm83F0Lz1blRoqtdvI6G/RjoT96p6GzptYcrdyLwh5/mJtuq49BAHeJQiGkN8XRybpz72dDYYfod9O4YW8L3xmjouIpMr9i/bw3PKC8PrynABo7ZCOLbBnBd/JpF45woTQhazTPqxWRVe0e6wFfFFKyG43jj4v2DVdHQ5xAsUPWdEsV2UfwwJFk71wPcjkXMC6I3ii19AUmhrM3CoWX4RKDR//P4h08kwfHm7/Jg+odCdoCc5qYhXYgRsR4HT2Vq3k8CtNZ/lPxEclPaMlQ=" - on_success: change - on_failure: always - on_pull_requests: true - -script: -- docker run -e "COVERALLS_TOKEN=$COVERALLS_TOKEN" -e "TRAVIS_BRANCH=$TRAVIS_BRANCH" --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/NVIDIA/proxyfs swiftstack/proxyfs_unit_tests diff --git a/test/container/launch.sh b/test/container/launch.sh index c22b9a90e..b487ee8e3 100755 --- a/test/container/launch.sh +++ b/test/container/launch.sh @@ -9,7 +9,7 @@ set -x # Build ProxyFS and run tests cd $GOPATH/src/github.com/NVIDIA/proxyfs make ci -# $COVERALLS_TOKEN must be configured in TravisCI -if [ -n "$COVERALLS_TOKEN" ] && [ -n "$TRAVIS_BRANCH" ]; then - GIT_BRANCH=$TRAVIS_BRANCH goveralls -coverprofile coverage.coverprofile -service travis-ci -repotoken $COVERALLS_TOKEN +# $COVERALLS_TOKEN must be configured in the CI/CD environment +if [ -n "$COVERALLS_TOKEN" ] && [ -n "$GIT_BRANCH" ]; then + goveralls -coverprofile coverage.coverprofile -service travis-ci -repotoken $COVERALLS_TOKEN fi diff --git a/utils/api_test.go b/utils/api_test.go index 661c5cbec..4c0c3d30b 100644 --- a/utils/api_test.go +++ b/utils/api_test.go @@ -180,14 +180,14 @@ func TestStopwatch(t *testing.T) { // Create stopwatch // sw1 := NewStopwatch() - //fmt.Printf("After creation: sw1 is %v\n", sw1) + now := time.Now() // check stuff - startTime1 := sw1.StartTime // Save startTime for later checks - assert.True(sw1.StartTime.Before(time.Now())) // Start time is in the past - assert.True(sw1.StopTime.IsZero()) // Stop time isn't set yet - assert.Equal(int64(sw1.ElapsedTime), int64(0)) // Elapsed time isn't set yet - assert.True(sw1.IsRunning) // stopwatch is running + startTime1 := sw1.StartTime // Save startTime for later checks + assert.True(sw1.StartTime.Before(now), "time stopped!", startTime1, now) // Start time is in the past + assert.True(sw1.StopTime.IsZero()) // Stop time isn't set yet + assert.Equal(int64(sw1.ElapsedTime), int64(0)) // Elapsed time isn't set yet + assert.True(sw1.IsRunning) // stopwatch is running // Delay then stop and check nonzero elapsed time sleepTime := 100 * time.Millisecond @@ -196,14 +196,14 @@ func TestStopwatch(t *testing.T) { // Stop it assert.True(sw1.IsRunning) // stopwatch is still running elapsed1 := sw1.Stop() - //fmt.Printf("After stop: sw1 is %v\n", sw1) + now = time.Now() // check stuff - assert.False(sw1.IsRunning) // stopwatch is not running - assert.False(sw1.StopTime.IsZero()) // Stop time is set - assert.True(sw1.StopTime.Before(time.Now())) // Stop time is in the past - assert.True(sw1.StartTime == startTime1) // StartTime hasn't changed - assert.True(elapsed1 >= sleepTime) // elapsed time is reasonable + assert.False(sw1.IsRunning) // stopwatch is not running + assert.False(sw1.StopTime.IsZero()) // Stop time is set + assert.True(sw1.StopTime.Before(now), "time stopped!", sw1.StopTime, now) // Stop time is in the past + assert.True(sw1.StartTime == startTime1) // StartTime hasn't changed + assert.True(elapsed1 >= sleepTime) // elapsed time is reasonable // // Call Elapsed() when stopped @@ -222,14 +222,14 @@ func TestStopwatch(t *testing.T) { // Create another stopwatch // sw2 := NewStopwatch() - //fmt.Printf("After creation: sw2 is %v\n", sw2) + now = time.Now() // check stuff - startTime2 := sw2.StartTime // Save startTime for later checks - assert.True(sw2.StartTime.Before(time.Now())) // Start time is in the past - assert.True(sw2.StopTime.IsZero()) // Stop time isn't set yet - assert.Equal(int64(sw2.ElapsedTime), int64(0)) // Elapsed time isn't set yet - assert.True(sw2.IsRunning) // stopwatch is running + startTime2 := sw2.StartTime // Save startTime for later checks + assert.True(sw2.StartTime.Before(now), "Time stopped!", startTime2, now) // Start time is in the past + assert.True(sw2.StopTime.IsZero()) // Stop time isn't set yet + assert.Equal(int64(sw2.ElapsedTime), int64(0)) // Elapsed time isn't set yet + assert.True(sw2.IsRunning) // stopwatch is running // Delay then check nonzero elapsed time (but don't stop) sleepTime = 856 * time.Millisecond @@ -251,28 +251,28 @@ func TestStopwatch(t *testing.T) { // Stop and then call Elapsed() again // elapsed3 := sw2.Stop() - //fmt.Printf("After stop: sw2 is %v\n", sw2) + now = time.Now() // check stuff - assert.False(sw2.IsRunning) // stopwatch is not running - assert.False(sw2.StopTime.IsZero()) // Stop time is set - assert.True(sw2.StopTime.Before(time.Now())) // Stop time is in the past - assert.True(sw2.StartTime == startTime2) // StartTime hasn't changed - assert.True(elapsed3 > elapsed2) // elapsed time is later than before + assert.False(sw2.IsRunning) // stopwatch is not running + assert.False(sw2.StopTime.IsZero()) // Stop time is set + assert.True(sw2.StopTime.Before(now), "time stopped?", sw2.StopTime, now) // Stop time is in the past + assert.True(sw2.StartTime == startTime2) // StartTime hasn't changed + assert.True(elapsed3 > elapsed2) // elapsed time is later than before assert.True(sw2.Elapsed() == elapsed3) // // restart a previously running stopwatch // sw2.Restart() - //fmt.Printf("After restart: sw2 is %v\n", sw2) + now = time.Now() // check stuff - startTime3 := sw2.StartTime // Save startTime for later checks - assert.True(sw2.StartTime.Before(time.Now())) // Start time is in the past - assert.True(sw2.StopTime.IsZero()) // Stop time isn't set yet - assert.Equal(int64(sw2.ElapsedTime), int64(0)) // Elapsed time isn't set yet - assert.True(sw2.IsRunning) // stopwatch is running + startTime3 := sw2.StartTime // Save startTime for later checks + assert.True(sw2.StartTime.Before(now), "time stopped?", sw2.StartTime, now) // Start time is in the past + assert.True(sw2.StopTime.IsZero()) // Stop time isn't set yet + assert.Equal(int64(sw2.ElapsedTime), int64(0)) // Elapsed time isn't set yet + assert.True(sw2.IsRunning) // stopwatch is running // Delay then stop and check nonzero elapsed time sleepTime = 912 * time.Microsecond @@ -283,14 +283,14 @@ func TestStopwatch(t *testing.T) { // assert.True(sw2.IsRunning) // stopwatch is still running elapsed4 := sw2.Stop() - //fmt.Printf("After stop: sw2 is %v\n", sw2) + now = time.Now() // check stuff - assert.False(sw2.IsRunning) // stopwatch is not running - assert.False(sw2.StopTime.IsZero()) // Stop time is set - assert.True(sw2.StopTime.Before(time.Now())) // Stop time is in the past - assert.True(sw2.StartTime == startTime3) // StartTime hasn't changed - assert.True(elapsed4 >= sleepTime) // elapsed time is reasonable + assert.False(sw2.IsRunning) // stopwatch is not running + assert.False(sw2.StopTime.IsZero()) // Stop time is set + assert.True(sw2.StopTime.Before(now), "time stopped?", sw2.StopTime, now) // Stop time is in the past + assert.True(sw2.StartTime == startTime3) // StartTime hasn't changed + assert.True(elapsed4 >= sleepTime) // elapsed time is reasonable // // Call Elapsed() when stopped @@ -307,14 +307,14 @@ func TestStopwatch(t *testing.T) { // restart a non-previously running stopwatch // sw3 := NewStopwatch() - //fmt.Printf("After creation: sw3 is %v\n", sw3) + now = time.Now() // check stuff - startTime4 := sw3.StartTime // Save startTime for later checks - assert.True(sw3.StartTime.Before(time.Now())) // Start time is in the past - assert.True(sw3.StopTime.IsZero()) // Stop time isn't set yet - assert.Equal(int64(sw3.ElapsedTime), int64(0)) // Elapsed time isn't set yet - assert.True(sw3.IsRunning) // stopwatch is running + startTime4 := sw3.StartTime // Save startTime for later checks + assert.True(sw3.StartTime.Before(now), sw3.StartTime, now) // Start time is in the past + assert.True(sw3.StopTime.IsZero()) // Stop time isn't set yet + assert.Equal(int64(sw3.ElapsedTime), int64(0)) // Elapsed time isn't set yet + assert.True(sw3.IsRunning) // stopwatch is running // Attempt restart, shouldn't do anything since not stopped sw3.Restart() From 49e7c10258314121cb9b60b10f4d5125a4c59ca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Fri, 16 Apr 2021 17:10:12 -0700 Subject: [PATCH 134/160] Update badge to point to GitHub Action status (#612) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb31e5004..bf1879aee 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.com/NVIDIA/proxyfs.svg?branch=development)](https://travis-ci.com/NVIDIA/proxyfs) +[![Build Status](https://github.com/NVIDIA/proxyfs/actions/workflows/run-unit-tests.yml/badge.svg)](https://github.com/NVIDIA/proxyfs/actions/workflows/run-unit-tests.yml?query=branch%3Adevelopment) [![Coverage Status](https://coveralls.io/repos/github/swiftstack/ProxyFS/badge.svg?branch=development)](https://coveralls.io/github/swiftstack/ProxyFS?branch=development) # ProxyFS From 0215d834248d9885535e762214d61daa7d15c161 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 20 Apr 2021 11:50:19 -0700 Subject: [PATCH 135/160] Clean up Makefile... no longer checks for armv71 nor "distro" --- Makefile | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index b532f071a..668f07a01 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ gopkgsubdirs = \ conf \ confgen \ dlm \ - etcdclient \ + etcdclient \ evtlog \ fs \ fuse \ @@ -85,26 +85,15 @@ gosubdirsforci = $(gopkgsubdirs) $(gobinsubdirsforci); gosubdirspathsforci = $(addprefix github.com/NVIDIA/proxyfs/,$(gosubdirsforci)) uname = $(shell uname) -machine = $(shell uname -m) ifeq ($(uname),Linux) - ifeq ($(machine),armv7l) - all: version fmt pre-generate generate install test + all: version fmt pre-generate generate install test python-test - ci: version fmt pre-generate generate install test cover + ci: version fmt pre-generate generate install test cover python-test - minimal: pre-generate generate install - else - distro := $(shell python -c "import platform; print platform.linux_distribution()[0]") + all-deb-builder: version fmt pre-generate generate install - all: version fmt pre-generate generate install test python-test - - ci: version fmt pre-generate generate install test cover python-test - - all-deb-builder: version fmt pre-generate generate install - - minimal: pre-generate generate install - endif + minimal: pre-generate generate install else all: version fmt pre-generate generate install test From da698e74d4f3377b40c5da874d2c0e1acaad0737 Mon Sep 17 00:00:00 2001 From: craig harmer Date: Wed, 21 Apr 2021 22:23:18 -0700 Subject: [PATCH 136/160] Fix unit tests in "utils". On the new "github actions" server it appears that some unit tests can fail because time "stops" for a short period of time (time.Now() returns the same value on two calls). Work around the problem. I also saw a one-time failure in TestTryLockMutex(). Tweak some of the timing paramaters to try and avoid that in the future even though it should not have failed in the first place. --- utils/api_test.go | 104 ++++++++++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 44 deletions(-) diff --git a/utils/api_test.go b/utils/api_test.go index 4c0c3d30b..362a0ae5d 100644 --- a/utils/api_test.go +++ b/utils/api_test.go @@ -25,16 +25,19 @@ func TestTryLockMutex(t *testing.T) { testTryLockMutex = NewTryLockMutex() testTryLockMutex.Lock() testTryLockMutex.Unlock() + shouldHaveGottonIt := testTryLockMutex.TryLock(100 * time.Millisecond) if !shouldHaveGottonIt { t.Fatalf("1st TryLock() should have succeeded") } + shouldNotHaveGottenIt := testTryLockMutex.TryLock(100 * time.Millisecond) if shouldNotHaveGottenIt { t.Fatalf("2nd TryLock() should have failed") } - _ = time.AfterFunc(50*time.Millisecond, testTryLockMutexAsyncUnlock) - shouldHaveGottonIt = testTryLockMutex.TryLock(100 * time.Millisecond) + + _ = time.AfterFunc(20*time.Millisecond, testTryLockMutexAsyncUnlock) + shouldHaveGottonIt = testTryLockMutex.TryLock(120 * time.Millisecond) if !shouldHaveGottonIt { t.Fatalf("3rd TryLock() should have succeeded") } @@ -174,20 +177,25 @@ func TestGetAFnName(t *testing.T) { func TestStopwatch(t *testing.T) { assert := assert.New(t) + var now time.Time + // Test normal operation // // Create stopwatch // sw1 := NewStopwatch() - now := time.Now() + + // insure time is not frozen + for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + } // check stuff - startTime1 := sw1.StartTime // Save startTime for later checks - assert.True(sw1.StartTime.Before(now), "time stopped!", startTime1, now) // Start time is in the past - assert.True(sw1.StopTime.IsZero()) // Stop time isn't set yet - assert.Equal(int64(sw1.ElapsedTime), int64(0)) // Elapsed time isn't set yet - assert.True(sw1.IsRunning) // stopwatch is running + startTime1 := sw1.StartTime + assert.True(sw1.StartTime.Before(now), "time stopped! %v %v", startTime1, now) // Start time is in the past + assert.True(sw1.StopTime.IsZero()) // Stop time isn't set yet + assert.Equal(int64(sw1.ElapsedTime), int64(0)) // Elapsed time isn't set yet + assert.True(sw1.IsRunning) // stopwatch is running // Delay then stop and check nonzero elapsed time sleepTime := 100 * time.Millisecond @@ -196,14 +204,17 @@ func TestStopwatch(t *testing.T) { // Stop it assert.True(sw1.IsRunning) // stopwatch is still running elapsed1 := sw1.Stop() - now = time.Now() + + // insure time doesn't stop + for now = time.Now(); sw1.StopTime.Equal(now); now = time.Now() { + } // check stuff - assert.False(sw1.IsRunning) // stopwatch is not running - assert.False(sw1.StopTime.IsZero()) // Stop time is set - assert.True(sw1.StopTime.Before(now), "time stopped!", sw1.StopTime, now) // Stop time is in the past - assert.True(sw1.StartTime == startTime1) // StartTime hasn't changed - assert.True(elapsed1 >= sleepTime) // elapsed time is reasonable + assert.False(sw1.IsRunning) // stopwatch is not running + assert.False(sw1.StopTime.IsZero()) // Stop time is set + assert.True(sw1.StopTime.Before(now), "time stopped! %v %v", sw1.StopTime, now) // Stop time is in the past + assert.True(sw1.StartTime == startTime1) // StartTime hasn't changed + assert.True(elapsed1 >= sleepTime) // elapsed time is reasonable // // Call Elapsed() when stopped @@ -222,14 +233,15 @@ func TestStopwatch(t *testing.T) { // Create another stopwatch // sw2 := NewStopwatch() - now = time.Now() + for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + } // check stuff - startTime2 := sw2.StartTime // Save startTime for later checks - assert.True(sw2.StartTime.Before(now), "Time stopped!", startTime2, now) // Start time is in the past - assert.True(sw2.StopTime.IsZero()) // Stop time isn't set yet - assert.Equal(int64(sw2.ElapsedTime), int64(0)) // Elapsed time isn't set yet - assert.True(sw2.IsRunning) // stopwatch is running + startTime2 := sw2.StartTime + assert.True(sw2.StartTime.Before(now), "Time stopped! %v %v", startTime2, now) // Start time is in the past + assert.True(sw2.StopTime.IsZero()) // Stop time isn't set yet + assert.Equal(int64(sw2.ElapsedTime), int64(0)) // Elapsed time isn't set yet + assert.True(sw2.IsRunning) // stopwatch is running // Delay then check nonzero elapsed time (but don't stop) sleepTime = 856 * time.Millisecond @@ -251,28 +263,30 @@ func TestStopwatch(t *testing.T) { // Stop and then call Elapsed() again // elapsed3 := sw2.Stop() - now = time.Now() + for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + } // check stuff - assert.False(sw2.IsRunning) // stopwatch is not running - assert.False(sw2.StopTime.IsZero()) // Stop time is set - assert.True(sw2.StopTime.Before(now), "time stopped?", sw2.StopTime, now) // Stop time is in the past - assert.True(sw2.StartTime == startTime2) // StartTime hasn't changed - assert.True(elapsed3 > elapsed2) // elapsed time is later than before + assert.False(sw2.IsRunning) // stopwatch is not running + assert.False(sw2.StopTime.IsZero()) // Stop time is set + assert.True(sw2.StopTime.Before(now), "time stopped? %v %v ", sw2.StopTime, now) // Stop time is in the past + assert.True(sw2.StartTime == startTime2) // StartTime hasn't changed + assert.True(elapsed3 > elapsed2) // elapsed time has grown assert.True(sw2.Elapsed() == elapsed3) // // restart a previously running stopwatch // sw2.Restart() - now = time.Now() + for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + } // check stuff - startTime3 := sw2.StartTime // Save startTime for later checks - assert.True(sw2.StartTime.Before(now), "time stopped?", sw2.StartTime, now) // Start time is in the past - assert.True(sw2.StopTime.IsZero()) // Stop time isn't set yet - assert.Equal(int64(sw2.ElapsedTime), int64(0)) // Elapsed time isn't set yet - assert.True(sw2.IsRunning) // stopwatch is running + startTime3 := sw2.StartTime + assert.True(sw2.StartTime.Before(now), "time stopped? %v %v", sw2.StartTime, now) // Start time is in the past + assert.True(sw2.StopTime.IsZero()) // Stop time isn't set yet + assert.Equal(int64(sw2.ElapsedTime), int64(0)) // Elapsed time isn't set yet + assert.True(sw2.IsRunning) // stopwatch is running // Delay then stop and check nonzero elapsed time sleepTime = 912 * time.Microsecond @@ -283,14 +297,15 @@ func TestStopwatch(t *testing.T) { // assert.True(sw2.IsRunning) // stopwatch is still running elapsed4 := sw2.Stop() - now = time.Now() + for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + } // check stuff - assert.False(sw2.IsRunning) // stopwatch is not running - assert.False(sw2.StopTime.IsZero()) // Stop time is set - assert.True(sw2.StopTime.Before(now), "time stopped?", sw2.StopTime, now) // Stop time is in the past - assert.True(sw2.StartTime == startTime3) // StartTime hasn't changed - assert.True(elapsed4 >= sleepTime) // elapsed time is reasonable + assert.False(sw2.IsRunning) // stopwatch is not running + assert.False(sw2.StopTime.IsZero()) // Stop time is set + assert.True(sw2.StopTime.Before(now), "time stopped? %v %v ", sw2.StopTime, now) // Stop time is in the past + assert.True(sw2.StartTime == startTime3) // StartTime hasn't changed + assert.True(elapsed4 >= sleepTime) // elapsed time is reasonable // // Call Elapsed() when stopped @@ -307,14 +322,15 @@ func TestStopwatch(t *testing.T) { // restart a non-previously running stopwatch // sw3 := NewStopwatch() - now = time.Now() + for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + } // check stuff - startTime4 := sw3.StartTime // Save startTime for later checks - assert.True(sw3.StartTime.Before(now), sw3.StartTime, now) // Start time is in the past - assert.True(sw3.StopTime.IsZero()) // Stop time isn't set yet - assert.Equal(int64(sw3.ElapsedTime), int64(0)) // Elapsed time isn't set yet - assert.True(sw3.IsRunning) // stopwatch is running + startTime4 := sw3.StartTime // Save startTime for later checks + assert.True(sw3.StartTime.Before(now), "%v %v", sw3.StartTime, now) // Start time is in the past + assert.True(sw3.StopTime.IsZero()) // Stop time isn't set yet + assert.Equal(int64(sw3.ElapsedTime), int64(0)) // Elapsed time isn't set yet + assert.True(sw3.IsRunning) // stopwatch is running // Attempt restart, shouldn't do anything since not stopped sw3.Restart() From 3b37f81434ebb54e492e369e741b6a958cf295a3 Mon Sep 17 00:00:00 2001 From: craig harmer Date: Thu, 22 Apr 2021 19:34:30 -0700 Subject: [PATCH 137/160] Fix typos in the checkin that fixed unit tests. the last utils/api_test.go didn't fix all of the problems with time standing still due to some typos. --- utils/api_test.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/utils/api_test.go b/utils/api_test.go index 362a0ae5d..4c39cd1fd 100644 --- a/utils/api_test.go +++ b/utils/api_test.go @@ -233,7 +233,7 @@ func TestStopwatch(t *testing.T) { // Create another stopwatch // sw2 := NewStopwatch() - for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + for now = time.Now(); sw2.StartTime.Equal(now); now = time.Now() { } // check stuff @@ -263,7 +263,9 @@ func TestStopwatch(t *testing.T) { // Stop and then call Elapsed() again // elapsed3 := sw2.Stop() - for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + + // wait for time to continue its march + for now = time.Now(); sw2.StopTime.Equal(now); now = time.Now() { } // check stuff @@ -278,7 +280,7 @@ func TestStopwatch(t *testing.T) { // restart a previously running stopwatch // sw2.Restart() - for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + for now = time.Now(); sw2.StartTime.Equal(now); now = time.Now() { } // check stuff @@ -297,7 +299,7 @@ func TestStopwatch(t *testing.T) { // assert.True(sw2.IsRunning) // stopwatch is still running elapsed4 := sw2.Stop() - for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + for now = time.Now(); sw2.StopTime.Equal(now); now = time.Now() { } // check stuff @@ -322,7 +324,9 @@ func TestStopwatch(t *testing.T) { // restart a non-previously running stopwatch // sw3 := NewStopwatch() - for now = time.Now(); sw1.StartTime.Equal(now); now = time.Now() { + + // wait for time to continue its march + for now = time.Now(); sw3.StartTime.Equal(now); now = time.Now() { } // check stuff From d362223eab755c89cbfac08f9fbedd429feb61ca Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Fri, 23 Apr 2021 09:16:44 -0700 Subject: [PATCH 138/160] Write a test tool to test how many RPCs the retryrpc layer can service (#614) * readme for the tool * Add option for a dbgport so we can start profiler * Document need to increase ulimit for file descriptors --- go.mod | 2 +- retryrpc/perfrpc/Makefile | 6 + retryrpc/perfrpc/README.md | 36 +++++ retryrpc/perfrpc/client.go | 281 +++++++++++++++++++++++++++++++++ retryrpc/perfrpc/dummy_test.go | 11 ++ retryrpc/perfrpc/perfrpc.go | 53 +++++++ retryrpc/perfrpc/ping_perf.go | 66 ++++++++ retryrpc/perfrpc/server.go | 149 +++++++++++++++++ retryrpc/perfrpc/tls.go | 95 +++++++++++ 9 files changed, 698 insertions(+), 1 deletion(-) create mode 100644 retryrpc/perfrpc/Makefile create mode 100644 retryrpc/perfrpc/README.md create mode 100644 retryrpc/perfrpc/client.go create mode 100644 retryrpc/perfrpc/dummy_test.go create mode 100644 retryrpc/perfrpc/perfrpc.go create mode 100644 retryrpc/perfrpc/ping_perf.go create mode 100644 retryrpc/perfrpc/server.go create mode 100644 retryrpc/perfrpc/tls.go diff --git a/go.mod b/go.mod index ffd849a2f..d77c281fb 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/stretchr/testify v1.5.1 go.etcd.io/etcd v0.5.0-alpha.5.0.20200401174654-e694b7bb0875 go.uber.org/multierr v1.5.0 // indirect - go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f // indirect + go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 golang.org/x/text v0.3.5 // indirect diff --git a/retryrpc/perfrpc/Makefile b/retryrpc/perfrpc/Makefile new file mode 100644 index 000000000..34cdec366 --- /dev/null +++ b/retryrpc/perfrpc/Makefile @@ -0,0 +1,6 @@ +# Copyright (c) 2015-2021, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +gosubdir := github.com/NVIDIA/proxyfs/retryrpc/perfrpc + +include ../../GoMakefile diff --git a/retryrpc/perfrpc/README.md b/retryrpc/perfrpc/README.md new file mode 100644 index 000000000..e147089de --- /dev/null +++ b/retryrpc/perfrpc/README.md @@ -0,0 +1,36 @@ +# PerfRPC +perfrpc is a tool to test and demonstrate the performance of the retryrpc layer. + +## Starting a perfrpc Server + + `# ./perfrpc server -ipaddr 127.0.0.1 -port 53167 -tlsdir ./tls` + + where `tls` is the directory where the server should *write* the TLS credentials + +## Starting a perfrpc Client + + `# ./perfrpc client -ipaddr 127.0.0.1 -port 53167 -tlsdir ./tls -clients 1000 -messages 100 -warmupcnt 10` + + `clients` is the number of clients the tool should create + + `messages` is the number of messages each client will send in parallel using goroutines + + `tls` is the directory where the client should *read* the TLS credentials + + `warmupcnt` is the number of messages each client will send to setup the connection and test the connection before running the performance test + +## What is produced? + +perfrpc clients will print a message such as + + `===== PERFRPC - Clients: 10 Messages per Client: 10 Total Messages: 100 ---- Test Duration: 6.207882ms` + +illustrating how many clients and total messages were sent followed by the length of time to run the test +## Tips for large number of clients + +It is important to have enough file descriptors for your clients and servers. + +We suggest running as `root` and setting the `ulimit` high such as + + `ulimit -n 1000000` + diff --git a/retryrpc/perfrpc/client.go b/retryrpc/perfrpc/client.go new file mode 100644 index 000000000..5c59fddb3 --- /dev/null +++ b/retryrpc/perfrpc/client.go @@ -0,0 +1,281 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "math/rand" + "net" + "net/http" + _ "net/http/pprof" + "sync" + "time" + + "github.com/NVIDIA/proxyfs/retryrpc" +) + +type globalsStruct struct { + cs *ClientSubcommand + tlsCerts *tlsCertsStruct + tlsDir string // Directory containing TLS info + useTLS bool +} + +var globals globalsStruct + +func ping(client *retryrpc.Client, method string, i int, agentID uint64) { + // Send a ping RPC and print the results + msg := fmt.Sprintf("Ping Me - %v", i) + pingRequest := &PerfPingReq{Message: msg} + pingReply := &PerfPingReply{} + expectedReply := fmt.Sprintf("pong %d bytes", len(msg)) + err := client.Send(method, pingRequest, pingReply) + if err != nil { + panic(err) + } + if expectedReply != pingReply.Message { + fmt.Printf(" client - AGENTID: %v\n", agentID) + fmt.Printf(" client.Send(RpcPing) reply '%+v'\n", pingReply) + fmt.Printf(" client.Send(RpcPing) expected '%s' but received '%s'\n", expectedReply, pingReply.Message) + fmt.Printf(" client.Send(RpcPing) SENT: msg '%v' but received '%s'\n", msg, pingReply.Message) + fmt.Printf(" client.Send(RpcPing) len(pingRequest.Message): '%d' i: %v\n", len(pingRequest.Message), i) + } +} + +func sendIt(client *retryrpc.Client, i int, sendWg *sync.WaitGroup, agentID uint64, method string) { + + defer sendWg.Done() + + switch method { + case "RpcPerfPing": + ping(client, method, i, agentID) + break + default: + err := fmt.Errorf("Invalid method: %v", method) + panic(err) + } +} + +type stressMyClient struct { + sync.Mutex + cond *sync.Cond // Signal that received Interrupt() callback + sawCallback bool // True if Interrupt() was called + interruptCnt int // Count of Interrupt() calls received (best effort) +} + +func (cb *stressMyClient) Interrupt(payload []byte) { + cb.Lock() + cb.sawCallback = true + cb.interruptCnt++ + cb.cond.Broadcast() + cb.Unlock() + return +} + +// Represents a pfsagent - sepearate client +func pfsagent(agentID uint64, method string, agentWG *sync.WaitGroup, warmUpCompleteWG *sync.WaitGroup, fence chan int) { + var ( + clientConfig *retryrpc.ClientConfig + ) + + defer agentWG.Done() + + cb := &stressMyClient{} + cb.cond = sync.NewCond(&cb.Mutex) + if globals.useTLS { + clientConfig = &retryrpc.ClientConfig{ + IPAddr: globals.cs.ipAddr, + Port: globals.cs.port, + RootCAx509CertificatePEM: globals.tlsCerts.caCertPEMBlock, + Callbacks: cb, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } + } else { + clientConfig = &retryrpc.ClientConfig{ + IPAddr: globals.cs.ipAddr, + Port: globals.cs.port, + RootCAx509CertificatePEM: nil, + Callbacks: cb, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + } + } + client, err := retryrpc.NewClient(clientConfig) + if err != nil { + err := fmt.Errorf("Dial() failed with err: %v\n", err) + panic(err) + } + defer client.Close() + + // Send messages to create connection and warm up client/server + var warmUpWg sync.WaitGroup + for i := 0; i < globals.cs.warmUpCnt; i++ { + + warmUpWg.Add(1) + go func(i int) { + sendIt(client, i, &warmUpWg, agentID, method) + }(i) + } + warmUpWg.Wait() + + // Signal caller that this agent has completed warmup cycle + warmUpCompleteWG.Done() + + // We have sent and received all our messages used to warm up client/server. + // Block here until the channel is closed which signifies that all goroutines + // should run the test + <-fence + + // Now run the real test + var sendWG sync.WaitGroup + for i := 0; i < globals.cs.messages; i++ { + + sendWG.Add(1) + go func(i int) { + sendIt(client, i, &sendWG, agentID, method) + }(i) + } + sendWG.Wait() +} + +// Start a bunch of "clients" and send messages +// +// We first start all the goroutines ("clients") and let them send +// some messages to the server. This allows us to "warmup" the +// clients so we are only measuring the steady state. +func parallelClientSenders(method string) (duration time.Duration) { + var ( + agentWG sync.WaitGroup + warmUpCompleteWG sync.WaitGroup + err error + ) + + // Figure out random seed for runs + r := rand.New(rand.NewSource(99)) + clientSeed := r.Uint64() + + globals.tlsCerts = &tlsCertsStruct{} + tlsSetFileNames(globals.tlsCerts, globals.tlsDir) + + globals.tlsCerts.caCertPEMBlock, err = ioutil.ReadFile(globals.tlsCerts.caCertFile) + if err != nil { + panic(err) + } + + // Start parallel pfsagents - each agent setup connection to server, + // send RPCs to warm up client/server and then block on channel named fence + var agentID uint64 + fence := make(chan int) + for i := 0; i < globals.cs.clients; i++ { + agentID = clientSeed + uint64(i) + + agentWG.Add(1) + warmUpCompleteWG.Add(1) + go pfsagent(agentID, method, &agentWG, &warmUpCompleteWG, fence) + } + + // TODO --- bucketstats to do measurement?? + + // Wait for all agents to warm up + warmUpCompleteWG.Wait() + + // At this point, we know that the goroutines/clients have started, + // established their connection to the server and sent some warm up + // messages. The goroutines then block on fence channel. + // + // We note the current time and then close fence channel. This in + // effect releases all of the goroutines at once. + + start := time.Now() + close(fence) + + // Now wait for all messages to be sent + agentWG.Wait() + duration = time.Since(start) + return +} + +type ClientSubcommand struct { + fs *flag.FlagSet + clients int // Number of clients which will be sending messages + ipAddr string // IP Address of server + warmUpCnt int // Number of messages to send to "warm up" + messages int // Number of messages to send + port int // Port on which the server is listening + dbgport string // Debug port for pprof webserver + tlsDir string // Directory to write TLS info +} + +func NewClientCommand() *ClientSubcommand { + cs := &ClientSubcommand{ + fs: flag.NewFlagSet("client", flag.ContinueOnError), + } + + cs.fs.IntVar(&cs.clients, "clients", 0, "Number of clients which will be sending a subset of messages") + cs.fs.StringVar(&cs.ipAddr, "ipaddr", "", "IP Address of server") + cs.fs.IntVar(&cs.messages, "messages", 0, "Number of total messages to send") + cs.fs.IntVar(&cs.warmUpCnt, "warmupcnt", 0, "Number of total messages to send warm up client/server") + cs.fs.IntVar(&cs.port, "port", 0, "Port on which the server is listening") + cs.fs.StringVar(&cs.dbgport, "dbgport", "", "Debug port for pprof webserver (optional)") + cs.fs.StringVar(&cs.tlsDir, "tlsdir", "", "Directory to write TLS info") + + return cs +} + +func (cs *ClientSubcommand) Init(args []string) error { + return cs.fs.Parse(args) +} + +func (cs *ClientSubcommand) Name() string { + return cs.fs.Name() +} + +func (cs *ClientSubcommand) Run() (err error) { + if cs.ipAddr == "" { + err = fmt.Errorf("IP Address cannot be blank") + cs.fs.PrintDefaults() + return + } + if cs.port == 0 { + err = fmt.Errorf("Port cannot be 0") + cs.fs.PrintDefaults() + return + } + if cs.messages == 0 { + err = fmt.Errorf("Messages cannot be 0") + cs.fs.PrintDefaults() + return + } + if cs.clients == 0 { + err = fmt.Errorf("Clients cannot be 0") + cs.fs.PrintDefaults() + return + } + if cs.tlsDir == "" { + err = fmt.Errorf("tlsdir cannot be empty") + cs.fs.PrintDefaults() + return + } + if cs.dbgport != "" { + hostPort := net.JoinHostPort("localhost", cs.dbgport) + go http.ListenAndServe(hostPort, nil) + } + + fmt.Printf("clients: %v messages: %v\n", cs.clients, cs.messages) + + // Use global structure to store settings and certificate instead of + // passing on stack + globals.cs = cs + globals.useTLS = true // TODO - make option? + globals.tlsDir = cs.tlsDir + + // Run the performance test + duration := parallelClientSenders("RpcPerfPing") + fmt.Printf("\n===== PERFRPC - Clients: %v Messages per Client: %v Total Messages: %v ---- Test Duration: %v\n", + globals.cs.clients, globals.cs.messages, globals.cs.clients*globals.cs.messages, duration) + return nil +} diff --git a/retryrpc/perfrpc/dummy_test.go b/retryrpc/perfrpc/dummy_test.go new file mode 100644 index 000000000..d42991728 --- /dev/null +++ b/retryrpc/perfrpc/dummy_test.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" +) + +func TestDummy(t *testing.T) { +} diff --git a/retryrpc/perfrpc/perfrpc.go b/retryrpc/perfrpc/perfrpc.go new file mode 100644 index 000000000..0e19ffbef --- /dev/null +++ b/retryrpc/perfrpc/perfrpc.go @@ -0,0 +1,53 @@ +package main + +import ( + "errors" + "fmt" + "os" +) + +// Interface defining the subcommands for this command +// +// Allows us to organize all variables for a subcommand in one place +// +type Runner interface { + Init([]string) error // Based on subcommand - init arguments before parsing + Name() string // Return name of subcommand + Run() error // Perform subcommand +} + +// Find subcommand, init flagSet, parse and and run subcommand +func main() { + var ( + err error + ) + + if len(os.Args) < 2 { + err = errors.New("You must pass a sub-command - either 'client' or 'server'") + fmt.Println(err) + os.Exit(1) + } + + cmds := []Runner{ + NewClientCommand(), + NewServerCommand(), + } + + subcommand := os.Args[1] + for _, cmd := range cmds { + if cmd.Name() == subcommand { + cmd.Init(os.Args[2:]) + err = cmd.Run() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + os.Exit(0) + } + } + + err = fmt.Errorf("Unknown subcommand: %s - must be either 'client' or 'server'", subcommand) + fmt.Println(err) + os.Exit(1) + +} diff --git a/retryrpc/perfrpc/ping_perf.go b/retryrpc/perfrpc/ping_perf.go new file mode 100644 index 000000000..21b3714a2 --- /dev/null +++ b/retryrpc/perfrpc/ping_perf.go @@ -0,0 +1,66 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "fmt" + + "github.com/NVIDIA/proxyfs/blunder" +) + +// PerfPingServer is a struct with pointer receivers implementing RpcPerfPing*() +type PerfPingServer struct{} + +// PerfPingReq is the request object for RpcPerfPing*() +type PerfPingReq struct { + Message string +} + +// PerfPingReply is the response object for RpcPerfPing*() +type PerfPingReply struct { + Message string +} + +// Simple ping for performance testing the RPC layer +func perfEncodeErrno(e *error) { + if *e != nil { + *e = fmt.Errorf("errno: %d", blunder.Errno(*e)) + } +} + +// RpcTestPing simply does a len on the message path and returns the result +func (s *PerfPingServer) RpcPerfPing(in *PerfPingReq, reply *PerfPingReply) (err error) { + reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) + return nil +} + +var largeStr string = "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + +// RpcPerfPingLarge simply does a len on the message path and returns the result +// along with a larger buffer. +func (s *PerfPingServer) RpcPerfPingLarge(in *PerfPingReq, reply *PerfPingReply) (err error) { + buf := bytes.Buffer{} + p := fmt.Sprintf("pong %d bytes", len(in.Message)) + buf.WriteString(p) + for i := 0; i < 1000; i++ { + buf.WriteString(largeStr) + } + reply.Message = fmt.Sprintf("%v", buf.String()) + return nil +} + +// RpcPerfPingWithError returns an error +func (s *PerfPingServer) RpcPerfPingWithError(in *PerfPingReq, reply *PerfPingReply) (err error) { + err = blunder.AddError(err, blunder.NotFoundError) + perfEncodeErrno(&err) + reply.Message = fmt.Sprintf("pong %d bytes", len(in.Message)) + return err +} + +// RpcPerfPingWithClientID simply does a len on the message path and returns the clientID & result +func (s *PerfPingServer) RpcPerfPingWithClientID(clientID uint64, in *PerfPingReq, reply *PerfPingReply) (err error) { + reply.Message = fmt.Sprintf("Client ID: %v pong %d bytes", clientID, len(in.Message)) + return nil +} diff --git a/retryrpc/perfrpc/server.go b/retryrpc/perfrpc/server.go new file mode 100644 index 000000000..48786a62e --- /dev/null +++ b/retryrpc/perfrpc/server.go @@ -0,0 +1,149 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/tls" + "flag" + "fmt" + "net" + "net/http" + _ "net/http/pprof" + "os" + "time" + + "github.com/NVIDIA/proxyfs/retryrpc" +) + +func configNewServer(ipAddr string, port int, lt time.Duration, tlsDir string, dontStartTrimmers bool, useTLS bool) (rrSvr *retryrpc.Server, err error) { + var ( + config *retryrpc.ServerConfig + ) + + if useTLS { + _ = os.Mkdir(tlsDir, os.ModePerm) + tlsCert := tlsCertsAllocate(ipAddr, tlsDir) + config = &retryrpc.ServerConfig{ + LongTrim: lt, + ShortTrim: 100 * time.Millisecond, + IPAddr: ipAddr, + Port: port, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + TLSCertificate: tlsCert.endpointTLSCert, + } + } else { + config = &retryrpc.ServerConfig{ + LongTrim: lt, + ShortTrim: 100 * time.Millisecond, + IPAddr: ipAddr, + Port: port, + DeadlineIO: 60 * time.Second, + KeepAlivePeriod: 60 * time.Second, + TLSCertificate: tls.Certificate{}, + } + } + + // Create a new RetryRPC Server. Completed request will live on + // completedRequests for 10 seconds. + rrSvr = retryrpc.NewServer(config) + + return +} + +// TODO - should this be a goroutine with WG????? +// want way to shutdown with curl/WS? how dump stats? +func becomeAServer(ipAddr string, port int, tlsDir string, useTLS bool) error { + + // Create new TestPingServer - needed for calling RPCs + myJrpcfs := &PerfPingServer{} + + rrSvr, err := configNewServer(ipAddr, port, 10*time.Minute, tlsDir, false, useTLS) + if err != nil { + return err + } + + // Register the Server - sets up the methods supported by the + // server + err = rrSvr.Register(myJrpcfs) + if err != nil { + return err + } + + // Start listening for requests on the ipaddr/port + err = rrSvr.Start() + if err != nil { + return err + } + + // Tell server to start accepting and processing requests + rrSvr.Run() + + // Block indefinitely + for { + time.Sleep(1 * time.Hour) + } + + /* + // TODO - decide if we want a way to shutdown server + // Stop the server before exiting + rrSvr.Close() + return nil + */ +} + +type ServerSubcommand struct { + fs *flag.FlagSet + ipAddr string // IP Address server will use + port int // Port on which to listen + tlsDir string // Directory to read for TLS info + dbgport string // Debug port for pprof webserver +} + +func NewServerCommand() *ServerSubcommand { + s := &ServerSubcommand{ + fs: flag.NewFlagSet("server", flag.ContinueOnError), + } + s.fs.StringVar(&s.ipAddr, "ipaddr", "", "IP Address server will use") + s.fs.IntVar(&s.port, "port", 0, "Port on which to listen") + s.fs.StringVar(&s.tlsDir, "tlsdir", "", "Directory containing TLS info") + s.fs.StringVar(&s.dbgport, "dbgport", "", "Debug port for pprof webserver (optional)") + + return s +} + +func (s *ServerSubcommand) Init(args []string) error { + return s.fs.Parse(args) +} + +func (s *ServerSubcommand) Name() string { + return s.fs.Name() +} + +// TODO - how dump bucketstats, etc???? do from webserver? +func (s *ServerSubcommand) Run() (err error) { + if s.ipAddr == "" { + err = fmt.Errorf("IP Address cannot be blank") + s.fs.PrintDefaults() + return + } + if s.port == 0 { + err = fmt.Errorf("Port cannot be 0") + s.fs.PrintDefaults() + return + } + if s.tlsDir == "" { + err = fmt.Errorf("tlsdir cannot be empty") + s.fs.PrintDefaults() + return + } + + if s.dbgport != "" { + hostPort := net.JoinHostPort("localhost", s.dbgport) + go http.ListenAndServe(hostPort, nil) + } + + err = becomeAServer(s.ipAddr, s.port, s.tlsDir, true) + return err +} diff --git a/retryrpc/perfrpc/tls.go b/retryrpc/perfrpc/tls.go new file mode 100644 index 000000000..7720b614a --- /dev/null +++ b/retryrpc/perfrpc/tls.go @@ -0,0 +1,95 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/tls" + "crypto/x509/pkix" + "fmt" + "net" + "os" + "time" + + "github.com/NVIDIA/proxyfs/icert/icertpkg" +) + +type tlsCertsStruct struct { + caCertPEMBlock []byte + caKeyPEMBlock []byte + endpointCertPEMBlock []byte + endpointKeyPEMBlock []byte + endpointTLSCert tls.Certificate + caCertFile string + caKeyFile string + endpointCertFile string + endpointKeyFile string +} + +var tlsCerts *tlsCertsStruct + +func tlsSetFileNames(tlsCerts *tlsCertsStruct, tlsDir string) { + tlsCerts.caCertFile = tlsDir + "/caCertFile" + tlsCerts.caKeyFile = tlsDir + "/caKeyFile" + tlsCerts.endpointCertFile = tlsDir + "/endpointCertFile" + tlsCerts.endpointKeyFile = tlsDir + "/endpointKeyFile" +} + +// Utility function to initialize tlsCerts +func tlsCertsAllocate(ipAddr string, tlsDir string) (tlsCerts *tlsCertsStruct) { + var ( + err error + ) + + tlsCerts = &tlsCertsStruct{} + tlsSetFileNames(tlsCerts, tlsDir) + + tlsCerts.caCertPEMBlock, tlsCerts.caKeyPEMBlock, err = icertpkg.GenCACert( + icertpkg.GenerateKeyAlgorithmEd25519, + pkix.Name{ + Organization: []string{"Test Organization CA"}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + 24*time.Hour, + tlsCerts.caCertFile, + tlsCerts.caKeyFile) + + if err != nil { + fmt.Printf("icertpkg.GenCACert() failed: %v", err) + os.Exit(1) + } + + tlsCerts.endpointCertPEMBlock, tlsCerts.endpointKeyPEMBlock, err = icertpkg.GenEndpointCert( + icertpkg.GenerateKeyAlgorithmEd25519, + pkix.Name{ + Organization: []string{"Test Organization Endpoint"}, + Country: []string{}, + Province: []string{}, + Locality: []string{}, + StreetAddress: []string{}, + PostalCode: []string{}, + }, + []string{}, + []net.IP{net.ParseIP(ipAddr)}, + time.Hour, + tlsCerts.caCertPEMBlock, + tlsCerts.caKeyPEMBlock, + tlsCerts.endpointCertFile, + tlsCerts.endpointKeyFile) + + if err != nil { + fmt.Printf("icertpkg.genEndpointCert() failed: %v", err) + os.Exit(1) + } + + tlsCerts.endpointTLSCert, err = tls.X509KeyPair(tlsCerts.endpointCertPEMBlock, tlsCerts.endpointKeyPEMBlock) + if err != nil { + fmt.Printf("tls.LoadX509KeyPair() failed: %v", err) + os.Exit(1) + } + return +} From 68b7a1a6daa9d00daa3f57d252a9e822fdf0fa93 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 23 Apr 2021 17:30:08 -0700 Subject: [PATCH 139/160] Cleaned up TestTryLockMutex() to make success more reliably reported ...avoids (by seriously elongating the wait time) that 3rd TryLock() call that is supposed to succeed --- utils/api.go | 6 ++++-- utils/api_test.go | 7 +++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/utils/api.go b/utils/api.go index d5874391d..45b3b6d11 100644 --- a/utils/api.go +++ b/utils/api.go @@ -38,9 +38,11 @@ func (tryLockMutex *TryLockMutex) TryLock(timeout time.Duration) (gotIt bool) { timer := time.NewTimer(timeout) select { case tryLockMutex.c <- struct{}{}: - timer.Stop() + if !timer.Stop() { + <-timer.C + } gotIt = true - case <-time.After(time.Duration(timeout)): + case <-timer.C: gotIt = false } return diff --git a/utils/api_test.go b/utils/api_test.go index 4c39cd1fd..d78bb4d2c 100644 --- a/utils/api_test.go +++ b/utils/api_test.go @@ -23,10 +23,11 @@ func testTryLockMutexAsyncUnlock() { func TestTryLockMutex(t *testing.T) { testTryLockMutex = NewTryLockMutex() + testTryLockMutex.Lock() testTryLockMutex.Unlock() - shouldHaveGottonIt := testTryLockMutex.TryLock(100 * time.Millisecond) + shouldHaveGottonIt := testTryLockMutex.TryLock(10 * time.Second) if !shouldHaveGottonIt { t.Fatalf("1st TryLock() should have succeeded") } @@ -37,10 +38,12 @@ func TestTryLockMutex(t *testing.T) { } _ = time.AfterFunc(20*time.Millisecond, testTryLockMutexAsyncUnlock) - shouldHaveGottonIt = testTryLockMutex.TryLock(120 * time.Millisecond) + + shouldHaveGottonIt = testTryLockMutex.TryLock(10 * time.Second) if !shouldHaveGottonIt { t.Fatalf("3rd TryLock() should have succeeded") } + testTryLockMutex.Unlock() } From 02ff70b629309dbad78419a0444668f7b5f1e2e0 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Mon, 26 Apr 2021 13:42:14 -0700 Subject: [PATCH 140/160] Add support for open()/create() calls setting O_APPEND option It still matters for create() since this means seek's will be ignored... --- pfsagentd/fission.go | 62 +++++++++++++++++++++++++++++++++++++++----- pfsagentd/globals.go | 3 +++ 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/pfsagentd/fission.go b/pfsagentd/fission.go index adc028d63..251494d9d 100644 --- a/pfsagentd/fission.go +++ b/pfsagentd/fission.go @@ -1172,6 +1172,7 @@ func (dummy *globalsStruct) DoOpen(inHeader *fission.InHeader, openIn *fission.O globals.lastFH++ + globals.fhToOpenRequestMap[globals.lastFH] = openIn.Flags globals.fhToInodeNumberMap[globals.lastFH] = inHeader.NodeID fhSet, ok = globals.inodeNumberToFHMap[inHeader.NodeID] @@ -1209,6 +1210,7 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R err error fhInodeNumber uint64 fileInode *fileInodeStruct + fOpenRequestFlags uint32 getStatReply *jrpcfs.StatStruct getStatRequest *jrpcfs.GetStatRequest logSegmentCacheElement *logSegmentCacheElementStruct @@ -1252,9 +1254,14 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R globals.Lock() + fOpenRequestFlags, ok = globals.fhToOpenRequestMap[readIn.FH] + if !ok { + logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH (in fhToOpenRequestMap)", inHeader.NodeID, readIn.FH) + } + fhInodeNumber, ok = globals.fhToInodeNumberMap[readIn.FH] if !ok { - logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH", inHeader.NodeID, readIn.FH) + logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH (in fhToInodeNumberMap)", inHeader.NodeID, readIn.FH) } if fhInodeNumber != inHeader.NodeID { logFatalf("DoRead(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, readIn.FH, fhInodeNumber) @@ -1262,6 +1269,12 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R globals.Unlock() + if 0 != (fOpenRequestFlags & fission.FOpenRequestWRONLY) { + fileInode.unlock(true) + errno = syscall.EINVAL + return + } + // fileInode = referenceFileInode(inode.InodeNumber(inHeader.NodeID)) if nil == fileInode { logFatalf("DoRead(NodeID=%v,FH=%v) called for non-FileInode", inHeader.NodeID, readIn.FH) @@ -1367,11 +1380,13 @@ func (dummy *globalsStruct) DoRead(inHeader *fission.InHeader, readIn *fission.R func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission.WriteIn) (writeOut *fission.WriteOut, errno syscall.Errno) { var ( + actualOffset uint64 chunkedPutContext *chunkedPutContextStruct chunkedPutContextElement *list.Element err error fhInodeNumber uint64 fileInode *fileInodeStruct + fOpenRequestFlags uint32 ok bool singleObjectExtent *singleObjectExtentStruct ) @@ -1382,9 +1397,14 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission globals.Lock() + fOpenRequestFlags, ok = globals.fhToOpenRequestMap[writeIn.FH] + if !ok { + logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH (in fhToOpenRequestMap)", inHeader.NodeID, writeIn.FH) + } + fhInodeNumber, ok = globals.fhToInodeNumberMap[writeIn.FH] if !ok { - logFatalf("DoWrite(NodeID=%v,FH=%v) called for unknown FH", inHeader.NodeID, writeIn.FH) + logFatalf("DoWrite(NodeID=%v,FH=%v) called for unknown FH (in fhToInodeNumberMap)", inHeader.NodeID, writeIn.FH) } if fhInodeNumber != inHeader.NodeID { logFatalf("DoWrite(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, writeIn.FH, fhInodeNumber) @@ -1392,6 +1412,12 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission globals.Unlock() + if 0 != (fOpenRequestFlags & fission.FOpenRequestRDONLY) { + fileInode.unlock(true) + errno = syscall.EINVAL + return + } + // fileInode = referenceFileInode(inode.InodeNumber(inHeader.NodeID)) if nil == fileInode { logFatalf("DoWrite(NodeID=%v,FH=%v) called for non-FileInode", inHeader.NodeID, writeIn.FH) @@ -1403,6 +1429,12 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission return } + if 0 == (fOpenRequestFlags & fission.FOpenRequestAPPEND) { + actualOffset = writeIn.Offset + } else { + actualOffset = fileInode.cachedStat.Size + } + // Grab quota to start a fresh chunkedPutContext before calling fileInode.getExclusiveLock() // since, in the pathologic case where only fileInode has any outstanding chunkedPutContext's, // they can only be complete()'d while holding fileInode.getExclusiveLock() and we would @@ -1476,7 +1508,7 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission } singleObjectExtent = &singleObjectExtentStruct{ - fileOffset: uint64(writeIn.Offset), + fileOffset: actualOffset, objectOffset: uint64(len(chunkedPutContext.buf)), length: uint64(len(writeIn.Data)), } @@ -1571,9 +1603,16 @@ func (dummy *globalsStruct) DoRelease(inHeader *fission.InHeader, releaseIn *fis globals.Lock() + _, ok = globals.fhToOpenRequestMap[releaseIn.FH] + if !ok { + logFatalf("DoRelease(NodeID=%v,FH=%v) called for unknown FH (fhToOpenRequestMap)", inHeader.NodeID, releaseIn.FH) + } + + delete(globals.fhToOpenRequestMap, releaseIn.FH) + fhInodeNumber, ok = globals.fhToInodeNumberMap[releaseIn.FH] if !ok { - logFatalf("DoRelease(NodeID=%v,FH=%v) called for unknown FH", inHeader.NodeID, releaseIn.FH) + logFatalf("DoRelease(NodeID=%v,FH=%v) called for unknown FH (fhToInodeNumberMap)", inHeader.NodeID, releaseIn.FH) } if fhInodeNumber != inHeader.NodeID { logFatalf("DoRelease(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, releaseIn.FH, fhInodeNumber) @@ -1616,9 +1655,14 @@ func (dummy *globalsStruct) DoFSync(inHeader *fission.InHeader, fSyncIn *fission globals.Lock() + _, ok = globals.fhToOpenRequestMap[fSyncIn.FH] + if !ok { + logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH (in fhToOpenRequestMap)", inHeader.NodeID, fSyncIn.FH) + } + fhInodeNumber, ok = globals.fhToInodeNumberMap[fSyncIn.FH] if !ok { - logFatalf("DoFSync(NodeID=%v,FH=%v) called for unknown FH", inHeader.NodeID, fSyncIn.FH) + logFatalf("DoFSync(NodeID=%v,FH=%v) called for unknown FH (in fhToInodeNumberMap)", inHeader.NodeID, fSyncIn.FH) } if fhInodeNumber != inHeader.NodeID { logFatalf("DoFSync(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, fSyncIn.FH, fhInodeNumber) @@ -1844,9 +1888,14 @@ func (dummy *globalsStruct) DoFlush(inHeader *fission.InHeader, flushIn *fission globals.Lock() + _, ok = globals.fhToOpenRequestMap[flushIn.FH] + if !ok { + logFatalf("DoRead(NodeID=%v,FH=%v) called for unknown FH (in fhToOpenRequestMap)", inHeader.NodeID, flushIn.FH) + } + fhInodeNumber, ok = globals.fhToInodeNumberMap[flushIn.FH] if !ok { - logFatalf("DoFlush(NodeID=%v,FH=%v) called for unknown FH", inHeader.NodeID, flushIn.FH) + logFatalf("DoFlush(NodeID=%v,FH=%v) called for unknown FH (in fhToInodeNumberMap)", inHeader.NodeID, flushIn.FH) } if fhInodeNumber != inHeader.NodeID { logFatalf("DoFlush(NodeID=%v,FH=%v) called for FH associated with NodeID=%v", inHeader.NodeID, flushIn.FH, fhInodeNumber) @@ -2205,6 +2254,7 @@ func (dummy *globalsStruct) DoCreate(inHeader *fission.InHeader, createIn *fissi globals.lastFH++ + globals.fhToOpenRequestMap[globals.lastFH] = createIn.Flags globals.fhToInodeNumberMap[globals.lastFH] = uint64(createReply.InodeNumber) fhSet, ok = globals.inodeNumberToFHMap[uint64(createReply.InodeNumber)] diff --git a/pfsagentd/globals.go b/pfsagentd/globals.go index a73c5ce48..fb873156a 100644 --- a/pfsagentd/globals.go +++ b/pfsagentd/globals.go @@ -358,6 +358,7 @@ type globalsStruct struct { unleasedFileInodeCacheLRU *list.List // Front() is oldest fileInodeStruct.leaseListElement sharedLeaseFileInodeCacheLRU *list.List // Front() is oldest fileInodeStruct.leaseListElement exclusiveLeaseFileInodeCacheLRU *list.List // Front() is oldest fileInodeStruct.leaseListElement + fhToOpenRequestMap map[uint64]uint32 // Key == FH; Value == {Create|Open}In.Flags fhToInodeNumberMap map[uint64]uint64 // Key == FH; Value == InodeNumber inodeNumberToFHMap map[uint64]fhSetType // Key == InodeNumber; Value == set of FH's lastFH uint64 // Valid FH's start at 1 @@ -739,6 +740,7 @@ func initializeGlobals(confMap conf.ConfMap) { globals.sharedLeaseFileInodeCacheLRU = list.New() globals.exclusiveLeaseFileInodeCacheLRU = list.New() + globals.fhToOpenRequestMap = make(map[uint64]uint32) globals.fhToInodeNumberMap = make(map[uint64]uint64) globals.inodeNumberToFHMap = make(map[uint64]fhSetType) @@ -780,6 +782,7 @@ func uninitializeGlobals() { globals.unleasedFileInodeCacheLRU = nil // TODO: Obsolete this globals.sharedLeaseFileInodeCacheLRU = nil // TODO: Obsolete this globals.exclusiveLeaseFileInodeCacheLRU = nil // TODO: Obsolete this + globals.fhToOpenRequestMap = nil globals.fhToInodeNumberMap = nil globals.inodeNumberToFHMap = nil globals.lastFH = 0 From 9f67ab082fde109f62fe8302e136d3bb2262b0ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Mon, 26 Apr 2021 15:00:30 -0700 Subject: [PATCH 141/160] Add logging for noauth proxy server in Runway --- .../etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cookbooks/proxyfs_swift/files/default/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf b/cookbooks/proxyfs_swift/files/default/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf index 35d248f44..8e5916a44 100644 --- a/cookbooks/proxyfs_swift/files/default/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf +++ b/cookbooks/proxyfs_swift/files/default/etc/swift/proxy-server/proxy-noauth.conf.d/20_settings.conf @@ -1,6 +1,8 @@ [DEFAULT] bind_ip = 127.0.0.1 bind_port = 8090 +log_facility = LOG_LOCAL1 +log_name = noauth-proxy-server [pipeline:main] pipeline = catch_errors healthcheck proxy-logging cache list-endpoints bulk tempurl slo dlo proxy-logging proxy-server From 8e58047364811e289f81992f0750b49a589fb294 Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Tue, 27 Apr 2021 17:22:08 -0700 Subject: [PATCH 142/160] Don't dump bucketstats, fix hang and lengthen cert (#619) --- retryrpc/client_tracking.go | 6 ------ retryrpc/perfrpc/server.go | 5 ++++- retryrpc/perfrpc/tls.go | 6 +++--- retryrpc/server.go | 4 ++++ 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/retryrpc/client_tracking.go b/retryrpc/client_tracking.go index 615a88c1c..0198d3543 100644 --- a/retryrpc/client_tracking.go +++ b/retryrpc/client_tracking.go @@ -8,7 +8,6 @@ import ( "strconv" "github.com/NVIDIA/proxyfs/bucketstats" - "github.com/NVIDIA/proxyfs/logger" ) // This file contains functions in the server which @@ -55,12 +54,7 @@ func (ci *clientInfo) setMethodStats(method string, deltaTime uint64) { func (ci *clientInfo) unregsiterMethodStats(server *Server) { idAsStr := strconv.FormatInt(int64(ci.myUniqueID), 10) - logger.Infof("bucketstats for myUniqueID: '%v' - %s\n", ci.myUniqueID, - bucketstats.SprintStats(bucketstats.StatFormatParsable1, bucketStatsPkgName, "*")) for m := range server.svrMap { - logger.Infof("bucketstats myUniqueID: '%v' method: '%v'- %s\n", ci.myUniqueID, m, - bucketstats.SprintStats(bucketstats.StatFormatParsable1, bucketStatsPkgName, - methodAndName(strconv.FormatInt(int64(ci.myUniqueID), 10), m))) bucketstats.UnRegister(bucketStatsPkgName, methodAndName(strconv.FormatInt(int64(ci.myUniqueID), 10), m)) } bucketstats.UnRegister(bucketStatsPkgName, idAsStr) diff --git a/retryrpc/perfrpc/server.go b/retryrpc/perfrpc/server.go index 48786a62e..bcf98b92c 100644 --- a/retryrpc/perfrpc/server.go +++ b/retryrpc/perfrpc/server.go @@ -23,7 +23,10 @@ func configNewServer(ipAddr string, port int, lt time.Duration, tlsDir string, d if useTLS { _ = os.Mkdir(tlsDir, os.ModePerm) - tlsCert := tlsCertsAllocate(ipAddr, tlsDir) + + // Create certificate/CA that last for 30 days + tlsCert := tlsCertsAllocate(ipAddr, 30*24*time.Hour, tlsDir) + config = &retryrpc.ServerConfig{ LongTrim: lt, ShortTrim: 100 * time.Millisecond, diff --git a/retryrpc/perfrpc/tls.go b/retryrpc/perfrpc/tls.go index 7720b614a..2c0d41869 100644 --- a/retryrpc/perfrpc/tls.go +++ b/retryrpc/perfrpc/tls.go @@ -36,7 +36,7 @@ func tlsSetFileNames(tlsCerts *tlsCertsStruct, tlsDir string) { } // Utility function to initialize tlsCerts -func tlsCertsAllocate(ipAddr string, tlsDir string) (tlsCerts *tlsCertsStruct) { +func tlsCertsAllocate(ipAddr string, ttl time.Duration, tlsDir string) (tlsCerts *tlsCertsStruct) { var ( err error ) @@ -54,7 +54,7 @@ func tlsCertsAllocate(ipAddr string, tlsDir string) (tlsCerts *tlsCertsStruct) { StreetAddress: []string{}, PostalCode: []string{}, }, - 24*time.Hour, + ttl, tlsCerts.caCertFile, tlsCerts.caKeyFile) @@ -75,7 +75,7 @@ func tlsCertsAllocate(ipAddr string, tlsDir string) (tlsCerts *tlsCertsStruct) { }, []string{}, []net.IP{net.ParseIP(ipAddr)}, - time.Hour, + ttl, tlsCerts.caCertPEMBlock, tlsCerts.caKeyPEMBlock, tlsCerts.endpointCertFile, diff --git a/retryrpc/server.go b/retryrpc/server.go index 8ee6894b4..944551963 100644 --- a/retryrpc/server.go +++ b/retryrpc/server.go @@ -238,6 +238,10 @@ func (server *Server) getClientIDAndWait(cCtx *connCtx) (ci *clientInfo, err err server.Lock() lci, ok := server.perClientInfo[connUniqueID] if !ok { + // This could happen if the TTL is set to low and then client has not really died. + + // TODO - tell the client they need to panic since it is better for the client to + // panic then the server. err = fmt.Errorf("Server - msgType PassID but can't find uniqueID in perClientInfo") logger.PanicfWithError(err, "getClientIDAndWait() buf: %v connUniqueID: %v err: %+v", buf, connUniqueID, err) server.Unlock() From 11369414205c1a1188fa1325622883b28e31e300 Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Fri, 30 Apr 2021 11:56:24 -0700 Subject: [PATCH 143/160] Pass reference to the connection in the call to the goroutine (#620) Pass the connection as a reference when starting the goroutine. Previously, the goroutine would find the connection directly which would fail if another goroutine and niled the connection. This solution allows the goroutine to get the EOF and gracefully exit. --- retryrpc/api.go | 4 +--- retryrpc/client.go | 15 +++++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/retryrpc/api.go b/retryrpc/api.go index 4b8d45fd4..509d627e5 100644 --- a/retryrpc/api.go +++ b/retryrpc/api.go @@ -345,7 +345,7 @@ func NewClient(config *ClientConfig) (client *Client, err error) { client.outstandingRequest = make(map[requestID]*reqCtx) client.bt = btree.New(2) - if nil == config.RootCAx509CertificatePEM { + if config.RootCAx509CertificatePEM == nil { client.connection.useTLS = false client.connection.tlsConn = nil client.connection.x509CertPool = nil @@ -395,7 +395,5 @@ func (client *Client) Close() { // Wait for the goroutines to return client.goroutineWG.Wait() - logger.Infof("bucketstats for myUniqueID: '%v' - %s\n", client.myUniqueID, - bucketstats.SprintStats(bucketstats.StatFormatParsable1, bucketStatsPkgName, client.GetStatsGroupName())) bucketstats.UnRegister(bucketStatsPkgName, client.GetStatsGroupName()) } diff --git a/retryrpc/client.go b/retryrpc/client.go index 6400a79f3..8d1323cc1 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -289,7 +289,7 @@ func (client *Client) notifyReply(buf []byte, genNum uint64, recvResponse time.T // // As soon as it reads a complete response, it launches a goroutine to process // the response and notify the blocked Send(). -func (client *Client) readReplies(callingGenNum uint64) { +func (client *Client) readReplies(nC net.Conn, callingGenNum uint64) { defer client.goroutineWG.Done() var localCnt int @@ -300,7 +300,6 @@ func (client *Client) readReplies(callingGenNum uint64) { client.Unlock() return } - nC := client.connection.castToNetConn() client.Unlock() if nC == nil { @@ -554,8 +553,12 @@ func (client *Client) reDial() (err error) { } // Start readResponse goroutine to read responses from server + // + // Pass copy of connection used since a new readReplies() will be + // started with a new connection as needed. + nC := client.connection.castToNetConn() client.goroutineWG.Add(1) - go client.readReplies(client.connection.genNum) + go client.readReplies(nC, client.connection.genNum) return } @@ -653,8 +656,12 @@ func (client *Client) initialDial() (err error) { } // Start readResponse goroutine to read responses from server + // + // Pass copy of connection used since a new readReplies() will be + // started with a new connection as needed. + nC := client.connection.castToNetConn() client.goroutineWG.Add(1) - go client.readReplies(client.connection.genNum) + go client.readReplies(nC, client.connection.genNum) return } From 92cd760c8119c2dfbc7c41ecc7e4c3a92aedc8f4 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 13 Apr 2021 14:33:18 -0700 Subject: [PATCH 144/160] Go Module updated following rebase merge --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d77c281fb..5f7058608 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/ory/go-acc v0.2.6 // indirect github.com/pborman/uuid v1.2.1 // indirect - github.com/pelletier/go-toml v1.8.1 // indirect + github.com/pelletier/go-toml v1.9.0 // indirect github.com/sirupsen/logrus v1.2.0 github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cobra v1.1.3 // indirect @@ -28,7 +28,7 @@ require ( go.uber.org/multierr v1.5.0 // indirect go.uber.org/zap v1.10.1-0.20190619185213-853ac185800f golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 - golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 - golang.org/x/text v0.3.5 // indirect + golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 + golang.org/x/text v0.3.6 // indirect gopkg.in/ini.v1 v1.62.0 // indirect ) diff --git a/go.sum b/go.sum index 239a370fc..49c89bb11 100644 --- a/go.sum +++ b/go.sum @@ -221,6 +221,8 @@ github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMF github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.0 h1:NOd0BRdOKpPf0SxkL3HxSQOG7rNh+4kl6PHcBPFs7Q0= +github.com/pelletier/go-toml v1.9.0/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -391,6 +393,8 @@ golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSK golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210326220804-49726bf1d181 h1:64ChN/hjER/taL4YJuA+gpLfIMT+/NFherRZixbxOhg= golang.org/x/sys v0.0.0-20210326220804-49726bf1d181/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750 h1:ZBu6861dZq7xBnG1bn5SRU0vA8nx42at4+kP07FMTog= +golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -398,6 +402,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From e00615d0c124e218bf1c3cdd89db87d2f7171fcd Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 16 Apr 2021 15:16:38 -0700 Subject: [PATCH 145/160] A start on TODOs for imgr RPC testing --- imgr/imgrpkg/retry-rpc_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 imgr/imgrpkg/retry-rpc_test.go diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go new file mode 100644 index 000000000..e3e2fb024 --- /dev/null +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -0,0 +1,19 @@ +// Copyright (c) 2015-2021, NVIDIA CORPORATION. +// SPDX-License-Identifier: Apache-2.0 + +package imgrpkg + +import ( + "testing" +) + +func TestRetryRPC(t *testing.T) { + testSetup(t) + + // TODO: Attempt a FetchNonceRange() without a prior Mount()... which should fail (no Mount) + // TODO: Perform a Mount() + // TODO: Attempt a GetInodeTableEntry() for RootDirInode... which should fail (no Lease) + // TODO: Fetch a Shared Lease + + testTeardown(t) +} From e573f304803b1ecd39e8f4947424af4836384470 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 16 Apr 2021 17:23:47 -0700 Subject: [PATCH 146/160] Flushed out the remaining TODOs in imgr/imgrpkg/retry-rpc_test.go --- imgr/imgr.conf | 2 +- imgr/imgrpkg/api.go | 4 +++- imgr/imgrpkg/retry-rpc_test.go | 28 +++++++++++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/imgr/imgr.conf b/imgr/imgr.conf index 0c5eee0fa..c3e7c8c8b 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -15,7 +15,7 @@ RetryRPCKeepAlivePeriod: 60s RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used -FetchNonceRangeToReturn: 100 +FetchNonceRangeToReturn: 100 MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index eb1b5d2a2..b57305055 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -287,12 +287,14 @@ func (dummy *RetryRPCServerStruct) DeleteInodeTableEntry(deleteInodeTableEntryRe type AdjustInodeTableEntryOpenCountRequestStruct struct { MountID string InodeNumber uint64 + Adjustment int64 } // AdjustInodeTableEntryOpenCountReplyStruct is the reply object for AdjustInodeTableEntryOpenCount. // type AdjustInodeTableEntryOpenCountReplyStruct struct { - // TODO + CurrentOpenCountThisMount uint64 + CurrentOpenCountAllMounts uint64 } // AdjustInodeTableEntryOpenCount requests the specified Inode's OpenCount be diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index e3e2fb024..75669796e 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -13,7 +13,33 @@ func TestRetryRPC(t *testing.T) { // TODO: Attempt a FetchNonceRange() without a prior Mount()... which should fail (no Mount) // TODO: Perform a Mount() // TODO: Attempt a GetInodeTableEntry() for RootDirInode... which should fail (no Lease) - // TODO: Fetch a Shared Lease + // TODO: Force a need for a re-auth + // TODO: Attempt a GetInodeTableEntry() for RootDirInode... which should fail (re-auth required) + // TODO: Perform a RenewMount() + // TODO: Fetch a Shared Lease on RootDirInode + // TODO: Perform a GetInodeTableEntry() for RootDirInode + // TODO: Attempt a PutInodeTableEntries() for RootDirInode... which should fail (only Shared Lease) + // TODO: Perform a Lease Promote on RootDirInode + // TODO: Perform a PutInodeTableEntries() on RootDirInode + // TODO: Create a FileInode (set LinkCount to 0 & no dir entry)... with small amount of data + // TODO: Fetch an Exclusive Lease on FileInode + // TODO: Perform a PutInodeTableEntries() for FileInode + // TODO: Append some data (to new Object) for FileInode... and new stat + // TODO: Perform a PutInodeTableEntries() for FileInode + // TODO: Perform a Flush() + // TODO: Overwrite the original data in FileInode (to new Object)... and new stat dereferencing 1st Object + // TODO: Perform a Flush() + // TODO: Verify that 1st Object for FileInode gets deleted... but not 2nd nor 3rd + // TODO: Perform a AdjustInodeTableEntryOpenCount(+1) for FileInode + // TODO: Perform a DeleteInodeTableEntry() on FileInode + // TODO: Perform a Flush() + // TODO: Verify that FileInode is still in InodeTable + // TODO: Perform a AdjustInodeTableEntryOpenCount(-1) for FileInode + // TODO: Perform a Flush() + // TODO: Verify that FileInode is no longer in InodeTable and 2nd and 3rd Objects are deleted + // TODO: Perform a Lease Release on FileInode + // TODO: Perform a Unmount()... without first releasing Exclusive Lease on RootDirInode + // TODO: Verify that Exclusive Lease on RootDirInode is implicitly released testTeardown(t) } From cce9b08c516f06370096ae7b1cf0d4c008b76ec7 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 16 Apr 2021 17:30:29 -0700 Subject: [PATCH 147/160] Switched imgrpkg RPC "replies" to "responses"... to get alphabetization it be better in godoc --- imgr/imgrpkg/api.go | 98 +++++++++++++++++++-------------------- imgr/imgrpkg/retry-rpc.go | 22 ++++----- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index b57305055..8a2e6205e 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -155,17 +155,17 @@ type MountRequestStruct struct { AuthToken string } -// MountReplyStruct is the reply object for Mount. +// MountResponseStruct is the response object for Mount. // -type MountReplyStruct struct { +type MountResponseStruct struct { MountID string } // Mount performs a mount of the specified Volume and returns a MountID to be used // in all subsequent RPCs to reference this Volume by this Client. // -func (dummy *RetryRPCServerStruct) Mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { - return mount(retryRPCClientID, mountRequest, mountReply) +func (dummy *RetryRPCServerStruct) Mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountResponse *MountResponseStruct) (err error) { + return mount(retryRPCClientID, mountRequest, mountResponse) } // RenewMountRequestStruct is the request object for RenewMount. @@ -175,14 +175,14 @@ type RenewMountRequestStruct struct { AuthToken string } -// RenewMountReplyStruct is the reply object for RenewMount. +// RenewMountResponseStruct is the response object for RenewMount. // -type RenewMountReplyStruct struct{} +type RenewMountResponseStruct struct{} // RenewMount updates the AuthToken for the specified MountID. // -func (dummy *RetryRPCServerStruct) RenewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { - return renewMount(renewMountRequest, renewMountReply) +func (dummy *RetryRPCServerStruct) RenewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse *RenewMountResponseStruct) (err error) { + return renewMount(renewMountRequest, renewMountResponse) } // UnmountRequestStruct is the request object for Unmount. @@ -191,15 +191,15 @@ type UnmountRequestStruct struct { MountID string } -// UnmountReplyStruct is the reply object for Unmount. +// UnmountResponseStruct is the response object for Unmount. // -type UnmountReplyStruct struct{} +type UnmountResponseStruct struct{} // Unmount requests that the given MountID be released (and implicitly releases // any Leases held by the MountID). // -func (dummy *RetryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { - return unmount(unmountRequest, unmountReply) +func (dummy *RetryRPCServerStruct) Unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountResponseStruct) (err error) { + return unmount(unmountRequest, unmountResponse) } // FetchNonceRangeRequestStruct is the request object for FetchNonceRange. @@ -208,9 +208,9 @@ type FetchNonceRangeRequestStruct struct { MountID string } -// FetchNonceRangeReplyStruct is the reply object for FetchNonceRange. +// FetchNonceRangeResponseStruct is the response object for FetchNonceRange. // -type FetchNonceRangeReplyStruct struct { +type FetchNonceRangeResponseStruct struct { NextNonce uint64 NumNoncesFetched uint64 } @@ -218,8 +218,8 @@ type FetchNonceRangeReplyStruct struct { // FetchNonceRange requests a range of uint64 nonce values (i.e. values that will // never be reused). // -func (dummy *RetryRPCServerStruct) FetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { - return fetchNonceRange(fetchNonceRangeRequest, fetchNonceRangeReply) +func (dummy *RetryRPCServerStruct) FetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeResponse *FetchNonceRangeResponseStruct) (err error) { + return fetchNonceRange(fetchNonceRangeRequest, fetchNonceRangeResponse) } // GetInodeTableEntryRequestStruct is the request object for GetInodeTableEntry. @@ -229,17 +229,17 @@ type GetInodeTableEntryRequestStruct struct { InodeNumber uint64 } -// GetInodeTableEntryReplyStruct is the reply object for GetInodeTableEntry. +// GetInodeTableEntryResponseStruct is the response object for GetInodeTableEntry. // -type GetInodeTableEntryReplyStruct struct { +type GetInodeTableEntryResponseStruct struct { // TODO } // GetInodeTableEntry requests the Inode information for the specified Inode // (which must have an active Shared or Exclusive Lease granted to the MountID). // -func (dummy *RetryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { - return getInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryReply) +func (dummy *RetryRPCServerStruct) GetInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryResponse *GetInodeTableEntryResponseStruct) (err error) { + return getInodeTableEntry(getInodeTableEntryRequest, getInodeTableEntryResponse) } // PutInodeTableEntriesRequestStruct is the request object for PutInodeTableEntries. @@ -249,15 +249,15 @@ type PutInodeTableEntriesRequestStruct struct { // TODO } -// PutInodeTableEntriesReplyStruct is the reply object for PutInodeTableEntries. +// PutInodeTableEntriesResponseStruct is the response object for PutInodeTableEntries. // -type PutInodeTableEntriesReplyStruct struct{} +type PutInodeTableEntriesResponseStruct struct{} // PutInodeTableEntries requests an atomic update of the listed Inodes (which must // each have an active Exclusive Lease granted to the MountID). // -func (dummy *RetryRPCServerStruct) PutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { - return putInodeTableEntries(putInodeTableEntriesRequest, putInodeTableEntriesReply) +func (dummy *RetryRPCServerStruct) PutInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesResponse *PutInodeTableEntriesResponseStruct) (err error) { + return putInodeTableEntries(putInodeTableEntriesRequest, putInodeTableEntriesResponse) } // DeleteInodeTableEntryRequestStruct is the request object for DeleteInodeTableEntry. @@ -267,9 +267,9 @@ type DeleteInodeTableEntryRequestStruct struct { InodeNumber uint64 } -// DeleteInodeTableEntryReplyStruct is the reply object for DeleteInodeTableEntry. +// DeleteInodeTableEntryResponseStruct is the response object for DeleteInodeTableEntry. // -type DeleteInodeTableEntryReplyStruct struct { +type DeleteInodeTableEntryResponseStruct struct { // TODO } @@ -278,8 +278,8 @@ type DeleteInodeTableEntryReplyStruct struct { // unless/until the OpenCount for the Inode drops to zero, the Inode will // still exist. // -func (dummy *RetryRPCServerStruct) DeleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryReply *DeleteInodeTableEntryReplyStruct) (err error) { - return deleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryReply) +func (dummy *RetryRPCServerStruct) DeleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct) (err error) { + return deleteInodeTableEntry(deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) } // AdjustInodeTableEntryOpenCountRequestStruct is the request object for AdjustInodeTableEntryOpenCount. @@ -290,9 +290,9 @@ type AdjustInodeTableEntryOpenCountRequestStruct struct { Adjustment int64 } -// AdjustInodeTableEntryOpenCountReplyStruct is the reply object for AdjustInodeTableEntryOpenCount. +// AdjustInodeTableEntryOpenCountResponseStruct is the response object for AdjustInodeTableEntryOpenCount. // -type AdjustInodeTableEntryOpenCountReplyStruct struct { +type AdjustInodeTableEntryOpenCountResponseStruct struct { CurrentOpenCountThisMount uint64 CurrentOpenCountAllMounts uint64 } @@ -303,8 +303,8 @@ type AdjustInodeTableEntryOpenCountReplyStruct struct { // for deletion by a prior call to DeleteInodeTableEntry, the Inode will be // deleted. // -func (dummy *RetryRPCServerStruct) AdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountReply *AdjustInodeTableEntryOpenCountReplyStruct) (err error) { - return adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountReply) +func (dummy *RetryRPCServerStruct) AdjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct) (err error) { + return adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) } // FlushRequestStruct is the request object for Flush. @@ -314,14 +314,14 @@ type FlushRequestStruct struct { // TODO } -// FlushReplyStruct is the reply object for Flush. +// FlushResponseStruct is the response object for Flush. // -type FlushReplyStruct struct{} +type FlushResponseStruct struct{} // Flush that the results of prior PutInodeTableEntries requests be persisted. // -func (dummy *RetryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err error) { - return flush(flushRequest, flushReply) +func (dummy *RetryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) (err error) { + return flush(flushRequest, flushResponse) } // LeaseRequestType specifies the requested lease operation. @@ -344,31 +344,31 @@ type LeaseRequestStruct struct { LeaseRequestType // One of LeaseRequestType* } -// LeaseReplyType specifies the acknowledgement that the requested lease operation +// LeaseResponseType specifies the acknowledgement that the requested lease operation // has been completed or denied (e.g. when a Promotion request cannot be satisfied // and the client will soon be receiving a LeaseInterruptTypeRelease). // -type LeaseReplyType uint32 +type LeaseResponseType uint32 const ( - LeaseReplyTypeDenied LeaseReplyType = iota // Request denied (e.g. Promotion deadlock avoidance) - LeaseReplyTypeShared // SharedLease granted - LeaseReplyTypePromoted // SharedLease promoted to ExclusiveLease - LeaseReplyTypeExclusive // ExclusiveLease granted - LeaseReplyTypeDemoted // ExclusiveLease demoted to SharedLease - LeaseReplyTypeReleased // SharedLease or ExclusiveLease released + LeaseResponseTypeDenied LeaseResponseType = iota // Request denied (e.g. Promotion deadlock avoidance) + LeaseResponseTypeShared // SharedLease granted + LeaseResponseTypePromoted // SharedLease promoted to ExclusiveLease + LeaseResponseTypeExclusive // ExclusiveLease granted + LeaseResponseTypeDemoted // ExclusiveLease demoted to SharedLease + LeaseResponseTypeReleased // SharedLease or ExclusiveLease released ) -// LeaseReplyStruct is the reply object for Lease. +// LeaseResponseStruct is the response object for Lease. // -type LeaseReplyStruct struct { - LeaseReplyType // One of LeaseReplyType* +type LeaseResponseStruct struct { + LeaseResponseType // One of LeaseResponseType* } // Lease is a blocking Lease Request. // -func (dummy *RetryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err error) { - return lease(leaseRequest, leaseReply) +func (dummy *RetryRPCServerStruct) Lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) (err error) { + return lease(leaseRequest, leaseResponse) } // RPCInterruptType specifies the action (unmount, demotion, or release) requested by ProxyFS diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 233ca0f7d..dc01ccb73 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -66,7 +66,7 @@ func stopRetryRPCServer() (err error) { return nil } -func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply *MountReplyStruct) (err error) { +func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountResponse *MountResponseStruct) (err error) { var ( startTime time.Time ) @@ -80,7 +80,7 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountReply return nil // TODO } -func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *RenewMountReplyStruct) (err error) { +func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse *RenewMountResponseStruct) (err error) { var ( startTime time.Time ) @@ -94,7 +94,7 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountReply *Ren return nil // TODO } -func unmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStruct) (err error) { +func unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountResponseStruct) (err error) { var ( startTime time.Time ) @@ -108,7 +108,7 @@ func unmount(unmountRequest *UnmountRequestStruct, unmountReply *UnmountReplyStr return nil // TODO } -func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeReply *FetchNonceRangeReplyStruct) (err error) { +func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeResponse *FetchNonceRangeResponseStruct) (err error) { var ( startTime time.Time ) @@ -122,7 +122,7 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch return nil // TODO } -func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryReply *GetInodeTableEntryReplyStruct) (err error) { +func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryResponse *GetInodeTableEntryResponseStruct) (err error) { var ( startTime time.Time ) @@ -136,7 +136,7 @@ func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStru return nil // TODO } -func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesReply *PutInodeTableEntriesReplyStruct) (err error) { +func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesResponse *PutInodeTableEntriesResponseStruct) (err error) { var ( startTime time.Time ) @@ -150,7 +150,7 @@ func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesReque return nil // TODO } -func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryReply *DeleteInodeTableEntryReplyStruct) (err error) { +func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct) (err error) { var ( startTime time.Time ) @@ -164,7 +164,7 @@ func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRe return nil // TODO } -func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountReply *AdjustInodeTableEntryOpenCountReplyStruct) (err error) { +func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct) (err error) { var ( startTime time.Time ) @@ -178,7 +178,7 @@ func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *Adjus return nil // TODO } -func flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err error) { +func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) (err error) { var ( startTime time.Time ) @@ -192,7 +192,7 @@ func flush(flushRequest *FlushRequestStruct, flushReply *FlushReplyStruct) (err return nil // TODO } -func lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err error) { +func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) (err error) { var ( startTime time.Time ) @@ -225,7 +225,7 @@ func lease(leaseRequest *LeaseRequestStruct, leaseReply *LeaseReplyStruct) (err globals.stats.ReleaseLeaseRequestUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() default: - leaseReply.LeaseReplyType = LeaseReplyTypeDenied + leaseResponse.LeaseResponseType = LeaseResponseTypeDenied err = fmt.Errorf("LeaseRequestType %v not supported", leaseRequest.LeaseRequestType) err = blunder.AddError(err, blunder.BadLeaseRequest) return From 5a0610b5f36402f5de98aa073d9da2cf9a3f894f Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 16 Apr 2021 20:03:20 -0700 Subject: [PATCH 148/160] Re-worked ilayout to append an ObjType|Version|Length tuple to SuperBlock & InodeHead instead of just Version --- ilayout/api.go | 66 ++++++++----- ilayout/api_test.go | 44 +++++---- ilayout/impl.go | 225 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 246 insertions(+), 89 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index d4ceaa111..5003c8e65 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -110,22 +110,50 @@ func UnmarshalCheckPointHeaderV1(checkPointHeaderV1String string) (checkPointHea return } -// SuperBlockVersionV* specifies the format of all preceeding bytes -// referenced by CheckPointHeaderV1Struct.SuperBlock{ObjectNumber|Length}. +// ObjectTrailerStruct specifies the layout of a trailer found in each Object +// that identifies the objType, version, and size of a structure immediately +// proceeding it. // -// The value is stored in LittleEndian format. +// The struct is serialized as a sequence of LittleEndian formatted fields. // -const ( - SuperBlockVersionV1 uint64 = 1 -) +type ObjectTrailerStruct struct { + ObjType uint16 + Version uint16 + Length uint32 +} + +// MarshalObjectTrailer encodes objectTrailer to objectTrailerBuf. +// +func (objectTrailer *ObjectTrailerStruct) MarshalObjectTrailer() (objectTrailerBuf []byte, err error) { + objectTrailerBuf, err = objectTrailer.marshalObjectTrailer() + return +} -// UnmarshalSuperBlockVersion extracts superBlockVersion from superBlockBuf. +// UnmarshalObjectTrailer decodes objectTrailer from objectTrailerBuf. // -func UnmarshalSuperBlockVersion(superBlockBuf []byte) (superBlockVersion uint64, err error) { - superBlockVersion, err = unmarshalSuperBlockVersion(superBlockBuf) +// Note that the last 8 bytes of objectTrailerBuf are decoded. The entire +// objectTrailerBuf is expected to contain precisely objectTrailer.Length +// bytes before the ObjectTrailerStruct. +// +func UnmarshalObjectTrailer(objectTrailerBuf []byte) (objectTrailer *ObjectTrailerStruct, err error) { + objectTrailer, err = unmarshalObjectTrailer(objectTrailerBuf) return } +// SuperBlockType specifies that this ObjectTrailerStruct refers to +// a SuperBlockV*Struct immediately preceeding it. +// +const ( + SuperBlockType uint16 = 0x5342 // 'S' 'B' +) + +// SuperBlockVersionV* specifies, for an ObjectTrailerStruct of Type SuperBlockType, +// the Version of the SuperBlockV*Struct immediately preceeding the ObjectTrailerStruct. +// +const ( + SuperBlockVersionV1 uint16 = 1 +) + // InodeTableLayoutEntryV1Struct specifies the layout of the InodeTable B+Tree in Objects. // Since any modification of the Volume will result in a fresh SuperBlockV1Struct // being written to a new Object, when BytesReferenced drops to zero, the Object @@ -142,8 +170,6 @@ type InodeTableLayoutEntryV1Struct struct { // SuperBlockStruct specifies the format of the SuperBlock found at // the CheckPointHeaderV1Struct.SuperBlockLength trailing bytes of // the Object indicated by CheckPointHeaderV1Struct.SuperBlockObjectNumber. -// Note that the ...Length includes the SuperBlockVersionV1 value appended -// to it. // // The InodeTable is a B+Tree where the Key is the uint64 InodeNumber. The Value // is a InodeTableEntryValueV1Struct. @@ -222,21 +248,19 @@ func UnmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTab return } -// InodeHeadVersionV* specifies the format of all preceeding bytes referenced -// by InodeTableEntryValueV1Struct.InodeHead{ObjectNumber|Length}. -// -// The value is stored in LittleEndian format. +// InodeHeadType specifies that this ObjectTrailerStruct refers to +// a InodeHeadV*Struct immediately preceeding it. // const ( - InodeHeadVersionV1 uint64 = 1 + InodeHeadType uint16 = 0x4948 // 'I' 'H' ) -// UnmarshalInodeHeadVersion extracts inodeHeadVersion from inodeHeadBuf. +// InodeHeadVersionV* specifies, for an ObjectTrailerStruct of Type InodeHeadType, +// the Version of InodeHeadV*Struct immediately preceeding the ObjectTrailerStruct. // -func UnmarshalInodeHeadVersion(inodeHeadBuf []byte) (inodeHeadVersion uint64, err error) { - inodeHeadVersion, err = unmarshalInodeHeadVersion(inodeHeadBuf) - return -} +const ( + InodeHeadVersionV1 uint16 = 1 +) // RootDirInodeNumber is the InodeNumber for the directory at the root of the file system. // diff --git a/ilayout/api_test.go b/ilayout/api_test.go index efd61e520..4ae507a16 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -24,6 +24,15 @@ func TestAPI(t *testing.T) { unmarshaledCheckPointHeaderVersion uint64 unmarshaledCheckPointHeaderV1 *CheckPointHeaderV1Struct + testObjectTrailer = &ObjectTrailerStruct{ + ObjType: 1, + Version: 2, + Length: 0, // Appropriate if marshaledObjectTrailer is used to unmarshal (with no prepended payload) + } + + marshaledObjectTrailer []byte + unmarshaledObjectTrailer *ObjectTrailerStruct + testSuperBlockV1 = &SuperBlockV1Struct{ InodeTableRootObjectNumber: 1, InodeTableRootObjectOffset: 2, @@ -45,9 +54,8 @@ func TestAPI(t *testing.T) { InodeBytesReferenced: 6, } - marshaledSuperBlockV1 []byte - unmarshaledSuperBlockVersion uint64 - unmarshaledSuperBlockV1 *SuperBlockV1Struct + marshaledSuperBlockV1 []byte + unmarshaledSuperBlockV1 *SuperBlockV1Struct inodeTableLayoutIndex int @@ -111,9 +119,8 @@ func TestAPI(t *testing.T) { }, } - marshaledInodeHeadV1 []byte - unmarshaledInodeHeadVersion uint64 - unmarshaledInodeHeadV1 *InodeHeadV1Struct + marshaledInodeHeadV1 []byte + unmarshaledInodeHeadV1 *InodeHeadV1Struct linkTableIndex int streamTableIndex int @@ -159,17 +166,24 @@ func TestAPI(t *testing.T) { t.Fatalf("Bad unmarshaledCheckPointHeaderV1 (%+v) - expected testCheckPointHeaderV1 (%+v)", unmarshaledCheckPointHeaderV1, testCheckPointHeaderV1) } - marshaledSuperBlockV1, err = testSuperBlockV1.MarshalSuperBlockV1() + marshaledObjectTrailer, err = testObjectTrailer.marshalObjectTrailer() if nil != err { t.Fatal(err) } - unmarshaledSuperBlockVersion, err = UnmarshalSuperBlockVersion(marshaledSuperBlockV1) + unmarshaledObjectTrailer, err = UnmarshalObjectTrailer(marshaledObjectTrailer) if nil != err { t.Fatal(err) } - if SuperBlockVersionV1 != unmarshaledSuperBlockVersion { - t.Fatalf("Bad unmarshaledSuperBlockVersion (%016X) - expected SuperBlockVersionV1 (%016X)", unmarshaledSuperBlockVersion, SuperBlockVersionV1) + if (testObjectTrailer.ObjType != unmarshaledObjectTrailer.ObjType) || + (testObjectTrailer.Version != unmarshaledObjectTrailer.Version) || + (testObjectTrailer.Length != unmarshaledObjectTrailer.Length) { + t.Fatalf("Bad unmarshaledObjectTrailer (%+v) - expected testObjectTrailer (%+v)", unmarshaledObjectTrailer, testObjectTrailer) + } + + marshaledSuperBlockV1, err = testSuperBlockV1.MarshalSuperBlockV1() + if nil != err { + t.Fatal(err) } unmarshaledSuperBlockV1, err = UnmarshalSuperBlockV1(marshaledSuperBlockV1) @@ -198,7 +212,7 @@ func TestAPI(t *testing.T) { t.Fatal(err) } if InodeTableEntryValueVersionV1 != unmarshaledInodeTableEntryValueVersion { - t.Fatalf("Bad unmarshaledSuperBlockVersion (%016X) - expected SuperBlockVersionV1 (%016X)", unmarshaledSuperBlockVersion, SuperBlockVersionV1) + t.Fatalf("Bad unmarshaledInodeTableEntryValueVersion (%016X) - expected InodeTableEntryValueVersionV1 (%016X)", unmarshaledInodeTableEntryValueVersion, InodeTableEntryValueVersionV1) } unmarshaledInodeTableEntryValueV1, err = UnmarshalInodeTableEntryValueV1(marshaledInodeTableEntryValueV1) @@ -215,14 +229,6 @@ func TestAPI(t *testing.T) { t.Fatal(err) } - unmarshaledInodeHeadVersion, err = UnmarshalInodeHeadVersion(marshaledInodeHeadV1) - if nil != err { - t.Fatal(err) - } - if InodeHeadVersionV1 != unmarshaledInodeHeadVersion { - t.Fatalf("Bad unmarshaledInodeHeadVersion (%016X) - expected InodeHeadVersionV1 (%016X)", unmarshaledInodeHeadVersion, InodeHeadVersionV1) - } - unmarshaledInodeHeadV1, err = UnmarshalInodeHeadV1(marshaledInodeHeadV1) if nil != err { t.Fatal(err) diff --git a/ilayout/impl.go b/ilayout/impl.go index 70d7184b9..80c219db7 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -5,6 +5,7 @@ package ilayout import ( "fmt" + "math" "time" ) @@ -32,20 +33,80 @@ func unmarshalCheckPointHeaderV1(checkPointHeaderV1String string) (checkPointHea return } -func unmarshalSuperBlockVersion(superBlockBuf []byte) (superBlockVersion uint64, err error) { +func (objectTrailer *ObjectTrailerStruct) marshalObjectTrailer() (objectTrailerBuf []byte, err error) { var ( - superBlockBufLen int + curPos int ) - superBlockBufLen = len(superBlockBuf) + objectTrailerBuf = make([]byte, 2+2+4) + + curPos = 0 - if superBlockBufLen < 8 { - err = fmt.Errorf("Insufficient space in superBlockBuf for superBlockVersion") + curPos, err = putLEUint16ToBuf(objectTrailerBuf, curPos, objectTrailer.ObjType) + if nil != err { return } - superBlockVersion, _, err = getLEUint64FromBuf(superBlockBuf, superBlockBufLen-8) + curPos, err = putLEUint16ToBuf(objectTrailerBuf, curPos, objectTrailer.Version) + if nil != err { + return + } + curPos, err = putLEUint32ToBuf(objectTrailerBuf, curPos, objectTrailer.Length) + if nil != err { + return + } + + err = nil + return +} + +func unmarshalObjectTrailer(objectTrailerBuf []byte) (objectTrailer *ObjectTrailerStruct, err error) { + var ( + curPos int + expectedLength uint32 + ) + + curPos = len(objectTrailerBuf) - (2 + 2 + 4) + if curPos < 0 { + err = fmt.Errorf("No room for ObjectTrailerStruct at end of objectTrailerBuf") + return + } + if curPos > math.MaxUint32 { + err = fmt.Errorf("Cannot parse an objectTrailerBuf with > math.MaxUint32 (0x%8X) payload preceeding ObjectTrailerStruct", math.MaxUint32) + return + } + + expectedLength = uint32(curPos) + + objectTrailer = &ObjectTrailerStruct{} + + objectTrailer.ObjType, curPos, err = getLEUint16FromBuf(objectTrailerBuf, curPos) + if nil != err { + return + } + + objectTrailer.Version, curPos, err = getLEUint16FromBuf(objectTrailerBuf, curPos) + if nil != err { + return + } + + objectTrailer.Length, curPos, err = getLEUint32FromBuf(objectTrailerBuf, curPos) + if nil != err { + return + } + + if curPos != len(objectTrailerBuf) { + err = fmt.Errorf("Logic error extracting ObjectTrailerStruct from end of objectTrailerBuf") + return + } + + if objectTrailer.Length != expectedLength { + err = fmt.Errorf("Payload of objectTrailerBuf preceeding ObjectTrailerStruct length mismatch") + return + } + + err = nil return } @@ -53,9 +114,11 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ var ( curPos int inodeTableLayoutIndex int + objectTrailer *ObjectTrailerStruct + objectTrailerBuf []byte ) - superBlockV1Buf = make([]byte, 8+8+8+8+(len(superBlockV1.InodeTableLayout)*(8+8+8))+8+8+8+8) + superBlockV1Buf = make([]byte, 8+8+8+8+(len(superBlockV1.InodeTableLayout)*(8+8+8))+8+8+8+(2+2+4)) curPos = 0 @@ -111,7 +174,23 @@ func (superBlockV1 *SuperBlockV1Struct) marshalSuperBlockV1() (superBlockV1Buf [ return } - curPos, err = putLEUint64ToBuf(superBlockV1Buf, curPos, SuperBlockVersionV1) + if curPos > math.MaxUint32 { + err = fmt.Errorf("Cannot marshal an superBlockV1Buf with > math.MaxUint32 (0x%8X) payload preceeding ObjectTrailerStruct", math.MaxUint32) + return + } + + objectTrailer = &ObjectTrailerStruct{ + ObjType: SuperBlockType, + Version: SuperBlockVersionV1, + Length: uint32(curPos), + } + + objectTrailerBuf, err = objectTrailer.MarshalObjectTrailer() + if nil != err { + return + } + + curPos, err = putFixedByteSliceToBuf(superBlockV1Buf, curPos, objectTrailerBuf) if nil != err { return } @@ -125,9 +204,22 @@ func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St curPos int inodeTableLayoutIndex uint64 inodeTableLayoutLen uint64 - superBlockVersion uint64 + objectTrailer *ObjectTrailerStruct ) + objectTrailer, err = unmarshalObjectTrailer(superBlockV1Buf) + if nil != err { + return + } + if objectTrailer.ObjType != SuperBlockType { + err = fmt.Errorf("superBlockV1Buf does not contain a SuperBlockV1Struct - wrong ObjType") + return + } + if objectTrailer.Version != SuperBlockVersionV1 { + err = fmt.Errorf("superBlockV1Buf does not contain a SuperBlockV1Struct - wrong Version") + return + } + superBlockV1 = &SuperBlockV1Struct{} curPos = 0 @@ -186,16 +278,7 @@ func unmarshalSuperBlockV1(superBlockV1Buf []byte) (superBlockV1 *SuperBlockV1St return } - superBlockVersion, curPos, err = getLEUint64FromBuf(superBlockV1Buf, curPos) - if nil != err { - return - } - if superBlockVersion != SuperBlockVersionV1 { - err = fmt.Errorf("Incorrect Version for superBlockV1Buf") - return - } - - if curPos != len(superBlockV1Buf) { + if curPos != int(objectTrailer.Length) { err = fmt.Errorf("Incorrect size for superBlockV1Buf") return } @@ -276,29 +359,14 @@ func unmarshalInodeTableEntryValueV1(inodeTableEntryValueV1Buf []byte) (inodeTab return } -func unmarshalInodeHeadVersion(inodeHeadBuf []byte) (inodeHeadVersion uint64, err error) { - var ( - inodeHeadBufLen int - ) - - inodeHeadBufLen = len(inodeHeadBuf) - - if inodeHeadBufLen < 8 { - err = fmt.Errorf("Insufficient space in inodeHeadBuf for inodeHeadVersion") - return - } - - inodeHeadVersion, _, err = getLEUint64FromBuf(inodeHeadBuf, inodeHeadBufLen-8) - - return -} - func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byte, err error) { var ( curPos int inodeHeadV1BufLen int layoutIndex int linkTableIndex int + objectTrailer *ObjectTrailerStruct + objectTrailerBuf []byte streamTableIndex int ) @@ -324,7 +392,7 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt inodeHeadV1BufLen += 8 + (len(inodeHeadV1.Layout) * (8 + 8 + 8)) - inodeHeadV1BufLen += 8 + inodeHeadV1BufLen += (2 + 2 + 4) inodeHeadV1Buf = make([]byte, inodeHeadV1BufLen) @@ -456,7 +524,23 @@ func (inodeHeadV1 *InodeHeadV1Struct) marshalInodeHeadV1() (inodeHeadV1Buf []byt } } - curPos, err = putLEUint64ToBuf(inodeHeadV1Buf, curPos, InodeHeadVersionV1) + if curPos > math.MaxUint32 { + err = fmt.Errorf("Cannot marshal an inodeHeadV1Buf with > math.MaxUint32 (0x%8X) payload preceeding ObjectTrailerStruct", math.MaxUint32) + return + } + + objectTrailer = &ObjectTrailerStruct{ + ObjType: InodeHeadType, + Version: InodeHeadVersionV1, + Length: uint32(curPos), + } + + objectTrailerBuf, err = objectTrailer.MarshalObjectTrailer() + if nil != err { + return + } + + curPos, err = putFixedByteSliceToBuf(inodeHeadV1Buf, curPos, objectTrailerBuf) if nil != err { return } @@ -471,16 +555,29 @@ func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct attrChangeTimeAsUnixTimeInNs uint64 creationTimeAsUnixTimeInNs uint64 curPos int - inodeHeadVersion uint64 layoutIndex uint64 layoutLen uint64 linkTableIndex uint64 linkTableLen uint64 modificationTimeAsUnixTimeInNs uint64 + objectTrailer *ObjectTrailerStruct streamTableIndex uint64 streamTableLen uint64 ) + objectTrailer, err = unmarshalObjectTrailer(inodeHeadV1Buf) + if nil != err { + return + } + if objectTrailer.ObjType != InodeHeadType { + err = fmt.Errorf("inodeHeadV1Buf does not contain a InodeHeadV1Struct - wrong ObjType") + return + } + if objectTrailer.Version != InodeHeadVersionV1 { + err = fmt.Errorf("inodeHeadV1Buf does not contain a InodeHeadV1Struct - wrong Version") + return + } + inodeHeadV1 = &InodeHeadV1Struct{} curPos = 0 @@ -625,16 +722,7 @@ func unmarshalInodeHeadV1(inodeHeadV1Buf []byte) (inodeHeadV1 *InodeHeadV1Struct } } - inodeHeadVersion, curPos, err = getLEUint64FromBuf(inodeHeadV1Buf, curPos) - if nil != err { - return - } - if inodeHeadVersion != InodeHeadVersionV1 { - err = fmt.Errorf("Incorrect Version for inodeHeadV1Buf") - return - } - - if curPos != len(inodeHeadV1Buf) { + if curPos != int(objectTrailer.Length) { err = fmt.Errorf("Incorrect size for inodeHeadV1Buf") return } @@ -971,3 +1059,42 @@ func putLEByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err = nil return } + +func getFixedByteSliceFromBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { + var ( + byteSliceLen int + ) + + byteSliceLen = len(byteSlice) + + if byteSliceLen < (len(buf) - curPos) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for []byte of len(byteSlice) length") + return + } + + copy(byteSlice, buf[nextPos:nextPos+byteSliceLen]) + nextPos = curPos + byteSliceLen + + err = nil + return +} + +func putFixedByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { + var ( + byteSliceLen int + ) + + byteSliceLen = len(byteSlice) + + nextPos = curPos + byteSliceLen + + if nextPos > len(buf) { + err = fmt.Errorf("Insufficient space in buf[curPos:] for []byte") + return + } + + copy(buf[curPos:nextPos], byteSlice) + + err = nil + return +} From 15405567fbf35947ff85c42a3741565ae1d0475c Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 16 Apr 2021 20:11:36 -0700 Subject: [PATCH 149/160] Fixed missing public wrappers around Fixed ByteSlice Get|Put marshalers in ilayout --- ilayout/api.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/ilayout/api.go b/ilayout/api.go index 5003c8e65..f916c8284 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -538,7 +538,7 @@ func GetLEByteSliceFromBuf(buf []byte, curPos int) (byteSlice []byte, nextPos in // PutLEByteSliceToBuf writes a []byte to buf starting at curPos. // -// The []byte is written aas a LittleEndian byte order uint64 length followed by the +// The []byte is written as a LittleEndian byte order uint64 length followed by the // bytes that make up the []byte. The returned nextPost indicates where the next field // (if any) should be written. // @@ -546,3 +546,22 @@ func PutLEByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, nextPos, err = putLEByteSliceToBuf(buf, curPos, byteSlice) return } + +// GetFixedByteSliceFromBuf fetches a []byte from buf starting at curPos. +// +// The []byte is assumed to have been written with the same length as byteSlice. +// The returned nextPos indicates where the next field (if any) should be read from. +// +func GetFixedByteSliceFromBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { + nextPos, err = getFixedByteSliceFromBuf(buf, curPos, byteSlice) + return +} + +// PutFixedByteSliceToBuf writes a []byte to buf starting at curPos. +// +// The returned nextPost indicates where the next field (if any) should be written. +// +func PutFixedByteSliceToBuf(buf []byte, curPos int, byteSlice []byte) (nextPos int, err error) { + nextPos, err = putFixedByteSliceToBuf(buf, curPos, byteSlice) + return +} From 51ae9d62762eb34ea82632f54a659661eaab3b33 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 16 Apr 2021 20:21:03 -0700 Subject: [PATCH 150/160] Fixed up a few comments in ilayout --- ilayout/api.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index f916c8284..ce72b2b2b 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -180,7 +180,8 @@ type InodeTableLayoutEntryV1Struct struct { // serialization of each one. // // Note that the CheckPointHeaderV1Struct.SuperBlockLength also includes the bytes for -// holding SuperBlockVersionV1 that is appended. +// holding the ObjectTrailerStruct{ObjType: SuperBlockType, Version: SuperBlockVersionV1} +// that is appended. // type SuperBlockV1Struct struct { InodeTableRootObjectNumber uint64 // Identifies the Object containing the root of the InodeTable @@ -326,14 +327,15 @@ type InodeHeadLayoutEntryV1Struct struct { // InodeHeadV1Struct specifies the layout of an Inode. // -// The struct is serializes as a sequence of fields. +// The struct is serializes as a sequence of fields: // For uint* fields, LittleEndian format is used. // For table fields, a uint64 length in LittleEndian format is followed by the serialization // specified in the table entry struct. // For time.Time fields, a uint64 in LittleEndian is used to hold the UnixNano() equivalent. // // Note that the InodeTableEntryValueV1Struct.InodeHeadLength also includes the bytes for -// holding InodeHeadVersionV1 that is appended. +// holding the ObjectTrailerStruct{ObjType: InodeHeadType, Version: InodeHeadVersionV1} +// that is appended. // type InodeHeadV1Struct struct { InodeNumber uint64 From a9d74059e1725edcb688f233ebcce2c18a458c13 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 21 Apr 2021 16:28:04 -0700 Subject: [PATCH 151/160] First test case implemented in imgr/imgrpkg/retry-rpc_test.go Provides the template for the rest of the steps... --- imgr/imgrpkg/http-server_test.go | 2 +- imgr/imgrpkg/retry-rpc.go | 35 ++++++++++----- imgr/imgrpkg/retry-rpc_test.go | 44 +++++++++++++++++- imgr/imgrpkg/utils_test.go | 76 ++++++++++++++++++++++---------- 4 files changed, 120 insertions(+), 37 deletions(-) diff --git a/imgr/imgrpkg/http-server_test.go b/imgr/imgrpkg/http-server_test.go index fb3f98202..709732985 100644 --- a/imgr/imgrpkg/http-server_test.go +++ b/imgr/imgrpkg/http-server_test.go @@ -23,7 +23,7 @@ func TestHTTPServer(t *testing.T) { responseBodyExpected string ) - testSetup(t) + testSetup(t, nil) _, _, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/config", nil, nil) if nil != err { diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index dc01ccb73..fa3255905 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -77,7 +77,7 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo globals.stats.MountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return nil // TODO + return fmt.Errorf("TODO") } func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse *RenewMountResponseStruct) (err error) { @@ -91,7 +91,7 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * globals.stats.RenewMountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return nil // TODO + return fmt.Errorf("TODO") } func unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountResponseStruct) (err error) { @@ -105,11 +105,13 @@ func unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountRespo globals.stats.UnmountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return nil // TODO + return fmt.Errorf("TODO") } func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeResponse *FetchNonceRangeResponseStruct) (err error) { var ( + mount *mountStruct + ok bool startTime time.Time ) @@ -119,7 +121,20 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch globals.stats.FetchNonceRangeUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return nil // TODO + globals.RLock() + + mount, ok = globals.mountMap[fetchNonceRangeRequest.MountID] + if !ok { + globals.RUnlock() + err = fmt.Errorf("MountID not recognized") + return + } + + fmt.Printf("TODO: Perform fetchNonceRange() for mountStruct @ %p\n", mount) + + globals.RUnlock() + + return fmt.Errorf("TODO") } func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryResponse *GetInodeTableEntryResponseStruct) (err error) { @@ -133,7 +148,7 @@ func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStru globals.stats.GetInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return nil // TODO + return fmt.Errorf("TODO") } func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesResponse *PutInodeTableEntriesResponseStruct) (err error) { @@ -147,7 +162,7 @@ func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesReque globals.stats.PutInodeTableEntriesUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return nil // TODO + return fmt.Errorf("TODO") } func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct) (err error) { @@ -161,7 +176,7 @@ func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRe globals.stats.DeleteInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return nil // TODO + return fmt.Errorf("TODO") } func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct) (err error) { @@ -175,7 +190,7 @@ func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *Adjus globals.stats.AdjustInodeTableEntryOpenCountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return nil // TODO + return fmt.Errorf("TODO") } func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) (err error) { @@ -189,7 +204,7 @@ func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) globals.stats.FlushUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return nil // TODO + return fmt.Errorf("TODO") } func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) (err error) { @@ -231,5 +246,5 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) return } - return nil // TODO + return fmt.Errorf("TODO") } diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index 75669796e..1743822fc 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -5,12 +5,50 @@ package imgrpkg import ( "testing" + + "github.com/NVIDIA/proxyfs/retryrpc" ) +type testRetryRPCClientCallbacksStruct struct { + interruptPayloadChan chan []byte +} + +func (retryrpcClientCallbacks *testRetryRPCClientCallbacksStruct) Interrupt(payload []byte) { + retryrpcClientCallbacks.interruptPayloadChan <- payload +} + func TestRetryRPC(t *testing.T) { - testSetup(t) + var ( + err error + fetchNonceRangeRequest *FetchNonceRangeRequestStruct + fetchNonceRangeResponse *FetchNonceRangeResponseStruct + retryrpcClient *retryrpc.Client + retryrpcClientCallbacks *testRetryRPCClientCallbacksStruct + ) + + retryrpcClientCallbacks = &testRetryRPCClientCallbacksStruct{ + interruptPayloadChan: make(chan []byte), + } + + testSetup(t, retryrpcClientCallbacks) + + retryrpcClient, err = retryrpc.NewClient(testGlobals.retryrpcClientConfig) + if nil != err { + t.Fatalf("retryrpc.NewClient() failed: %v", err) + } + + // Attempt a FetchNonceRange() without a prior Mount()... which should fail (no Mount) + + fetchNonceRangeRequest = &FetchNonceRangeRequestStruct{ + MountID: "", // An invalid MountID... so will not be found in globals.mountMap + } + fetchNonceRangeResponse = &FetchNonceRangeResponseStruct{} + + err = retryrpcClient.Send("FetchNonceRange", fetchNonceRangeRequest, fetchNonceRangeResponse) + if nil == err { + t.Fatalf("retryrpcClient.Send(\"FetchNonceRange\",,) should have failed") + } - // TODO: Attempt a FetchNonceRange() without a prior Mount()... which should fail (no Mount) // TODO: Perform a Mount() // TODO: Attempt a GetInodeTableEntry() for RootDirInode... which should fail (no Lease) // TODO: Force a need for a re-auth @@ -41,5 +79,7 @@ func TestRetryRPC(t *testing.T) { // TODO: Perform a Unmount()... without first releasing Exclusive Lease on RootDirInode // TODO: Verify that Exclusive Lease on RootDirInode is implicitly released + retryrpcClient.Close() + testTeardown(t) } diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index ac141f142..b27a39b38 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -17,36 +17,44 @@ import ( "github.com/NVIDIA/proxyfs/conf" "github.com/NVIDIA/proxyfs/icert/icertpkg" "github.com/NVIDIA/proxyfs/iswift/iswiftpkg" + "github.com/NVIDIA/proxyfs/retryrpc" ) const ( - testIPAddr = "127.0.0.1" - testRetryRPCPort = 32357 - testHTTPServerPort = 15347 - testSwiftProxyTCPPort = 24367 - testSwiftAuthUser = "test" - testSwiftAuthKey = "test" - testContainer = "testContainer" - testVolume = "testVolume" + testIPAddr = "127.0.0.1" + testRetryRPCPort = 32357 + testHTTPServerPort = 15347 + testSwiftProxyTCPPort = 24367 + testSwiftAuthUser = "test" + testSwiftAuthKey = "test" + testContainer = "testContainer" + testVolume = "testVolume" + testRPCDeadlineIO = "60s" + testRPCKeepAlivePeriod = "60s" ) type testGlobalsStruct struct { - tempDir string - caCertFile string - caKeyFile string - endpointCertFile string - endpointKeyFile string - confMap conf.ConfMap - httpServerURL string - authURL string - authToken string - accountURL string - containerURL string + tempDir string + caCertFile string + caKeyFile string + caCertPEMBlock []byte + caKeyPEMBlock []byte + endpointCertFile string + endpointKeyFile string + endpointCertPEMBlock []byte + endpointKeyPEMBlock []byte + confMap conf.ConfMap + httpServerURL string + authURL string + authToken string + accountURL string + containerURL string + retryrpcClientConfig *retryrpc.ClientConfig } var testGlobals *testGlobalsStruct -func testSetup(t *testing.T) { +func testSetup(t *testing.T, retryrpcCallbacks interface{}) { var ( authRequestHeaders http.Header authResponseHeaders http.Header @@ -54,6 +62,8 @@ func testSetup(t *testing.T) { err error putAccountRequestHeaders http.Header putContainerRequestHeaders http.Header + retryrpcDeadlineIO time.Duration + retryrpcKeepAlivePeriod time.Duration tempDir string ) @@ -72,7 +82,7 @@ func testSetup(t *testing.T) { authURL: fmt.Sprintf("http://%s:%d/auth/v1.0", testIPAddr, testSwiftProxyTCPPort), } - _, _, err = icertpkg.GenCACert( + testGlobals.caCertPEMBlock, testGlobals.caKeyPEMBlock, err = icertpkg.GenCACert( icertpkg.GenerateKeyAlgorithmEd25519, pkix.Name{ Organization: []string{"Test Organization CA"}, @@ -89,7 +99,7 @@ func testSetup(t *testing.T) { t.Fatalf("icertpkg.GenCACert() failed: %v", err) } - _, _, err = icertpkg.GenEndpointCert( + testGlobals.endpointCertPEMBlock, testGlobals.endpointKeyPEMBlock, err = icertpkg.GenEndpointCert( icertpkg.GenerateKeyAlgorithmEd25519, pkix.Name{ Organization: []string{"Test Organization Endpoint"}, @@ -102,8 +112,8 @@ func testSetup(t *testing.T) { []string{}, []net.IP{net.ParseIP(testIPAddr)}, time.Hour, - testGlobals.caCertFile, - testGlobals.caKeyFile, + testGlobals.caCertPEMBlock, + testGlobals.caKeyPEMBlock, testGlobals.endpointCertFile, testGlobals.endpointKeyFile) if nil != err { @@ -204,6 +214,24 @@ func testSetup(t *testing.T) { if nil != err { t.Fatalf("Start(testGlobals.confMap) failed: %v", err) } + + retryrpcDeadlineIO, err = time.ParseDuration(testRPCDeadlineIO) + if nil != err { + t.Fatalf("time.ParseDuration(\"%s\") failed: %v", testRPCDeadlineIO, err) + } + retryrpcKeepAlivePeriod, err = time.ParseDuration(testRPCKeepAlivePeriod) + if nil != err { + t.Fatalf("time.ParseDuration(\"%s\") failed: %v", retryrpcKeepAlivePeriod, err) + } + + testGlobals.retryrpcClientConfig = &retryrpc.ClientConfig{ + IPAddr: testIPAddr, + Port: testRetryRPCPort, + RootCAx509CertificatePEM: testGlobals.caCertPEMBlock, + Callbacks: retryrpcCallbacks, + DeadlineIO: retryrpcDeadlineIO, + KeepAlivePeriod: retryrpcKeepAlivePeriod, + } } func testTeardown(t *testing.T) { From b80e11085b37a107920ebcaf5e1956f4e1f29cb4 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 23 Apr 2021 15:59:47 -0700 Subject: [PATCH 152/160] Flushed out nearly all of the test steps in imgr/imgrpkg/retry-rpc_test.go Basically what's missing: PutInodeTableEntries() calls & verification steps For now, almost all steps are skipped until their underlying functionality is complete --- imgr/imgrpkg/api.go | 1 - imgr/imgrpkg/retry-rpc_test.go | 336 ++++++++++++++++++++++++++++++--- 2 files changed, 313 insertions(+), 24 deletions(-) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 8a2e6205e..0f4719401 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -311,7 +311,6 @@ func (dummy *RetryRPCServerStruct) AdjustInodeTableEntryOpenCount(adjustInodeTab // type FlushRequestStruct struct { MountID string - // TODO } // FlushResponseStruct is the response object for Flush. diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index 1743822fc..368802f10 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -19,11 +19,31 @@ func (retryrpcClientCallbacks *testRetryRPCClientCallbacksStruct) Interrupt(payl func TestRetryRPC(t *testing.T) { var ( - err error - fetchNonceRangeRequest *FetchNonceRangeRequestStruct - fetchNonceRangeResponse *FetchNonceRangeResponseStruct - retryrpcClient *retryrpc.Client - retryrpcClientCallbacks *testRetryRPCClientCallbacksStruct + adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct + adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct + deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct + deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct + err error + fetchNonceRangeRequest *FetchNonceRangeRequestStruct + fetchNonceRangeResponse *FetchNonceRangeResponseStruct + fileInodeNumber uint64 + fileInodeObjectA uint64 + fileInodeObjectB uint64 + fileInodeObjectC uint64 + flushRequest *FlushRequestStruct + flushResponse *FlushResponseStruct + getInodeTableEntryRequest *GetInodeTableEntryRequestStruct + getInodeTableEntryResponse *GetInodeTableEntryResponseStruct + leaseRequest *LeaseRequestStruct + leaseResponse *LeaseResponseStruct + mountRequest *MountRequestStruct + mountResponse *MountResponseStruct + renewMountRequest *RenewMountRequestStruct + renewMountResponse *RenewMountResponseStruct + retryrpcClient *retryrpc.Client + retryrpcClientCallbacks *testRetryRPCClientCallbacksStruct + unmountRequest *UnmountRequestStruct + unmountResponse *UnmountResponseStruct ) retryrpcClientCallbacks = &testRetryRPCClientCallbacksStruct{ @@ -46,37 +66,307 @@ func TestRetryRPC(t *testing.T) { err = retryrpcClient.Send("FetchNonceRange", fetchNonceRangeRequest, fetchNonceRangeResponse) if nil == err { - t.Fatalf("retryrpcClient.Send(\"FetchNonceRange\",,) should have failed") + t.Fatalf("retryrpcClient.Send(\"FetchNonceRange()\",,) should have failed") + } + + // TODO: Remove this early exit skipping of following TODOs + + if nil != err { + t.Logf("Exiting TestRetryRPC() early to skip following TODOs") + return + } + + // Perform a Mount() + + mountRequest = &MountRequestStruct{ + VolumeName: testVolume, + AuthToken: testGlobals.authToken, + } + mountResponse = &MountResponseStruct{} + + err = retryrpcClient.Send("Mount", mountRequest, mountResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"Mount(,)\",,) failed: %v", err) + } + + // Attempt a GetInodeTableEntry() for RootDirInode... which should fail (no Lease) + + getInodeTableEntryRequest = &GetInodeTableEntryRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: 1, + } + getInodeTableEntryResponse = &GetInodeTableEntryResponseStruct{} + + err = retryrpcClient.Send("GetInodeTableEntry", getInodeTableEntryRequest, getInodeTableEntryResponse) + if nil == err { + t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,1)\",,) should have failed") } - // TODO: Perform a Mount() - // TODO: Attempt a GetInodeTableEntry() for RootDirInode... which should fail (no Lease) // TODO: Force a need for a re-auth - // TODO: Attempt a GetInodeTableEntry() for RootDirInode... which should fail (re-auth required) - // TODO: Perform a RenewMount() - // TODO: Fetch a Shared Lease on RootDirInode - // TODO: Perform a GetInodeTableEntry() for RootDirInode + + // Attempt a GetInodeTableEntry() for RootDirInode... which should fail (re-auth required) + + getInodeTableEntryRequest = &GetInodeTableEntryRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: 1, + } + getInodeTableEntryResponse = &GetInodeTableEntryResponseStruct{} + + err = retryrpcClient.Send("GetInodeTableEntry", getInodeTableEntryRequest, getInodeTableEntryResponse) + if nil == err { + t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,1)\",,) should have failed") + } + + // Perform a RenewMount() + + renewMountRequest = &RenewMountRequestStruct{ + MountID: mountResponse.MountID, + AuthToken: testGlobals.authToken, + } + renewMountResponse = &RenewMountResponseStruct{} + + err = retryrpcClient.Send("RenewMount", renewMountRequest, renewMountResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"RenewMount(,)\",,) failed: %v", err) + } + + // Fetch a Shared Lease on RootDirInode + + leaseRequest = &LeaseRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: 1, + LeaseRequestType: LeaseRequestTypeShared, + } + leaseResponse = &LeaseResponseStruct{} + + err = retryrpcClient.Send("Lease", leaseRequest, leaseResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"Lease(,1,LeaseRequestTypeShared)\",,) failed: %v", err) + } + + // Perform a GetInodeTableEntry() for RootDirInode + + getInodeTableEntryRequest = &GetInodeTableEntryRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: 1, + } + getInodeTableEntryResponse = &GetInodeTableEntryResponseStruct{} + + err = retryrpcClient.Send("GetInodeTableEntry", getInodeTableEntryRequest, getInodeTableEntryResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"GetInodeTableEntry(,1)\",,) failed: %v", err) + } + // TODO: Attempt a PutInodeTableEntries() for RootDirInode... which should fail (only Shared Lease) - // TODO: Perform a Lease Promote on RootDirInode + + // Perform a Lease Promote on RootDirInode + + leaseRequest = &LeaseRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: 1, + LeaseRequestType: LeaseRequestTypePromote, + } + leaseResponse = &LeaseResponseStruct{} + + err = retryrpcClient.Send("Lease", leaseRequest, leaseResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"Lease(,1,LeaseRequestTypePromote)\",,) failed: %v", err) + } + // TODO: Perform a PutInodeTableEntries() on RootDirInode + + // Perform a FetchNonceRange() + + fetchNonceRangeRequest = &FetchNonceRangeRequestStruct{ + MountID: mountResponse.MountID, + } + fetchNonceRangeResponse = &FetchNonceRangeResponseStruct{} + + err = retryrpcClient.Send("FetchNonceRange", fetchNonceRangeRequest, fetchNonceRangeResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"FetchNonceRange()\",,) failed: %v", err) + } + // TODO: Create a FileInode (set LinkCount to 0 & no dir entry)... with small amount of data - // TODO: Fetch an Exclusive Lease on FileInode + + if 2 > fetchNonceRangeResponse.NumNoncesFetched { + t.Fatalf("fetchNonceRangeResponse contained insufficient NumNoncesFetched") + } + + fileInodeNumber = fetchNonceRangeResponse.NextNonce + fileInodeObjectA = fileInodeNumber + 1 + + fetchNonceRangeResponse.NextNonce += 2 + fetchNonceRangeResponse.NumNoncesFetched -= 2 + + t.Logf(" fileInodeNumber: %016X", fileInodeNumber) + t.Logf("fileInodeObjectA: %016X", fileInodeObjectA) + + // Fetch an Exclusive Lease on FileInode + + leaseRequest = &LeaseRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: fileInodeNumber, + LeaseRequestType: LeaseRequestTypeExclusive, + } + leaseResponse = &LeaseResponseStruct{} + + err = retryrpcClient.Send("Lease", leaseRequest, leaseResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"Lease(,fileInodeNumber,LeaseRequestTypeExclusive)\",,) failed: %v", err) + } + // TODO: Perform a PutInodeTableEntries() for FileInode + // TODO: Append some data (to new Object) for FileInode... and new stat + + if 1 > fetchNonceRangeResponse.NumNoncesFetched { + t.Fatalf("fetchNonceRangeResponse contained insufficient NumNoncesFetched") + } + + fileInodeObjectB = fileInodeObjectA + 1 + + fetchNonceRangeResponse.NextNonce++ + fetchNonceRangeResponse.NumNoncesFetched-- + + t.Logf("fileInodeObjectB: %016X", fileInodeObjectB) + // TODO: Perform a PutInodeTableEntries() for FileInode - // TODO: Perform a Flush() + + // Perform a Flush() + + flushRequest = &FlushRequestStruct{ + MountID: mountResponse.MountID, + } + flushResponse = &FlushResponseStruct{} + + err = retryrpcClient.Send("Flush", flushRequest, flushResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"Flush()\",,) failed: %v", err) + } + // TODO: Overwrite the original data in FileInode (to new Object)... and new stat dereferencing 1st Object - // TODO: Perform a Flush() + + if 1 > fetchNonceRangeResponse.NumNoncesFetched { + t.Fatalf("fetchNonceRangeResponse contained insufficient NumNoncesFetched") + } + + fileInodeObjectC = fileInodeObjectB + 1 + + fetchNonceRangeResponse.NextNonce++ + fetchNonceRangeResponse.NumNoncesFetched-- + + t.Logf("fileInodeObjectC: %016X", fileInodeObjectC) + + // Perform a Flush() + + flushRequest = &FlushRequestStruct{ + MountID: mountResponse.MountID, + } + flushResponse = &FlushResponseStruct{} + + err = retryrpcClient.Send("Flush", flushRequest, flushResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"Flush()\",,) failed: %v", err) + } + // TODO: Verify that 1st Object for FileInode gets deleted... but not 2nd nor 3rd - // TODO: Perform a AdjustInodeTableEntryOpenCount(+1) for FileInode - // TODO: Perform a DeleteInodeTableEntry() on FileInode - // TODO: Perform a Flush() + + // Perform a AdjustInodeTableEntryOpenCount(+1) for FileInode + + adjustInodeTableEntryOpenCountRequest = &AdjustInodeTableEntryOpenCountRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: fileInodeNumber, + Adjustment: +1, + } + adjustInodeTableEntryOpenCountResponse = &AdjustInodeTableEntryOpenCountResponseStruct{} + + err = retryrpcClient.Send("AdjustInodeTableEntryOpenCount", adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"AdjustInodeTableEntryOpenCount(,fileInodeNumber,+1)\",,) failed: %v", err) + } + + // Perform a DeleteInodeTableEntry() on FileInode + + deleteInodeTableEntryRequest = &DeleteInodeTableEntryRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: fileInodeNumber, + } + deleteInodeTableEntryResponse = &DeleteInodeTableEntryResponseStruct{} + + err = retryrpcClient.Send("DeleteInodeTableEntry", deleteInodeTableEntryRequest, deleteInodeTableEntryResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"DeleteInodeTableEntry(,fileInodeNumber)\",,) failed: %v", err) + } + + // Perform a Flush() + + flushRequest = &FlushRequestStruct{ + MountID: mountResponse.MountID, + } + flushResponse = &FlushResponseStruct{} + + err = retryrpcClient.Send("Flush", flushRequest, flushResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"Flush()\",,) failed: %v", err) + } + // TODO: Verify that FileInode is still in InodeTable - // TODO: Perform a AdjustInodeTableEntryOpenCount(-1) for FileInode - // TODO: Perform a Flush() + + // Perform a AdjustInodeTableEntryOpenCount(-1) for FileInode + + adjustInodeTableEntryOpenCountRequest = &AdjustInodeTableEntryOpenCountRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: fileInodeNumber, + Adjustment: -1, + } + adjustInodeTableEntryOpenCountResponse = &AdjustInodeTableEntryOpenCountResponseStruct{} + + err = retryrpcClient.Send("AdjustInodeTableEntryOpenCount", adjustInodeTableEntryOpenCountRequest, adjustInodeTableEntryOpenCountResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"AdjustInodeTableEntryOpenCount(,fileInodeNumber,-1)\",,) failed: %v", err) + } + + // Perform a Flush() + + flushRequest = &FlushRequestStruct{ + MountID: mountResponse.MountID, + } + flushResponse = &FlushResponseStruct{} + + err = retryrpcClient.Send("Flush", flushRequest, flushResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"Flush()\",,) failed: %v", err) + } + // TODO: Verify that FileInode is no longer in InodeTable and 2nd and 3rd Objects are deleted - // TODO: Perform a Lease Release on FileInode - // TODO: Perform a Unmount()... without first releasing Exclusive Lease on RootDirInode + + // Perform a Lease Release on FileInode + + leaseRequest = &LeaseRequestStruct{ + MountID: mountResponse.MountID, + InodeNumber: fileInodeNumber, + LeaseRequestType: LeaseRequestTypeRelease, + } + leaseResponse = &LeaseResponseStruct{} + + err = retryrpcClient.Send("Lease", leaseRequest, leaseResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"Lease(,fileInodeNumber,LeaseRequestTypeRelease)\",,) failed: %v", err) + } + + // Perform a Unmount()... without first releasing Exclusive Lease on RootDirInode + + unmountRequest = &UnmountRequestStruct{ + MountID: mountResponse.MountID, + } + unmountResponse = &UnmountResponseStruct{} + + err = retryrpcClient.Send("Unmount", unmountRequest, unmountResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"Unmount()\",,) failed: %v", err) + } + // TODO: Verify that Exclusive Lease on RootDirInode is implicitly released retryrpcClient.Close() From 06e4a12291e0de4d68df2eebf674db4b2dad1bcb Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 27 Apr 2021 14:13:30 -0700 Subject: [PATCH 153/160] First mount in imgrpkg completed --- imgr/imgrpkg/globals.go | 42 ++++++++++------- imgr/imgrpkg/http-server_test.go | 2 +- imgr/imgrpkg/retry-rpc.go | 81 +++++++++++++++++++++++++++++++- imgr/imgrpkg/retry-rpc_test.go | 44 ++++++++++++++--- imgr/imgrpkg/volume.go | 51 +++++++++++--------- 5 files changed, 169 insertions(+), 51 deletions(-) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index e75beca3a..368fb6e2a 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -117,28 +117,34 @@ type statsStruct struct { SwiftObjectPutUsecs bucketstats.BucketLog2Round } +const ( + mountIDByteArrayLen = 15 // actual mountID will be Base64-encoded form +) + type mountStruct struct { - // reentrancy covered by volumeStruct's sync.RWMutex - volume *volumeStruct // volume.{R|}Lock() also protects each mountStruct - mountID string // - authTokenValid bool // if true, mount is "healthy"; if false, mount is "expired" - authToken string // - lastAuthTime time.Time // used to periodically check TTL of authToken - listElement *list.Element // LRU element on either volumeStruct.{healthy|expired}MountList + // reentrancy covered by volumeStruct's sync.RWMutex + volume *volumeStruct // volume.{R|}Lock() also protects each mountStruct + mountID string // + leasesExpired bool // if true, leases are being expired prior to auto-deletion of mountStruct + authTokenExpired bool // if true, authToken has been rejected... needing a renewMount() to update + authToken string // + lastAuthTime time.Time // used to periodically check TTL of authToken + listElement *list.Element // LRU element on either volumeStruct.{healthy|expired}MountList } type volumeStruct struct { - sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() - name string // - storageURL string // - mountMap map[string]*mountStruct // key == mountStruct.mountID - healthyMountList *list.List // LRU of mountStruct's with .authTokenValid == true - expiredMountList *list.List // LRU of mountStruct's with .authTokenValid == false - deleting bool // - checkPointHeader *ilayout.CheckPointHeaderV1Struct // == nil if not currently mounted and/or checkpointing - superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing - inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing - pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber + sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() + name string // + storageURL string // + mountMap map[string]*mountStruct // key == mountStruct.mountID + healthyMountList *list.List // LRU of mountStruct's with .{leases|authToken}Expired == false + leasesExpiredMountList *list.List // List of mountStruct's with .leasesExpired == true (regardless of .authTokenExpired) value + authTokenExpiredMountList *list.List // List of mountStruct's with at .authTokenExpired == true (& .leasesExpired == false) + deleting bool // + checkPointHeader *ilayout.CheckPointHeaderV1Struct // == nil if not currently mounted and/or checkpointing + superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing + inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing + pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber } type globalsStruct struct { diff --git a/imgr/imgrpkg/http-server_test.go b/imgr/imgrpkg/http-server_test.go index 709732985..3f47a6972 100644 --- a/imgr/imgrpkg/http-server_test.go +++ b/imgr/imgrpkg/http-server_test.go @@ -72,7 +72,7 @@ func TestHTTPServer(t *testing.T) { t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) failed: %v", err) } - responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"HealthyMounts\":0,\"ExpiredMounts\":0}", testVolume, testGlobals.containerURL) + responseBodyExpected = fmt.Sprintf("{\"Name\":\"%s\",\"StorageURL\":\"%s\",\"HealthyMounts\":0,\"LeasesExpiredMounts\":0,\"AuthTokenExpiredMounts\":0}", testVolume, testGlobals.containerURL) _, responseBody, err = testDoHTTPRequest("GET", testGlobals.httpServerURL+"/volume/"+testVolume, nil, nil) if nil != err { diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index fa3255905..eea174967 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -5,11 +5,16 @@ package imgrpkg import ( "crypto/tls" + "encoding/base64" "fmt" "time" + "github.com/NVIDIA/sortedmap" + "github.com/NVIDIA/proxyfs/blunder" + "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/proxyfs/retryrpc" + "github.com/NVIDIA/proxyfs/utils" ) func startRetryRPCServer() (err error) { @@ -68,7 +73,14 @@ func stopRetryRPCServer() (err error) { func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountResponse *MountResponseStruct) (err error) { var ( - startTime time.Time + alreadyInGlobalsMountMap bool + mount *mountStruct + mountIDAsByteArray []byte + mountIDAsString string + ok bool + startTime time.Time + volume *volumeStruct + volumeAsValue sortedmap.Value ) startTime = time.Now() @@ -77,7 +89,72 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo globals.stats.MountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf("TODO") + globals.Lock() + + volumeAsValue, ok, err = globals.volumeMap.GetByKey(mountRequest.VolumeName) + if nil != err { + logFatalf("globals.volumeMap.GetByKey() failed: %v", err) + } + if !ok { + globals.Unlock() + err = fmt.Errorf("Unknown VolumeName (\"%s\")", mountRequest.VolumeName) + return + } + + volume, ok = volumeAsValue.(*volumeStruct) + if !ok { + logFatalf("volumeAsValue.(*volumeStruct) returned !ok") + } + + volume.Lock() + + if volume.deleting { + volume.Unlock() + globals.Unlock() + err = fmt.Errorf("Volume (\"%s\") being deleted", mountRequest.VolumeName) + return + } + + _, err = swiftContainerHeaderGet(volume.storageURL, mountRequest.AuthToken, ilayout.CheckPointHeaderName) + if nil != err { + volume.Unlock() + globals.Unlock() + err = fmt.Errorf("AuthToken (\"%s\") rejected", mountRequest.AuthToken) + return + } + +retryGenerateMountID: + + mountIDAsByteArray = utils.FetchRandomByteSlice(mountIDByteArrayLen) + mountIDAsString = base64.StdEncoding.EncodeToString(mountIDAsByteArray[:]) + + _, alreadyInGlobalsMountMap = globals.mountMap[mountIDAsString] + if alreadyInGlobalsMountMap { + goto retryGenerateMountID + } + + mount = &mountStruct{ + volume: volume, + mountID: mountIDAsString, + leasesExpired: false, + authTokenExpired: false, + authToken: mountRequest.AuthToken, + lastAuthTime: time.Now(), + } + + volume.mountMap[mountIDAsString] = mount + mount.listElement = volume.healthyMountList.PushBack(mount) + globals.mountMap[mountIDAsString] = mount + + // TODO: if first healthy mount, kick off checkpoint goroutine + + volume.Unlock() + globals.Unlock() + + mountResponse.MountID = mountIDAsString + + err = nil + return } func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse *RenewMountResponseStruct) (err error) { diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index 368802f10..db620a003 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -4,6 +4,8 @@ package imgrpkg import ( + "fmt" + "strings" "testing" "github.com/NVIDIA/proxyfs/retryrpc" @@ -38,6 +40,8 @@ func TestRetryRPC(t *testing.T) { leaseResponse *LeaseResponseStruct mountRequest *MountRequestStruct mountResponse *MountResponseStruct + postRequestBody string + putRequestBody string renewMountRequest *RenewMountRequestStruct renewMountResponse *RenewMountResponseStruct retryrpcClient *retryrpc.Client @@ -46,10 +50,14 @@ func TestRetryRPC(t *testing.T) { unmountResponse *UnmountResponseStruct ) + // Setup RetryRPC Client + retryrpcClientCallbacks = &testRetryRPCClientCallbacksStruct{ interruptPayloadChan: make(chan []byte), } + // Setup test environment + testSetup(t, retryrpcClientCallbacks) retryrpcClient, err = retryrpc.NewClient(testGlobals.retryrpcClientConfig) @@ -57,6 +65,24 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpc.NewClient() failed: %v", err) } + // Format testVolume + + postRequestBody = fmt.Sprintf("{\"StorageURL\":\"%s\",\"AuthToken\":\"%s\"}", testGlobals.containerURL, testGlobals.authToken) + + _, _, err = testDoHTTPRequest("POST", testGlobals.httpServerURL+"/volume", nil, strings.NewReader(postRequestBody)) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"POST\", testGlobals.httpServerURL+\"/volume\", nil, strings.NewReader(postRequestBody)) failed: %v", err) + } + + // Start serving testVolume + + putRequestBody = fmt.Sprintf("{\"StorageURL\":\"%s\"}", testGlobals.containerURL) + + _, _, err = testDoHTTPRequest("PUT", testGlobals.httpServerURL+"/volume/"+testVolume, nil, strings.NewReader(putRequestBody)) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"PUT\", testGlobals.httpServerURL+\"/volume\"+testVolume, nil, strings.NewReader(putRequestBody)) failed: %v", err) + } + // Attempt a FetchNonceRange() without a prior Mount()... which should fail (no Mount) fetchNonceRangeRequest = &FetchNonceRangeRequestStruct{ @@ -69,13 +95,6 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"FetchNonceRange()\",,) should have failed") } - // TODO: Remove this early exit skipping of following TODOs - - if nil != err { - t.Logf("Exiting TestRetryRPC() early to skip following TODOs") - return - } - // Perform a Mount() mountRequest = &MountRequestStruct{ @@ -89,6 +108,13 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"Mount(,)\",,) failed: %v", err) } + // TODO: Remove this early exit skipping of following TODOs + + if nil == err { + t.Logf("Exiting TestRetryRPC() early to skip following TODOs") + return + } + // Attempt a GetInodeTableEntry() for RootDirInode... which should fail (no Lease) getInodeTableEntryRequest = &GetInodeTableEntryRequestStruct{ @@ -369,7 +395,11 @@ func TestRetryRPC(t *testing.T) { // TODO: Verify that Exclusive Lease on RootDirInode is implicitly released + // Teardown RetryRPC Client + retryrpcClient.Close() + // And teardown test environment + testTeardown(t) } diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 748819efd..21a8b3ffa 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -10,8 +10,9 @@ import ( "io" "time" - "github.com/NVIDIA/proxyfs/ilayout" "github.com/NVIDIA/sortedmap" + + "github.com/NVIDIA/proxyfs/ilayout" ) func startVolumeManagement() (err error) { @@ -110,10 +111,11 @@ func deleteVolume(volumeName string) (err error) { } type volumeGETStruct struct { - Name string - StorageURL string - HealthyMounts uint64 - ExpiredMounts uint64 + Name string + StorageURL string + HealthyMounts uint64 + LeasesExpiredMounts uint64 + AuthTokenExpiredMounts uint64 } func getVolumeAsJSON(volumeName string) (volume []byte, err error) { @@ -145,10 +147,11 @@ func getVolumeAsJSON(volumeName string) (volume []byte, err error) { globals.RUnlock() volumeToReturn = &volumeGETStruct{ - Name: volumeAsStruct.name, - StorageURL: volumeAsStruct.storageURL, - HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), - ExpiredMounts: uint64(volumeAsStruct.expiredMountList.Len()), + Name: volumeAsStruct.name, + StorageURL: volumeAsStruct.storageURL, + HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), + LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), + AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), } volumeAsStruct.RUnlock() @@ -199,10 +202,11 @@ func getVolumeListAsJSON() (volumeList []byte) { volumeAsStruct.RLock() volumeListToReturn[volumeListIndex] = &volumeGETStruct{ - Name: volumeAsStruct.name, - StorageURL: volumeAsStruct.storageURL, - HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), - ExpiredMounts: uint64(volumeAsStruct.expiredMountList.Len()), + Name: volumeAsStruct.name, + StorageURL: volumeAsStruct.storageURL, + HealthyMounts: uint64(volumeAsStruct.healthyMountList.Len()), + LeasesExpiredMounts: uint64(volumeAsStruct.leasesExpiredMountList.Len()), + AuthTokenExpiredMounts: uint64(volumeAsStruct.authTokenExpiredMountList.Len()), } volumeAsStruct.RUnlock() @@ -716,16 +720,17 @@ func putVolume(name string, storageURL string) (err error) { ) volume = &volumeStruct{ - name: name, - storageURL: storageURL, - mountMap: make(map[string]*mountStruct), - healthyMountList: list.New(), - expiredMountList: list.New(), - deleting: false, - checkPointHeader: nil, - superBlock: nil, - inodeTable: nil, - pendingObjectDeleteSet: make(map[uint64]struct{}), + name: name, + storageURL: storageURL, + mountMap: make(map[string]*mountStruct), + healthyMountList: list.New(), + leasesExpiredMountList: list.New(), + authTokenExpiredMountList: list.New(), + deleting: false, + checkPointHeader: nil, + superBlock: nil, + inodeTable: nil, + pendingObjectDeleteSet: make(map[uint64]struct{}), } globals.Lock() From 39008295db459c9bb5c76d5ec0ee3943e07f4949 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 27 Apr 2021 14:40:56 -0700 Subject: [PATCH 154/160] Converted all (current) imgrpkg RetryRPC error strings to be parseable for specific failure --- imgr/imgrpkg/api.go | 10 ++++++++++ imgr/imgrpkg/globals.go | 5 +++-- imgr/imgrpkg/retry-rpc.go | 28 +++++++++++++++------------- imgr/imgrpkg/retry-rpc_test.go | 24 ++++++++++++------------ 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 0f4719401..3077e123c 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -144,6 +144,16 @@ func LogInfof(format string, args ...interface{}) { logInfof(format, args...) } +// E* specifies the prefix of an error string returned by any RetryRPC API +// +const ( + EUnknownVolumeName = "EUnknownVolumeName:" + EVolumeBeingDeleted = "EVolumeBeingDeleted:" + EAuthTokenRejected = "EAuthTokenRejected:" + + ETODO = "ETODO:" +) + type RetryRPCServerStruct struct{} var retryRPCServer *RetryRPCServerStruct diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 368fb6e2a..5b0392b53 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -138,12 +138,13 @@ type volumeStruct struct { storageURL string // mountMap map[string]*mountStruct // key == mountStruct.mountID healthyMountList *list.List // LRU of mountStruct's with .{leases|authToken}Expired == false - leasesExpiredMountList *list.List // List of mountStruct's with .leasesExpired == true (regardless of .authTokenExpired) value - authTokenExpiredMountList *list.List // List of mountStruct's with at .authTokenExpired == true (& .leasesExpired == false) + leasesExpiredMountList *list.List // list of mountStruct's with .leasesExpired == true (regardless of .authTokenExpired) value + authTokenExpiredMountList *list.List // list of mountStruct's with at .authTokenExpired == true (& .leasesExpired == false) deleting bool // checkPointHeader *ilayout.CheckPointHeaderV1Struct // == nil if not currently mounted and/or checkpointing superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing + lastCheckpointHeader string // most recent (read or written) ilayout.CheckPointHeaderName value pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber } diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index eea174967..16af280fe 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -74,6 +74,7 @@ func stopRetryRPCServer() (err error) { func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountResponse *MountResponseStruct) (err error) { var ( alreadyInGlobalsMountMap bool + lastCheckpointHeader string mount *mountStruct mountIDAsByteArray []byte mountIDAsString string @@ -97,7 +98,7 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo } if !ok { globals.Unlock() - err = fmt.Errorf("Unknown VolumeName (\"%s\")", mountRequest.VolumeName) + err = fmt.Errorf("%s %s", EUnknownVolumeName, mountRequest.VolumeName) return } @@ -111,15 +112,15 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo if volume.deleting { volume.Unlock() globals.Unlock() - err = fmt.Errorf("Volume (\"%s\") being deleted", mountRequest.VolumeName) + err = fmt.Errorf("%s %s", EVolumeBeingDeleted, mountRequest.VolumeName) return } - _, err = swiftContainerHeaderGet(volume.storageURL, mountRequest.AuthToken, ilayout.CheckPointHeaderName) + lastCheckpointHeader, err = swiftContainerHeaderGet(volume.storageURL, mountRequest.AuthToken, ilayout.CheckPointHeaderName) if nil != err { volume.Unlock() globals.Unlock() - err = fmt.Errorf("AuthToken (\"%s\") rejected", mountRequest.AuthToken) + err = fmt.Errorf("%s %s", EAuthTokenRejected, mountRequest.AuthToken) return } @@ -147,6 +148,7 @@ retryGenerateMountID: globals.mountMap[mountIDAsString] = mount // TODO: if first healthy mount, kick off checkpoint goroutine + fmt.Printf("UNDO: lastCheckpointHeader: %s\n", lastCheckpointHeader) volume.Unlock() globals.Unlock() @@ -168,7 +170,7 @@ func renewMount(renewMountRequest *RenewMountRequestStruct, renewMountResponse * globals.stats.RenewMountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf("TODO") + return fmt.Errorf(ETODO + " renewMount") } func unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountResponseStruct) (err error) { @@ -182,7 +184,7 @@ func unmount(unmountRequest *UnmountRequestStruct, unmountResponse *UnmountRespo globals.stats.UnmountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf("TODO") + return fmt.Errorf(ETODO + " unmount") } func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetchNonceRangeResponse *FetchNonceRangeResponseStruct) (err error) { @@ -211,7 +213,7 @@ func fetchNonceRange(fetchNonceRangeRequest *FetchNonceRangeRequestStruct, fetch globals.RUnlock() - return fmt.Errorf("TODO") + return fmt.Errorf(ETODO + " fetchNonceRange") } func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStruct, getInodeTableEntryResponse *GetInodeTableEntryResponseStruct) (err error) { @@ -225,7 +227,7 @@ func getInodeTableEntry(getInodeTableEntryRequest *GetInodeTableEntryRequestStru globals.stats.GetInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf("TODO") + return fmt.Errorf(ETODO + " getInodeTableEntry") } func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesRequestStruct, putInodeTableEntriesResponse *PutInodeTableEntriesResponseStruct) (err error) { @@ -239,7 +241,7 @@ func putInodeTableEntries(putInodeTableEntriesRequest *PutInodeTableEntriesReque globals.stats.PutInodeTableEntriesUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf("TODO") + return fmt.Errorf(ETODO + " putInodeTableEntries") } func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRequestStruct, deleteInodeTableEntryResponse *DeleteInodeTableEntryResponseStruct) (err error) { @@ -253,7 +255,7 @@ func deleteInodeTableEntry(deleteInodeTableEntryRequest *DeleteInodeTableEntryRe globals.stats.DeleteInodeTableEntryUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf("TODO") + return fmt.Errorf(ETODO + " deleteInodeTableEntry") } func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *AdjustInodeTableEntryOpenCountRequestStruct, adjustInodeTableEntryOpenCountResponse *AdjustInodeTableEntryOpenCountResponseStruct) (err error) { @@ -267,7 +269,7 @@ func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *Adjus globals.stats.AdjustInodeTableEntryOpenCountUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf("TODO") + return fmt.Errorf(ETODO + " adjustInodeTableEntryOpenCount") } func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) (err error) { @@ -281,7 +283,7 @@ func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) globals.stats.FlushUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf("TODO") + return fmt.Errorf(ETODO + " flush") } func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) (err error) { @@ -323,5 +325,5 @@ func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) return } - return fmt.Errorf("TODO") + return fmt.Errorf(ETODO + " lease") } diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index db620a003..21c4595bd 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -115,6 +115,18 @@ func TestRetryRPC(t *testing.T) { return } + // Perform a FetchNonceRange() + + fetchNonceRangeRequest = &FetchNonceRangeRequestStruct{ + MountID: mountResponse.MountID, + } + fetchNonceRangeResponse = &FetchNonceRangeResponseStruct{} + + err = retryrpcClient.Send("FetchNonceRange", fetchNonceRangeRequest, fetchNonceRangeResponse) + if nil != err { + t.Fatalf("retryrpcClient.Send(\"FetchNonceRange()\",,) failed: %v", err) + } + // Attempt a GetInodeTableEntry() for RootDirInode... which should fail (no Lease) getInodeTableEntryRequest = &GetInodeTableEntryRequestStruct{ @@ -201,18 +213,6 @@ func TestRetryRPC(t *testing.T) { // TODO: Perform a PutInodeTableEntries() on RootDirInode - // Perform a FetchNonceRange() - - fetchNonceRangeRequest = &FetchNonceRangeRequestStruct{ - MountID: mountResponse.MountID, - } - fetchNonceRangeResponse = &FetchNonceRangeResponseStruct{} - - err = retryrpcClient.Send("FetchNonceRange", fetchNonceRangeRequest, fetchNonceRangeResponse) - if nil != err { - t.Fatalf("retryrpcClient.Send(\"FetchNonceRange()\",,) failed: %v", err) - } - // TODO: Create a FileInode (set LinkCount to 0 & no dir entry)... with small amount of data if 2 > fetchNonceRangeResponse.NumNoncesFetched { From 9e2ac8e3b0f7719e911966e4dcf5f059df1ff46e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 27 Apr 2021 16:35:20 -0700 Subject: [PATCH 155/160] Implemented Flush and all but actual CheckPoint in imgrpkg CheckPoint's both by interval and explicit --- ilayout/api.go | 2 +- imgr/imgr.conf | 2 + imgr/imgrpkg/api.go | 7 +++- imgr/imgrpkg/globals.go | 16 ++++++-- imgr/imgrpkg/retry-rpc.go | 73 +++++++++++++++++++++++++++------- imgr/imgrpkg/retry-rpc_test.go | 6 +-- imgr/imgrpkg/utils_test.go | 2 + imgr/imgrpkg/volume.go | 55 +++++++++++++++++++++++++ 8 files changed, 140 insertions(+), 23 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index ce72b2b2b..8fc3f372d 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -67,7 +67,7 @@ const ( CheckPointHeaderName = "X-Container-Meta-Check-Point" ) -// CheckpointHeaderVersionV* specifies the format of the CheckPointHeader. +// CheckPointHeaderVersionV* specifies the format of the CheckPointHeader. // The CheckPointHeaderVersion must always be fetched by scanning the entire // CheckPointHeader using a %016X format specifier. This value will then be // used to interpret the remaining characters of the CheckPointHeader string. diff --git a/imgr/imgr.conf b/imgr/imgr.conf index c3e7c8c8b..390fa1391 100644 --- a/imgr/imgr.conf +++ b/imgr/imgr.conf @@ -17,6 +17,8 @@ RetryRPCKeyFilePath: # non-TLS RetryRPC will be FetchNonceRangeToReturn: 100 +CheckPointInterval: 10s + MinLeaseDuration: 250ms LeaseInterruptInterval: 250ms LeaseInterruptLimit: 20 diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index 3077e123c..fbc7d25ad 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -24,6 +24,8 @@ // RetryRPCCertFilePath: # If both RetryRPC{Cert|Key}FilePath are missing or empty, // RetryRPCKeyFilePath: # non-TLS RetryRPC will be selected; otherwise TLS will be used // +// CheckPointInterval: 10s +// // FetchNonceRangeToReturn: 100 // // MinLeaseDuration: 250ms @@ -147,9 +149,10 @@ func LogInfof(format string, args ...interface{}) { // E* specifies the prefix of an error string returned by any RetryRPC API // const ( - EUnknownVolumeName = "EUnknownVolumeName:" - EVolumeBeingDeleted = "EVolumeBeingDeleted:" EAuthTokenRejected = "EAuthTokenRejected:" + EVolumeBeingDeleted = "EVolumeBeingDeleted:" + EUnknownMountID = "EUnknownMountID:" + EUnknownVolumeName = "EUnknownVolumeName:" ETODO = "ETODO:" ) diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 5b0392b53..474365301 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -34,6 +34,8 @@ type configStruct struct { RetryRPCCertFilePath string RetryRPCKeyFilePath string + CheckPointInterval time.Duration + FetchNonceRangeToReturn uint64 MinLeaseDuration time.Duration @@ -94,6 +96,8 @@ type statsStruct struct { RenewMountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).RenewMount() UnmountUsecs bucketstats.BucketLog2Round // (*RetryRPCServerStruct).Unmount() + VolumeCheckPointUsecs bucketstats.BucketLog2Round + SharedLeaseRequestUsecs bucketstats.BucketLog2Round PromoteLeaseRequestUsecs bucketstats.BucketLog2Round ExclusiveLeaseRequestUsecs bucketstats.BucketLog2Round @@ -104,8 +108,6 @@ type statsStruct struct { DemoteLeaseInterrupts bucketstats.Totaler RevokeLeaseInterrupts bucketstats.Totaler - VolumeCheckpointUsecs bucketstats.BucketLog2Round - InodeTableCacheHits bucketstats.Totaler InodeTableCacheMisses bucketstats.Totaler @@ -144,8 +146,9 @@ type volumeStruct struct { checkPointHeader *ilayout.CheckPointHeaderV1Struct // == nil if not currently mounted and/or checkpointing superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing - lastCheckpointHeader string // most recent (read or written) ilayout.CheckPointHeaderName value pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber + checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() + checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG } type globalsStruct struct { @@ -261,6 +264,11 @@ func initializeGlobals(confMap conf.ConfMap) (err error) { } } + globals.config.CheckPointInterval, err = confMap.FetchOptionValueDuration("IMGR", "CheckPointInterval") + if nil != err { + logFatal(err) + } + globals.config.FetchNonceRangeToReturn, err = confMap.FetchOptionValueUint64("IMGR", "FetchNonceRangeToReturn") if nil != err { logFatal(err) @@ -363,6 +371,8 @@ func uninitializeGlobals() (err error) { globals.config.RetryRPCCertFilePath = "" globals.config.RetryRPCKeyFilePath = "" + globals.config.CheckPointInterval = time.Duration(0) + globals.config.FetchNonceRangeToReturn = 0 globals.config.MinLeaseDuration = time.Duration(0) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index 16af280fe..eb213556e 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -73,15 +73,16 @@ func stopRetryRPCServer() (err error) { func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountResponse *MountResponseStruct) (err error) { var ( - alreadyInGlobalsMountMap bool - lastCheckpointHeader string - mount *mountStruct - mountIDAsByteArray []byte - mountIDAsString string - ok bool - startTime time.Time - volume *volumeStruct - volumeAsValue sortedmap.Value + alreadyInGlobalsMountMap bool + lastCheckPointHeader *ilayout.CheckPointHeaderV1Struct + lastCheckPointHeaderAsString string + mount *mountStruct + mountIDAsByteArray []byte + mountIDAsString string + ok bool + startTime time.Time + volume *volumeStruct + volumeAsValue sortedmap.Value ) startTime = time.Now() @@ -116,7 +117,7 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo return } - lastCheckpointHeader, err = swiftContainerHeaderGet(volume.storageURL, mountRequest.AuthToken, ilayout.CheckPointHeaderName) + lastCheckPointHeaderAsString, err = swiftContainerHeaderGet(volume.storageURL, mountRequest.AuthToken, ilayout.CheckPointHeaderName) if nil != err { volume.Unlock() globals.Unlock() @@ -147,8 +148,20 @@ retryGenerateMountID: mount.listElement = volume.healthyMountList.PushBack(mount) globals.mountMap[mountIDAsString] = mount - // TODO: if first healthy mount, kick off checkpoint goroutine - fmt.Printf("UNDO: lastCheckpointHeader: %s\n", lastCheckpointHeader) + if nil == volume.checkPointControlChan { + lastCheckPointHeader, err = ilayout.UnmarshalCheckPointHeaderV1(lastCheckPointHeaderAsString) + if nil != err { + logFatalf("ilayout.UnmarshalCheckPointHeaderV1(lastCheckPointHeaderAsString==\"%s\") failed: %v", lastCheckPointHeaderAsString, err) + } + + volume.checkPointHeader = lastCheckPointHeader + + volume.checkPointControlChan = make(chan chan error) + + volume.checkPointControlWG.Add(1) + + go volume.checkPointDaemon(volume.checkPointControlChan) + } volume.Unlock() globals.Unlock() @@ -274,7 +287,11 @@ func adjustInodeTableEntryOpenCount(adjustInodeTableEntryOpenCountRequest *Adjus func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) (err error) { var ( - startTime time.Time + checkPointResponseChan chan error + mount *mountStruct + ok bool + startTime time.Time + volume *volumeStruct ) startTime = time.Now() @@ -283,7 +300,35 @@ func flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) globals.stats.FlushUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - return fmt.Errorf(ETODO + " flush") + checkPointResponseChan = make(chan error) + + globals.RLock() + + mount, ok = globals.mountMap[flushRequest.MountID] + if !ok { + globals.RUnlock() + err = fmt.Errorf("%s %s", EUnknownMountID, flushRequest.MountID) + return + } + + volume = mount.volume + + volume.RLock() + globals.RUnlock() + + if nil == volume.checkPointControlChan { + volume.RUnlock() + err = nil + return + } + + volume.checkPointControlChan <- checkPointResponseChan + + volume.RUnlock() + + err = <-checkPointResponseChan + + return } func lease(leaseRequest *LeaseRequestStruct, leaseResponse *LeaseResponseStruct) (err error) { diff --git a/imgr/imgrpkg/retry-rpc_test.go b/imgr/imgrpkg/retry-rpc_test.go index 21c4595bd..de29e6c16 100644 --- a/imgr/imgrpkg/retry-rpc_test.go +++ b/imgr/imgrpkg/retry-rpc_test.go @@ -298,7 +298,7 @@ func TestRetryRPC(t *testing.T) { // TODO: Verify that 1st Object for FileInode gets deleted... but not 2nd nor 3rd - // Perform a AdjustInodeTableEntryOpenCount(+1) for FileInode + // Perform an AdjustInodeTableEntryOpenCount(+1) for FileInode adjustInodeTableEntryOpenCountRequest = &AdjustInodeTableEntryOpenCountRequestStruct{ MountID: mountResponse.MountID, @@ -339,7 +339,7 @@ func TestRetryRPC(t *testing.T) { // TODO: Verify that FileInode is still in InodeTable - // Perform a AdjustInodeTableEntryOpenCount(-1) for FileInode + // Perform an AdjustInodeTableEntryOpenCount(-1) for FileInode adjustInodeTableEntryOpenCountRequest = &AdjustInodeTableEntryOpenCountRequestStruct{ MountID: mountResponse.MountID, @@ -381,7 +381,7 @@ func TestRetryRPC(t *testing.T) { t.Fatalf("retryrpcClient.Send(\"Lease(,fileInodeNumber,LeaseRequestTypeRelease)\",,) failed: %v", err) } - // Perform a Unmount()... without first releasing Exclusive Lease on RootDirInode + // Perform an Unmount()... without first releasing Exclusive Lease on RootDirInode unmountRequest = &UnmountRequestStruct{ MountID: mountResponse.MountID, diff --git a/imgr/imgrpkg/utils_test.go b/imgr/imgrpkg/utils_test.go index b27a39b38..f00d9047d 100644 --- a/imgr/imgrpkg/utils_test.go +++ b/imgr/imgrpkg/utils_test.go @@ -134,6 +134,8 @@ func testSetup(t *testing.T, retryrpcCallbacks interface{}) { "IMGR.RetryRPCCertFilePath=" + testGlobals.endpointCertFile, "IMGR.RetryRPCKeyFilePath=" + testGlobals.endpointKeyFile, + "IMGR.CheckPointInterval=10s", + "IMGR.FetchNonceRangeToReturn=100", "IMGR.MinLeaseDuration=250ms", diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 21a8b3ffa..2a18072c3 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -731,6 +731,7 @@ func putVolume(name string, storageURL string) (err error) { superBlock: nil, inodeTable: nil, pendingObjectDeleteSet: make(map[uint64]struct{}), + checkPointControlChan: nil, } globals.Lock() @@ -750,3 +751,57 @@ func putVolume(name string, storageURL string) (err error) { return } + +func (volume *volumeStruct) checkPointDaemon(checkPointControlChan chan chan error) { + var ( + checkPointIntervalTimer *time.Timer + checkPointResponseChan chan error + err error + more bool + ) + + for { + checkPointIntervalTimer = time.NewTimer(globals.config.CheckPointInterval) + + select { + case _ = <-checkPointIntervalTimer.C: + err = volume.doCheckPoint() + if nil != err { + logWarnf("checkPointIntervalTimer-triggered doCheckPoint() failed: %v", err) + } + case checkPointResponseChan, more = <-checkPointControlChan: + if !checkPointIntervalTimer.Stop() { + _ = <-checkPointIntervalTimer.C + } + + if more { + err = volume.doCheckPoint() + if nil != err { + logWarnf("requested doCheckPoint() failed: %v", err) + } + + checkPointResponseChan <- err + } else { + volume.checkPointControlWG.Done() + } + } + } +} + +func (volume *volumeStruct) doCheckPoint() (err error) { + var ( + startTime time.Time + ) + + volume.Lock() + + startTime = time.Now() + + err = nil // TODO: perform the actual doCheckPoint() + + globals.stats.VolumeCheckPointUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + + volume.Unlock() + + return +} From 30f4533c9f21c2c8bbc4f3f769ad08232bc99653 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Tue, 27 Apr 2021 16:44:09 -0700 Subject: [PATCH 156/160] Updated imgrpkg API doc to set pattern for listing possible RetryRPC errors --- imgr/imgrpkg/api.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/imgr/imgrpkg/api.go b/imgr/imgrpkg/api.go index fbc7d25ad..3125f8e0a 100644 --- a/imgr/imgrpkg/api.go +++ b/imgr/imgrpkg/api.go @@ -177,6 +177,8 @@ type MountResponseStruct struct { // Mount performs a mount of the specified Volume and returns a MountID to be used // in all subsequent RPCs to reference this Volume by this Client. // +// Possible errors: EAuthTokenRejected EVolumeBeingDeleted EUnknownVolumeName +// func (dummy *RetryRPCServerStruct) Mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountResponse *MountResponseStruct) (err error) { return mount(retryRPCClientID, mountRequest, mountResponse) } @@ -332,6 +334,8 @@ type FlushResponseStruct struct{} // Flush that the results of prior PutInodeTableEntries requests be persisted. // +// Possible errors: EUnknownMountID +// func (dummy *RetryRPCServerStruct) Flush(flushRequest *FlushRequestStruct, flushResponse *FlushResponseStruct) (err error) { return flush(flushRequest, flushResponse) } From a572366a8b26204813db998955caba650822f126 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 30 Apr 2021 11:36:58 -0700 Subject: [PATCH 157/160] Switched to use a CheckPointHeader "Object" instead of a Container Header since S3 Buckets don't support that --- ilayout/api.go | 70 +++++++++++++++-------------- ilayout/api_test.go | 24 +++++----- ilayout/impl.go | 18 ++++---- imgr/imgrpkg/globals.go | 39 ++++++++-------- imgr/imgrpkg/http-server_test.go | 16 ++++--- imgr/imgrpkg/retry-rpc.go | 30 +++++++------ imgr/imgrpkg/swift-client.go | 77 ++++---------------------------- imgr/imgrpkg/volume.go | 17 +++---- 8 files changed, 121 insertions(+), 170 deletions(-) diff --git a/ilayout/api.go b/ilayout/api.go index 8fc3f372d..71b9a813f 100644 --- a/ilayout/api.go +++ b/ilayout/api.go @@ -10,14 +10,19 @@ // The structure of the file system assumes an underlying object store system // (e.g. an S3 Bucket or OpenStack Swift Container) that may or may not support // consistency. As such, no Object (or any other numerically identified entity -// in the file system) is ever written twice. This means that the file system -// operates in a write journaled mode. +// in the file system) is ever written twice except for the CheckPoint described +// below. // // At the top level, the file system periodically checkpoints. This enables the // file system to be complemented by the extreme scale of such Object Store // systems despite such often suffering long latencies and coarse granularity -// of writes. The CheckPointHeader is the changing indication of the most -// recent checkpoint performed. +// of writes. The CheckPoint is the changing indication of the most recent +// checkpoint performed. As a default, the CheckPoint is written to a specific +// Object in the Bucket/Container. This practice violates the safe practice of +// using an eventually consistent object store, but is provided for cases where +// the object store can deliver consistency. For deployments where the object +// store cannot guarantee consistency, the authoritative copy of the CheckPoint +// should also be written to a consistent store alternative. // // From the latest checkpoint, the file system's SuperBlock is located. This // SuperBlock is responsible for tracking the location, in the Object Store, @@ -60,53 +65,52 @@ import ( "time" ) -// CheckPointHeaderName specifies the name of the CheckPointHeader applied -// to the Container to identify the SuperBlock. +// CheckPointObjectNumber specifies the ObjectNumber of the Object to hold the +// default/backup copy of the CheckPoint. // const ( - CheckPointHeaderName = "X-Container-Meta-Check-Point" + CheckPointObjectNumber uint64 = 0 ) -// CheckPointHeaderVersionV* specifies the format of the CheckPointHeader. -// The CheckPointHeaderVersion must always be fetched by scanning the entire -// CheckPointHeader using a %016X format specifier. This value will then be -// used to interpret the remaining characters of the CheckPointHeader string. +// CheckPointVersionV* specifies the format of the CheckPoint. The CheckPointVersion +// must always be fetched by scanning the entire CheckPoint using a %016X format +// specifier. This value will then be used to interpret the remaining characters of +// the CheckPoint string. // const ( - CheckPointHeaderVersionV1 uint64 = 1 + CheckPointVersionV1 uint64 = 1 ) -// UnmarshalCheckPointHeaderVersion extracts checkPointHeaderVersion from checkpointHeaderString. +// UnmarshalCheckPointVersion extracts checkPointVersion from checkpointString. // -func UnmarshalCheckPointHeaderVersion(checkpointHeaderString string) (checkPointHeaderVersion uint64, err error) { - checkPointHeaderVersion, err = unmarshalCheckPointHeaderVersion(checkpointHeaderString) +func UnmarshalCheckPointVersion(checkpointString string) (checkPointVersion uint64, err error) { + checkPointVersion, err = unmarshalCheckPointVersion(checkpointString) return } -// CheckPointHeaderV1Struct specifies the format of the CheckPointHeader -// as of V1. +// CheckPointV1Struct specifies the format of the CheckPoint as of V1. // // The contents of the struct are serialized as space separated fields formatted // via %016X numbers. // -type CheckPointHeaderV1Struct struct { - Version uint64 // == CheckPointHeaderVersionV1 +type CheckPointV1Struct struct { + Version uint64 // == CheckPointVersionV1 SuperBlockObjectNumber uint64 // Identifies the Object containing the SuperBlock at the end SuperBlockLength uint64 // Total length of the SuperBlock found at the end of the Object indicated by SuperBlockObjectNumber ReservedToNonce uint64 // Ensures all numbers requiring uniqueness (e.g. Object numbers, Inode numbers) are never reused } -// MarshalCheckPointHeaderV1 encodes checkPointHeaderV1 to checkpointHeaderString. +// MarshalCheckPointV1 encodes checkPointV1 to checkpointString. // -func (checkPointHeaderV1 *CheckPointHeaderV1Struct) MarshalCheckPointHeaderV1() (checkPointHeaderV1String string, err error) { - checkPointHeaderV1String, err = checkPointHeaderV1.marshalCheckPointHeaderV1() +func (checkPointV1 *CheckPointV1Struct) MarshalCheckPointV1() (checkPointV1String string, err error) { + checkPointV1String, err = checkPointV1.marshalCheckPointV1() return } -// UnmarshalCheckPointHeaderV1 decodes checkPointHeaderV1 from checkpointHeaderString. +// UnmarshalCheckPointV1 decodes checkPointV1 from checkpointString. // -func UnmarshalCheckPointHeaderV1(checkPointHeaderV1String string) (checkPointHeaderV1 *CheckPointHeaderV1Struct, err error) { - checkPointHeaderV1, err = unmarshalCheckPointHeaderV1(checkPointHeaderV1String) +func UnmarshalCheckPointV1(checkPointV1String string) (checkPointV1 *CheckPointV1Struct, err error) { + checkPointV1, err = unmarshalCheckPointV1(checkPointV1String) return } @@ -167,21 +171,21 @@ type InodeTableLayoutEntryV1Struct struct { BytesReferenced uint64 // Number of bytes currently referenced in the Object } -// SuperBlockStruct specifies the format of the SuperBlock found at -// the CheckPointHeaderV1Struct.SuperBlockLength trailing bytes of -// the Object indicated by CheckPointHeaderV1Struct.SuperBlockObjectNumber. +// SuperBlockStruct specifies the format of the SuperBlock found at the +// CheckPointV1Struct.SuperBlockLength trailing bytes of the Object +// indicated by CheckPointV1Struct.SuperBlockObjectNumber. // -// The InodeTable is a B+Tree where the Key is the uint64 InodeNumber. The Value -// is a InodeTableEntryValueV1Struct. +// The InodeTable is a B+Tree where the Key is the uint64 InodeNumber. +// The Value is a InodeTableEntryValueV1Struct. // // The struct is serialized as a sequence of LittleEndian formatted fields. // The InodeTableLayout slice is serialized by a preceeding LittleEndian // count of the number of InodeTableLayoutEntryV1Struct's followed by the // serialization of each one. // -// Note that the CheckPointHeaderV1Struct.SuperBlockLength also includes the bytes for -// holding the ObjectTrailerStruct{ObjType: SuperBlockType, Version: SuperBlockVersionV1} -// that is appended. +// Note that the CheckPointV1Struct.SuperBlockLength also includes the bytes for holding +// the ObjectTrailerStruct{ObjType: SuperBlockType, Version: SuperBlockVersionV1} that is +// appended. // type SuperBlockV1Struct struct { InodeTableRootObjectNumber uint64 // Identifies the Object containing the root of the InodeTable diff --git a/ilayout/api_test.go b/ilayout/api_test.go index 4ae507a16..5e2b3c5ea 100644 --- a/ilayout/api_test.go +++ b/ilayout/api_test.go @@ -13,16 +13,16 @@ func TestAPI(t *testing.T) { var ( err error - testCheckPointHeaderV1 = &CheckPointHeaderV1Struct{ - Version: CheckPointHeaderVersionV1, + testCheckPointV1 = &CheckPointV1Struct{ + Version: CheckPointVersionV1, SuperBlockObjectNumber: 2, SuperBlockLength: 3, ReservedToNonce: 4, } - marshaledCheckPointHeaderV1 string - unmarshaledCheckPointHeaderVersion uint64 - unmarshaledCheckPointHeaderV1 *CheckPointHeaderV1Struct + marshaledCheckPointV1 string + unmarshaledCheckPointVersion uint64 + unmarshaledCheckPointV1 *CheckPointV1Struct testObjectTrailer = &ObjectTrailerStruct{ ObjType: 1, @@ -145,25 +145,25 @@ func TestAPI(t *testing.T) { unmarshaledExtentMapEntryValueV1 *ExtentMapEntryValueV1Struct ) - marshaledCheckPointHeaderV1, err = testCheckPointHeaderV1.MarshalCheckPointHeaderV1() + marshaledCheckPointV1, err = testCheckPointV1.MarshalCheckPointV1() if nil != err { t.Fatal(err) } - unmarshaledCheckPointHeaderVersion, err = UnmarshalCheckPointHeaderVersion(marshaledCheckPointHeaderV1) + unmarshaledCheckPointVersion, err = UnmarshalCheckPointVersion(marshaledCheckPointV1) if nil != err { t.Fatal(err) } - if CheckPointHeaderVersionV1 != unmarshaledCheckPointHeaderVersion { - t.Fatalf("Bad unmarshaledCheckPointHeaderVersion (%016X) - expected CheckPointHeaderVersionV1 (%016X)", unmarshaledCheckPointHeaderVersion, CheckPointHeaderVersionV1) + if CheckPointVersionV1 != unmarshaledCheckPointVersion { + t.Fatalf("Bad unmarshaledCheckPointVersion (%016X) - expected CheckPointVersionV1 (%016X)", unmarshaledCheckPointVersion, CheckPointVersionV1) } - unmarshaledCheckPointHeaderV1, err = UnmarshalCheckPointHeaderV1(marshaledCheckPointHeaderV1) + unmarshaledCheckPointV1, err = UnmarshalCheckPointV1(marshaledCheckPointV1) if nil != err { t.Fatal(err) } - if *testCheckPointHeaderV1 != *unmarshaledCheckPointHeaderV1 { - t.Fatalf("Bad unmarshaledCheckPointHeaderV1 (%+v) - expected testCheckPointHeaderV1 (%+v)", unmarshaledCheckPointHeaderV1, testCheckPointHeaderV1) + if *testCheckPointV1 != *unmarshaledCheckPointV1 { + t.Fatalf("Bad unmarshaledCheckPointV1 (%+v) - expected testCheckPointV1 (%+v)", unmarshaledCheckPointV1, testCheckPointV1) } marshaledObjectTrailer, err = testObjectTrailer.marshalObjectTrailer() diff --git a/ilayout/impl.go b/ilayout/impl.go index 80c219db7..464493499 100644 --- a/ilayout/impl.go +++ b/ilayout/impl.go @@ -9,25 +9,25 @@ import ( "time" ) -func unmarshalCheckPointHeaderVersion(checkpointHeaderString string) (checkPointHeaderVersion uint64, err error) { - _, err = fmt.Sscanf(checkpointHeaderString, "%016X", &checkPointHeaderVersion) +func unmarshalCheckPointVersion(checkpointString string) (checkPointVersion uint64, err error) { + _, err = fmt.Sscanf(checkpointString, "%016X", &checkPointVersion) return } -func (checkPointHeaderV1 *CheckPointHeaderV1Struct) marshalCheckPointHeaderV1() (checkPointHeaderV1String string, err error) { - checkPointHeaderV1String = fmt.Sprintf("%016X %016X %016X %016X", checkPointHeaderV1.Version, checkPointHeaderV1.SuperBlockObjectNumber, checkPointHeaderV1.SuperBlockLength, checkPointHeaderV1.ReservedToNonce) +func (checkPointV1 *CheckPointV1Struct) marshalCheckPointV1() (checkPointV1String string, err error) { + checkPointV1String = fmt.Sprintf("%016X %016X %016X %016X", checkPointV1.Version, checkPointV1.SuperBlockObjectNumber, checkPointV1.SuperBlockLength, checkPointV1.ReservedToNonce) err = nil return } -func unmarshalCheckPointHeaderV1(checkPointHeaderV1String string) (checkPointHeaderV1 *CheckPointHeaderV1Struct, err error) { - checkPointHeaderV1 = &CheckPointHeaderV1Struct{} +func unmarshalCheckPointV1(checkPointV1String string) (checkPointV1 *CheckPointV1Struct, err error) { + checkPointV1 = &CheckPointV1Struct{} - _, err = fmt.Sscanf(checkPointHeaderV1String, "%016X %016X %016X %016X", &checkPointHeaderV1.Version, &checkPointHeaderV1.SuperBlockObjectNumber, &checkPointHeaderV1.SuperBlockLength, &checkPointHeaderV1.ReservedToNonce) - if (nil == err) && (CheckPointHeaderVersionV1 != checkPointHeaderV1.Version) { - err = fmt.Errorf("Version mismatch... found %016X... expected %016X", checkPointHeaderV1.Version, CheckPointHeaderVersionV1) + _, err = fmt.Sscanf(checkPointV1String, "%016X %016X %016X %016X", &checkPointV1.Version, &checkPointV1.SuperBlockObjectNumber, &checkPointV1.SuperBlockLength, &checkPointV1.ReservedToNonce) + if (nil == err) && (CheckPointVersionV1 != checkPointV1.Version) { + err = fmt.Errorf("Version mismatch... found %016X... expected %016X", checkPointV1.Version, CheckPointVersionV1) } return diff --git a/imgr/imgrpkg/globals.go b/imgr/imgrpkg/globals.go index 474365301..dc6c17cb1 100644 --- a/imgr/imgrpkg/globals.go +++ b/imgr/imgrpkg/globals.go @@ -111,12 +111,11 @@ type statsStruct struct { InodeTableCacheHits bucketstats.Totaler InodeTableCacheMisses bucketstats.Totaler - SwiftContainerHeaderGetUsecs bucketstats.BucketLog2Round - SwiftContainerHeaderSetUsecs bucketstats.BucketLog2Round - SwiftObjectDeleteUsecs bucketstats.BucketLog2Round - SwiftObjectGetRangeUsecs bucketstats.BucketLog2Round - SwiftObjectGetTailUsecs bucketstats.BucketLog2Round - SwiftObjectPutUsecs bucketstats.BucketLog2Round + SwiftObjectDeleteUsecs bucketstats.BucketLog2Round + SwiftObjectGetUsecs bucketstats.BucketLog2Round + SwiftObjectGetRangeUsecs bucketstats.BucketLog2Round + SwiftObjectGetTailUsecs bucketstats.BucketLog2Round + SwiftObjectPutUsecs bucketstats.BucketLog2Round } const ( @@ -135,20 +134,20 @@ type mountStruct struct { } type volumeStruct struct { - sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() - name string // - storageURL string // - mountMap map[string]*mountStruct // key == mountStruct.mountID - healthyMountList *list.List // LRU of mountStruct's with .{leases|authToken}Expired == false - leasesExpiredMountList *list.List // list of mountStruct's with .leasesExpired == true (regardless of .authTokenExpired) value - authTokenExpiredMountList *list.List // list of mountStruct's with at .authTokenExpired == true (& .leasesExpired == false) - deleting bool // - checkPointHeader *ilayout.CheckPointHeaderV1Struct // == nil if not currently mounted and/or checkpointing - superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing - inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing - pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber - checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() - checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG + sync.RWMutex // must globals.{R|}Lock() before volume.{R|}Lock() + name string // + storageURL string // + mountMap map[string]*mountStruct // key == mountStruct.mountID + healthyMountList *list.List // LRU of mountStruct's with .{leases|authToken}Expired == false + leasesExpiredMountList *list.List // list of mountStruct's with .leasesExpired == true (regardless of .authTokenExpired) value + authTokenExpiredMountList *list.List // list of mountStruct's with at .authTokenExpired == true (& .leasesExpired == false) + deleting bool // + checkPoint *ilayout.CheckPointV1Struct // == nil if not currently mounted and/or checkpointing + superBlock *ilayout.SuperBlockV1Struct // == nil if not currently mounted and/or checkpointing + inodeTable sortedmap.BPlusTree // == nil if not currently mounted and/or checkpointing + pendingObjectDeleteSet map[uint64]struct{} // key == objectNumber + checkPointControlChan chan chan error // send chan error to chan to request a CheckPoint; close it to terminate checkPointDaemon() + checkPointControlWG sync.WaitGroup // checkPointDeamon() indicates it is done by calling .Done() on this WG } type globalsStruct struct { diff --git a/imgr/imgrpkg/http-server_test.go b/imgr/imgrpkg/http-server_test.go index 3f47a6972..bb8140897 100644 --- a/imgr/imgrpkg/http-server_test.go +++ b/imgr/imgrpkg/http-server_test.go @@ -16,7 +16,6 @@ func TestHTTPServer(t *testing.T) { var ( err error getRequestHeaders http.Header - getResponseHeaders http.Header postRequestBody string putRequestBody string responseBody []byte @@ -54,17 +53,22 @@ func TestHTTPServer(t *testing.T) { getRequestHeaders["X-Auth-Token"] = []string{testGlobals.authToken} - getResponseHeaders, responseBody, err = testDoHTTPRequest("GET", testGlobals.containerURL, getRequestHeaders, nil) + _, responseBody, err = testDoHTTPRequest("GET", testGlobals.containerURL, getRequestHeaders, nil) if nil != err { t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL, getRequestHeaders, nil) failed: %v", err) } - if getResponseHeaders.Get(ilayout.CheckPointHeaderName) != "0000000000000001 0000000000000003 0000000000000058 0000000000000003" { - t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL, getRequestHeaders, nil) returned unexpected %s: \"%s\"", ilayout.CheckPointHeaderName, getResponseHeaders.Get(ilayout.CheckPointHeaderName)) - } - if "0000000000000002\n0000000000000003\n" != string(responseBody[:]) { + if "0000000000000000\n0000000000000002\n0000000000000003\n" != string(responseBody[:]) { t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL, getRequestHeaders, nil) returned unexpected Object List: \"%s\"", string(responseBody[:])) } + _, responseBody, err = testDoHTTPRequest("GET", fmt.Sprintf("%s/%016X", testGlobals.containerURL, ilayout.CheckPointObjectNumber), getRequestHeaders, nil) + if nil != err { + t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL/ilayout.CheckPointObjectNumber, getRequestHeaders, nil) failed: %v", err) + } + if "0000000000000001 0000000000000003 0000000000000058 0000000000000003" != string(responseBody[:]) { + t.Fatalf("testDoHTTPRequest(\"GET\", testGlobals.containerURL/ilayout.CheckPointObjectNumber, getRequestHeaders, nil) returned unexpected Object List: \"%s\"", string(responseBody[:])) + } + putRequestBody = fmt.Sprintf("{\"StorageURL\":\"%s\"}", testGlobals.containerURL) _, _, err = testDoHTTPRequest("PUT", testGlobals.httpServerURL+"/volume/"+testVolume, nil, strings.NewReader(putRequestBody)) diff --git a/imgr/imgrpkg/retry-rpc.go b/imgr/imgrpkg/retry-rpc.go index eb213556e..db0f6fea0 100644 --- a/imgr/imgrpkg/retry-rpc.go +++ b/imgr/imgrpkg/retry-rpc.go @@ -73,16 +73,17 @@ func stopRetryRPCServer() (err error) { func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountResponse *MountResponseStruct) (err error) { var ( - alreadyInGlobalsMountMap bool - lastCheckPointHeader *ilayout.CheckPointHeaderV1Struct - lastCheckPointHeaderAsString string - mount *mountStruct - mountIDAsByteArray []byte - mountIDAsString string - ok bool - startTime time.Time - volume *volumeStruct - volumeAsValue sortedmap.Value + alreadyInGlobalsMountMap bool + lastCheckPoint *ilayout.CheckPointV1Struct + lastCheckPointAsByteSlice []byte + lastCheckPointAsString string + mount *mountStruct + mountIDAsByteArray []byte + mountIDAsString string + ok bool + startTime time.Time + volume *volumeStruct + volumeAsValue sortedmap.Value ) startTime = time.Now() @@ -117,13 +118,14 @@ func mount(retryRPCClientID uint64, mountRequest *MountRequestStruct, mountRespo return } - lastCheckPointHeaderAsString, err = swiftContainerHeaderGet(volume.storageURL, mountRequest.AuthToken, ilayout.CheckPointHeaderName) + lastCheckPointAsByteSlice, err = swiftObjectGet(volume.storageURL, mountRequest.AuthToken, ilayout.CheckPointObjectNumber) if nil != err { volume.Unlock() globals.Unlock() err = fmt.Errorf("%s %s", EAuthTokenRejected, mountRequest.AuthToken) return } + lastCheckPointAsString = string(lastCheckPointAsByteSlice[:]) retryGenerateMountID: @@ -149,12 +151,12 @@ retryGenerateMountID: globals.mountMap[mountIDAsString] = mount if nil == volume.checkPointControlChan { - lastCheckPointHeader, err = ilayout.UnmarshalCheckPointHeaderV1(lastCheckPointHeaderAsString) + lastCheckPoint, err = ilayout.UnmarshalCheckPointV1(lastCheckPointAsString) if nil != err { - logFatalf("ilayout.UnmarshalCheckPointHeaderV1(lastCheckPointHeaderAsString==\"%s\") failed: %v", lastCheckPointHeaderAsString, err) + logFatalf("ilayout.UnmarshalCheckPointV1(lastCheckPointAsString==\"%s\") failed: %v", lastCheckPointAsString, err) } - volume.checkPointHeader = lastCheckPointHeader + volume.checkPoint = lastCheckPoint volume.checkPointControlChan = make(chan chan error) diff --git a/imgr/imgrpkg/swift-client.go b/imgr/imgrpkg/swift-client.go index dcd620653..66e539d07 100644 --- a/imgr/imgrpkg/swift-client.go +++ b/imgr/imgrpkg/swift-client.go @@ -58,91 +58,32 @@ func stopSwiftClient() (err error) { return } -func swiftContainerHeaderGet(storageURL string, authToken string, headerName string) (headerValue string, err error) { +func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) (err error) { var ( httpRequest *http.Request httpResponse *http.Response nextSwiftRetryDelay time.Duration numSwiftRetries uint32 + objectURL string startTime time.Time ) startTime = time.Now() defer func() { - globals.stats.SwiftContainerHeaderGetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + globals.stats.SwiftObjectDeleteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() - nextSwiftRetryDelay = globals.config.SwiftRetryDelay - - for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - httpRequest, err = http.NewRequest("HEAD", storageURL, nil) - if nil != err { - return - } - - if "" != authToken { - httpRequest.Header["X-Auth-Token"] = []string{authToken} - } - - httpResponse, err = globals.httpClient.Do(httpRequest) - if nil != err { - err = fmt.Errorf("globals.httpClient.Do(HEAD %s) failed: %v\n", storageURL, err) - return - } - - _, err = ioutil.ReadAll(httpResponse.Body) - if nil != err { - err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) - return - } - err = httpResponse.Body.Close() - if nil != err { - err = fmt.Errorf("httpResponse.Body.Close() failed: %v\n", err) - return - } - - if (200 <= httpResponse.StatusCode) && (299 >= httpResponse.StatusCode) { - headerValue = httpResponse.Header.Get(headerName) - - err = nil - return - } - - time.Sleep(nextSwiftRetryDelay) - - nextSwiftRetryDelay = time.Duration(float64(nextSwiftRetryDelay) * globals.config.SwiftRetryExpBackoff) - } - - err = fmt.Errorf("globals.config.SwiftRetryLimit exceeded") - return -} - -func swiftContainerHeaderSet(storageURL string, authToken string, headerName string, headerValue string) (err error) { - var ( - httpRequest *http.Request - httpResponse *http.Response - nextSwiftRetryDelay time.Duration - numSwiftRetries uint32 - startTime time.Time - ) - - startTime = time.Now() - - defer func() { - globals.stats.SwiftContainerHeaderSetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) - }() + objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - httpRequest, err = http.NewRequest("POST", storageURL, nil) + httpRequest, err = http.NewRequest("DELETE", objectURL, nil) if nil != err { return } - httpRequest.Header[headerName] = []string{headerValue} - if "" != authToken { httpRequest.Header["X-Auth-Token"] = []string{authToken} } @@ -178,7 +119,7 @@ func swiftContainerHeaderSet(storageURL string, authToken string, headerName str return } -func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) (err error) { +func swiftObjectGet(storageURL string, authToken string, objectNumber uint64) (buf []byte, err error) { var ( httpRequest *http.Request httpResponse *http.Response @@ -191,7 +132,7 @@ func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) startTime = time.Now() defer func() { - globals.stats.SwiftObjectDeleteUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) + globals.stats.SwiftObjectGetUsecs.Add(uint64(time.Since(startTime) / time.Microsecond)) }() objectURL = fmt.Sprintf("%s/%016X", storageURL, objectNumber) @@ -199,7 +140,7 @@ func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) nextSwiftRetryDelay = globals.config.SwiftRetryDelay for numSwiftRetries = 0; numSwiftRetries <= globals.config.SwiftRetryLimit; numSwiftRetries++ { - httpRequest, err = http.NewRequest("DELETE", objectURL, nil) + httpRequest, err = http.NewRequest("GET", objectURL, nil) if nil != err { return } @@ -214,7 +155,7 @@ func swiftObjectDelete(storageURL string, authToken string, objectNumber uint64) return } - _, err = ioutil.ReadAll(httpResponse.Body) + buf, err = ioutil.ReadAll(httpResponse.Body) if nil != err { err = fmt.Errorf("ioutil.ReadAll(httpResponse.Body) failed: %v\n", err) return diff --git a/imgr/imgrpkg/volume.go b/imgr/imgrpkg/volume.go index 2a18072c3..9830b043a 100644 --- a/imgr/imgrpkg/volume.go +++ b/imgr/imgrpkg/volume.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "io" + "strings" "time" "github.com/NVIDIA/sortedmap" @@ -507,8 +508,8 @@ func (postVolumeSuperBlockInodeTableCallbacks *postVolumeSuperBlockInodeTableCal func postVolume(storageURL string, authToken string) (err error) { var ( - checkPointHeaderV1 *ilayout.CheckPointHeaderV1Struct - checkPointHeaderV1String string + checkPointV1 *ilayout.CheckPointV1Struct + checkPointV1String string inodeTable sortedmap.BPlusTree ok bool postVolumeRootDirDirectoryCallbacks *postVolumeRootDirDirectoryCallbacksStruct @@ -690,21 +691,21 @@ func postVolume(storageURL string, authToken string) (err error) { return } - // Create CheckPointHeader + // Create CheckPoint - checkPointHeaderV1 = &ilayout.CheckPointHeaderV1Struct{ - Version: ilayout.CheckPointHeaderVersionV1, + checkPointV1 = &ilayout.CheckPointV1Struct{ + Version: ilayout.CheckPointVersionV1, SuperBlockObjectNumber: superBlockObjectNumber, SuperBlockLength: uint64(len(superBlockV1Buf)), ReservedToNonce: reservedToNonce, } - checkPointHeaderV1String, err = checkPointHeaderV1.MarshalCheckPointHeaderV1() + checkPointV1String, err = checkPointV1.MarshalCheckPointV1() if nil != err { return } - err = swiftContainerHeaderSet(storageURL, authToken, ilayout.CheckPointHeaderName, checkPointHeaderV1String) + err = swiftObjectPut(storageURL, authToken, ilayout.CheckPointObjectNumber, strings.NewReader(checkPointV1String)) if nil != err { return } @@ -727,7 +728,7 @@ func putVolume(name string, storageURL string) (err error) { leasesExpiredMountList: list.New(), authTokenExpiredMountList: list.New(), deleting: false, - checkPointHeader: nil, + checkPoint: nil, superBlock: nil, inodeTable: nil, pendingObjectDeleteSet: make(map[uint64]struct{}), From baadd2fd54455b87d5a49e86ed78746717f97676 Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Thu, 6 May 2021 14:07:40 -0700 Subject: [PATCH 158/160] When reacquiring client lock, check again if generation number changed (#622) --- retryrpc/client.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/retryrpc/client.go b/retryrpc/client.go index 8d1323cc1..5f5d307ac 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -269,6 +269,15 @@ func (client *Client) notifyReply(buf []byte, genNum uint64, recvResponse time.T } client.Lock() + + // We dropped the lock above so we have to again check if we hit a retransmit case + // and the socket was closed. + // + // If this message is from an old socket - throw it away since the request was resent. + if client.connection.genNum != genNum { + client.Unlock() + return + } delete(client.outstandingRequest, crID) client.Unlock() From 5bf47fdfd8d68b48c54e169a902ae90ceb9b8457 Mon Sep 17 00:00:00 2001 From: bschatz-swift Date: Fri, 7 May 2021 17:18:04 -0700 Subject: [PATCH 159/160] Close race condition window (#623) * Add additional check when reacquiring lock. --- retryrpc/client.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/retryrpc/client.go b/retryrpc/client.go index 5f5d307ac..a7e20caff 100644 --- a/retryrpc/client.go +++ b/retryrpc/client.go @@ -278,6 +278,14 @@ func (client *Client) notifyReply(buf []byte, genNum uint64, recvResponse time.T client.Unlock() return } + + _, ok = client.outstandingRequest[crID] + if !ok { + // Saw reply for request which is no longer on outstandingRequest list + // Can happen if handling retransmit + client.Unlock() + return + } delete(client.outstandingRequest, crID) client.Unlock() From 64e1e98545009cff30dedc650c613cd0dd49c29e Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 12 May 2021 10:59:49 -0700 Subject: [PATCH 160/160] Release notes for 1.19.0 --- release_notes.md | 115 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/release_notes.md b/release_notes.md index 81d93d26f..0d888f88d 100644 --- a/release_notes.md +++ b/release_notes.md @@ -1,5 +1,120 @@ # ProxyFS Release Notes +## 1.19.0 (May 12, 2021) + +### Bug Fixes: + +TOX fixes for WSGI middleware in support of move to Python3. + +Closed file state issues in PFSAgent when lookup(), readdirplus(), +and stat() operations are performed alongside write() activity. +Similar issues are resolved between lookup() and readdirplus() +operations in the presence of parallel directory modifying operations. +There was also a case where doing a simple file read() would lose +stat() information when performed in the presence of write() operations. + +### Features: + +New support for TLS connections to etcd added. + +PFSAgent's connection to ProxyFS for metadata operations was +previously transitioned to TLS. Now, TLS is optional (though, +of course, highly recommended). + +Replaced the auth plugin model with a new/simpler Golang plugin +mechanism. Previously, the auth plugin was actually a standalone +process launched by PFSAgent. As such, it could be written in a +variety of languages. With the new Golang plugin model, the plugin +must, of course, be written in Golang (or, at least, provide a +Golang-callable function that, perhaps, wraps an implementation +in another language). + +A new implementation of Swift emulation, complete with Swift Auth +support, has been added (`iswift`). The tool is implemented with the +pattern of its implementation being in a package, `iswiftpkg`, below +`iswift` in the repository source tree enabling easy integration +with Golang code (e.g. tests) that need easy Swift emulation. + +For easy generation of CA and Endpoint certificates (along the lines +of what openssl provides), a new `icert` tool has been added. Similar +to that of `iswift`, the implementation of `icert` is in subdirectory +package `icertpkg`. + +A new RpcLease benchmark has been added. It is anticipated that +Lease Management becomes the scale out bottleneck in the move to +PFSAgent usage. Hence, it is important to quantify Lease performance +in the target environment in order to anticipate file system +performance limits at least in metadata heavy workloads. Presumably, +bandwidth limitations will be proportional to the scale out of +PFSAgent instances as well as the underlying Swift Object Store +solution. + +### Notes: + +Many changes relate to the migration of this and other repositories +from the old github.com/swiftstack/ tree to github.com/NVIDIA/ tree. + +As the move to only support PFSAgent access to file systems (perhaps +eventually restoring BiModal access as well), support for SMB and NFS +is deprecated. This removes the tight linkage between ProxyFS and Samba +necessitated by the former vfs submodule's support for now obsolete +versions of Samba. + +To restore SMB & NFS support, simply deploy PFSAgent (perhaps adjacent +to ProxyFS) and use your distribution's Samba and NFSd support to export +the mount point presented by PFSAgent. + +This release introduces a lot of functionality development work targeted +at the 2.0 version of ProxyFS. In short, with the move to supporting +file systems only exposed by PFSAgent, the architecture of ProxyFS has +proved inappriate in that it majorly supported data path operations that +will no longer be used. Even metadata operations triggered by PFSAgent +will now be moved to PFSAgent. All that remains in ProxyFS going forward +is the centralized Lease Management functionality and the checkpoint +operation. Further, the checkpoints only involve sync'ing the Inode Table +that is, itself, greatly simplified (now only holding a "pointer" to +where the state of an Inode can be found). + +These changes, set to become operational in the 2.0 version of ProxyFS, +are being implemented in a parallel set of subdirectories. These mostly +follow the patter described for `iswift` and `icert` above. Specifically, +`imgr` is the upcoming implementation of the server side (replacing +`proxyfsd`) while `iclient` is the upcoming implementation of the client +side (replacing `pfsagentd`). The work is well underway and, indeed, the +bulk of the commits since 1.18.0, cover that effort. At this time, only +a small subset of the `imgr` work is operational. None of `iclient` has +been implemented. + +One big switch in the 2.0 model is a move away from Swift Accounts as +the top level of a ProxyFS file system. At the time of its adoption, +the Swift Account was chosen so that ProxyFS could load balance its +creation of Objects across many Swift Containers. At that time, Swift +had modest limits on the number of Objects that could reside in a single +Container, so this choice was then necessary. But Swift has dramatically +improved support for very large numbers of Objects in a Container since +that time. Making this move to a ProxyFS file system matching a Swift +Container instead of a Swift Account adjusts to this new capability. It +also avoids the unfortunate confusion over files and symlinks placed in +the root of a ProxyFS file system were not visible via BiModal (Swift or +S3 API) access. Finally, this move better aligns with supporting deployments +where only S3 API access is used and, hence, the whole concept of a Swift +Account is not involved (since S3 Buckets map to Swift Accounts). + +To get a good handle on how the 2.0 file system will "look" in terms +of "on disk" representation, a new `ilayout` package is provided. It +is believed to be complete. It describes how the *CheckPoint* Object +(now Object Zero; previously a Header on the checkpoint container) +points to the file system *SuperBlock*. From there, the *InodeTable* +is described as well as the format of the metadata portion of each +*Inode*. The `ilayout` package also introduces a new pattern of heavy +use of Golang *godoc* mechanisms to make browsing of the data structures +and APIs more convenient. This pattern will be applied across all of the +new 2.0 components (see in particular `imgrpkg` underneath `imgr` for a +full description of how to interact with the new server side of ProxyFS). + +In the meantime, the pre-2.0 functionality will be maintained in their +usual subdirectories (i.e. `proxyfsd` and `pfsagentd`). + ## 1.18.0 (January 21, 2021) ### Bug Fixes: