Merge branch 'release-1.16.0' into stable

.travis.yml

@@ -8,7 +8,7 @@ git:
 notifications:
   slack:
     rooms:
-      - secure: "wd0TnzxzZpO1hSihz6tyAbSJNijYru7HvCLIT5dq8N0JS0Bu6qnSuMD/mKs6oXegWR2hMoujtn64YLrnWUNImZFJEIK6s/Ai4bYTvCfI4z+KJ+R3/MliDCqylLO2Loxpreojji8BW76BwzwRrmAxAwhZKJFUfdE6zVLdkqWQWDTneHgOx8BX+rtEbxJ2/zpFBys5Ilq5gLsQbnuUugLEDnAxGlstDT5odZEAbwaW33P0xIlUnyZ4fI8U/CUlF1Vfifg+mJEjDHa+xO6JThZudoymyyQ7ODDfJ9rjPUCXiIhX+iAItNq3qQC6KXuXPPcvm8q4zf1lUTLW80aao4EikpGkceBdZzDIa5X1/9AV0GkIFfoYp5h1WVe7PpbYGrtFY1fJNIELqWrAZPyf7lzNHc9Tw/JLq+MB5zAJI4uH3eh98y90DoCrw3Nj32nCo7KJDIFEzJWiuT3VcYQG1ZoLVuAfkWt9uDYbWUvqJ/STFmPkfzAcmtFDU70twpaBgZIM+XVoUl5CjlxK3Lhrq5xwBIqQOyjpAVx0X0ZmVH5OsWtlT7qq863TunWc638fGnzkZZ+CEm1ekeMNzuxUx27cqkWDQr72yGzuqf+x3lvhZjqnMYjXwcg8SEck2zkwoeLuBFWZFMabNm+L/G04AHfCjAJ1wR0pf1g3tpRp9cIiI4I="
+      - secure: "qd9CCq5BJXolPXPvqXaeUcz1l1PchWO25efOhEyH0fE3SJfZVfAymUHLXblgd7GDD/lMJCJOngpBBiaDw/D2lBbTxHsZUliSKo5esg7Bxwm3MVNK3J5EKl63TPiLsbxrhAqcmrTf5DJHclJSnrZggv9RePZbZGNx+ByinC0DexZDps86kDK0KSlX2cox3wiuw4kRDyj37J+MY2o3OFk6riMqGBjnY0+ct3N5QN9NBkfWUQZ6T2pZ5X3+v12e6Q013wexfzOuRR/sJc4QXIfG6cJkR7JgE7CGxXVbqK1GGwHiKa1r0MIvOZQ5ggiCuHQq6ryowWLIk+0fu9y62HJ8tpCCNYz538XZAGDPxKQpxBGRtikgHwp3yfKZbQcHOJzFOLmtK9oYVV7WC6q6/dzYLpZLMm83F0Lz1blRoqtdvI6G/RjoT96p6GzptYcrdyLwh5/mJtuq49BAHeJQiGkN8XRybpz72dDYYfod9O4YW8L3xmjouIpMr9i/bw3PKC8PrynABo7ZCOLbBnBd/JpF45woTQhazTPqxWRVe0e6wFfFFKyG43jj4v2DVdHQ5xAsUPWdEsV2UfwwJFk71wPcjkXMC6I3ii19AUmhrM3CoWX4RKDR//P4h08kwfHm7/Jg+odCdoCc5qYhXYgRsR4HT2Vq3k8CtNZ/lPxEclPaMlQ="
     on_success: change
     on_failure: always
     on_pull_requests: true

CONTRIBUTORS.md

@@ -1,11 +1,10 @@
-
-
 In addition to the open-source contributors who can be found in the revision history of this project, the ProxyFS team would also like 
 to thank the following contributors to the ProxyFS project:
 
-- The indefatigable **Kim Malone** for her extensive work in the early, pre-open-source phases of development on the jrrpc layer and other areas of the code. 
+- The abductive **Balaji Rao** for his insightful design and implementation in the early, pre-open-source phases of development in many areas core to the file system.
+- The indefatigable **Kim Malone** for her extensive work in the early, pre-open-source phases of development on the jrrpc layer and other areas of the code.
 - The incomparable **Zack M. Davis** for his effort in protoyping and investigations in early phases of the project and implementations and
 testing in later phases.
 - **Didier M. Contis'** team at Georgia Tech, who display a rare combination of inquisitiveness and patience, and have been (and continue to be) a great 
 help throughout the development of this software.  We would like in particular to call out **Asu Ogork** and **Paul Manno** for their help testing,
-benchmarking, and representing the concerns of real administrators with complex deployment scenarios. 
+benchmarking, and representing the concerns of real administrators with complex deployment scenarios.

Makefile

@@ -36,11 +36,13 @@ gobinsubdirs = \
 	inodeworkout \
 	pfs-crash \
 	pfs-fsck \
-	pfs-retryrpc \
 	pfs-restart-test \
 	pfs-stress \
 	pfs-swift-load \
 	pfsagentd \
+	pfsagentd/pfsagentd-swift-auth-plugin \
+	pfsagentConfig \
+	pfsagentConfig/pfsagentConfig \
 	pfsconfjson \
 	pfsconfjsonpacked \
 	pfsworkout \
@@ -51,7 +53,6 @@ gobinsubdirs = \
 	ramswift/ramswift
 
 gobinsubdirsforci = \
-	pfsagentd \
 	pfsconfjson \
 	pfsconfjsonpacked \
 	confgen/confgen \
@@ -193,7 +194,12 @@ test:
 		$(MAKE) --no-print-directory -C $$gosubdir test; \
 	done; \
 	for gosubdir in $(gobinsubdirs); do \
-		$(MAKE) --no-print-directory -C $$gosubdir test; \
+		if [ $$gosubdir == "pfsagentd" ]; \
+		then \
+			echo "Skipping pfsagentd"; \
+		else \
+			$(MAKE) --no-print-directory -C $$gosubdir test; \
+		fi \
 	done
 
 version:

ci/ansible/chef_files/packages/rhel_ss.json

@@ -5,12 +5,12 @@
   "samba_deps": [
     ["gcc", "4.8.5-39.el7"],
     ["gcc-c++", "4.8.5-39.el7"],
-    ["python-devel", "2.7.5-86.el7"],
+    ["python-devel", "2.7.5-88.el7"],
     ["gnutls-devel", "3.3.29-9.el7_6"],
-    ["libacl-devel", "2.2.51-14.el7"],
+    ["libacl-devel", "2.2.51-15.el7"],
     ["openldap-devel", "2.4.44-21.el7_6"],
     ["cifs-utils", "6.2-10.el7"],
-    ["pam-devel", "1.1.8-22.el7"]
+    ["pam-devel", "1.1.8-23.el7"]
   ],
   "proxyfs_packages": [
     ["json-c-devel", "0.11-4.el7_0"],
@@ -24,10 +24,10 @@
     ["sshpass"]
   ],
   "nfs_packages": [
-    ["nfs-utils", "1.3.0-0.65.el7"]
+    ["nfs-utils", "1.3.0-0.66.el7"]
   ],
   "gdb_packages": [
-    ["gdb", "7.6.1-115.el7"],
+    ["gdb", "7.6.1-119.el7"],
     ["yum-utils"]
   ],
   "utils_packages": [

ci/ansible/tasks/main.yml

@@ -4,7 +4,7 @@
     - "../vars/{{ env }}.yml"
   vars:
     pkg_ver:
-      etcd: "3.3.10"
+      etcd: "3.4.7"
     golang_versions:
       current: "go1.13.6"
       candidate: "go1.13.6"

conf/api.go

@@ -58,7 +58,7 @@ const rightBracket = "(\\])"
 const sectionName = "([0-9A-Za-z_\\-/:\\.]+)"
 const separator = "([ \t]+|([ \t]*,[ \t]*))"
 const token = "(([0-9A-Za-z_\\*\\-/:\\.\\[\\]]+)\\$?)"
-const value = "(([0-9A-Za-z_=\\*\\-\\+/:\\.\\[\\]]+)\\$?)"
+const value = "(([0-9A-Za-z_=\\*\\-\\+/:\\.\\[\\]\\\"\\{\\}\\\\]+)\\$?)"
 const whiteSpace = "([ \t]+)"
 
 // A string to load looks like:
@@ -159,7 +159,7 @@ func (confMap ConfMap) UpdateFromString(confString string) (err error) {
 		confMap[sectionName] = section
 	}
 
-	section[optionName] = optionValuesSplit
+	section[optionName] = replaceUTF8SpacesAndCommasInStrings(optionValuesSplit)
 
 	// If we reach here, confString successfully processed
 
@@ -229,7 +229,7 @@ func (confMap ConfMap) UpdateFromFile(confFilePath string) (err error) {
 		if '\n' == lastRune {
 			// Terminate currentLine adding (non-empty) trimmed version to confFileLines
 
-			currentLineNumber += 1
+			currentLineNumber++
 
 			if confFileBytesLineOffsetStart < confFileBytesOffset {
 				currentLine = string(confFileBytes[confFileBytesLineOffsetStart:confFileBytesOffset])
@@ -314,7 +314,7 @@ func (confMap ConfMap) UpdateFromFile(confFilePath string) (err error) {
 							confMap[currentSectionName] = currentSection
 						}
 
-						currentSection[optionName] = optionValuesSplit
+						currentSection[optionName] = replaceUTF8SpacesAndCommasInStrings(optionValuesSplit)
 					}
 				}
 			}
@@ -411,7 +411,25 @@ func (confMap ConfMap) Dump() (buf string) {
 	return
 }
 
-// VerifyOptionValueIsEmpty returns an error if [sectionName]valueName's string value is not empty
+// VerifyOptionIsMissing returns an error if [sectionName]optionName exists
+func (confMap ConfMap) VerifyOptionIsMissing(sectionName string, optionName string) (err error) {
+	section, ok := confMap[sectionName]
+	if !ok {
+		err = fmt.Errorf("[%v] missing", sectionName)
+		return
+	}
+
+	_, ok = section[optionName]
+	if ok {
+		err = fmt.Errorf("[%v]%v exists", sectionName, optionName)
+	} else {
+		err = nil
+	}
+
+	return
+}
+
+// VerifyOptionValueIsEmpty returns an error if [sectionName]optionName's value is not empty
 func (confMap ConfMap) VerifyOptionValueIsEmpty(sectionName string, optionName string) (err error) {
 	section, ok := confMap[sectionName]
 	if !ok {
@@ -434,7 +452,7 @@ func (confMap ConfMap) VerifyOptionValueIsEmpty(sectionName string, optionName s
 	return
 }
 
-// FetchOptionValueStringSlice returns [sectionName]valueName's string values as a []string
+// FetchOptionValueStringSlice returns [sectionName]optionName's string values as a []string
 func (confMap ConfMap) FetchOptionValueStringSlice(sectionName string, optionName string) (optionValue []string, err error) {
 	optionValue = []string{}
 
@@ -455,7 +473,7 @@ func (confMap ConfMap) FetchOptionValueStringSlice(sectionName string, optionNam
 	return
 }
 
-// FetchOptionValueString returns [sectionName]valueName's single string value
+// FetchOptionValueString returns [sectionName]optionName's single string value
 func (confMap ConfMap) FetchOptionValueString(sectionName string, optionName string) (optionValue string, err error) {
 	optionValue = ""
 
@@ -475,7 +493,7 @@ func (confMap ConfMap) FetchOptionValueString(sectionName string, optionName str
 	return
 }
 
-// FetchOptionValueBase64String returns [sectionName]valueName's single string value Base64-decoded
+// FetchOptionValueBase64String returns [sectionName]optionName's single string value Base64-decoded
 func (confMap ConfMap) FetchOptionValueBase64String(sectionName string, optionName string) (optionValue string, err error) {
 	var (
 		base64DecodedOptionValue []byte
@@ -497,7 +515,7 @@ func (confMap ConfMap) FetchOptionValueBase64String(sectionName string, optionNa
 	return
 }
 
-// FetchOptionValueBase64StringSlice returns [sectionName]valueName's string values as a []string each element Base64-decoded
+// FetchOptionValueBase64StringSlice returns [sectionName]optionName's string values as a []string each element Base64-decoded
 func (confMap ConfMap) FetchOptionValueBase64StringSlice(sectionName string, optionName string) (optionValue []string, err error) {
 	var (
 		base64DecodedOptionValueElement []byte
@@ -525,7 +543,7 @@ func (confMap ConfMap) FetchOptionValueBase64StringSlice(sectionName string, opt
 	return
 }
 
-// FetchOptionValueBool returns [sectionName]valueName's single string value converted to a bool
+// FetchOptionValueBool returns [sectionName]optionName's single string value converted to a bool
 func (confMap ConfMap) FetchOptionValueBool(sectionName string, optionName string) (optionValue bool, err error) {
 	optionValueString, err := confMap.FetchOptionValueString(sectionName, optionName)
 	if nil != err {
@@ -556,7 +574,7 @@ func (confMap ConfMap) FetchOptionValueBool(sectionName string, optionName strin
 	return
 }
 
-// FetchOptionValueUint8 returns [sectionName]valueName's single string value converted to a uint8
+// FetchOptionValueUint8 returns [sectionName]optionName's single string value converted to a uint8
 func (confMap ConfMap) FetchOptionValueUint8(sectionName string, optionName string) (optionValue uint8, err error) {
 	optionValue = 0
 
@@ -577,7 +595,7 @@ func (confMap ConfMap) FetchOptionValueUint8(sectionName string, optionName stri
 	return
 }
 
-// FetchOptionValueUint16 returns [sectionName]valueName's single string value converted to a uint16
+// FetchOptionValueUint16 returns [sectionName]optionName's single string value converted to a uint16
 func (confMap ConfMap) FetchOptionValueUint16(sectionName string, optionName string) (optionValue uint16, err error) {
 	optionValue = 0
 
@@ -598,7 +616,7 @@ func (confMap ConfMap) FetchOptionValueUint16(sectionName string, optionName str
 	return
 }
 
-// FetchOptionValueUint32 returns [sectionName]valueName's single string value converted to a uint32
+// FetchOptionValueUint32 returns [sectionName]optionName's single string value converted to a uint32
 func (confMap ConfMap) FetchOptionValueUint32(sectionName string, optionName string) (optionValue uint32, err error) {
 	optionValue = 0
 
@@ -619,7 +637,7 @@ func (confMap ConfMap) FetchOptionValueUint32(sectionName string, optionName str
 	return
 }
 
-// FetchOptionValueUint64 returns [sectionName]valueName's single string value converted to a uint64
+// FetchOptionValueUint64 returns [sectionName]optionName's single string value converted to a uint64
 func (confMap ConfMap) FetchOptionValueUint64(sectionName string, optionName string) (optionValue uint64, err error) {
 	optionValue = 0
 
@@ -640,7 +658,7 @@ func (confMap ConfMap) FetchOptionValueUint64(sectionName string, optionName str
 	return
 }
 
-// FetchOptionValueFloat32 returns [sectionName]valueName's single string value converted to a float32
+// FetchOptionValueFloat32 returns [sectionName]optionName's single string value converted to a float32
 func (confMap ConfMap) FetchOptionValueFloat32(sectionName string, optionName string) (optionValue float32, err error) {
 	optionValueString, err := confMap.FetchOptionValueString(sectionName, optionName)
 	if nil != err {
@@ -658,7 +676,7 @@ func (confMap ConfMap) FetchOptionValueFloat32(sectionName string, optionName st
 	return
 }
 
-// FetchOptionValueFloat64 returns [sectionName]valueName's single string value converted to a float32
+// FetchOptionValueFloat64 returns [sectionName]optionName's single string value converted to a float32
 func (confMap ConfMap) FetchOptionValueFloat64(sectionName string, optionName string) (optionValue float64, err error) {
 	optionValueString, err := confMap.FetchOptionValueString(sectionName, optionName)
 	if nil != err {
@@ -675,7 +693,7 @@ func (confMap ConfMap) FetchOptionValueFloat64(sectionName string, optionName st
 	return
 }
 
-// FetchOptionValueFloatScaledToUint32 returns [sectionName]valueName's single string value converted to a float64, multiplied by the uint32 multiplier, as a uint32
+// FetchOptionValueFloatScaledToUint32 returns [sectionName]optionName's single string value converted to a float64, multiplied by the uint32 multiplier, as a uint32
 func (confMap ConfMap) FetchOptionValueFloatScaledToUint32(sectionName string, optionName string, multiplier uint32) (optionValue uint32, err error) {
 	optionValue = 0
 
@@ -708,7 +726,7 @@ func (confMap ConfMap) FetchOptionValueFloatScaledToUint32(sectionName string, o
 	return
 }
 
-// FetchOptionValueFloatScaledToUint64 returns [sectionName]valueName's single string value converted to a float64, multiplied by the uint64 multiplier, as a uint64
+// FetchOptionValueFloatScaledToUint64 returns [sectionName]optionName's single string value converted to a float64, multiplied by the uint64 multiplier, as a uint64
 func (confMap ConfMap) FetchOptionValueFloatScaledToUint64(sectionName string, optionName string, multiplier uint64) (optionValue uint64, err error) {
 	optionValue = 0
 
@@ -741,7 +759,7 @@ func (confMap ConfMap) FetchOptionValueFloatScaledToUint64(sectionName string, o
 	return
 }
 
-// FetchOptionValueDuration returns [sectionName]valueName's single string value converted to a time.Duration
+// FetchOptionValueDuration returns [sectionName]optionName's single string value converted to a time.Duration
 func (confMap ConfMap) FetchOptionValueDuration(sectionName string, optionName string) (optionValue time.Duration, err error) {
 	optionValueString, err := confMap.FetchOptionValueString(sectionName, optionName)
 	if nil != err {
@@ -763,7 +781,7 @@ func (confMap ConfMap) FetchOptionValueDuration(sectionName string, optionName s
 	return
 }
 
-// FetchOptionValueUUID returns [sectionName]valueName's single string value converted to a UUID ([16]byte)
+// FetchOptionValueUUID returns [sectionName]optionName's single string value converted to a UUID ([16]byte)
 //
 // From RFC 4122, a UUID string is defined as follows:
 //
@@ -936,3 +954,29 @@ func (confMap ConfMap) DumpConfMapToFile(confFilePath string, perm os.FileMode)
 
 	return // err as returned from ioutil.WriteFile() suffices here
 }
+
+func replaceUTF8SpacesAndCommasInString(src string) (dst string) {
+	dst = strings.ReplaceAll(src, "\\u0020", " ")
+	dst = strings.ReplaceAll(dst, "\\U0020", " ")
+	dst = strings.ReplaceAll(dst, "\\u002C", ",")
+	dst = strings.ReplaceAll(dst, "\\U002C", ",")
+	dst = strings.ReplaceAll(dst, "\\u002c", ",")
+	dst = strings.ReplaceAll(dst, "\\U002c", ",")
+
+	return
+}
+
+func replaceUTF8SpacesAndCommasInStrings(src []string) (dst []string) {
+	var (
+		element      string
+		elementIndex int
+	)
+
+	dst = make([]string, len(src))
+
+	for elementIndex, element = range src {
+		dst[elementIndex] = replaceUTF8SpacesAndCommasInString(element)
+	}
+
+	return
+}

conf/api_test.go

@@ -397,13 +397,22 @@ func TestFetch(t *testing.T) {
 		t.Fatalf("Couldn't add TestNamespace:Test_-_Section.Test_-_OptionGUIDString=12345678-1234-1234-1234-123456789ABC: %v", err)
 	}
 
+	err = confMap.VerifyOptionIsMissing("TestNamespace:Test_-_Section", "Test_-_OptionStringSlice0")
+	if nil != err {
+		t.Fatalf("TestNamespace:Test_-_Section.Test_-_OptionStringSlice0 should have verified as missing")
+	}
+	err = confMap.VerifyOptionIsMissing("TestNamespace:Test_-_Section", "Test_-_OptionStringSlice1")
+	if nil == err {
+		t.Fatalf("TestNamespace:Test_-_Section.Test_-_OptionStringSlice1 should have verified as not missing")
+	}
+
 	err = confMap.VerifyOptionValueIsEmpty("TestNamespace:Test_-_Section", "Test_-_OptionStringSlice1")
 	if nil != err {
 		t.Fatalf("TestNamespace:Test_-_Section.Test_-_OptionStringSlice1 should have verified as empty")
 	}
 	err = confMap.VerifyOptionValueIsEmpty("TestNamespace:Test_-_Section", "Test_-_OptionStringSlice2")
 	if nil == err {
-		t.Fatalf("TestNamespace:Test_-_Section.Test_-_OptionStringSlice2 should have verified as empty")
+		t.Fatalf("TestNamespace:Test_-_Section.Test_-_OptionStringSlice2 should have verified as not empty")
 	}
 
 	testStringSlice1, err := confMap.FetchOptionValueStringSlice("TestNamespace:Test_-_Section", "Test_-_OptionStringSlice1")

confgen/api_internal.go

@@ -320,17 +320,11 @@ func computeInitial(envMap EnvMap, confFilePath string, confOverrides []string,
 
 func computePhases(envMap EnvMap, initialDirPath string, confFilePath string, confOverrides []string, phaseOneDirPath string, phaseTwoDirPath string) (err error) {
 	var (
-		envSettings     *envSettingsStruct
 		initialConfMap  conf.ConfMap
 		phaseOneConfMap conf.ConfMap
 		phaseTwoConfMap conf.ConfMap
 	)
 
-	// Fetch environ settings
-
-	envSettings = fetchEnvironSettings(envMap)
-	fmt.Println("UNDO:", envSettings)
-
 	// Load config from initialDirPath
 
 	initialConfMap, err = conf.MakeConfMapFromFile(initialDirPath + "/" + proxyFSConfFileName)

cookbooks/proxyfs/recipes/default.rb

@@ -499,6 +499,13 @@
   group proxyfs_group
 end
 
+link '/usr/bin/pfsagentd-swift-auth-plugin' do
+  to "#{source_root}/bin/pfsagentd-swift-auth-plugin"
+  link_type :symbolic
+  owner proxyfs_user
+  group proxyfs_group
+end
+
 if ss_packages
   # Creating link to jrpcclient's libs into the new /opt/ss path
   link '/opt/ss/lib64/libproxyfs.so.1.0.0' do