Merge branch 'release-1.15.4.1' into stable

.gitignore

@@ -54,6 +54,7 @@ _testmain.go
 # test coverage artifacts (see https://blog.golang.org/cover#TOC_5.)
 coverage.out
 count.out
+coverage.coverprofile
 
 # log files
 *.log

.travis.yml

@@ -18,4 +18,4 @@ before_install:
 - git submodule update --init --recursive
 
 script:
-- docker run --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/swiftstack/ProxyFS swiftstack/proxyfs_unit_tests
+- docker run -e "COVERALLS_TOKEN=$COVERALLS_TOKEN" -e "TRAVIS_BRANCH=$TRAVIS_BRANCH" --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/swiftstack/ProxyFS swiftstack/proxyfs_unit_tests

GoMakefile

@@ -15,7 +15,7 @@ clean:
 	done
 
 cover:
-	go test -cover $(gosubdir)
+	go test $(gosubdir) -covermode=atomic -coverprofile=coverage.coverprofile
 
 fmt:
 	go fmt $(gosubdir)

Makefile

@@ -32,7 +32,6 @@ gopkgsubdirs = \
 
 gobinsubdirs = \
 	cleanproxyfs \
-	confgen/confgen \
 	fsworkout \
 	inodeworkout \
 	pfs-crash \
@@ -45,35 +44,53 @@ gobinsubdirs = \
 	pfsconfjson \
 	pfsconfjsonpacked \
 	pfsworkout \
+	confgen/confgen \
 	evtlog/pfsevtlogd \
 	mkproxyfs/mkproxyfs \
 	proxyfsd/proxyfsd \
 	ramswift/ramswift
 
+gobinsubdirsforci = \
+	pfsagentd \
+	pfsconfjson \
+	pfsconfjsonpacked \
+	confgen/confgen \
+	mkproxyfs/mkproxyfs \
+	proxyfsd/proxyfsd
+
+gosubdirsforci = $(gopkgsubdirs) $(gobinsubdirsforci);
+gosubdirspathsforci = $(addprefix github.com/swiftstack/ProxyFS/,$(gosubdirsforci))
+
 uname = $(shell uname)
 machine = $(shell uname -m)
 
 ifeq ($(uname),Linux)
     ifeq ($(machine),armv7l)
         all: version fmt pre-generate generate install test
 
+        ci: version fmt pre-generate generate install test cover
+
         minimal: pre-generate generate install
     else
         distro := $(shell python -c "import platform; print platform.linux_distribution()[0]")
 
         all: version fmt pre-generate generate install test python-test c-clean c-build c-install c-test
 
+        ci: version fmt pre-generate generate install test cover python-test c-clean c-build c-install c-test
+
         all-deb-builder: version fmt pre-generate generate install c-clean c-build c-install-deb-builder
 
         minimal: pre-generate generate install c-clean c-build c-install
     endif
 else
     all: version fmt pre-generate generate install test
 
+    ci: version fmt pre-generate generate install test cover
+
     minimal: pre-generate generate install
 endif
 
-.PHONY: all all-deb-builder bench c-build c-clean c-install c-install-deb-builder c-test clean cover fmt generate install pre-generate python-test test version
+.PHONY: all all-deb-builder bench c-build c-clean c-install c-install-deb-builder c-test ci clean cover fmt generate install pre-generate python-test test version
 
 bench:
 	@set -e; \
@@ -128,12 +145,14 @@ clean:
 
 cover:
 	@set -e; \
-	for gosubdir in $(gopkgsubdirs); do \
-		$(MAKE) --no-print-directory -C $$gosubdir cover; \
-	done; \
-	for gosubdir in $(gobinsubdirs); do \
-		$(MAKE) --no-print-directory -C $$gosubdir cover; \
-	done
+	go-acc -o coverage.coverprofile $(gosubdirspathsforci)
+# TODO: We're not sure yet how we want to run coverage. Once decided, remove any extra code/comments
+#	for gosubdir in $(gopkgsubdirs); do \
+#		$(MAKE) --no-print-directory -C $$gosubdir cover; \
+#	done; \
+#	for gosubdir in $(gobinsubdirsforci); do \
+#		$(MAKE) --no-print-directory -C $$gosubdir cover; \
+#	done
 
 fmt:
 	@set -e; \
@@ -166,6 +185,7 @@ install:
 pre-generate:
 	@set -e; \
 	go install github.com/swiftstack/ProxyFS/vendor/golang.org/x/tools/cmd/stringer; \
+	go get -u github.com/ory/go-acc; \
 	for gosubdir in $(gopregeneratesubdirs); do \
 		$(MAKE) --no-print-directory -C $$gosubdir install; \
 	done

README.md

@@ -1,4 +1,5 @@
 [![Build Status](https://travis-ci.com/swiftstack/ProxyFS.svg?branch=development)](https://travis-ci.com/swiftstack/ProxyFS)
+[![Coverage Status](https://coveralls.io/repos/github/swiftstack/ProxyFS/badge.svg?branch=development)](https://coveralls.io/github/swiftstack/ProxyFS?branch=development)
 
 # ProxyFS
 Integrated File and Object Access for Swift Object Storage

pfsagentd/file_inode.go

@@ -182,9 +182,14 @@ func (chunkedPutContext *chunkedPutContextStruct) sendDaemon() {
 
 			goto PerformFlush
 		case _, sendChanOpenOrNonEmpty = <-chunkedPutContext.sendChan:
-			// Send non-flushing chunk to *chunkedPutContextStruct.Read()
+			// Send chunk to *chunkedPutContextStruct.Read()
 
-			chunkedPutContext.wakeChan <- false
+			select {
+			case chunkedPutContext.wakeChan <- struct{}{}:
+				// We just notified Read()
+			default:
+				// We didn't need to notify Read()
+			}
 
 			if !sendChanOpenOrNonEmpty {
 				goto PerformFlush
@@ -194,9 +199,9 @@ func (chunkedPutContext *chunkedPutContextStruct) sendDaemon() {
 
 PerformFlush:
 
-	// Send flushing chunk to *chunkedPutContextStruct.Read() & wait for it to finish
+	// Tell *chunkedPutContextStruct.Read() to finish & wait for it to finish
 
-	chunkedPutContext.wakeChan <- true
+	close(chunkedPutContext.wakeChan)
 	chunkedPutContext.Wait()
 
 	// Chunked PUT is complete
@@ -395,13 +400,11 @@ func (chunkedPutContext *chunkedPutContextStruct) complete() {
 
 func (chunkedPutContext *chunkedPutContextStruct) Read(p []byte) (n int, err error) {
 	var (
-		grantedLock *fileInodeLockRequestStruct
+		grantedLock            *fileInodeLockRequestStruct
+		wakeChanOpenOrNonEmpty bool
 	)
 
-	chunkedPutContext.inRead = true
-	defer func() {
-		chunkedPutContext.inRead = false
-	}()
+	_, wakeChanOpenOrNonEmpty = <-chunkedPutContext.wakeChan
 
 	grantedLock = chunkedPutContext.fileInode.getExclusiveLock()
 
@@ -430,29 +433,32 @@ func (chunkedPutContext *chunkedPutContextStruct) Read(p []byte) (n int, err err
 
 	grantedLock.release()
 
-	if chunkedPutContext.flushRequested {
-		// Return io.EOF to indicate all chunks have been sent and to close this Chunked PUT
+	// At this point, n == 0... do we need to send EOF?
+
+	if wakeChanOpenOrNonEmpty {
+		// Nope... we were awoken to send bytes but we'd already sent them
 
-		err = io.EOF
+		err = nil
 		return
 	}
 
-	// At this point:
-	//   There was no data in buf to send
-	//   We need to wait for sendDaemon() to wake us up
-	//   Then simply return (n == 0; err == nil) to cause us to be re-entered
+	// Time to send EOF
 
-	chunkedPutContext.flushRequested = <-chunkedPutContext.wakeChan
+	err = io.EOF
 
-	err = nil
 	return
 }
 
 func (chunkedPutContext *chunkedPutContextStruct) Close() (err error) {
 	// Make sure Read() gets a chance to cleanly exit
 
 	if chunkedPutContext.inRead {
-		chunkedPutContext.wakeChan <- false
+		select {
+		case chunkedPutContext.wakeChan <- struct{}{}:
+			// We just notified Read()
+		default:
+			// We didn't need to notify Read()
+		}
 
 		for chunkedPutContext.inRead {
 			time.Sleep(chunkedPutContextExitReadPollingRate)

pfsagentd/fission.go

@@ -346,6 +346,7 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis
 		chownRequest             *jrpcfs.ChownRequest
 		chunkedPutContext        *chunkedPutContextStruct
 		chunkedPutContextElement *list.Element
+		chunkedPutContextLast    *chunkedPutContextStruct
 		err                      error
 		fileInode                *fileInodeStruct
 		getStatReply             *jrpcfs.StatStruct
@@ -459,6 +460,8 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis
 
 		pruneExtentMap(fileInode.extentMap, setAttrIn.Size)
 
+		chunkedPutContextLast = nil
+
 		chunkedPutContextElement = fileInode.chunkedPutList.Front()
 
 		for nil != chunkedPutContextElement {
@@ -467,6 +470,8 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis
 				logFatalf("chunkedPutContextElement.Value.(*chunkedPutContextStruct) returned !ok")
 			}
 
+			chunkedPutContextLast = chunkedPutContext
+
 			if chunkedPutContext.fileSize > setAttrIn.Size {
 				chunkedPutContext.fileSize = setAttrIn.Size
 			}
@@ -476,6 +481,16 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis
 			chunkedPutContextElement = chunkedPutContextElement.Next()
 		}
 
+		if nil == chunkedPutContextLast {
+			if fileInode.extentMapFileSize < setAttrIn.Size {
+				fileInode.extentMapFileSize = setAttrIn.Size
+			}
+		} else {
+			if chunkedPutContext.fileSize < setAttrIn.Size {
+				chunkedPutContext.fileSize = setAttrIn.Size
+			}
+		}
+
 		resizeRequest = &jrpcfs.ResizeRequest{
 			InodeHandle: jrpcfs.InodeHandle{
 				MountID:     globals.mountID,
@@ -1183,8 +1198,8 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission
 			buf:            make([]byte, 0),
 			fileInode:      fileInode,
 			state:          chunkedPutContextStateOpen,
-			sendChan:       make(chan struct{}),
-			wakeChan:       make(chan bool),
+			sendChan:       make(chan struct{}, 1),
+			wakeChan:       make(chan struct{}, 1),
 			inRead:         false,
 			flushRequested: false,
 		}
@@ -1222,8 +1237,8 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission
 				buf:            make([]byte, 0),
 				fileInode:      fileInode,
 				state:          chunkedPutContextStateOpen,
-				sendChan:       make(chan struct{}),
-				wakeChan:       make(chan bool),
+				sendChan:       make(chan struct{}, 1),
+				wakeChan:       make(chan struct{}, 1),
 				inRead:         false,
 				flushRequested: false,
 			}

pfsagentd/globals.go

@@ -153,11 +153,12 @@ type chunkedPutContextStruct struct {
 	fileInode             *fileInodeStruct //
 	state                 uint8            // One of chunkedPutContextState{Open|Closing|Closed}
 	pos                   int              // ObjectOffset just after last sent chunk
-	sendChan              chan struct{}    // Single element buffered chan to wake up sendDaemon()
+	sendChan              chan struct{}    // Single element buffered chan to wake up *chunkedPutContextStruct.sendDaemon()
 	//                                          will be closed to indicate a flush is requested
-	wakeChan       chan bool //               Wake-up Read callback to respond with a chunk and/or return EOF
-	inRead         bool      //               Set when in Read() as a hint to Close() to help Read() cleanly exit
-	flushRequested bool      //               Set to remember that a flush has been requested of *chunkedPutContextStruct.Read()
+	wakeChan chan struct{} //                 Single element buffered chan to wake up *chunkedPutContextStruct.Read()
+	//                                          will be closed to indicate a flush is requested
+	inRead         bool //                    Set when in Read() as a hint to Close() to help Read() cleanly exit
+	flushRequested bool //                    Set to remember that a flush has been requested of *chunkedPutContextStruct.Read()
 }
 
 type fileInodeStruct struct {

test/container/Dockerfile

@@ -60,14 +60,19 @@ RUN rm -rf /var/cache/yum
 
 RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
 
+# goveralls needs a more recent version of git than what comes with CentOS 7
+RUN yum -y remove git*
+RUN yum -y install  https://centos7.iuscommunity.org/ius-release.rpm
+RUN yum -y install  git2u-all
+RUN git --version
+
 RUN yum -y --disableexcludes=all install gcc
 RUN yum -y install \
     wget \
     epel-release \
     sudo \
     json-c-devel \
     fuse \
-    git \
     gcc-c++-4.8.5-16.el7_4.2 \
     python-devel-2.7.5-58.el7 \
     gnutls-devel-3.3.26-9.el7 \
@@ -117,4 +122,6 @@ RUN cd /liberasurecode && ./autogen.sh && ./configure && make && sudo make insta
 RUN git clone https://github.com/openstack/pyeclib.git
 RUN cd /pyeclib && pip install -e . && pip install -r test-requirements.txt
 
+RUN go get github.com/ory/go-acc github.com/mattn/goveralls
+
 CMD ["/bin/bash", "/gopathroot/src/github.com/swiftstack/ProxyFS/test/container/launch.sh"]

test/container/launch.sh

@@ -37,4 +37,8 @@ export SAMBA_SOURCE=$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba
 
 # Build ProxyFS and run tests
 cd $GOPATH/src/github.com/swiftstack/ProxyFS
-make
+make ci
+# $COVERALLS_TOKEN must be configured in TravisCI
+if [ -n "$COVERALLS_TOKEN" ] && [ -n "$TRAVIS_BRANCH" ]; then
+    GIT_BRANCH=$TRAVIS_BRANCH goveralls -coverprofile coverage.coverprofile -service travis-ci -repotoken $COVERALLS_TOKEN
+fi