From a2125de969e39f711a4266ec2d005878a9e760e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Tue, 18 Feb 2020 14:57:44 -0800 Subject: [PATCH 1/5] Add coverage checks on branch pushes (#450) Using coveralls service, go-acc, and goveralls --- .travis.yml | 2 +- GoMakefile | 2 +- Makefile | 30 +++++++++++++++++++++++------- test/container/Dockerfile | 9 ++++++++- test/container/launch.sh | 6 +++++- 5 files changed, 38 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4c4ce0666..09f97d9c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,4 @@ before_install: - git submodule update --init --recursive script: -- docker run --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/swiftstack/ProxyFS swiftstack/proxyfs_unit_tests +- docker run -e "COVERALLS_TOKEN=$COVERALLS_TOKEN" --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/swiftstack/ProxyFS swiftstack/proxyfs_unit_tests diff --git a/GoMakefile b/GoMakefile index da7cc33a8..cba57a29d 100644 --- a/GoMakefile +++ b/GoMakefile @@ -15,7 +15,7 @@ clean: done cover: - go test -cover $(gosubdir) + go test $(gosubdir) -covermode=atomic -coverprofile=coverage.coverprofile fmt: go fmt $(gosubdir) diff --git a/Makefile b/Makefile index 1b9d6f03e..739756750 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,14 @@ gobinsubdirs = \ proxyfsd/proxyfsd \ ramswift/ramswift +gobinsubdirsforci = \ + pfsagentd \ + mkproxyfs/mkproxyfs \ + proxyfsd/proxyfsd + +gosubdirsforci = $(gopkgsubdirs) $(gobinsubdirsforci); +gosubdirspathsforci = $(addprefix github.com/swiftstack/ProxyFS/,$(gosubdirsforci)) + uname = $(shell uname) machine = $(shell uname -m) @@ -57,12 +65,16 @@ ifeq ($(uname),Linux) ifeq ($(machine),armv7l) all: version fmt pre-generate generate install test + ci: version fmt pre-generate generate install test cover + minimal: pre-generate generate install else distro := $(shell python -c "import platform; print platform.linux_distribution()[0]") all: version fmt pre-generate generate install test python-test c-clean c-build c-install c-test + ci: version fmt pre-generate generate install test cover python-test c-clean c-build c-install c-test + all-deb-builder: version fmt pre-generate generate install c-clean c-build c-install-deb-builder minimal: pre-generate generate install c-clean c-build c-install @@ -70,10 +82,12 @@ ifeq ($(uname),Linux) else all: version fmt pre-generate generate install test + ci: version fmt pre-generate generate install test cover + minimal: pre-generate generate install endif -.PHONY: all all-deb-builder bench c-build c-clean c-install c-install-deb-builder c-test clean cover fmt generate install pre-generate python-test test version +.PHONY: all all-deb-builder bench c-build c-clean c-install c-install-deb-builder c-test ci clean cover fmt generate install pre-generate python-test test version bench: @set -e; \ @@ -128,12 +142,14 @@ clean: cover: @set -e; \ - for gosubdir in $(gopkgsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir cover; \ - done; \ - for gosubdir in $(gobinsubdirs); do \ - $(MAKE) --no-print-directory -C $$gosubdir cover; \ - done + go-acc -o coverage.coverprofile $(gosubdirspathsforci) +# TODO: We're not sure yet how we want to run coverage. Once decided, remove any extra code/comments +# for gosubdir in $(gopkgsubdirs); do \ +# $(MAKE) --no-print-directory -C $$gosubdir cover; \ +# done; \ +# for gosubdir in $(gobinsubdirsforci); do \ +# $(MAKE) --no-print-directory -C $$gosubdir cover; \ +# done fmt: @set -e; \ diff --git a/test/container/Dockerfile b/test/container/Dockerfile index 20399d2e6..f33fbd08a 100644 --- a/test/container/Dockerfile +++ b/test/container/Dockerfile @@ -60,6 +60,12 @@ RUN rm -rf /var/cache/yum RUN rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 +# goveralls needs a more recent version of git than what comes with CentOS 7 +RUN yum -y remove git* +RUN yum -y install https://centos7.iuscommunity.org/ius-release.rpm +RUN yum -y install git2u-all +RUN git --version + RUN yum -y --disableexcludes=all install gcc RUN yum -y install \ wget \ @@ -67,7 +73,6 @@ RUN yum -y install \ sudo \ json-c-devel \ fuse \ - git \ gcc-c++-4.8.5-16.el7_4.2 \ python-devel-2.7.5-58.el7 \ gnutls-devel-3.3.26-9.el7 \ @@ -117,4 +122,6 @@ RUN cd /liberasurecode && ./autogen.sh && ./configure && make && sudo make insta RUN git clone https://github.com/openstack/pyeclib.git RUN cd /pyeclib && pip install -e . && pip install -r test-requirements.txt +RUN go get github.com/ory/go-acc github.com/mattn/goveralls + CMD ["/bin/bash", "/gopathroot/src/github.com/swiftstack/ProxyFS/test/container/launch.sh"] diff --git a/test/container/launch.sh b/test/container/launch.sh index f03e8db54..150a1616d 100755 --- a/test/container/launch.sh +++ b/test/container/launch.sh @@ -37,4 +37,8 @@ export SAMBA_SOURCE=$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba # Build ProxyFS and run tests cd $GOPATH/src/github.com/swiftstack/ProxyFS -make +make ci +# $COVERALLS_TOKEN must be configured in TravisCI +if [ -n "$COVERALLS_TOKEN" ]; then + goveralls -coverprofile coverage.coverprofile -service travis-ci -repotoken $COVERALLS_TOKEN +fi From 48a32e459b85009d5c106cfb4ce20642475eb73b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Wed, 19 Feb 2020 14:24:33 -0800 Subject: [PATCH 2/5] Fix coveralls reports (#451) We weren't properly sending the branch name, so all the reports were shown as detached from HEAD. Also: - Add some binary dirs to coverage - Go get go-acc on pre-generate make target --- .gitignore | 1 + .travis.yml | 2 +- Makefile | 6 +++++- test/container/launch.sh | 4 ++-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index aa99ca417..3c02a9a96 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,7 @@ _testmain.go # test coverage artifacts (see https://blog.golang.org/cover#TOC_5.) coverage.out count.out +coverage.coverprofile # log files *.log diff --git a/.travis.yml b/.travis.yml index 09f97d9c0..4159a0f14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,4 @@ before_install: - git submodule update --init --recursive script: -- docker run -e "COVERALLS_TOKEN=$COVERALLS_TOKEN" --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/swiftstack/ProxyFS swiftstack/proxyfs_unit_tests +- docker run -e "COVERALLS_TOKEN=$COVERALLS_TOKEN" -e "TRAVIS_BRANCH=$TRAVIS_BRANCH" --cap-add SYS_ADMIN --device /dev/fuse -it -v `pwd`:/gopathroot/src/github.com/swiftstack/ProxyFS swiftstack/proxyfs_unit_tests diff --git a/Makefile b/Makefile index 739756750..535688334 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,6 @@ gopkgsubdirs = \ gobinsubdirs = \ cleanproxyfs \ - confgen/confgen \ fsworkout \ inodeworkout \ pfs-crash \ @@ -45,6 +44,7 @@ gobinsubdirs = \ pfsconfjson \ pfsconfjsonpacked \ pfsworkout \ + confgen/confgen \ evtlog/pfsevtlogd \ mkproxyfs/mkproxyfs \ proxyfsd/proxyfsd \ @@ -52,6 +52,9 @@ gobinsubdirs = \ gobinsubdirsforci = \ pfsagentd \ + pfsconfjson \ + pfsconfjsonpacked \ + confgen/confgen \ mkproxyfs/mkproxyfs \ proxyfsd/proxyfsd @@ -182,6 +185,7 @@ install: pre-generate: @set -e; \ go install github.com/swiftstack/ProxyFS/vendor/golang.org/x/tools/cmd/stringer; \ + go get -u github.com/ory/go-acc; \ for gosubdir in $(gopregeneratesubdirs); do \ $(MAKE) --no-print-directory -C $$gosubdir install; \ done diff --git a/test/container/launch.sh b/test/container/launch.sh index 150a1616d..0dad42052 100755 --- a/test/container/launch.sh +++ b/test/container/launch.sh @@ -39,6 +39,6 @@ export SAMBA_SOURCE=$GOPATH/src/github.com/swiftstack/ProxyFS/vfs/samba cd $GOPATH/src/github.com/swiftstack/ProxyFS make ci # $COVERALLS_TOKEN must be configured in TravisCI -if [ -n "$COVERALLS_TOKEN" ]; then - goveralls -coverprofile coverage.coverprofile -service travis-ci -repotoken $COVERALLS_TOKEN +if [ -n "$COVERALLS_TOKEN" ] && [ -n "$TRAVIS_BRANCH" ]; then + GIT_BRANCH=$TRAVIS_BRANCH goveralls -coverprofile coverage.coverprofile -service travis-ci -repotoken $COVERALLS_TOKEN fi From 87957a7eec6afc0885475ae18bcd0fe1219d18e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerard=20Gin=C3=A9?= Date: Wed, 19 Feb 2020 14:36:48 -0800 Subject: [PATCH 3/5] Add coveralls badge in README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4a91e2d28..5aef2dfa7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Build Status](https://travis-ci.com/swiftstack/ProxyFS.svg?branch=development)](https://travis-ci.com/swiftstack/ProxyFS) +[![Coverage Status](https://coveralls.io/repos/github/swiftstack/ProxyFS/badge.svg?branch=development)](https://coveralls.io/github/swiftstack/ProxyFS?branch=development) # ProxyFS Integrated File and Object Access for Swift Object Storage From bd2b76ebe74a32d30c67e4ab85ff9fa27d72adc4 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Wed, 19 Feb 2020 15:50:07 -0800 Subject: [PATCH 4/5] Fix remaining hangs in PFSAgent involving {send|wake}Chan semantics --- pfsagentd/file_inode.go | 44 +++++++++++++++++++++++------------------ pfsagentd/fission.go | 8 ++++---- pfsagentd/globals.go | 9 +++++---- 3 files changed, 34 insertions(+), 27 deletions(-) diff --git a/pfsagentd/file_inode.go b/pfsagentd/file_inode.go index 727505792..4a45d75ab 100644 --- a/pfsagentd/file_inode.go +++ b/pfsagentd/file_inode.go @@ -182,9 +182,14 @@ func (chunkedPutContext *chunkedPutContextStruct) sendDaemon() { goto PerformFlush case _, sendChanOpenOrNonEmpty = <-chunkedPutContext.sendChan: - // Send non-flushing chunk to *chunkedPutContextStruct.Read() + // Send chunk to *chunkedPutContextStruct.Read() - chunkedPutContext.wakeChan <- false + select { + case chunkedPutContext.wakeChan <- struct{}{}: + // We just notified Read() + default: + // We didn't need to notify Read() + } if !sendChanOpenOrNonEmpty { goto PerformFlush @@ -194,9 +199,9 @@ func (chunkedPutContext *chunkedPutContextStruct) sendDaemon() { PerformFlush: - // Send flushing chunk to *chunkedPutContextStruct.Read() & wait for it to finish + // Tell *chunkedPutContextStruct.Read() to finish & wait for it to finish - chunkedPutContext.wakeChan <- true + close(chunkedPutContext.wakeChan) chunkedPutContext.Wait() // Chunked PUT is complete @@ -395,13 +400,11 @@ func (chunkedPutContext *chunkedPutContextStruct) complete() { func (chunkedPutContext *chunkedPutContextStruct) Read(p []byte) (n int, err error) { var ( - grantedLock *fileInodeLockRequestStruct + grantedLock *fileInodeLockRequestStruct + wakeChanOpenOrNonEmpty bool ) - chunkedPutContext.inRead = true - defer func() { - chunkedPutContext.inRead = false - }() + _, wakeChanOpenOrNonEmpty = <-chunkedPutContext.wakeChan grantedLock = chunkedPutContext.fileInode.getExclusiveLock() @@ -430,21 +433,19 @@ func (chunkedPutContext *chunkedPutContextStruct) Read(p []byte) (n int, err err grantedLock.release() - if chunkedPutContext.flushRequested { - // Return io.EOF to indicate all chunks have been sent and to close this Chunked PUT + // At this point, n == 0... do we need to send EOF? + + if wakeChanOpenOrNonEmpty { + // Nope... we were awoken to send bytes but we'd already sent them - err = io.EOF + err = nil return } - // At this point: - // There was no data in buf to send - // We need to wait for sendDaemon() to wake us up - // Then simply return (n == 0; err == nil) to cause us to be re-entered + // Time to send EOF - chunkedPutContext.flushRequested = <-chunkedPutContext.wakeChan + err = io.EOF - err = nil return } @@ -452,7 +453,12 @@ func (chunkedPutContext *chunkedPutContextStruct) Close() (err error) { // Make sure Read() gets a chance to cleanly exit if chunkedPutContext.inRead { - chunkedPutContext.wakeChan <- false + select { + case chunkedPutContext.wakeChan <- struct{}{}: + // We just notified Read() + default: + // We didn't need to notify Read() + } for chunkedPutContext.inRead { time.Sleep(chunkedPutContextExitReadPollingRate) diff --git a/pfsagentd/fission.go b/pfsagentd/fission.go index 17c73cdeb..2a0542f09 100644 --- a/pfsagentd/fission.go +++ b/pfsagentd/fission.go @@ -1183,8 +1183,8 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission buf: make([]byte, 0), fileInode: fileInode, state: chunkedPutContextStateOpen, - sendChan: make(chan struct{}), - wakeChan: make(chan bool), + sendChan: make(chan struct{}, 1), + wakeChan: make(chan struct{}, 1), inRead: false, flushRequested: false, } @@ -1222,8 +1222,8 @@ func (dummy *globalsStruct) DoWrite(inHeader *fission.InHeader, writeIn *fission buf: make([]byte, 0), fileInode: fileInode, state: chunkedPutContextStateOpen, - sendChan: make(chan struct{}), - wakeChan: make(chan bool), + sendChan: make(chan struct{}, 1), + wakeChan: make(chan struct{}, 1), inRead: false, flushRequested: false, } diff --git a/pfsagentd/globals.go b/pfsagentd/globals.go index da16f7dcf..3c851aa28 100644 --- a/pfsagentd/globals.go +++ b/pfsagentd/globals.go @@ -153,11 +153,12 @@ type chunkedPutContextStruct struct { fileInode *fileInodeStruct // state uint8 // One of chunkedPutContextState{Open|Closing|Closed} pos int // ObjectOffset just after last sent chunk - sendChan chan struct{} // Single element buffered chan to wake up sendDaemon() + sendChan chan struct{} // Single element buffered chan to wake up *chunkedPutContextStruct.sendDaemon() // will be closed to indicate a flush is requested - wakeChan chan bool // Wake-up Read callback to respond with a chunk and/or return EOF - inRead bool // Set when in Read() as a hint to Close() to help Read() cleanly exit - flushRequested bool // Set to remember that a flush has been requested of *chunkedPutContextStruct.Read() + wakeChan chan struct{} // Single element buffered chan to wake up *chunkedPutContextStruct.Read() + // will be closed to indicate a flush is requested + inRead bool // Set when in Read() as a hint to Close() to help Read() cleanly exit + flushRequested bool // Set to remember that a flush has been requested of *chunkedPutContextStruct.Read() } type fileInodeStruct struct { From bd5e5ff220b404f5976a5222977ed92f621c1301 Mon Sep 17 00:00:00 2001 From: Ed McClanahan Date: Fri, 21 Feb 2020 15:13:40 -0800 Subject: [PATCH 5/5] Fix attr's returns when doing `truncate --size` to extend a file --- pfsagentd/fission.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pfsagentd/fission.go b/pfsagentd/fission.go index 2a0542f09..be146d738 100644 --- a/pfsagentd/fission.go +++ b/pfsagentd/fission.go @@ -346,6 +346,7 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis chownRequest *jrpcfs.ChownRequest chunkedPutContext *chunkedPutContextStruct chunkedPutContextElement *list.Element + chunkedPutContextLast *chunkedPutContextStruct err error fileInode *fileInodeStruct getStatReply *jrpcfs.StatStruct @@ -459,6 +460,8 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis pruneExtentMap(fileInode.extentMap, setAttrIn.Size) + chunkedPutContextLast = nil + chunkedPutContextElement = fileInode.chunkedPutList.Front() for nil != chunkedPutContextElement { @@ -467,6 +470,8 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis logFatalf("chunkedPutContextElement.Value.(*chunkedPutContextStruct) returned !ok") } + chunkedPutContextLast = chunkedPutContext + if chunkedPutContext.fileSize > setAttrIn.Size { chunkedPutContext.fileSize = setAttrIn.Size } @@ -476,6 +481,16 @@ func (dummy *globalsStruct) DoSetAttr(inHeader *fission.InHeader, setAttrIn *fis chunkedPutContextElement = chunkedPutContextElement.Next() } + if nil == chunkedPutContextLast { + if fileInode.extentMapFileSize < setAttrIn.Size { + fileInode.extentMapFileSize = setAttrIn.Size + } + } else { + if chunkedPutContext.fileSize < setAttrIn.Size { + chunkedPutContext.fileSize = setAttrIn.Size + } + } + resizeRequest = &jrpcfs.ResizeRequest{ InodeHandle: jrpcfs.InodeHandle{ MountID: globals.mountID,