Added [FSGlobals]CoalesceElementChunkSize to cap duration of each Coa…

…lesce lock phase duration

CONFIGURING.md

@@ -109,6 +109,7 @@ For each of the keys supported, the following table will list whether or not its
 |                                           | TryLockBackoffMax                        | No           | 50ms               | Yes                      | No                           |
 |                                           | TryLockSerializationThreshhold           | No           | 5                  | Yes                      | No                           |
 |                                           | SymlinkMax                               | No           | 32                 | Yes                      | No                           |
+|                                           | CoalesceElementChunkSize                 | No           | 16                 | Yes                      | No                           |
 |                                           | InodeRecCacheEvictLowLimit               | Yes          |                    | Yes                      | No                           |
 |                                           | InodeRecCacheEvictHighLimit              | Yes          |                    | Yes                      | No                           |
 |                                           | LogSegmentRecCacheEvictLowLimit          | Yes          |                    | Yes                      | No                           |

blunder/api_test.go

@@ -36,6 +36,7 @@ func testSetup(t *testing.T) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 	}
 
 	testConfMap, err = conf.MakeConfMapFromStrings(testConfStrings)

dlm/llm_test.go

@@ -96,6 +96,7 @@ func testSetup() (err error) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 	}
 
 	testConfMap, err = conf.MakeConfMapFromStrings(testConfMapStrings)

evtlog/api_test.go

@@ -48,6 +48,7 @@ func TestAPI(t *testing.T) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 		"EventLog.Enabled=true",
 		"EventLog.BufferKey=9876",     // Don't conflict with a running instance
 		"EventLog.BufferLength=65536", // 64KiB

evtlog/benchmark_test.go

@@ -34,6 +34,7 @@ func benchmarkSetup(b *testing.B, enable bool) {
 			"FSGlobals.TryLockBackoffMax=50ms",
 			"FSGlobals.TryLockSerializationThreshhold=5",
 			"FSGlobals.SymlinkMax=32",
+			"FSGlobals.CoalesceElementChunkSize=16",
 			"EventLog.Enabled=true",
 			"EventLog.BufferKey=1234",
 			"EventLog.BufferLength=65536", //64KiB
@@ -57,6 +58,7 @@ func benchmarkSetup(b *testing.B, enable bool) {
 			"FSGlobals.TryLockBackoffMax=50ms",
 			"FSGlobals.TryLockSerializationThreshhold=5",
 			"FSGlobals.SymlinkMax=32",
+			"FSGlobals.CoalesceElementChunkSize=16",
 			"EventLog.Enabled=false",
 		}
 	}

fs/api_internal.go

@@ -1381,19 +1381,20 @@ func (vS *volumeStruct) MiddlewareCoalesce(destPath string, metaData []byte, ele
 	ino uint64, numWrites uint64, attrChangeTime uint64, modificationTime uint64, err error) {
 
 	var (
-		coalesceElementList      []*inode.CoalesceElement
-		coalesceElementListIndex int
-		coalesceSize             uint64
-		ctime                    time.Time
-		mtime                    time.Time
-		destFileInodeNumber      inode.InodeNumber
-		dirEntryBasename         string
-		dirEntryInodeNumber      inode.InodeNumber
-		dirInodeNumber           inode.InodeNumber
-		elementPath              string
-		heldLocks                *heldLocksStruct
-		retryRequired            bool
-		tryLockBackoffContext    *tryLockBackoffContextStruct
+		coalesceElementList          []*inode.CoalesceElement
+		coalesceSize                 uint64
+		ctime                        time.Time
+		destFileInodeNumber          inode.InodeNumber
+		dirEntryBasename             string
+		dirEntryInodeNumber          inode.InodeNumber
+		dirInodeNumber               inode.InodeNumber
+		elementPathIndex             int
+		elementPathIndexAtChunkEnd   int
+		elementPathIndexAtChunkStart int
+		heldLocks                    *heldLocksStruct
+		mtime                        time.Time
+		retryRequired                bool
+		tryLockBackoffContext        *tryLockBackoffContextStruct
 	)
 
 	startTime := time.Now()
@@ -1408,54 +1409,17 @@ func (vS *volumeStruct) MiddlewareCoalesce(destPath string, metaData []byte, ele
 	vS.jobRWMutex.RLock()
 	defer vS.jobRWMutex.RUnlock()
 
-	// Retry until done or failure (starting with ZERO backoff)
+	// First create the destination file if necessary and ensure that it is empty
 
 	tryLockBackoffContext = &tryLockBackoffContextStruct{}
 
-Restart:
-
-	// Perform backoff and update for each restart (starting with ZERO backoff of course)
+RestartDestinationFileCreation:
 
 	tryLockBackoffContext.backoff()
 
-	// Construct fresh heldLocks for this restart
-
 	heldLocks = newHeldLocks()
 
-	// Assemble WriteLock on each FileInode and their containing DirInode in elementPaths
-
-	coalesceElementList = make([]*inode.CoalesceElement, len(elementPaths))
-
-	for coalesceElementListIndex, elementPath = range elementPaths {
-		dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, _, retryRequired, err =
-			vS.resolvePath(
-				inode.RootDirInodeNumber,
-				elementPath,
-				heldLocks,
-				resolvePathFollowDirSymlinks|
-					resolvePathRequireExclusiveLockOnDirEntryInode|
-					resolvePathRequireExclusiveLockOnDirInode)
-
-		if nil != err {
-			heldLocks.free()
-			return
-		}
-
-		if retryRequired {
-			heldLocks.free()
-			goto Restart
-		}
-
-		// Record dirInode & dirEntryInode (fileInode) in elementList
-
-		coalesceElementList[coalesceElementListIndex] = &inode.CoalesceElement{
-			ContainingDirectoryInodeNumber: dirInodeNumber,
-			ElementInodeNumber:             dirEntryInodeNumber,
-			ElementName:                    dirEntryBasename,
-		}
-	}
-
-	_, dirEntryInodeNumber, _, _, retryRequired, err =
+	_, destFileInodeNumber, _, _, retryRequired, err =
 		vS.resolvePath(
 			inode.RootDirInodeNumber,
 			destPath,
@@ -1472,18 +1436,94 @@ Restart:
 
 	if retryRequired {
 		heldLocks.free()
-		goto Restart
+		goto RestartDestinationFileCreation
 	}
 
-	// Invoke package inode to actually perform the Coalesce operation
+	vS.inodeVolumeHandle.SetSize(destFileInodeNumber, 0)
 
-	destFileInodeNumber = dirEntryInodeNumber
-	ctime, mtime, numWrites, coalesceSize, err = vS.inodeVolumeHandle.Coalesce(
-		destFileInodeNumber, MiddlewareStream, metaData, coalesceElementList)
+	heldLocks.free()
 
-	// We can now release all the WriteLocks we are currently holding
+	// Now setup for looping through elementPaths with fresh locks
+	// every globals.coalesceElementChunkSize elements holding an
+	// Exclusive Lock on each FileInode and their containing DirInode
 
-	heldLocks.free()
+	elementPathIndexAtChunkStart = 0
+
+	for elementPathIndexAtChunkStart < len(elementPaths) {
+		elementPathIndexAtChunkEnd = elementPathIndexAtChunkStart + int(globals.coalesceElementChunkSize)
+		if elementPathIndexAtChunkEnd > len(elementPaths) {
+			elementPathIndexAtChunkEnd = len(elementPaths)
+		}
+
+		// Coalesce elementPaths[elementPathIndexAtChunkStart:elementPathIndexAtChunkEnd)
+
+		tryLockBackoffContext = &tryLockBackoffContextStruct{}
+
+	RestartCoalesceChunk:
+
+		tryLockBackoffContext.backoff()
+
+		heldLocks = newHeldLocks()
+
+		coalesceElementList = make([]*inode.CoalesceElement, 0, (elementPathIndexAtChunkEnd - elementPathIndexAtChunkStart))
+
+		for elementPathIndex = elementPathIndexAtChunkStart; elementPathIndex < elementPathIndexAtChunkEnd; elementPathIndex++ {
+			dirInodeNumber, dirEntryInodeNumber, dirEntryBasename, _, retryRequired, err =
+				vS.resolvePath(
+					inode.RootDirInodeNumber,
+					elementPaths[elementPathIndex],
+					heldLocks,
+					resolvePathFollowDirSymlinks|
+						resolvePathRequireExclusiveLockOnDirEntryInode|
+						resolvePathRequireExclusiveLockOnDirInode)
+
+			if nil != err {
+				heldLocks.free()
+				return
+			}
+
+			if retryRequired {
+				heldLocks.free()
+				goto RestartCoalesceChunk
+			}
+
+			coalesceElementList = append(coalesceElementList, &inode.CoalesceElement{
+				ContainingDirectoryInodeNumber: dirInodeNumber,
+				ElementInodeNumber:             dirEntryInodeNumber,
+				ElementName:                    dirEntryBasename,
+			})
+		}
+
+		_, destFileInodeNumber, _, _, retryRequired, err =
+			vS.resolvePath(
+				inode.RootDirInodeNumber,
+				destPath,
+				heldLocks,
+				resolvePathFollowDirEntrySymlinks|
+					resolvePathFollowDirSymlinks|
+					resolvePathRequireExclusiveLockOnDirEntryInode)
+
+		if nil != err {
+			heldLocks.free()
+			return
+		}
+
+		if retryRequired {
+			heldLocks.free()
+			goto RestartCoalesceChunk
+		}
+
+		ctime, mtime, numWrites, coalesceSize, err = vS.inodeVolumeHandle.Coalesce(
+			destFileInodeNumber, MiddlewareStream, metaData, coalesceElementList)
+
+		heldLocks.free()
+
+		if nil != err {
+			return
+		}
+
+		elementPathIndexAtChunkStart = elementPathIndexAtChunkEnd
+	}
 
 	// Regardless of err return, fill in other return values
 

fs/config.go

@@ -57,6 +57,7 @@ type globalsStruct struct {
 	tryLockBackoffMax              time.Duration
 	tryLockSerializationThreshhold uint64
 	symlinkMax                     uint16
+	coalesceElementChunkSize       uint16
 
 	volumeMap map[string]*volumeStruct // key == volumeStruct.volumeName
 
@@ -204,6 +205,10 @@ func (dummy *globalsStruct) Up(confMap conf.ConfMap) (err error) {
 	if nil != err {
 		globals.symlinkMax = 32 // TODO: Eventually, just return
 	}
+	globals.coalesceElementChunkSize, err = confMap.FetchOptionValueUint16("FSGlobbals", "CoalesceElementChunkSize")
+	if nil != err {
+		globals.coalesceElementChunkSize = 16 // TODO: Eventually, just return
+	}
 
 	globals.volumeMap = make(map[string]*volumeStruct)
 

fs/setup_teardown_test.go

@@ -106,6 +106,7 @@ func testSetup(t *testing.T, starvationMode bool) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 		"FSGlobals.InodeRecCacheEvictLowLimit=10000",
 		"FSGlobals.InodeRecCacheEvictHighLimit=10010",
 		"FSGlobals.LogSegmentRecCacheEvictLowLimit=10000",

halter/api_test.go

@@ -26,6 +26,7 @@ func TestAPI(t *testing.T) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 	}
 
 	testConfMap, err := conf.MakeConfMapFromStrings(testConfMapStrings)

headhunter/api_test.go

@@ -148,6 +148,7 @@ func TestHeadHunterAPI(t *testing.T) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 		"FSGlobals.InodeRecCacheEvictLowLimit=10000",
 		"FSGlobals.InodeRecCacheEvictHighLimit=10010",
 		"FSGlobals.LogSegmentRecCacheEvictLowLimit=10000",

headhunter/stress_test.go

@@ -185,6 +185,7 @@ func TestHeadHunterStress(t *testing.T) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 		"FSGlobals.InodeRecCacheEvictLowLimit=10000",
 		"FSGlobals.InodeRecCacheEvictHighLimit=10010",
 		"FSGlobals.LogSegmentRecCacheEvictLowLimit=10000",

httpserver/setup_teardown_test.go

@@ -67,6 +67,7 @@ func testSetup(t *testing.T) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 		"FSGlobals.InodeRecCacheEvictLowLimit=10000",
 		"FSGlobals.InodeRecCacheEvictHighLimit=10010",
 		"FSGlobals.LogSegmentRecCacheEvictLowLimit=10000",

inode/file.go

@@ -1138,24 +1138,17 @@ func (vS *volumeStruct) Coalesce(destInodeNumber InodeNumber, metaDataName strin
 		return
 	}
 
-	// Now truncate destInode & "append" each Element's extents to destInode (creating duplicate references to LogSegments for now)
+	// Now "append" each Element's extents to destInode (creating duplicate references to LogSegments for now)
 
 	destInodeExtentMap = destInode.payload.(sortedmap.BPlusTree)
 
 	destInode.dirty = true
 
-	err = setSizeInMemory(destInode, 0)
-	if nil != err {
-		err = blunder.NewError(blunder.InvalidArgError, "Coalesce() unable to truncate destInodeNumber 0x%016X: %v", destInodeNumber, err)
-		return
-	}
-
-	destInodeOffsetBeforeElementAppend = 0
-	destInode.NumWrites = 0
+	destInodeOffsetBeforeElementAppend = fileLen(destInodeExtentMap)
 
 	for _, element = range elements {
 		elementInode = inodeMap[element.ElementInodeNumber]
-		destInode.NumWrites += 1
+		destInode.NumWrites++
 		elementInodeExtentMap = elementInode.payload.(sortedmap.BPlusTree)
 		elementInodeExtentMapLen, err = elementInodeExtentMap.Len()
 		for elementInodeExtentMapIndex = 0; elementInodeExtentMapIndex < elementInodeExtentMapLen; elementInodeExtentMapIndex++ {

inode/setup_teardown_test.go

@@ -103,6 +103,7 @@ func testSetup(t *testing.T, starvationMode bool) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 		"FSGlobals.InodeRecCacheEvictLowLimit=10000",
 		"FSGlobals.InodeRecCacheEvictHighLimit=10010",
 		"FSGlobals.LogSegmentRecCacheEvictLowLimit=10000",

jrpcfs/middleware_test.go

@@ -63,6 +63,7 @@ func testSetup() []func() {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=8",
+		"FSGlobals.CoalesceElementChunkSize=16",
 		"FSGlobals.InodeRecCacheEvictLowLimit=10000",
 		"FSGlobals.InodeRecCacheEvictHighLimit=10010",
 		"FSGlobals.LogSegmentRecCacheEvictLowLimit=10000",

pfsagentd/setup_teardown_test.go

@@ -221,6 +221,7 @@ func testSetup(t *testing.T) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 		"FSGlobals.InodeRecCacheEvictLowLimit=10000",
 		"FSGlobals.InodeRecCacheEvictHighLimit=10010",
 		"FSGlobals.LogSegmentRecCacheEvictLowLimit=10000",

proxyfsd/daemon_test.go

@@ -139,6 +139,7 @@ func TestDaemon(t *testing.T) {
 		"FSGlobals.TryLockBackoffMax=50ms",
 		"FSGlobals.TryLockSerializationThreshhold=5",
 		"FSGlobals.SymlinkMax=32",
+		"FSGlobals.CoalesceElementChunkSize=16",
 		"FSGlobals.InodeRecCacheEvictLowLimit=10000",
 		"FSGlobals.InodeRecCacheEvictHighLimit=10010",
 		"FSGlobals.LogSegmentRecCacheEvictLowLimit=10000",

proxyfsd/default.conf

@@ -184,6 +184,7 @@ TryLockBackoffMin:                        10ms
 TryLockBackoffMax:                        50ms
 TryLockSerializationThreshhold:           5
 SymlinkMax:                               32
+CoalesceElementChunkSize:                 16
 InodeRecCacheEvictLowLimit:               10000
 InodeRecCacheEvictHighLimit:              10010
 LogSegmentRecCacheEvictLowLimit:          10000

proxyfsd/file_server.conf

@@ -119,6 +119,7 @@ TryLockBackoffMin:                        10ms
 TryLockBackoffMax:                        50ms
 TryLockSerializationThreshhold:           5
 SymlinkMax:                               32
+CoalesceElementChunkSize:                 16
 InodeRecCacheEvictLowLimit:               10000
 InodeRecCacheEvictHighLimit:              10010
 LogSegmentRecCacheEvictLowLimit:          10000

proxyfsd/file_server_mac_3_peers.conf

@@ -127,6 +127,7 @@ TryLockBackoffMin:                        10ms
 TryLockBackoffMax:                        50ms
 TryLockSerializationThreshhold:           5
 SymlinkMax:                               32
+CoalesceElementChunkSize:                 16
 InodeRecCacheEvictLowLimit:               10000
 InodeRecCacheEvictHighLimit:              10010
 LogSegmentRecCacheEvictLowLimit:          10000

ramswift/daemon_test.go

@@ -37,6 +37,7 @@ func TestViaNoAuthClient(t *testing.T) {
 			"FSGlobals.TryLockBackoffMax=50ms",
 			"FSGlobals.TryLockSerializationThreshhold=5",
 			"FSGlobals.SymlinkMax=32",
+			"FSGlobals.CoalesceElementChunkSize=16",
 			"Peer:Peer0.ReadCacheQuotaFraction=0.20",
 			"RamSwiftInfo.MaxAccountNameLength=256",
 			"RamSwiftInfo.MaxContainerNameLength=256",