From aadf0c77cd33611af29e52fde0e9200882ee5eba Mon Sep 17 00:00:00 2001 From: Ben Boyter Date: Mon, 30 Sep 2024 15:29:44 +1000 Subject: [PATCH] Remove redundant code --- processor/processor.go | 3 -- processor/processor_unix.go | 69 -------------------------------- processor/processor_unix_test.go | 24 ----------- 3 files changed, 96 deletions(-) delete mode 100644 processor/processor_unix.go delete mode 100644 processor/processor_unix_test.go diff --git a/processor/processor.go b/processor/processor.go index ef07c663b..82ff25483 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -156,9 +156,6 @@ var FileOutput = "" // PathDenyList sets the paths that should be skipped var PathDenyList = []string{} -// DirectoryWalkerJobWorkers is the number of workers which will walk the directory tree -var DirectoryWalkerJobWorkers = runtime.NumCPU() - // FileListQueueSize is the queue of files found and ready to be read into memory var FileListQueueSize = runtime.NumCPU() diff --git a/processor/processor_unix.go b/processor/processor_unix.go deleted file mode 100644 index 4f08157d8..000000000 --- a/processor/processor_unix.go +++ /dev/null @@ -1,69 +0,0 @@ -//go:build linux || darwin -// +build linux darwin - -// SPDX-License-Identifier: MIT - -package processor - -import ( - "syscall" -) - -func init() { - // Doing the limits configuration in init() directly is possible, but it - // means that we wouldn't have access to user flags like Verbose. - ConfigureLimits = ConfigureLimitsUnix -} - -// ConfigureLimitsUnix attempts to control ulimits on unix like OS -func ConfigureLimitsUnix() { - margin := 16 - - // High water mark of how many open files we may need. - // Each FileReadJobWorker needs one + DirectoryWorker may need up to one per runtime CPU - // We pad it out a bit because every process needs a few handles just to exist - highWaterMark := uint64(FileProcessJobWorkers + DirectoryWalkerJobWorkers + margin) - - limit := syscall.Rlimit{} - err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit) - if err != nil { - printWarnf("Unable to determine open file limit: %v", err) - return - } - - originalLimit := limit.Cur - - printDebugf("Open file limit: current=%d max=%d", limit.Cur, limit.Max) - - // limit.Cur is the current (soft) limit. If this is too low, we may be - // able to raise it with Setrlimit - if limit.Cur < highWaterMark { - limit.Cur = highWaterMark - - // limit.Max is the hard limit. If this is still too low, we'll scale - // it as high as we can, but we also have to scale back how many workers - // we launch. - if limit.Max < highWaterMark { - printWarn("Scaling down workers to fit open file ulimit - performance may be sub-optimal") - limit.Cur = limit.Max - scaleWorkersToLimit(int(limit.Max), margin) - } - - if originalLimit < limit.Max { - err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit) - if err != nil { - printWarnf("Adjusting file limit failed: %v", err) - scaleWorkersToLimit(int(originalLimit), margin) - } else { - printDebugf("Adjusted open file limit to %d", limit.Cur) - } - } - } -} - -func scaleWorkersToLimit(max, margin int) { - scalingBase := (max - margin) / 5 - - DirectoryWalkerJobWorkers = scalingBase - FileProcessJobWorkers = scalingBase * 4 -} diff --git a/processor/processor_unix_test.go b/processor/processor_unix_test.go deleted file mode 100644 index e7908140e..000000000 --- a/processor/processor_unix_test.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build linux || darwin -// +build linux darwin - -// SPDX-License-Identifier: MIT - -package processor - -import ( - "testing" -) - -func TestScaleWorkersToLimit(t *testing.T) { - scaleWorkersToLimit(10, 10) -} - -func TestConfigureLimitsUnix(t *testing.T) { - ConfigureLimitsUnix() -} - -func TestConfigureLimitsUnixHighWater(t *testing.T) { - FileProcessJobWorkers = 1 - DirectoryWalkerJobWorkers = 1 - ConfigureLimitsUnix() -}