Replace remove files activity

cmd/worker/workercmd/cmd.go

@@ -13,7 +13,7 @@
 
 	"github.com/artefactual-sdps/preprocessing-moma/internal/config"
 	"github.com/artefactual-sdps/preprocessing-moma/internal/workflow"
-	remove "github.com/artefactual-sdps/remove-files-activity"
+	"github.com/artefactual-sdps/temporal-activities/removefiles"
 )
 
 const Name = "preprocessing-worker"
@@ -58,8 +58,8 @@
 		temporalsdk_workflow.RegisterOptions{Name: m.cfg.Temporal.WorkflowName},
 	)
 	w.RegisterActivityWithOptions(
-		remove.NewRemoveFilesActivity().Execute,
-		temporalsdk_activity.RegisterOptions{Name: remove.RemoveFilesName},
+		removefiles.NewActivity(m.cfg.RemoveFiles).Execute,
+		temporalsdk_activity.RegisterOptions{Name: removefiles.ActivityName},
 	)
 
 	if err := w.Start(); err != nil {

go.mod

@@ -4,9 +4,12 @@ go 1.22.2
 
 require (
 	github.com/artefactual-sdps/remove-files-activity v0.0.0-20240301164716-575fb8b7b834
+	github.com/artefactual-sdps/temporal-activities v0.0.0-20240418211119-e97b26fed41b
 	github.com/go-logr/logr v1.4.1
+	github.com/otiai10/copy v1.14.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.18.2
+	github.com/stretchr/testify v1.9.0
 	go.artefactual.dev/tools v0.8.0
 	go.temporal.io/sdk v1.26.1
 	gotest.tools/v3 v3.5.1
@@ -37,13 +40,13 @@ require (
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
-	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	go.temporal.io/api v1.32.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect
 	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect

go.sum

@@ -4,6 +4,8 @@ github.com/Diogenesoftoronto/go-gitignore v1.1.0 h1:G8jh5GxjmCidSTvotSiNeBMxMLma
 github.com/Diogenesoftoronto/go-gitignore v1.1.0/go.mod h1:mmVD39ljrmsXQECsd829osXfYy+nzbxqYEH2rCLE+XA=
 github.com/artefactual-sdps/remove-files-activity v0.0.0-20240301164716-575fb8b7b834 h1:EJzGpWDLTrx/FbXfwewHqXsGeIihz1CyKTjCdAO67mU=
 github.com/artefactual-sdps/remove-files-activity v0.0.0-20240301164716-575fb8b7b834/go.mod h1:5U5uQ147WRiusG3LMDopp9MOZEcOJ649TUnccLefhKU=
+github.com/artefactual-sdps/temporal-activities v0.0.0-20240418211119-e97b26fed41b h1:gfScS2NGX6STL9BkTuuDkCp9/f4PUqG4QITbsTKyuJA=
+github.com/artefactual-sdps/temporal-activities v0.0.0-20240418211119-e97b26fed41b/go.mod h1:oviveqOfixKKhUlNYb/gRq/NceVLdseOB/GUK8Rinfk=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -69,6 +71,10 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
+github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
+github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks=
+github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM=
 github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
 github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
@@ -168,6 +174,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

hack/kube/overlays/dev/preprocessing-moma-secret.yaml

@@ -18,5 +18,6 @@ stringData:
 
     [worker]
     maxConcurrentSessions = 1
-  .remove: |
-    .DS_Store
+
+    [removeFiles]
+    removeNames = ".DS_Store"

internal/config/config.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/artefactual-sdps/temporal-activities/removefiles"
 	"github.com/spf13/viper"
 )
 
@@ -12,11 +13,13 @@ type ConfigurationValidator interface {
 }
 
 type Configuration struct {
-	Verbosity  int
 	Debug      bool
+	Verbosity  int
 	SharedPath string
 	Temporal   Temporal
 	Worker     WorkerConfig
+
+	RemoveFiles removefiles.Config
 }
 
 type Temporal struct {

internal/config/config_test.go

@@ -11,8 +11,17 @@ import (
 
 const testConfig = `# Config
 debug = true
+verbosity = 2
+sharedPath = "/tmp"
 [temporal]
 address = "host:port"
+namespace = "default"
+taskQueue = "preprocessing"
+workflowName = "preprocessing"
+[worker]
+maxConcurrentSessions = 1
+[removeFiles]
+removeNames = ".DS_Store, Thumbs.db"
 `
 
 func TestConfig(t *testing.T) {
@@ -31,5 +40,17 @@ func TestConfig(t *testing.T) {
 	assert.NilError(t, err)
 	assert.Equal(t, found, true)
 	assert.Equal(t, configFileUsed, configFile)
+
+	assert.Equal(t, c.Debug, true)
+	assert.Equal(t, c.Verbosity, 2)
+	assert.Equal(t, c.SharedPath, "/tmp")
+
 	assert.Equal(t, c.Temporal.Address, "host:port")
+	assert.Equal(t, c.Temporal.Namespace, "default")
+	assert.Equal(t, c.Temporal.TaskQueue, "preprocessing")
+	assert.Equal(t, c.Temporal.WorkflowName, "preprocessing")
+
+	assert.Equal(t, c.Worker.MaxConcurrentSessions, 1)
+
+	assert.Equal(t, c.RemoveFiles.RemoveNames, ".DS_Store, Thumbs.db")
 }

internal/workflow/preprocessing.go

@@ -5,8 +5,7 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/artefactual-sdps/preprocessing-moma/internal/activities"
-	remove "github.com/artefactual-sdps/remove-files-activity"
+	"github.com/artefactual-sdps/temporal-activities/removefiles"
 	"go.artefactual.dev/tools/temporal"
 	temporalsdk_temporal "go.temporal.io/sdk/temporal"
 	temporalsdk_workflow "go.temporal.io/sdk/workflow"
@@ -44,16 +43,13 @@ func (w *PreprocessingWorkflow) Execute(
 
 	localPath := filepath.Join(w.sharedPath, filepath.Clean(params.RelativePath))
 
-	// TODO Make the file path a part of the enduro config or check the configuration later.
-	// A remove file works like a .gitignore file.
-	removePath := "/home/preprocessing-moma/.config/.remove"
-
-	// Remove hidden files.
-	var removedPaths activities.RemovePathsResult
-	e = temporalsdk_workflow.ExecuteActivity(withLocalActOpts(ctx), remove.RemoveFilesName, &remove.RemoveFilesParams{
-		RemovePath: localPath,
-		IgnorePath: removePath,
-	}).Get(ctx, &removedPaths)
+	// Remove unwanted files.
+	var removeFilesResult removefiles.ActivityResult
+	e = temporalsdk_workflow.ExecuteActivity(
+		withLocalActOpts(ctx),
+		removefiles.ActivityName,
+		&removefiles.ActivityParams{Path: localPath},
+	).Get(ctx, &removeFilesResult)
 	if e != nil {
 		return nil, e
 	}

internal/workflow/preprocessing_test.go

@@ -0,0 +1,105 @@
+package workflow_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/artefactual-sdps/preprocessing-moma/internal/config"
+	"github.com/artefactual-sdps/preprocessing-moma/internal/workflow"
+	remove "github.com/artefactual-sdps/remove-files-activity"
+
+	"github.com/artefactual-sdps/temporal-activities/removefiles"
+	cp "github.com/otiai10/copy"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/suite"
+	temporalsdk_activity "go.temporal.io/sdk/activity"
+	"go.temporal.io/sdk/testsuite"
+	temporalsdk_worker "go.temporal.io/sdk/worker"
+)
+
+type PreprocessingTestSuite struct {
+	suite.Suite
+	testsuite.WorkflowTestSuite
+
+	env *testsuite.TestWorkflowEnvironment
+
+	// Each test creates its own temporary transfer directory.
+	testDir string
+
+	// Each test registers the workflow with a different name to avoid
+	// duplicates.
+	workflow *workflow.PreprocessingWorkflow
+}
+
+func (s *PreprocessingTestSuite) copyTestTransfer(src, relPath string) {
+	err := cp.Copy(src, filepath.Join(s.testDir, relPath))
+	if err != nil {
+		s.T().Fatalf("copyTestTransfer: %v", err)
+	}
+}
+
+func (s *PreprocessingTestSuite) SetupTest(
+	cfg config.Configuration,
+	transferPath, relPath string,
+) {
+	s.env = s.NewTestWorkflowEnvironment()
+	s.env.SetWorkerOptions(temporalsdk_worker.Options{EnableSessionWorker: true})
+	s.testDir = s.T().TempDir()
+
+	// Register activities.
+	s.env.RegisterActivityWithOptions(
+		removefiles.NewActivity(cfg.RemoveFiles).Execute,
+		temporalsdk_activity.RegisterOptions{Name: remove.RemoveFilesName},
+	)
+
+	s.workflow = workflow.NewPreprocessingWorkflow(s.testDir)
+
+	// Copy test files to testDir.
+	s.copyTestTransfer(transferPath, relPath)
+}
+
+func (s *PreprocessingTestSuite) AfterTest(suiteName, testName string) {
+	s.env.AssertExpectations(s.T())
+}
+
+func TestPreprocessingWorkflow(t *testing.T) {
+	suite.Run(t, new(PreprocessingTestSuite))
+}
+
+func (s *PreprocessingTestSuite) TestExecute() {
+	relPath := "transfer"
+	cfg := config.Configuration{
+		RemoveFiles: removefiles.Config{RemoveNames: ".DS_Store"},
+	}
+
+	s.SetupTest(
+		cfg,
+		filepath.Join("testdata", "transfer_with_ds_store"),
+		relPath,
+	)
+
+	// Mock activities.
+	sessionCtx := mock.AnythingOfType("*context.timerCtx")
+	s.env.OnActivity(
+		removefiles.ActivityName,
+		sessionCtx,
+		&removefiles.ActivityParams{Path: filepath.Join(s.testDir, relPath)},
+	).Return(
+		&removefiles.ActivityResult{Count: 1}, nil,
+	)
+
+	s.env.ExecuteWorkflow(
+		s.workflow.Execute,
+		&workflow.PreprocessingWorkflowParams{RelativePath: relPath},
+	)
+
+	s.True(s.env.IsWorkflowCompleted())
+
+	var result workflow.PreprocessingWorkflowResult
+	err := s.env.GetWorkflowResult(&result)
+	s.NoError(err)
+	s.Equal(
+		&result,
+		&workflow.PreprocessingWorkflowResult{RelativePath: relPath},
+	)
+}

internal/workflow/testdata/transfer_with_ds_store/small.txt

@@ -0,0 +1 @@
+I am a small file.