delete empty files

docs/config.md

@@ -178,6 +178,8 @@ odfi:
 
     # Should we delete the remote file on an ODFI's server after downloading and processing of each file.
     [ keepRemoteFiles: <boolean> | default = false ]
+    # Time to wait before deleting a zero byte file
+    [ removeZeroByteFilesAfter: <time.Duration> ]
 
     local:
       [ directory: <filename> ]

examples/config.yaml

@@ -46,6 +46,7 @@ odfi:
   storage:
     cleanupLocalDirectory: true
     keepRemoteFiles: false
+    removeZeroByteFilesAfter: 10m
     local:
       directory: "./storage/"
 transfers:

pkg/config/odfi.go

@@ -245,6 +245,9 @@ type Storage struct {
 	// after downloading and processing of each file.
 	KeepRemoteFiles bool
 
+	// RemoveZeroByteFilesAfter The amount of time to wait before deleting zero byte files.
+	RemoveZeroByteFilesAfter time.Duration
+
 	Local *Local
 }
 

pkg/transfers/inbound/cleanup.go

@@ -7,7 +7,9 @@ package inbound
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/moov-io/base"
 
@@ -16,6 +18,7 @@ import (
 	"github.com/moov-io/base/log"
 )
 
+// Cleanup deletes files if enabled via config
 func Cleanup(logger log.Logger, agent upload.Agent, dl *downloadedFiles) error {
 	var el base.ErrorList
 
@@ -25,13 +28,34 @@ func Cleanup(logger log.Logger, agent upload.Agent, dl *downloadedFiles) error {
 	if err := deleteFilesOnRemote(logger, agent, dl.dir, agent.ReturnPath()); err != nil {
 		el.Add(err)
 	}
+	if el.Empty() {
+		return nil
+	}
+	return el
+}
+
+// CleanupEmptyFiles deletes empty ACH files if file is older than value in config
+func CleanupEmptyFiles(logger log.Logger, agent upload.Agent, dl *downloadedFiles, sourceTime time.Time, after time.Duration) error {
+	var el base.ErrorList
+
+	if after <= 0*time.Second {
+		logger.Log(fmt.Sprintf("deleting empty file requires after > 0. currently: %s", after))
+		return nil
+	}
 
+	if err := deleteEmptyFiles(logger, agent, dl.dir, agent.InboundPath(), sourceTime, after); err != nil {
+		el.Add(err)
+	}
+	if err := deleteEmptyFiles(logger, agent, dl.dir, agent.ReturnPath(), sourceTime, after); err != nil {
+		el.Add(err)
+	}
 	if el.Empty() {
 		return nil
 	}
 	return el
 }
 
+// deleteFilesOnRemote deletes all files for a given directory
 func deleteFilesOnRemote(logger log.Logger, agent upload.Agent, localDir, suffix string) error {
 	baseDir := filepath.Join(localDir, suffix)
 	infos, err := ioutil.ReadDir(baseDir)
@@ -54,3 +78,43 @@ func deleteFilesOnRemote(logger log.Logger, agent upload.Agent, localDir, suffix
 	}
 	return el
 }
+
+// deleteEmptyFiles deletes all empty files that are older than after (time.Duration)
+func deleteEmptyFiles(logger log.Logger, agent upload.Agent, localDir, suffix string, sourceTime time.Time, after time.Duration) error {
+	baseDir := filepath.Join(localDir, suffix)
+	infos, err := ioutil.ReadDir(baseDir)
+	if err != nil {
+		return fmt.Errorf("reading %s: %v", baseDir, err)
+	}
+
+	var el base.ErrorList
+	for i := range infos {
+		fileInfo := infos[i]
+		path := filepath.Join(suffix, filepath.Base(infos[i].Name()))
+		if shouldDeleteEmptyFile(fileInfo, sourceTime, after) {
+			err := agent.Delete(path)
+			if err != nil {
+				el.Add(err)
+			} else {
+				logger.Log(fmt.Sprintf("deleted zero byte file %s", path))
+			}
+		} else {
+			logger.Log(fmt.Sprintf("zero byte file not deleted %s", path))
+		}
+	}
+
+	if el.Empty() {
+		return nil
+	}
+	return el
+}
+
+// shouldDeleteEmptyFile determines if a file is empty and if it should be deleted
+// per the config setting RemoveEmptyFileAfter
+func shouldDeleteEmptyFile(info os.FileInfo, sourceTime time.Time, removeEmptyFileAfter time.Duration) bool {
+	if info.Size() != 0 {
+		return false
+	}
+	diff := sourceTime.Sub(info.ModTime())
+	return diff.Minutes() >= removeEmptyFileAfter.Minutes()
+}

pkg/transfers/inbound/cleanup_test.go

@@ -10,12 +10,41 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
-
-	"github.com/moov-io/paygate/pkg/upload"
+	"time"
 
 	"github.com/moov-io/base/log"
+	"github.com/moov-io/paygate/pkg/upload"
 )
 
+type mockedFS struct {
+	fileSize int64
+	modTime  time.Time
+}
+
+func (m mockedFS) Name() string {
+	panic("implement me")
+}
+
+func (m mockedFS) Size() int64 {
+	return m.fileSize
+}
+
+func (m mockedFS) Mode() os.FileMode {
+	panic("implement me")
+}
+
+func (m mockedFS) ModTime() time.Time {
+	return m.modTime
+}
+
+func (m mockedFS) IsDir() bool {
+	panic("implement me")
+}
+
+func (m mockedFS) Sys() interface{} {
+	panic("implement me")
+}
+
 func TestCleanupErr(t *testing.T) {
 	agent := &upload.MockAgent{
 		Err: errors.New("bad error"),
@@ -43,3 +72,149 @@ func TestCleanupErr(t *testing.T) {
 		t.Errorf("unexpected deleted file: %s", agent.DeletedFile)
 	}
 }
+
+func Test_CleanupEmptyFiles_InboundPath_Success(t *testing.T) {
+	agent := &upload.MockAgent{}
+
+	dir, _ := ioutil.TempDir("", "clenaup-testing")
+	dl := &downloadedFiles{dir: dir}
+	defer dl.deleteFiles()
+
+	// write a test file to attempt deletion
+	path := filepath.Join(dl.dir, agent.InboundPath())
+	if err := os.MkdirAll(path, 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(path, "empty_file.ach"), []byte(""), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fileInfo := files[0]
+	mockTimeNow := fileInfo.ModTime().Add(10 * time.Minute)
+	minutesAfterToDelete := 5 * time.Minute
+
+	if err := CleanupEmptyFiles(log.NewNopLogger(), agent, dl, mockTimeNow, minutesAfterToDelete); err == nil {
+		t.Error("expected error")
+	}
+
+	if agent.DeletedFile != "inbound/empty_file.ach" {
+		t.Errorf("unexpected deleted file: %s", agent.DeletedFile)
+	}
+}
+
+func Test_CleanupEmptyFiles_ReturnPath_Success(t *testing.T) {
+	agent := &upload.MockAgent{}
+
+	dir, _ := ioutil.TempDir("", "clenaup-testing")
+	dl := &downloadedFiles{dir: dir}
+	defer dl.deleteFiles()
+
+	// write a test file to attempt deletion
+	path := filepath.Join(dl.dir, agent.ReturnPath())
+	if err := os.MkdirAll(path, 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(path, "empty_file.ach"), []byte(""), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fileInfo := files[0]
+	mockTimeNow := fileInfo.ModTime().Add(10 * time.Minute)
+	minutesAfterToDelete := 5 * time.Minute
+
+	if err := CleanupEmptyFiles(log.NewNopLogger(), agent, dl, mockTimeNow, minutesAfterToDelete); err == nil {
+		t.Error("expected error")
+	}
+
+	if agent.DeletedFile != "return/empty_file.ach" {
+		t.Errorf("unexpected deleted file: %s", agent.DeletedFile)
+	}
+}
+
+func Test_CleanupEmptyFiles_Not_Run_When_Config_Is_Zero(t *testing.T) {
+	agent := &upload.MockAgent{}
+
+	dir, _ := ioutil.TempDir("", "clenaup-testing")
+	dl := &downloadedFiles{dir: dir}
+	defer dl.deleteFiles()
+
+	// write a test file to attempt deletion
+	path := filepath.Join(dl.dir, agent.InboundPath())
+	if err := os.MkdirAll(path, 0777); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := ioutil.WriteFile(filepath.Join(path, "empty_file.ach"), []byte(""), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fileInfo := files[0]
+	mockTimeNow := fileInfo.ModTime().Add(10 * time.Minute)
+	minutesAfterToDelete := 0 * time.Minute
+
+	if err := CleanupEmptyFiles(log.NewNopLogger(), agent, dl, mockTimeNow, minutesAfterToDelete); err != nil {
+		t.Error("expected nil")
+	}
+
+	if agent.DeletedFile == "inbound/empty_file.ach" {
+		t.Errorf("unexpected deleted file: %s", agent.DeletedFile)
+	}
+}
+
+func Test_ShouldDeleteEmptyFile_Success(t *testing.T) {
+	now := time.Now()
+	mfs := &mockedFS{
+		fileSize: 0,
+		modTime:  now.Add(time.Minute * -10),
+	}
+	actual := shouldDeleteEmptyFile(mfs, now, 10)
+	if actual == false {
+		t.Error("expected true")
+	}
+}
+
+func Test_ShouldDeleteEmptyFile_Fails_File_Size_Greater_Than_Zero(t *testing.T) {
+	now := time.Now()
+	mfs := &mockedFS{
+		fileSize: 1,
+		modTime:  now.Add(time.Minute * 10),
+	}
+	actual := shouldDeleteEmptyFile(mfs, now, 10)
+	if actual == true {
+		t.Error("expected false")
+	}
+}
+
+func Test_ShouldDeleteEmptyFile_Returns_False_When_File_Size_Is_Not_Zero(t *testing.T) {
+	mfs := &mockedFS{
+		fileSize: 12,
+	}
+	actual := shouldDeleteEmptyFile(mfs, time.Now(), 10)
+	if actual == true {
+		t.Error("expected false")
+	}
+}
+
+func Test_ShouldDeleteEmptyFile_Returns_False_When_Under_Threshold(t *testing.T) {
+	mfs := &mockedFS{
+		fileSize: 12,
+	}
+	actual := shouldDeleteEmptyFile(mfs, time.Now(), 10)
+	if actual == true {
+		t.Error("expected false")
+	}
+}

pkg/transfers/inbound/scheduler.go

@@ -109,5 +109,9 @@ func (s *PeriodicScheduler) tick() error {
 		}
 	}
 
+	if err := CleanupEmptyFiles(s.logger, s.agent, dl, time.Now(), s.cfg.Storage.RemoveZeroByteFilesAfter); err != nil {
+		return fmt.Errorf("ERROR: deleting zero byte files: %v", err)
+	}
+
 	return nil
 }

pkg/transfers/inbound/scheduler_test.go

@@ -16,8 +16,9 @@ func TestScheduler(t *testing.T) {
 	cfg := config.Empty()
 	cfg.ODFI.Inbound.Interval = 10 * time.Second
 	cfg.ODFI.Storage = &config.Storage{
-		CleanupLocalDirectory: true,
-		KeepRemoteFiles:       false,
+		CleanupLocalDirectory:    true,
+		KeepRemoteFiles:          false,
+		RemoveZeroByteFilesAfter: 10 * time.Minute,
 	}
 
 	agent := &upload.MockAgent{}