Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix missing batch header partial files #160

Closed
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions configs/config.default.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,4 @@ ACHWebViewer:
HelpfulLinks:
Corrections: "https://moov-io.github.io/ach/changes/"
Returns: "https://moov-io.github.io/ach/returns/"
AllowMissingBatchHeader: true
jasonbornsteinMOOV marked this conversation as resolved.
Show resolved Hide resolved
14 changes: 7 additions & 7 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ go 1.20
require (
github.com/gorilla/mux v1.8.0
github.com/markbates/pkger v0.17.1
github.com/moov-io/ach v1.32.2
github.com/moov-io/ach v1.32.3-0.20231002183900-f656fc767101
github.com/moov-io/base v0.47.0
github.com/moov-io/cryptfs v0.7.0
github.com/stretchr/testify v1.8.4
Expand Down Expand Up @@ -56,8 +56,8 @@ require (
github.com/moov-io/iso4217 v0.3.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/client_golang v1.17.0 // indirect
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/rickar/cal/v2 v2.1.13 // indirect
Expand All @@ -69,12 +69,12 @@ require (
github.com/spf13/viper v1.16.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/crypto v0.12.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/crypto v0.13.0 // indirect
golang.org/x/net v0.15.0 // indirect
golang.org/x/oauth2 v0.11.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.134.0 // indirect
Expand Down
34 changes: 14 additions & 20 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -192,8 +192,6 @@ github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8=
Expand Down Expand Up @@ -264,10 +262,8 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/moov-io/ach v1.32.2 h1:uwBYSL1m9vV4GSK6v8m5dZmoGgtVEbkOiJETipb2A1g=
github.com/moov-io/ach v1.32.2/go.mod h1:cwLQSQgveLFgUCMU+nekIuQjFXKCCXDIPBDrbvelunM=
github.com/moov-io/base v0.46.0 h1:4sgFjfD9EELJ+i6cGfPtezoPAAxwE3imLJ64bbHtBwM=
github.com/moov-io/base v0.46.0/go.mod h1:AFoLPJA1uuylLp33g1nGzTnGk8zyqYH7KfBS7flc0P8=
github.com/moov-io/ach v1.32.3-0.20231002183900-f656fc767101 h1:lwiB+o5c/ghkxYvoWTLJ9Yr4l8YY+TlAqm2SSa/eDzc=
github.com/moov-io/ach v1.32.3-0.20231002183900-f656fc767101/go.mod h1:VK7fphrNkVyO+zmwaStWFAougHAzq6Z+dtU1BljLWE0=
github.com/moov-io/base v0.47.0 h1:r4PALYPSMwCB/smDUcMUk3a2JW6TA4mT17oOBx9ZpEE=
github.com/moov-io/base v0.47.0/go.mod h1:jz9m/WyTr6RTbm1DP0V5j4RVUuXqmq3bUnzJ6NwCPo8=
github.com/moov-io/cryptfs v0.7.0 h1:BTWK/PI1P8zhCGhLdD+Ha8DxkGG073P9GgR1FrjFerU=
Expand All @@ -281,11 +277,11 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
Expand Down Expand Up @@ -352,8 +348,8 @@ golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
Expand Down Expand Up @@ -426,8 +422,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down Expand Up @@ -496,8 +492,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
Expand All @@ -515,8 +511,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down Expand Up @@ -669,8 +665,6 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5
google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
google.golang.org/grpc v1.58.1 h1:OL+Vz23DTtrrldqHK49FUOPHyY75rvFqJfXC84NYW58=
google.golang.org/grpc v1.58.1/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
Expand Down
7 changes: 3 additions & 4 deletions pkg/filelist/achgateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"path/filepath"
"time"

"github.com/moov-io/ach"
"github.com/moov-io/ach-web-viewer/pkg/service"
)

Expand Down Expand Up @@ -96,7 +95,7 @@ func (a *achgatewayLister) getFiles(shard string) ([]File, error) {
return out, nil
}

func (a *achgatewayLister) GetFile(path string) (*File, error) {
func (a *achgatewayLister) GetFile(path string, cfg service.DisplayConfig) (*File, error) {
req, err := http.NewRequest("GET", a.endpoint+"/"+path, nil)
if err != nil {
return nil, err
Expand Down Expand Up @@ -124,15 +123,15 @@ func (a *achgatewayLister) GetFile(path string) (*File, error) {
return nil, err
}

file, err := ach.NewReader(bytes.NewReader(contents)).Read()
file, err := readFile(bytes.NewReader(contents), cfg)
if err != nil {
return nil, err
}

return &File{
Name: wrapper.Filename,
StoragePath: dir,
Contents: &file,
Contents: file,
CreatedAt: wrapper.ModTime,
Size: int64(len(contents)),
}, nil
Expand Down
4 changes: 2 additions & 2 deletions pkg/filelist/bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func (ls *bucketLister) GetFiles(opts ListOpts) (Files, error) {
return out, nil
}

func (ls *bucketLister) GetFile(path string) (*File, error) {
func (ls *bucketLister) GetFile(path string, cfg service.DisplayConfig) (*File, error) {
rdr, err := ls.buck.NewReader(context.Background(), path, nil)
if err != nil {
return nil, err
Expand All @@ -97,7 +97,7 @@ func (ls *bucketLister) GetFile(path string) (*File, error) {

_, name := filepath.Split(path)

file, err := readFile(bytes.NewReader(bs))
file, err := readFile(bytes.NewReader(bs), cfg)

return &File{
Name: name,
Expand Down
9 changes: 7 additions & 2 deletions pkg/filelist/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,15 @@ import (
"strings"

"github.com/moov-io/ach"
"github.com/moov-io/ach-web-viewer/pkg/service"
)

func readFile(r io.Reader) (*ach.File, error) {
file, err := ach.NewReader(r).Read()
func readFile(r io.Reader, cfg service.DisplayConfig) (*ach.File, error) {
reader := ach.NewReader(r)
reader.SetValidation(&ach.ValidateOpts{
AllowMissingBatchHeader: cfg.AllowMissingBatchHeader,
})
file, err := reader.Read()
if err != nil {
message := err.Error()
switch {
Expand Down
4 changes: 2 additions & 2 deletions pkg/filelist/filesystem.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func (ls *filesystemLister) GetFiles(opts ListOpts) (Files, error) {
return out, nil
}

func (ls *filesystemLister) GetFile(path string) (*File, error) {
func (ls *filesystemLister) GetFile(path string, cfg service.DisplayConfig) (*File, error) {
path = filepath.Clean(path)

if strings.Contains(path, "..") || strings.HasPrefix(path, "/") {
Expand All @@ -75,7 +75,7 @@ func (ls *filesystemLister) GetFile(path string) (*File, error) {

_, name := filepath.Split(fd.Name())

file, err := readFile(fd)
file, err := readFile(fd, cfg)

var stat fs.FileInfo
if fd != nil {
Expand Down
6 changes: 3 additions & 3 deletions pkg/filelist/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ type File struct {
type Lister interface {
SourceID() string

GetFile(path string) (*File, error)
GetFile(path string, cfg service.DisplayConfig) (*File, error)
GetFiles(opts ListOpts) (Files, error)
}

Expand Down Expand Up @@ -71,10 +71,10 @@ func createLister(src service.Source) (Lister, error) {
return nil, fmt.Errorf("unknown source: %#v", src)
}

func (ls Listers) GetFile(sourceID, path string) (*File, error) {
func (ls Listers) GetFile(sourceID, path string, cfg service.DisplayConfig) (*File, error) {
for i := range ls {
if ls[i].SourceID() == sourceID {
return ls[i].GetFile(path)
return ls[i].GetFile(path, cfg)
}
}
return nil, fmt.Errorf("%s not found for sourceID=%s", path, sourceID)
Expand Down
7 changes: 4 additions & 3 deletions pkg/service/model_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,10 @@ type BindAddress struct {
}

type DisplayConfig struct {
Format string // e.g. "human-readable"
Masking MaskingConfig
HelpfulLinks HelpfulLinks
Format string // e.g. "human-readable"
Masking MaskingConfig
HelpfulLinks HelpfulLinks
AllowMissingBatchHeader bool
}

type MaskingConfig struct {
Expand Down
2 changes: 1 addition & 1 deletion pkg/web/web.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ func getFile(logger log.Logger, cfg service.DisplayConfig, listers filelist.List
sourceID := mux.Vars(r)["sourceID"]
fullPath := strings.TrimPrefix(r.URL.Path, fmt.Sprintf("%s/sources/%s/", basePath, sourceID))

file, err := listers.GetFile(sourceID, fullPath)
file, err := listers.GetFile(sourceID, fullPath, cfg)
if err != nil {
logger.Warn().Logf("ERROR: %v\n", err)
w.WriteHeader(http.StatusInternalServerError)
Expand Down
Loading