fixed: error: bufio.Scanner token too long

pkg/config/config.go

@@ -24,6 +24,7 @@ type Config struct {
 	Target Resource
 	Silent bool
 	TTL    bool
+	MaxBuf int
 }
 
 // exit will exit and print the usage.
@@ -36,7 +37,7 @@ func exit(e error) {
 
 // validate makes sure from and to are Redis URIs or file paths,
 // and generates the final Config.
-func validate(from, to string, silent, ttl bool) (Config, error) {
+func validate(from, to string, silent, ttl bool, maxBuf int) (Config, error) {
 	cfg := Config{
 		Source: Resource{
 			URI: from,
@@ -46,6 +47,7 @@ func validate(from, to string, silent, ttl bool) (Config, error) {
 		},
 		Silent: silent,
 		TTL:    ttl,
+		MaxBuf: maxBuf,
 	}
 
 	if strings.HasPrefix(from, "redis://") {
@@ -76,10 +78,10 @@ func Parse() Config {
 	to := flag.String("to", "", example)
 	silent := flag.Bool("silent", false, "optional, no verbose output")
 	ttl := flag.Bool("ttl", false, "optional, enable ttl sync")
-
+	maxBuf := flag.Int("buffer", 64*1024, "the size of the buffer used when reading the file, uint:byte")
 	flag.Parse()
 
-	cfg, err := validate(*from, *to, *silent, *ttl)
+	cfg, err := validate(*from, *to, *silent, *ttl, *maxBuf)
 	if err != nil {
 		// we exit here instead of returning so that we can show
 		// the usage examples in case of an error.

pkg/file/file.go

@@ -18,6 +18,7 @@ type File struct {
 	Bus    message.Bus
 	Silent bool
 	TTL    bool
+	MaxBuf int
 }
 
 // splitCross is a double-cross (✝✝) custom Scanner Split.
@@ -38,12 +39,13 @@ func splitCross(data []byte, atEOF bool) (advance int, token []byte, err error)
 }
 
 // New creates the File struct, to be used for reading/writing.
-func New(path string, bus message.Bus, silent, ttl bool) *File {
+func New(path string, bus message.Bus, silent, ttl bool, maxBuf int) *File {
 	return &File{
 		Path:   path,
 		Bus:    bus,
 		Silent: silent,
 		TTL:    ttl,
+		MaxBuf: maxBuf,
 	}
 }
 
@@ -67,6 +69,8 @@ func (f *File) Read(ctx context.Context) error {
 
 	// Scan file, split by double-cross separator
 	scanner := bufio.NewScanner(d)
+	buf := make([]byte, 0, bufio.MaxScanTokenSize)
+	scanner.Buffer(buf, f.MaxBuf)
 	scanner.Split(splitCross)
 
 	// Scan line by line

pkg/run/run.go

@@ -49,7 +49,7 @@ func Run(cfg config.Config) {
 			return source.Read(gctx)
 		})
 	} else {
-		source := file.New(cfg.Source.URI, ch, cfg.Silent, cfg.TTL)
+		source := file.New(cfg.Source.URI, ch, cfg.Silent, cfg.TTL, cfg.MaxBuf)
 
 		g.Go(func() error {
 			return source.Read(gctx)
@@ -70,7 +70,7 @@ func Run(cfg config.Config) {
 			return target.Write(gctx)
 		})
 	} else {
-		target := file.New(cfg.Target.URI, ch, cfg.Silent, cfg.TTL)
+		target := file.New(cfg.Target.URI, ch, cfg.Silent, cfg.TTL, cfg.MaxBuf)
 
 		g.Go(func() error {
 			defer cancel()