diff --git a/file.go b/file.go index 78da232..7f4d54d 100644 --- a/file.go +++ b/file.go @@ -23,6 +23,7 @@ import ( "os" "bufio" "encoding/json" + "io" ) func checkFileIsExist(filename string) (bool) { @@ -40,13 +41,18 @@ func (m *Migrator) NewFileReadWorker(pb *pb.ProgressBar, wg *sync.WaitGroup) { log.Error(err) return } - scanner := bufio.NewScanner(f) - scanner.Split(bufio.ScanLines) + + defer f.Close() + r := bufio.NewReader(f) lineCount := 0 - for scanner.Scan() { - lineCount++ + for{ + line,err := r.ReadString('\n') + if io.EOF == err || nil != err{ + break + } + lineCount += 1 js := map[string]interface{}{} - line := scanner.Text() + //log.Trace("reading file,",lineCount,",", line) err = json.Unmarshal([]byte(line), &js) if(err!=nil){ diff --git a/main.go b/main.go index 0056110..46e9f60 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( goflags "github.com/jessevdk/go-flags" pb "gopkg.in/cheggaaa/pb.v1" "os" + "io" ) func main() { @@ -125,10 +126,15 @@ func main() { return } //get file lines - fileScanner := bufio.NewScanner(f) lineCount := 0 - for fileScanner.Scan() { - lineCount++ + defer f.Close() + r := bufio.NewReader(f) + for{ + _,err := r.ReadString('\n') + if io.EOF == err || nil != err{ + break + } + lineCount += 1 } log.Trace("file line,", lineCount) fetchBar = pb.New(lineCount).Prefix("Read")