Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Import json file #488

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions cmd/genji/dbutil/insert.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package dbutil

import (
"bufio"
"encoding/csv"
"encoding/json"
"fmt"
"io"
Expand Down Expand Up @@ -109,3 +110,36 @@ func readByteIgnoreWhitespace(r *bufio.Reader) (byte, error) {

return c, nil
}

// InsertCSV reads csv documents from r and inserts them into the selected table.
// The first line of the csv file should contain the field names.
func InsertCSV(db *genji.DB, table string, r io.Reader) error {
tx, err := db.Begin(true)
if err != nil {
return err
}
defer tx.Rollback()

rd := csv.NewReader(r)

headers, err := rd.Read()
if err != nil {
return err
}

for {
columns, err := rd.Read()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}
err = tx.Exec("INSERT INTO "+table+" VALUES ?", document.NewFromCSV(headers, columns))
if err != nil {
return err
}
}

return tx.Commit()
}
46 changes: 14 additions & 32 deletions cmd/genji/shell/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package shell
import (
"bytes"
"context"
"encoding/csv"
"fmt"
"io"
"os"
Expand Down Expand Up @@ -83,7 +82,7 @@ var commands = []command{
Name: ".import",
Options: "TYPE FILE table",
DisplayName: ".import",
Description: "Import data from a file. Only supported type is 'csv'",
Description: "Import data from a file. Supported types are 'csv' and 'json'.",
},
{
Name: ".timer",
Expand Down Expand Up @@ -201,8 +200,9 @@ func runSaveCmd(ctx context.Context, db *genji.DB, dbPath string) error {
}

func runImportCmd(db *genji.DB, fileType, path, table string) error {
if strings.ToLower(fileType) != "csv" {
return errors.New("TYPE should be csv")
fileType = strings.ToLower(fileType)
if fileType != "csv" && fileType != "json" {
return fmt.Errorf("unsupported TYPE: %q, should be csv or json", fileType)
}

f, err := os.Open(path)
Expand All @@ -211,37 +211,19 @@ func runImportCmd(db *genji.DB, fileType, path, table string) error {
}
defer f.Close()

tx, err := db.Begin(true)
err = db.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s", table))
if err != nil {
return err
}
defer tx.Rollback()

r := csv.NewReader(f)

err = tx.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s", table))
if err != nil {
return err
}

headers, err := r.Read()
if err != nil {
return err
}

for {
columns, err := r.Read()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}
err = tx.Exec("INSERT INTO "+table+" VALUES ?", document.NewFromCSV(headers, columns))
if err != nil {
return err
}
switch fileType {
case "csv":
return dbutil.InsertCSV(db, table, f)
case "json":
return dbutil.InsertJSON(db, table, f)
default:
return fmt.Errorf("unsupported TYPE %q", fileType)
}

return tx.Commit()
}

// Separated insert csv to