diff --git a/go.work.sum b/go.work.sum index e47e1ccf9..6c3208f3f 100644 --- a/go.work.sum +++ b/go.work.sum @@ -470,6 +470,7 @@ google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= honnef.co/go/js/dom v0.0.0-20210725211120-f030747120f2 h1:oomkgU6VaQDsV6qZby2uz1Lap0eXmku8+2em3A/l700= honnef.co/go/js/dom v0.0.0-20210725211120-f030747120f2/go.mod h1:sUMDUKNB2ZcVjt92UnLy3cdGs+wDAcrPdV3JP6sVgA4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= diff --git a/server/main.go b/server/main.go index df7f01e32..604217962 100644 --- a/server/main.go +++ b/server/main.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/eukarya-inc/reearth-plateauview/server/putil" + "github.com/eukarya-inc/reearth-plateauview/server/tool" "github.com/go-playground/validator/v10" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" @@ -27,7 +28,10 @@ func main() { conf := lo.Must(NewConfig()) if len(os.Args) > 1 && os.Args[1] != "" { - tool(conf, os.Args[1:]) + tool.Main(&tool.Config{ + CMS_BaseURL: conf.CMS_BaseURL, + CMS_Token: conf.CMS_Token, + }, os.Args[1:]) return } diff --git a/server/tool/migrate-v1.go b/server/tool/migrate-v1.go new file mode 100644 index 000000000..44f29021b --- /dev/null +++ b/server/tool/migrate-v1.go @@ -0,0 +1,196 @@ +package tool + +import ( + "archive/zip" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "strings" +) + +func migrateV1(conf *Config, args []string) error { + var listFilePath, output, baseURL string + var offset int + var wetrun bool + + flags := flag.NewFlagSet("migrate-v1", flag.ExitOnError) + flags.StringVar(&listFilePath, "list", "", "list file path") + flags.StringVar(&output, "output", "", "output file path") + flags.StringVar(&baseURL, "base", "", "base URL") + flags.BoolVar(&wetrun, "wetrun", false, "wet run") + flags.IntVar(&offset, "offset", 0, "offset") + if err := flags.Parse(args); err != nil { + return err + } + + if baseURL == "" { + return fmt.Errorf("base URL is required") + } + + if output != "" { + _ = os.MkdirAll(output, 0755) + } + + listFile, err := os.ReadFile(listFilePath) + if err != nil { + return fmt.Errorf("failed to read list file: %w", err) + } + + list, err := parseList(listFile) + + if offset > 0 { + if len(list) < offset { + return fmt.Errorf("offset is too large") + } + list = list[offset:] + } + + le := len(list) + if err != nil { + return fmt.Errorf("failed to parse list: %w", err) + } + + var file *os.File + var zw *zip.Writer + + exhangeZw := func(g string) error { + if zw != nil { + if err := zw.Close(); err != nil { + return fmt.Errorf("failed to close zip: %w", err) + } + } + + if file != nil { + if err := file.Close(); err != nil { + return fmt.Errorf("failed to close file: %w", err) + } + } + + if g == "" { + zw = nil + file = nil + return nil + } + + file, err = os.OpenFile(filepath.Join(output, g+".zip"), os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + + zw = zip.NewWriter(file) + return nil + } + + defer func() { + if zw != nil { + _ = zw.Close() + } + if file != nil { + _ = file.Close() + } + }() + + group := "" + for i, p := range list { + g := p[1] + filePath := path.Join(p[2:]...) + path := filepath.Join(p...) + u, err := url.JoinPath(baseURL, path) + if err != nil { + return fmt.Errorf("failed to join path: %w", err) + } + + fmt.Printf("%d/%d | %s | %s | %s\n", i+1, le, g, filePath, u) + + if group != g { + fmt.Printf("group: %s\n", g) + + if wetrun { + if err := exhangeZw(g); err != nil { + return err + } + } + + group = g + } + + if wetrun { + if err := downloadAndAddToZip(zw, path, u, filePath); err != nil { + return fmt.Errorf("failed to download: %w", err) + } + } + } + + if err := exhangeZw(""); err != nil { + return err + } + + return nil +} + +func downloadAndAddToZip(zw *zip.Writer, path, url, filePath string) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + res, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to request: %w", err) + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return fmt.Errorf("failed to request: %s", res.Status) + } + + f, err := zw.Create(filePath) + if err != nil { + return fmt.Errorf("failed to create zip entry: %w", err) + } + + if _, err := io.Copy(f, res.Body); err != nil { + return fmt.Errorf("failed to copy: %w", err) + } + + return nil +} + +func parseList(b []byte) (res [][]string, err error) { + start := 0 + for i, c := range b { + if c == '\n' { + r := string(b[start:i]) + start = i + 1 + + s := reSpace.Split(r, -1) + if len(s) <= 3 { + err = fmt.Errorf("invalid line at %d: %s", i, r) + return + } + + t := strings.Split(s[3], "/") + if strings.HasPrefix(t[len(t)-1], ".") { + continue + } + + if len(t) <= 2 { + err = fmt.Errorf("invalid line at %d: %s", i, r) + return + } + + res = append(res, t) + } + } + + return +} + +var reSpace = regexp.MustCompile(`\s+`) diff --git a/server/tool.go b/server/tool/setup-city-items.go similarity index 82% rename from server/tool.go rename to server/tool/setup-city-items.go index bcbf05070..c3e76c885 100644 --- a/server/tool.go +++ b/server/tool/setup-city-items.go @@ -1,4 +1,4 @@ -package main +package tool import ( "context" @@ -13,30 +13,6 @@ import ( "github.com/samber/lo" ) -func tool(conf *Config, args []string) { - subommand := args[0] - var err error - - switch subommand { - case "setup-city-items": - err = setupCityItems(conf, args[1:]) - case "help": - err = help(conf) - default: - err = errors.New("invalid subcommand") - } - - if err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func help(*Config) error { - fmt.Println(`Usage: plateauview [arguments] [options] [flags]`) - return nil -} - func setupCityItems(conf *Config, args []string) error { println("setup-city-items") diff --git a/server/tool/tool.go b/server/tool/tool.go new file mode 100644 index 000000000..b4b0e0f17 --- /dev/null +++ b/server/tool/tool.go @@ -0,0 +1,40 @@ +package tool + +import ( + "errors" + "fmt" + "os" +) + +type Config struct { + CMS_BaseURL string + CMS_Token string +} + +func Main(conf *Config, args []string) { + subommand := args[0] + var err error + + switch subommand { + case "setup-city-items": + err = setupCityItems(conf, args[1:]) + case "migrate-v1": + err = migrateV1(conf, args[1:]) + case "upload-assets": + err = uploadAssets(conf, args[1:]) + case "help": + err = help(conf) + default: + err = errors.New("invalid subcommand") + } + + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func help(*Config) error { + fmt.Println(`Usage: plateauview [arguments] [options] [flags]`) + return nil +} diff --git a/server/tool/upload-assets.go b/server/tool/upload-assets.go new file mode 100644 index 000000000..fd6310883 --- /dev/null +++ b/server/tool/upload-assets.go @@ -0,0 +1,69 @@ +package tool + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "path" + "path/filepath" + + cms "github.com/reearth/reearth-cms-api/go" +) + +func uploadAssets(conf *Config, args []string) error { + var base, token, pid, target string + + flags := flag.NewFlagSet("upload-assets", flag.ExitOnError) + flags.StringVar(&base, "base", conf.CMS_BaseURL, "CMS base URL") + flags.StringVar(&token, "token", conf.CMS_Token, "CMS token") + flags.StringVar(&pid, "project", "", "project ID") + flags.StringVar(&target, "target", "", "target dir path") + + if err := flags.Parse(args); err != nil { + return err + } + + if pid == "" { + return errors.New("project ID is required") + } + + c, err := cms.New(base, token) + if err != nil { + return err + } + + files, err := os.ReadDir(target) + if err != nil { + return err + } + + ctx := context.Background() + for _, f := range files { + if f.IsDir() { + continue + } + + ext := path.Ext(f.Name()) + if ext != ".zip" { + continue + } + + file, err := os.Open(filepath.Join(target, f.Name())) + if err != nil { + return err + } + + fmt.Printf("%s -> ", file.Name()) + + assetID, err := c.UploadAssetDirectly(ctx, pid, file.Name(), file) + if err != nil { + return err + } + + fmt.Printf("%s\n", assetID) + } + + return nil +} diff --git a/tools/src/args.rs b/tools/src/args.rs index a644a4a2c..ce2a1e3c5 100644 --- a/tools/src/args.rs +++ b/tools/src/args.rs @@ -23,13 +23,6 @@ pub enum Commands { #[clap(short, long)] output: Option, }, - /// PLATEAU VIEW 1.1 のS3内に格納されたユースケースデータのファイルを移行します。 - MigrateUC { - #[clap(short, long)] - list_path: PathBuf, - #[clap(short, long)] - output: Option, - }, } #[derive(Debug, Clone, ValueEnum)] diff --git a/tools/src/main.rs b/tools/src/main.rs index 06f9cb079..cf4ecfe76 100644 --- a/tools/src/main.rs +++ b/tools/src/main.rs @@ -1,5 +1,4 @@ mod args; -mod migrateuc; mod prepare; use args::{Cli, Commands}; @@ -18,9 +17,6 @@ fn main() { output, format: format.into(), }), - Commands::MigrateUC { list_path, output } => { - migrateuc::migrateuc(migrateuc::Config { list_path, output }) - } } { eprintln!("{}", err); std::process::exit(1); diff --git a/tools/src/migrateuc.rs b/tools/src/migrateuc.rs deleted file mode 100644 index 890eb6dba..000000000 --- a/tools/src/migrateuc.rs +++ /dev/null @@ -1,13 +0,0 @@ -use std::{fs::create_dir_all, path::PathBuf}; - -pub struct Config { - pub list_path: PathBuf, - pub output: Option, -} - -pub fn migrateuc(config: Config) -> anyhow::Result<()> { - let output = config.output.unwrap_or_else(|| PathBuf::from("uc")); - create_dir_all(output)?; - - todo!() -}