This repository has been archived by the owner on Apr 8, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(server): add tool to migrate uc v1 data and upload assets
- Loading branch information
Showing
9 changed files
with
312 additions
and
50 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,196 @@ | ||
| package tool | ||
|
|
||
| import ( | ||
| "archive/zip" | ||
| "flag" | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
| "net/url" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
| "regexp" | ||
| "strings" | ||
| ) | ||
|
|
||
| func migrateV1(conf *Config, args []string) error { | ||
| var listFilePath, output, baseURL string | ||
| var offset int | ||
| var wetrun bool | ||
|
|
||
| flags := flag.NewFlagSet("migrate-v1", flag.ExitOnError) | ||
| flags.StringVar(&listFilePath, "list", "", "list file path") | ||
| flags.StringVar(&output, "output", "", "output file path") | ||
| flags.StringVar(&baseURL, "base", "", "base URL") | ||
| flags.BoolVar(&wetrun, "wetrun", false, "wet run") | ||
| flags.IntVar(&offset, "offset", 0, "offset") | ||
| if err := flags.Parse(args); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if baseURL == "" { | ||
| return fmt.Errorf("base URL is required") | ||
| } | ||
|
|
||
| if output != "" { | ||
| _ = os.MkdirAll(output, 0755) | ||
| } | ||
|
|
||
| listFile, err := os.ReadFile(listFilePath) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to read list file: %w", err) | ||
| } | ||
|
|
||
| list, err := parseList(listFile) | ||
|
|
||
| if offset > 0 { | ||
| if len(list) < offset { | ||
| return fmt.Errorf("offset is too large") | ||
| } | ||
| list = list[offset:] | ||
| } | ||
|
|
||
| le := len(list) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to parse list: %w", err) | ||
| } | ||
|
|
||
| var file *os.File | ||
| var zw *zip.Writer | ||
|
|
||
| exhangeZw := func(g string) error { | ||
| if zw != nil { | ||
| if err := zw.Close(); err != nil { | ||
| return fmt.Errorf("failed to close zip: %w", err) | ||
| } | ||
| } | ||
|
|
||
| if file != nil { | ||
| if err := file.Close(); err != nil { | ||
| return fmt.Errorf("failed to close file: %w", err) | ||
| } | ||
| } | ||
|
|
||
| if g == "" { | ||
| zw = nil | ||
| file = nil | ||
| return nil | ||
| } | ||
|
|
||
| file, err = os.OpenFile(filepath.Join(output, g+".zip"), os.O_CREATE|os.O_WRONLY, 0644) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to open file: %w", err) | ||
| } | ||
|
|
||
| zw = zip.NewWriter(file) | ||
| return nil | ||
| } | ||
|
|
||
| defer func() { | ||
| if zw != nil { | ||
| _ = zw.Close() | ||
| } | ||
| if file != nil { | ||
| _ = file.Close() | ||
| } | ||
| }() | ||
|
|
||
| group := "" | ||
| for i, p := range list { | ||
| g := p[1] | ||
| filePath := path.Join(p[2:]...) | ||
| path := filepath.Join(p...) | ||
| u, err := url.JoinPath(baseURL, path) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to join path: %w", err) | ||
| } | ||
|
|
||
| fmt.Printf("%d/%d | %s | %s | %s\n", i+1, le, g, filePath, u) | ||
|
|
||
| if group != g { | ||
| fmt.Printf("group: %s\n", g) | ||
|
|
||
| if wetrun { | ||
| if err := exhangeZw(g); err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| group = g | ||
| } | ||
|
|
||
| if wetrun { | ||
| if err := downloadAndAddToZip(zw, path, u, filePath); err != nil { | ||
| return fmt.Errorf("failed to download: %w", err) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if err := exhangeZw(""); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func downloadAndAddToZip(zw *zip.Writer, path, url, filePath string) error { | ||
| req, err := http.NewRequest("GET", url, nil) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to create request: %w", err) | ||
| } | ||
|
|
||
| res, err := http.DefaultClient.Do(req) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to request: %w", err) | ||
| } | ||
|
|
||
| defer res.Body.Close() | ||
|
|
||
| if res.StatusCode != http.StatusOK { | ||
| return fmt.Errorf("failed to request: %s", res.Status) | ||
| } | ||
|
|
||
| f, err := zw.Create(filePath) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to create zip entry: %w", err) | ||
| } | ||
|
|
||
| if _, err := io.Copy(f, res.Body); err != nil { | ||
| return fmt.Errorf("failed to copy: %w", err) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func parseList(b []byte) (res [][]string, err error) { | ||
| start := 0 | ||
| for i, c := range b { | ||
| if c == '\n' { | ||
| r := string(b[start:i]) | ||
| start = i + 1 | ||
|
|
||
| s := reSpace.Split(r, -1) | ||
| if len(s) <= 3 { | ||
| err = fmt.Errorf("invalid line at %d: %s", i, r) | ||
| return | ||
| } | ||
|
|
||
| t := strings.Split(s[3], "/") | ||
| if strings.HasPrefix(t[len(t)-1], ".") { | ||
| continue | ||
| } | ||
|
|
||
| if len(t) <= 2 { | ||
| err = fmt.Errorf("invalid line at %d: %s", i, r) | ||
| return | ||
| } | ||
|
|
||
| res = append(res, t) | ||
| } | ||
| } | ||
|
|
||
| return | ||
| } | ||
|
|
||
| var reSpace = regexp.MustCompile(`\s+`) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package tool | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "os" | ||
| ) | ||
|
|
||
| type Config struct { | ||
| CMS_BaseURL string | ||
| CMS_Token string | ||
| } | ||
|
|
||
| func Main(conf *Config, args []string) { | ||
| subommand := args[0] | ||
| var err error | ||
|
|
||
| switch subommand { | ||
| case "setup-city-items": | ||
| err = setupCityItems(conf, args[1:]) | ||
| case "migrate-v1": | ||
| err = migrateV1(conf, args[1:]) | ||
| case "upload-assets": | ||
| err = uploadAssets(conf, args[1:]) | ||
| case "help": | ||
| err = help(conf) | ||
| default: | ||
| err = errors.New("invalid subcommand") | ||
| } | ||
|
|
||
| if err != nil { | ||
| fmt.Println(err) | ||
| os.Exit(1) | ||
| } | ||
| } | ||
|
|
||
| func help(*Config) error { | ||
| fmt.Println(`Usage: plateauview <command> [arguments] [options] [flags]`) | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| package tool | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "flag" | ||
| "fmt" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
|
|
||
| cms "github.com/reearth/reearth-cms-api/go" | ||
| ) | ||
|
|
||
| func uploadAssets(conf *Config, args []string) error { | ||
| var base, token, pid, target string | ||
|
|
||
| flags := flag.NewFlagSet("upload-assets", flag.ExitOnError) | ||
| flags.StringVar(&base, "base", conf.CMS_BaseURL, "CMS base URL") | ||
| flags.StringVar(&token, "token", conf.CMS_Token, "CMS token") | ||
| flags.StringVar(&pid, "project", "", "project ID") | ||
| flags.StringVar(&target, "target", "", "target dir path") | ||
|
|
||
| if err := flags.Parse(args); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if pid == "" { | ||
| return errors.New("project ID is required") | ||
| } | ||
|
|
||
| c, err := cms.New(base, token) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| files, err := os.ReadDir(target) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| ctx := context.Background() | ||
| for _, f := range files { | ||
| if f.IsDir() { | ||
| continue | ||
| } | ||
|
|
||
| ext := path.Ext(f.Name()) | ||
| if ext != ".zip" { | ||
| continue | ||
| } | ||
|
|
||
| file, err := os.Open(filepath.Join(target, f.Name())) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| fmt.Printf("%s -> ", file.Name()) | ||
|
|
||
| assetID, err := c.UploadAssetDirectly(ctx, pid, file.Name(), file) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| fmt.Printf("%s\n", assetID) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.