diff --git a/README.md b/README.md index 8d46c8a..65f0e41 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ proof, _ := delegation.Extract(b) // space to list uploads from space, _ := did.Parse("did:key:z6MkwDuRThQcyWjqNsK54yKAmzfsiH6BTkASyiucThMtHt1y") -rcpt, _ := client.List( +rcpt, _ := client.UploadList( signer, space, &uploadlist.Caveat{}, diff --git a/capability/storeadd/capability.go b/capability/storeadd/capability.go new file mode 100644 index 0000000..ba81dd5 --- /dev/null +++ b/capability/storeadd/capability.go @@ -0,0 +1,52 @@ +package storeadd + +import ( + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/web3-storage/go-ucanto/did" + "github.com/web3-storage/go-ucanto/ucan" +) + +const Ability = "store/add" + +type Caveat struct { + Link ipld.Link + Size uint64 + Origin *ipld.Link +} + +var _ ucan.MapBuilder = (*Caveat)(nil) + +func (c *Caveat) Build() (map[string]datamodel.Node, error) { + data := map[string]datamodel.Node{} + + b := basicnode.Prototype.Link.NewBuilder() + err := b.AssignLink(c.Link) + if err != nil { + return nil, err + } + data["link"] = b.Build() + + b = basicnode.Prototype.Int.NewBuilder() + err = b.AssignInt(int64(c.Size)) + if err != nil { + return nil, err + } + data["size"] = b.Build() + + if c.Origin != nil { + b = basicnode.Prototype.Link.NewBuilder() + err = b.AssignLink(c.Link) + if err != nil { + return nil, err + } + data["origin"] = b.Build() + } + + return data, nil +} + +func NewCapability(space did.DID, nb *Caveat) ucan.Capability[ucan.MapBuilder] { + return ucan.NewCapability(Ability, space.String(), ucan.MapBuilder(nb)) +} diff --git a/capability/storeadd/receipt.go b/capability/storeadd/receipt.go new file mode 100644 index 0000000..2b981df --- /dev/null +++ b/capability/storeadd/receipt.go @@ -0,0 +1,7 @@ +package storeadd + +import "github.com/web3-storage/go-ucanto/core/receipt" + +func NewReceiptReader() (receipt.ReceiptReader[*Success, *Failure], error) { + return receipt.NewReceiptReader[*Success, *Failure](ResultSchema) +} diff --git a/capability/storeadd/result.go b/capability/storeadd/result.go new file mode 100644 index 0000000..05f6855 --- /dev/null +++ b/capability/storeadd/result.go @@ -0,0 +1,29 @@ +package storeadd + +import ( + _ "embed" + + "github.com/ipld/go-ipld-prime" +) + +//go:embed result.ipldsch +var ResultSchema []byte + +type Success struct { + Status string + With string + Link ipld.Link + Url *string + Headers *Headers +} + +type Headers struct { + Keys []string + Values map[string]string +} + +type Failure struct { + Name *string + Message string + Stack *string +} diff --git a/capability/storeadd/result.ipldsch b/capability/storeadd/result.ipldsch new file mode 100644 index 0000000..ea9650f --- /dev/null +++ b/capability/storeadd/result.ipldsch @@ -0,0 +1,40 @@ +type Result union { + | Success "ok" + | Failure "error" +} representation keyed + +type Success struct { + status String + with DID + link Link + url optional URL + headers optional {String: String} +} + +# type Success union { +# | SuccessUpload "upload" +# | SuccessDone "done" +# } representation inline { +# discriminantKey "status" +# } + +# type SuccessUpload struct { +# with DID +# link Link +# url URL +# headers {String: String} +# } + +# type SuccessDone struct { +# with DID +# link Link +# } + +type DID string +type URL string + +type Failure struct { + name optional String + message String + stack optional String +} diff --git a/capability/uploadadd/capability.go b/capability/uploadadd/capability.go new file mode 100644 index 0000000..2f0c16b --- /dev/null +++ b/capability/uploadadd/capability.go @@ -0,0 +1,50 @@ +package uploadadd + +import ( + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/web3-storage/go-ucanto/did" + "github.com/web3-storage/go-ucanto/ucan" +) + +const Ability = "upload/add" + +type Caveat struct { + Root ipld.Link + Shards []ipld.Link +} + +var _ ucan.MapBuilder = (*Caveat)(nil) + +func (c *Caveat) Build() (map[string]datamodel.Node, error) { + data := map[string]datamodel.Node{} + + b := basicnode.Prototype.Link.NewBuilder() + err := b.AssignLink(c.Root) + if err != nil { + return nil, err + } + data["root"] = b.Build() + + if c.Shards != nil { + b := basicnode.Prototype.Any.NewBuilder() + la, err := b.BeginList(int64(len(c.Shards))) + if err != nil { + return nil, err + } + for _, s := range c.Shards { + err := la.AssembleValue().AssignLink(s) + if err != nil { + return nil, err + } + } + la.Finish() + data["shards"] = b.Build() + } + return data, nil +} + +func NewCapability(space did.DID, nb *Caveat) ucan.Capability[ucan.MapBuilder] { + return ucan.NewCapability(Ability, space.String(), ucan.MapBuilder(nb)) +} diff --git a/capability/uploadadd/receipt.go b/capability/uploadadd/receipt.go new file mode 100644 index 0000000..ab450ae --- /dev/null +++ b/capability/uploadadd/receipt.go @@ -0,0 +1,7 @@ +package uploadadd + +import "github.com/web3-storage/go-ucanto/core/receipt" + +func NewReceiptReader() (receipt.ReceiptReader[*Success, *Failure], error) { + return receipt.NewReceiptReader[*Success, *Failure](ResultSchema) +} diff --git a/capability/uploadadd/result.go b/capability/uploadadd/result.go new file mode 100644 index 0000000..eaa40de --- /dev/null +++ b/capability/uploadadd/result.go @@ -0,0 +1,21 @@ +package uploadadd + +import ( + _ "embed" + + "github.com/ipld/go-ipld-prime" +) + +//go:embed result.ipldsch +var ResultSchema []byte + +type Success struct { + Root ipld.Link + Shards []ipld.Link +} + +type Failure struct { + Name *string + Message string + Stack *string +} diff --git a/capability/uploadadd/result.ipldsch b/capability/uploadadd/result.ipldsch new file mode 100644 index 0000000..426ba08 --- /dev/null +++ b/capability/uploadadd/result.ipldsch @@ -0,0 +1,15 @@ +type Result union { + | Success "ok" + | Failure "error" +} representation keyed + +type Success struct { + root Link + shards optional [Link] +} + +type Failure struct { + name optional String + message String + stack optional String +} diff --git a/capability/uploadlist/capability.go b/capability/uploadlist/capability.go index 1f995c9..e530441 100644 --- a/capability/uploadlist/capability.go +++ b/capability/uploadlist/capability.go @@ -21,17 +21,26 @@ func (c *Caveat) Build() (map[string]datamodel.Node, error) { data := map[string]datamodel.Node{} if c.Cursor != "" { b := basicnode.Prototype.String.NewBuilder() - b.AssignString(c.Cursor) + err := b.AssignString(c.Cursor) + if err != nil { + return nil, err + } data["cursor"] = b.Build() } if c.Size != 0 { b := basicnode.Prototype.Int.NewBuilder() - b.AssignInt(c.Size) + err := b.AssignInt(c.Size) + if err != nil { + return nil, err + } data["size"] = b.Build() } if c.Pre { b := basicnode.Prototype.Bool.NewBuilder() - b.AssignBool(c.Pre) + err := b.AssignBool(c.Pre) + if err != nil { + return nil, err + } data["pre"] = b.Build() } return data, nil diff --git a/capability/uploadlist/model.go b/capability/uploadlist/model.go deleted file mode 100644 index 370b7ec..0000000 --- a/capability/uploadlist/model.go +++ /dev/null @@ -1,54 +0,0 @@ -package uploadlist - -import ( - _ "embed" - "fmt" - "sync" - - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/schema" -) - -//go:embed upload.ipldsch -var UploadSchema []byte - -var ( - once sync.Once - ts *schema.TypeSystem - err error -) - -func mustLoadSchema() *schema.TypeSystem { - once.Do(func() { - ts, err = ipld.LoadSchemaBytes(UploadSchema) - }) - if err != nil { - panic(fmt.Errorf("failed to load IPLD schema: %s", err)) - } - return ts -} - -func UploadListResultType() schema.Type { - return mustLoadSchema().TypeByName("Result") -} - -type UploadListSuccess struct { - Results []UploadListItem - Before *string - After *string - Cursor *string - Size uint64 -} - -type UploadListItem struct { - Root ipld.Link - Shards []ipld.Link - InsertedAt string - UpdatedAt string -} - -type UploadListFailure struct { - Name *string - Message string - Stack *string -} diff --git a/capability/uploadlist/receipt.go b/capability/uploadlist/receipt.go index b452ea9..d1883e5 100644 --- a/capability/uploadlist/receipt.go +++ b/capability/uploadlist/receipt.go @@ -2,6 +2,6 @@ package uploadlist import "github.com/web3-storage/go-ucanto/core/receipt" -func NewReceiptReader() (receipt.ReceiptReader[*UploadListSuccess, *UploadListFailure], error) { - return receipt.NewReceiptReader[*UploadListSuccess, *UploadListFailure](UploadSchema) +func NewReceiptReader() (receipt.ReceiptReader[*Success, *Failure], error) { + return receipt.NewReceiptReader[*Success, *Failure](ResultSchema) } diff --git a/capability/uploadlist/result.go b/capability/uploadlist/result.go new file mode 100644 index 0000000..9a40271 --- /dev/null +++ b/capability/uploadlist/result.go @@ -0,0 +1,31 @@ +package uploadlist + +import ( + _ "embed" + + "github.com/ipld/go-ipld-prime" +) + +//go:embed result.ipldsch +var ResultSchema []byte + +type Success struct { + Results []Item + Before *string + After *string + Cursor *string + Size uint64 +} + +type Item struct { + Root ipld.Link + Shards []ipld.Link + InsertedAt string + UpdatedAt string +} + +type Failure struct { + Name *string + Message string + Stack *string +} diff --git a/capability/uploadlist/upload.ipldsch b/capability/uploadlist/result.ipldsch similarity index 61% rename from capability/uploadlist/upload.ipldsch rename to capability/uploadlist/result.ipldsch index 98a3bb0..93bbbb3 100644 --- a/capability/uploadlist/upload.ipldsch +++ b/capability/uploadlist/result.ipldsch @@ -1,24 +1,24 @@ type Result union { - | UploadListSuccess "ok" - | UploadListFailure "error" + | Success "ok" + | Failure "error" } representation keyed -type UploadListSuccess struct { - results [UploadListItem] +type Success struct { + results [Item] cursor optional String before optional String after optional String size Int } -type UploadListItem struct { +type Item struct { root Link shards optional [Link] insertedAt String updatedAt String } -type UploadListFailure struct { +type Failure struct { name optional String message String stack optional String diff --git a/client/client.go b/client/client.go index bf75e00..2257acc 100644 --- a/client/client.go +++ b/client/client.go @@ -8,10 +8,12 @@ import ( "github.com/web3-storage/go-ucanto/core/receipt" "github.com/web3-storage/go-ucanto/did" "github.com/web3-storage/go-ucanto/principal" + "github.com/web3-storage/go-w3up/capability/storeadd" + "github.com/web3-storage/go-w3up/capability/uploadadd" "github.com/web3-storage/go-w3up/capability/uploadlist" ) -func List(signer principal.Signer, space did.DID, params *uploadlist.Caveat, options ...Option) (receipt.Receipt[*uploadlist.UploadListSuccess, *uploadlist.UploadListFailure], error) { +func StoreAdd(signer principal.Signer, space did.DID, params *storeadd.Caveat, options ...Option) (receipt.Receipt[*storeadd.Success, *storeadd.Failure], error) { cfg := ClientConfig{conn: DefaultConnection} for _, opt := range options { if err := opt(&cfg); err != nil { @@ -22,7 +24,85 @@ func List(signer principal.Signer, space did.DID, params *uploadlist.Caveat, opt inv, err := invocation.Invoke( signer, cfg.conn.ID(), - uploadlist.NewCapability(space, &uploadlist.Caveat{}), + storeadd.NewCapability(space, params), + convertToInvocationOptions(cfg)..., + ) + if err != nil { + return nil, err + } + + resp, err := client.Execute([]invocation.Invocation{inv}, cfg.conn) + if err != nil { + return nil, err + } + + rcptlnk, ok := resp.Get(inv.Link()) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("receipt not found: %s", inv.Link()) + } + + reader, err := storeadd.NewReceiptReader() + if err != nil { + return nil, err + } + + return reader.Read(rcptlnk, resp.Blocks()) +} + +func UploadAdd(signer principal.Signer, space did.DID, params *uploadadd.Caveat, options ...Option) (receipt.Receipt[*uploadadd.Success, *uploadadd.Failure], error) { + cfg := ClientConfig{conn: DefaultConnection} + for _, opt := range options { + if err := opt(&cfg); err != nil { + return nil, err + } + } + + inv, err := invocation.Invoke( + signer, + cfg.conn.ID(), + uploadadd.NewCapability(space, params), + convertToInvocationOptions(cfg)..., + ) + if err != nil { + return nil, err + } + + resp, err := client.Execute([]invocation.Invocation{inv}, cfg.conn) + if err != nil { + return nil, err + } + + rcptlnk, ok := resp.Get(inv.Link()) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("receipt not found: %s", inv.Link()) + } + + reader, err := uploadadd.NewReceiptReader() + if err != nil { + return nil, err + } + + return reader.Read(rcptlnk, resp.Blocks()) +} + +func UploadList(signer principal.Signer, space did.DID, params *uploadlist.Caveat, options ...Option) (receipt.Receipt[*uploadlist.Success, *uploadlist.Failure], error) { + cfg := ClientConfig{conn: DefaultConnection} + for _, opt := range options { + if err := opt(&cfg); err != nil { + return nil, err + } + } + + inv, err := invocation.Invoke( + signer, + cfg.conn.ID(), + uploadlist.NewCapability(space, params), convertToInvocationOptions(cfg)..., ) if err != nil { diff --git a/cmd/lib/lib.go b/cmd/lib/lib.go index 3eaaa15..ec6ea5d 100644 --- a/cmd/lib/lib.go +++ b/cmd/lib/lib.go @@ -3,7 +3,6 @@ package lib import ( "bytes" _ "embed" - "fmt" "io" "log" "net/url" @@ -124,7 +123,7 @@ func MustGetConnection() client.Connection { func MustParseDID(str string) did.DID { did, err := did.Parse(str) if err != nil { - log.Fatal(fmt.Errorf("parsing DID: %s", err)) + log.Fatalf("parsing DID: %s", err) } return did } @@ -132,7 +131,7 @@ func MustParseDID(str string) did.DID { func MustGetProof(path string) delegation.Delegation { b, err := os.ReadFile(path) if err != nil { - log.Fatal(fmt.Errorf("reading proof file: %s", err)) + log.Fatalf("reading proof file: %s", err) } proof, err := delegation.Extract(b) @@ -140,13 +139,13 @@ func MustGetProof(path string) delegation.Delegation { // try decode legacy format _, blocks, err := archive.Decode(bytes.NewReader(b)) if err != nil { - log.Fatal(fmt.Errorf("extracting proof: %s", err)) + log.Fatalf("extracting proof: %s", err) } var rt block.Block bs, err := blockstore.NewBlockStore() if err != nil { - log.Fatal(fmt.Errorf("creating blockstore: %s", err)) + log.Fatalf("creating blockstore: %s", err) } for { bl, err := blocks.Next() @@ -154,11 +153,11 @@ func MustGetProof(path string) delegation.Delegation { if err == io.EOF { break } - log.Fatal(fmt.Errorf("reading block: %s", err)) + log.Fatalf("reading block: %s", err) } err = bs.Put(bl) if err != nil { - log.Fatal(fmt.Errorf("putting block: %s", err)) + log.Fatalf("putting block: %s", err) } rt = bl } diff --git a/cmd/w3.go b/cmd/w3.go index e29c284..613f44c 100644 --- a/cmd/w3.go +++ b/cmd/w3.go @@ -3,10 +3,18 @@ package main import ( "fmt" "log" + "net/http" "os" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/multiformats/go-multihash" "github.com/urfave/cli/v2" + "github.com/web3-storage/go-ucanto/core/car" "github.com/web3-storage/go-ucanto/core/delegation" + "github.com/web3-storage/go-w3up/capability/storeadd" + "github.com/web3-storage/go-w3up/capability/uploadadd" "github.com/web3-storage/go-w3up/capability/uploadlist" "github.com/web3-storage/go-w3up/client" "github.com/web3-storage/go-w3up/cmd/lib" @@ -22,6 +30,30 @@ func main() { Usage: "Print information about the current agent.", Action: whoami, }, + { + Name: "up", + Aliases: []string{"upload"}, + Usage: "Store a file(s) to the service and register an upload.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "space", + Value: "", + Usage: "DID of space to list uploads from.", + }, + &cli.StringFlag{ + Name: "proof", + Value: "", + Usage: "Path to file containing UCAN proof(s) for the operation.", + }, + &cli.StringFlag{ + Name: "car", + Aliases: []string{"c"}, + Value: "", + Usage: "Path to CAR file to upload (max 4GB).", + }, + }, + Action: up, + }, { Name: "ls", Aliases: []string{"list"}, @@ -59,13 +91,130 @@ func whoami(cCtx *cli.Context) error { return nil } +func up(cCtx *cli.Context) error { + signer := lib.MustGetSigner() + conn := lib.MustGetConnection() + space := lib.MustParseDID(cCtx.String("space")) + proof := lib.MustGetProof(cCtx.String("proof")) + + f0, err := os.Open(cCtx.String("car")) + if err != nil { + log.Fatalf("opening file: %s", err) + } + + stat, err := f0.Stat() + if err != nil { + log.Fatalf("stat file: %s", err) + } + + mh, err := multihash.SumStream(f0, multihash.SHA2_256, -1) + if err != nil { + log.Fatalf("hashing CAR: %s", err) + } + + err = f0.Close() + if err != nil { + log.Fatalf("closing file: %s", err) + } + + link := cidlink.Link{Cid: cid.NewCidV1(0x0202, mh)} + fmt.Println(link.String()) + + rcpt, err := client.StoreAdd( + signer, + space, + &storeadd.Caveat{ + Link: link, + Size: uint64(stat.Size()), + }, + client.WithConnection(conn), + client.WithProofs([]delegation.Delegation{proof}), + ) + if err != nil { + return err + } + + if rcpt.Out().Error() != nil { + log.Fatalf("%+v\n", rcpt.Out().Error()) + } + + if rcpt.Out().Ok().Status == "upload" { + f2, err := os.Open(cCtx.String("car")) + if err != nil { + log.Fatalf("opening file: %s", err) + } + + fmt.Println(*rcpt.Out().Ok().Url) + hr, err := http.NewRequest("PUT", *rcpt.Out().Ok().Url, f2) + if err != nil { + log.Fatalf("creating HTTP request: %s", err) + } + + hdr := map[string][]string{} + for k, v := range rcpt.Out().Ok().Headers.Values { + fmt.Printf("%s: %s\n", k, v) + hdr[k] = []string{v} + } + + hr.Header = hdr + httpClient := http.Client{} + res, err := httpClient.Do(hr) + if err != nil { + log.Fatalf("doing HTTP request: %s", err) + } + fmt.Printf("%+v\n", res.Header) + if res.StatusCode != 200 { + log.Fatalf("non-200 status code while uploading file: %d", res.StatusCode) + } + err = f2.Close() + if err != nil { + log.Fatalf("closing file: %s", err) + } + } + + f3, err := os.Open(cCtx.String("car")) + if err != nil { + log.Fatalf("opening file: %s", err) + } + roots, _, err := car.Decode(f3) + if err != nil { + log.Fatalf("reading roots: %s", err) + } + err = f3.Close() + if err != nil { + log.Fatalf("closing file: %s", err) + } + if len(roots) > 0 { + rcpt, err := client.UploadAdd( + signer, + space, + &uploadadd.Caveat{ + Root: roots[0], + Shards: []ipld.Link{link}, + }, + client.WithConnection(conn), + client.WithProofs([]delegation.Delegation{proof}), + ) + if err != nil { + return err + } + if rcpt.Out().Error() != nil { + log.Fatalf("%+v\n", rcpt.Out().Error()) + } + + fmt.Printf("⁂ https://w3s.link/ipfs/%s", roots[0]) + } + + return nil +} + func ls(cCtx *cli.Context) error { signer := lib.MustGetSigner() conn := lib.MustGetConnection() space := lib.MustParseDID(cCtx.String("space")) proof := lib.MustGetProof(cCtx.String("proof")) - rcpt, err := client.List( + rcpt, err := client.UploadList( signer, space, &uploadlist.Caveat{},