Skip to content

Commit 7e5d161

Browse files
voyvodovvoyvodov
committed
Add multipart handling commands
Adding command to list active multipart uploads. Adding command to abort active multipart upload.
1 parent c280956 commit 7e5d161

File tree

14 files changed

+836
-49
lines changed

14 files changed

+836
-49
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ storage services and local filesystems.
3535
- Print object contents to stdout
3636
- Select JSON records from objects using SQL expressions
3737
- Create or remove buckets
38+
- List or abort multipart uploads
3839
- Summarize objects sizes, grouping by storage class
3940
- Wildcard support for all operations
4041
- Multiple arguments support for delete operation

‎command/abortmp.go‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package command
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/peak/s5cmd/v2/storage"
7+
"github.com/peak/s5cmd/v2/storage/url"
8+
"github.com/urfave/cli/v2"
9+
)
10+
11+
var abortmpHelpTemplate = `Name:
12+
{{.HelpName}} - {{.Usage}}
13+
14+
Usage:
15+
{{.HelpName}} [options] object-path upload-id
16+
17+
Options:
18+
{{range .VisibleFlags}}{{.}}
19+
{{end}}
20+
Examples:
21+
1. Abort multipart upload
22+
> s5cmd {{.HelpName}} s3://bucket/object 01000191-daf9-7547-5278-71bd81953ffe
23+
`
24+
25+
func NewAbortMultipartCommand() *cli.Command {
26+
cmd := &cli.Command{
27+
Name: "abortmp",
28+
HelpName: "abortmp",
29+
Usage: "abort multipart uploads",
30+
CustomHelpTemplate: abortmpHelpTemplate,
31+
Flags: []cli.Flag{},
32+
Before: func(c *cli.Context) error {
33+
err := validateAbortMultipartCommand(c)
34+
if err != nil {
35+
printError(commandFromContext(c), c.Command.Name, err)
36+
}
37+
return err
38+
},
39+
Action: func(c *cli.Context) (err error) {
40+
41+
// var merror error
42+
43+
fullCommand := commandFromContext(c)
44+
45+
objurl, err := url.New(c.Args().First())
46+
if err != nil {
47+
printError(fullCommand, c.Command.Name, err)
48+
return err
49+
}
50+
uploadID := c.Args().Get(1)
51+
52+
client, err := storage.NewRemoteClient(c.Context, objurl, NewStorageOpts(c))
53+
if err != nil {
54+
printError(fullCommand, c.Command.Name, err)
55+
return err
56+
}
57+
58+
err = client.AbortMultipartUpload(c.Context, objurl, uploadID)
59+
if err != nil && err != storage.ErrNoObjectFound {
60+
printError(fullCommand, c.Command.Name, err)
61+
return err
62+
}
63+
64+
return nil
65+
},
66+
}
67+
68+
cmd.BashComplete = getBashCompleteFn(cmd, false, false)
69+
return cmd
70+
}
71+
72+
func validateAbortMultipartCommand(c *cli.Context) error {
73+
if c.Args().Len() != 2 {
74+
return fmt.Errorf("expected object path and upload id arguments")
75+
}
76+
77+
objectPath := c.Args().Get(0)
78+
uploadID := c.Args().Get(1)
79+
80+
_, err := url.New(objectPath)
81+
if err != nil {
82+
return err
83+
}
84+
85+
if uploadID == "" {
86+
return fmt.Errorf("expected upload id, got empty string")
87+
}
88+
89+
return nil
90+
}

‎command/app.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,9 @@ func Commands() []*cli.Command {
200200
NewDeleteCommand(),
201201
NewMoveCommand(),
202202
NewMakeBucketCommand(),
203+
NewAbortMultipartCommand(),
204+
NewListMultipartCommand(),
205+
NewMultipartPartsCommand(),
203206
NewRemoveBucketCommand(),
204207
NewSelectCommand(),
205208
NewSizeCommand(),

‎command/cp.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -565,7 +565,9 @@ func (c Copy) Run(ctx context.Context) error {
565565
waiter.Wait()
566566
<-errDoneCh
567567

568-
return multierror.Append(merrorWaiter, merrorObjects).ErrorOrNil()
568+
err = multierror.Append(merrorWaiter, merrorObjects).ErrorOrNil()
569+
570+
return handleMultipartError(c.fullCommand, c.op, err)
569571
}
570572

571573
func (c Copy) prepareCopyTask(

‎command/error.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package command
22

33
import (
4+
"errors"
45
"fmt"
56
"strings"
67

8+
"github.com/aws/aws-sdk-go/service/s3/s3manager"
79
"github.com/hashicorp/go-multierror"
810

911
errorpkg "github.com/peak/s5cmd/v2/error"
@@ -94,3 +96,28 @@ func cleanupError(err error) string {
9496
s = strings.TrimSpace(s)
9597
return s
9698
}
99+
100+
func handleMultipartError(command, op string, err error) error {
101+
var pkgErr *errorpkg.Error
102+
if err == nil {
103+
return err
104+
}
105+
106+
if multiErr, ok := err.(*multierror.Error); ok {
107+
for _, merr := range multiErr.Errors {
108+
if errors.As(merr, &pkgErr) {
109+
if awsErr, ok := pkgErr.Err.(s3manager.MultiUploadFailure); ok {
110+
printError(command, op, fmt.Errorf("multipart upload fail. To resume use the following id: %s", awsErr.UploadID()))
111+
}
112+
}
113+
}
114+
} else {
115+
if errors.As(err, &pkgErr) {
116+
if awsErr, ok := pkgErr.Err.(s3manager.MultiUploadFailure); ok {
117+
printError(command, op, fmt.Errorf("multipart upload fail. To resume use the following id: %s", awsErr.UploadID()))
118+
}
119+
}
120+
}
121+
122+
return err
123+
}

‎command/lsmp.go‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
package command
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/hashicorp/go-multierror"
7+
"github.com/peak/s5cmd/v2/log"
8+
"github.com/peak/s5cmd/v2/storage"
9+
"github.com/peak/s5cmd/v2/storage/url"
10+
"github.com/peak/s5cmd/v2/strutil"
11+
"github.com/urfave/cli/v2"
12+
)
13+
14+
var lsmpHelpTemplate = `Name:
15+
{{.HelpName}} - {{.Usage}}
16+
17+
Usage:
18+
{{.HelpName}} [options] prefix
19+
20+
Options:
21+
{{range .VisibleFlags}}{{.}}
22+
{{end}}
23+
Examples:
24+
1. List multipart uploads for bucket
25+
> s5cmd {{.HelpName}} s3://bucket
26+
2. List multipart uploads for specific object
27+
> s5cmd {{.HelpName}} s3://bucket/object
28+
3. List multipart uploads with full path to the object
29+
> s5cmd {{.HelpName}} --show-fullpath s3://bucket/object
30+
`
31+
32+
func NewListMultipartCommand() *cli.Command {
33+
cmd := &cli.Command{
34+
Name: "lsmp",
35+
HelpName: "lsmp",
36+
Usage: "list multipart uploads",
37+
CustomHelpTemplate: lsmpHelpTemplate,
38+
Flags: []cli.Flag{
39+
&cli.BoolFlag{
40+
Name: "show-fullpath",
41+
Usage: "show the fullpath names of the object(s)",
42+
},
43+
},
44+
Before: func(c *cli.Context) error {
45+
err := validateListMultipartCommand(c)
46+
if err != nil {
47+
printError(commandFromContext(c), c.Command.Name, err)
48+
}
49+
return err
50+
},
51+
Action: func(c *cli.Context) (err error) {
52+
53+
var merror error
54+
55+
fullCommand := commandFromContext(c)
56+
57+
srcurl, err := url.New(c.Args().First())
58+
if err != nil {
59+
printError(fullCommand, c.Command.Name, err)
60+
return err
61+
}
62+
63+
client, err := storage.NewRemoteClient(c.Context, srcurl, NewStorageOpts(c))
64+
if err != nil {
65+
printError(fullCommand, c.Command.Name, err)
66+
return err
67+
}
68+
69+
for object := range client.ListMultipartUploads(c.Context, srcurl) {
70+
if err := object.Err; err != nil {
71+
merror = multierror.Append(merror, err)
72+
printError(fullCommand, c.Command.Name, err)
73+
continue
74+
}
75+
msg := ListMPUploadMessage{
76+
Object: object,
77+
showFullPath: c.Bool("show-fullpath"),
78+
}
79+
log.Info(msg)
80+
}
81+
82+
return nil
83+
},
84+
}
85+
86+
cmd.BashComplete = getBashCompleteFn(cmd, false, false)
87+
return cmd
88+
}
89+
90+
type ListMPUploadMessage struct {
91+
Object *storage.UploadObject `json:"object"`
92+
93+
showFullPath bool
94+
}
95+
96+
// String returns the string representation of ListMessage.
97+
func (l ListMPUploadMessage) String() string {
98+
// date and storage fields
99+
var listFormat = "%19s"
100+
101+
listFormat = listFormat + " %s %s"
102+
103+
var s string
104+
105+
var path string
106+
if l.showFullPath {
107+
path = l.Object.URL.String()
108+
} else {
109+
path = l.Object.URL.Relative()
110+
}
111+
112+
s = fmt.Sprintf(
113+
listFormat,
114+
l.Object.Initiated.Format(dateFormat),
115+
path,
116+
l.Object.UploadID,
117+
)
118+
119+
return s
120+
}
121+
122+
// JSON returns the JSON representation of ListMessage.
123+
func (l ListMPUploadMessage) JSON() string {
124+
return strutil.JSON(l.Object)
125+
}
126+
127+
func validateListMultipartCommand(c *cli.Context) error {
128+
if c.Args().Len() != 1 {
129+
return fmt.Errorf("expected 1 argument")
130+
}
131+
132+
_, err := url.New(c.Args().First())
133+
if err != nil {
134+
return err
135+
}
136+
return nil
137+
}

0 commit comments

Comments
 (0)