Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions agent/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ go 1.24.9

require (
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
github.com/aws/aws-sdk-go v1.55.0
github.com/compose-spec/compose-go/v2 v2.9.0
github.com/creack/pty v1.1.24
github.com/docker/cli v28.5.1+incompatible
github.com/docker/compose/v2 v2.40.2
github.com/docker/docker v28.5.1+incompatible
github.com/docker/go-connections v0.6.0
Expand All @@ -26,10 +26,10 @@ require (
github.com/jinzhu/copier v0.4.0
github.com/jinzhu/gorm v1.9.16
github.com/joho/godotenv v1.5.1
github.com/klauspost/compress v1.18.1
github.com/klauspost/compress v1.18.2
github.com/mholt/archiver/v4 v4.0.0-alpha.8
github.com/miekg/dns v1.1.69
github.com/minio/minio-go/v7 v7.0.74
github.com/minio/minio-go/v7 v7.0.98
github.com/nicksnyder/go-i18n/v2 v2.4.0
github.com/opencontainers/image-spec v1.1.1
github.com/oschwald/maxminddb-golang v1.13.1
Expand Down Expand Up @@ -113,7 +113,6 @@ require (
github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/buildx v0.29.1 // indirect
github.com/docker/cli v28.5.1+incompatible // indirect
github.com/docker/docker-credential-helpers v0.9.3 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/dsnet/compress v0.0.1 // indirect
Expand Down Expand Up @@ -159,9 +158,9 @@ require (
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
Expand All @@ -172,6 +171,7 @@ require (
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/mattn/go-shellwords v1.0.12 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
Expand Down Expand Up @@ -202,12 +202,13 @@ require (
github.com/ovh/go-ovh v1.9.0 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 // indirect
Expand All @@ -218,6 +219,7 @@ require (
github.com/spf13/pflag v1.0.10 // indirect
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.3.12 // indirect
github.com/therootcompany/xz v1.0.1 // indirect
github.com/tinylib/msgp v1.6.1 // indirect
github.com/tjfoc/gmsm v1.4.1 // indirect
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
Expand Down
28 changes: 16 additions & 12 deletions agent/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,6 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
github.com/aws/aws-sdk-go v1.55.0 h1:hVALKPjXz33kP1R9nTyJpUK7qF59dO2mleQxUW9mCVE=
github.com/aws/aws-sdk-go v1.55.0/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4=
github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4=
github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
Expand Down Expand Up @@ -588,9 +586,7 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
Expand All @@ -613,13 +609,15 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
Expand Down Expand Up @@ -683,12 +681,14 @@ github.com/miekg/dns v1.1.69 h1:Kb7Y/1Jo+SG+a2GtfoFUfDkG//csdRPwRLkCsxDG9Sc=
github.com/miekg/dns v1.1.69/go.mod h1:7OyjD9nEba5OkqQ/hB4fy3PIoxafSZJtducccIelz3g=
github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0=
github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
github.com/minio/minio-go/v7 v7.0.98 h1:MeAVKjLVz+XJ28zFcuYyImNSAh8Mq725uNW4beRisi0=
github.com/minio/minio-go/v7 v7.0.98/go.mod h1:cY0Y+W7yozf0mdIclrttzo1Iiu7mEf9y7nk2uXqMOvM=
github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
Expand Down Expand Up @@ -804,6 +804,8 @@ github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCko
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM=
github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
Expand Down Expand Up @@ -871,8 +873,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
Expand Down Expand Up @@ -951,6 +953,8 @@ github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+x
github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY=
github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c=
github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw=
github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY=
github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA=
github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w=
github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho=
github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE=
Expand Down
162 changes: 91 additions & 71 deletions agent/utils/cloud_storage/client/s3.go
Original file line number Diff line number Diff line change
@@ -1,20 +1,30 @@
package client

import (
"context"
"crypto/tls"
"net/http"
"os"
"strings"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)

const (
s3DefaultTimeout = 30 * time.Second
s3TransferTimeout = 24 * time.Hour
)

type s3Client struct {
scType string
bucket string
Sess session.Session
client *minio.Client
}

func (s *s3Client) ctx(timeout time.Duration) (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), timeout)
}

func NewS3Client(vars map[string]interface{}) (*s3Client, error) {
Expand All @@ -31,69 +41,80 @@ func NewS3Client(vars map[string]interface{}) (*s3Client, error) {
if len(mode) == 0 {
mode = "virtual hosted"
}
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
Endpoint: aws.String(endpoint),
Region: aws.String(region),
DisableSSL: aws.Bool(true), S3ForcePathStyle: aws.Bool(mode == "path"),

lookupStyle := minio.BucketLookupDNS
if mode == "path" {
lookupStyle = minio.BucketLookupPath
}

ssl := strings.Split(endpoint, ":")[0]
secure := false
tlsConfig := &tls.Config{}
if ssl == "https" {
secure = true
tlsConfig.InsecureSkipVerify = true
}
var transport http.RoundTripper = &http.Transport{
TLSClientConfig: tlsConfig,
}

endpoint = strings.TrimPrefix(endpoint, ssl+"://")

client, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Secure: secure,
Region: region,
BucketLookup: lookupStyle,
Transport: transport,
})
if err != nil {
return nil, err
}
return &s3Client{scType: scType, bucket: bucket, Sess: *sess}, nil
return &s3Client{scType: scType, bucket: bucket, client: client}, nil
}

func (s s3Client) ListBuckets() ([]interface{}, error) {
var result []interface{}
svc := s3.New(&s.Sess)
res, err := svc.ListBuckets(nil)
ctx, cancel := s.ctx(s3DefaultTimeout)
defer cancel()
buckets, err := s.client.ListBuckets(ctx)
if err != nil {
return nil, err
}
for _, b := range res.Buckets {
var result []interface{}
for _, b := range buckets {
result = append(result, b.Name)
}
return result, nil
}

func (s s3Client) Exist(path string) (bool, error) {
svc := s3.New(&s.Sess)
if _, err := svc.HeadObject(&s3.HeadObjectInput{
Bucket: &s.bucket,
Key: &path,
}); err != nil {
if aerr, ok := err.(awserr.RequestFailure); ok {
if aerr.StatusCode() == 404 {
return false, nil
}
} else {
return false, aerr
ctx, cancel := s.ctx(s3DefaultTimeout)
defer cancel()
_, err := s.client.StatObject(ctx, s.bucket, path, minio.StatObjectOptions{})
if err != nil {
resp := minio.ToErrorResponse(err)
if resp.StatusCode == 404 {
return false, nil
}
return false, err
}
return true, nil
}

func (s *s3Client) Size(path string) (int64, error) {
svc := s3.New(&s.Sess)
file, err := svc.GetObject(&s3.GetObjectInput{
Bucket: &s.bucket,
Key: &path,
})
ctx, cancel := s.ctx(s3DefaultTimeout)
defer cancel()
info, err := s.client.StatObject(ctx, s.bucket, path, minio.StatObjectOptions{})
if err != nil {
return 0, err
}
return *file.ContentLength, nil
return info.Size, nil
}

func (s s3Client) Delete(path string) (bool, error) {
svc := s3.New(&s.Sess)
if _, err := svc.DeleteObject(&s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(path)}); err != nil {
return false, err
}
if err := svc.WaitUntilObjectNotExists(&s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
}); err != nil {
ctx, cancel := s.ctx(s3DefaultTimeout)
defer cancel()
if err := s.client.RemoveObject(ctx, s.bucket, path, minio.RemoveObjectOptions{}); err != nil {
return false, err
}
return true, nil
Expand All @@ -110,16 +131,21 @@ func (s s3Client) Upload(src, target string) (bool, error) {
}
defer file.Close()

uploader := s3manager.NewUploader(&s.Sess)
if fileInfo.Size() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize {
uploader.PartSize = fileInfo.Size() / (s3manager.MaxUploadParts - 1)
opts := minio.PutObjectOptions{
StorageClass: s.scType,
}

const maxParts = 10000
const defaultPartSize = 64 * 1024 * 1024 // 64 MiB
partSize := uint64(defaultPartSize)
if fileInfo.Size() > int64(maxParts)*int64(defaultPartSize) {
partSize = uint64(fileInfo.Size()) / (maxParts - 1)
}
if _, err := uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(s.bucket),
Key: aws.String(target),
Body: file,
StorageClass: &s.scType,
}); err != nil {
opts.PartSize = partSize

ctx, cancel := s.ctx(s3TransferTimeout)
defer cancel()
if _, err := s.client.PutObject(ctx, s.bucket, target, file, fileInfo.Size(), opts); err != nil {
return false, err
}
return true, nil
Expand All @@ -129,34 +155,28 @@ func (s s3Client) Download(src, target string) (bool, error) {
if _, err := os.Stat(target); err == nil {
_ = os.Remove(target)
}
file, err := os.Create(target)
if err != nil {
return false, err
}
defer file.Close()
downloader := s3manager.NewDownloader(&s.Sess)
if _, err = downloader.Download(file, &s3.GetObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(src),
}); err != nil {
ctx, cancel := s.ctx(s3TransferTimeout)
defer cancel()
if err := s.client.FGetObject(ctx, s.bucket, src, target, minio.GetObjectOptions{}); err != nil {
os.Remove(target)
return false, err
}
return true, nil
}

func (s *s3Client) ListObjects(prefix string) ([]string, error) {
svc := s3.New(&s.Sess)
var result []string
outputs, err := svc.ListObjects(&s3.ListObjectsInput{
Bucket: &s.bucket,
Prefix: &prefix,
})
if err != nil {
return result, err
opts := minio.ListObjectsOptions{
Recursive: true,
Prefix: prefix,
}
for _, item := range outputs.Contents {
result = append(result, *item.Key)
var result []string
ctx, cancel := s.ctx(s3DefaultTimeout)
defer cancel()
for object := range s.client.ListObjects(ctx, s.bucket, opts) {
if object.Err != nil {
return result, object.Err
}
result = append(result, object.Key)
}
return result, nil
}
Loading