Backup_Repos/filestash
1
0
Fork 0
mirror of https://github.com/mickael-kerjean/filestash synced 2025-12-06 08:22:24 +01:00
Code Issues Projects Releases Wiki Activity

feature (s3): add timeout option for very large buckets

Browse source
This commit is contained in:
MickaelK 2025-06-10 21:42:17 +10:00
parent 55af393f7f
commit 2616c078b5
1 changed files with 18 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
server/plugin/plg_backend_s3/index.go
View file

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"strconv" "strconv"
"sync" "sync"
"time"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/awserr"
@ -31,6 +32,7 @@ type S3Backend struct {
params map[string]string params map[string]string
Context context.Context Context context.Context
threadSize int threadSize int
timeout time.Duration
} }
func init() { func init() {
@ -78,6 +80,10 @@ func (this S3Backend) Init(params map[string]string, app *App) (IBackend, error)
if params["endpoint"] != "" { if params["endpoint"] != "" {
config.Endpoint = aws.String(params["endpoint"]) config.Endpoint = aws.String(params["endpoint"])
} }
var timeout time.Duration
if secs, err := strconv.Atoi(params["timeout"]); err == nil {
timeout = time.Duration(secs) * time.Second
}
threadSize, err := strconv.Atoi(params["number_thread"]) threadSize, err := strconv.Atoi(params["number_thread"])
if err != nil { if err != nil {
threadSize = 50 threadSize = 50
@ -90,6 +96,7 @@ func (this S3Backend) Init(params map[string]string, app *App) (IBackend, error)
client: s3.New(session.New(config)), client: s3.New(session.New(config)),
Context: app.Context, Context: app.Context,
threadSize: threadSize, threadSize: threadSize,
timeout: timeout,
} }
return backend, nil return backend, nil
} }
@ -118,7 +125,7 @@ func (this S3Backend) LoginForm() Form {
Placeholder: "Advanced", Placeholder: "Advanced",
Target: []string{ Target: []string{
"s3_region", "s3_endpoint", "s3_role_arn", "s3_session_token", "s3_region", "s3_endpoint", "s3_role_arn", "s3_session_token",
"s3_path", "s3_encryption_key", "s3_number_thread", "s3_path", "s3_encryption_key", "s3_number_thread", "s3_timeout",
}, },
}, },
FormElement{ FormElement{
@ -163,6 +170,12 @@ func (this S3Backend) LoginForm() Form {
Type: "text", Type: "text",
Placeholder: "Num. Thread", Placeholder: "Num. Thread",
}, },
FormElement{
Id: "s3_timeout",
Name: "timeout",
Type: "number",
Placeholder: "List Object Timeout",
},
}, },
} }
} }
@ -197,6 +210,7 @@ func (this S3Backend) Ls(path string) (files []os.FileInfo, err error) {
return files, nil return files, nil
} }
client := s3.New(this.createSession(p.bucket)) client := s3.New(this.createSession(p.bucket))
start := time.Now()
err = client.ListObjectsV2PagesWithContext( err = client.ListObjectsV2PagesWithContext(
this.Context, this.Context,
&s3.ListObjectsV2Input{ &s3.ListObjectsV2Input{
@ -232,6 +246,9 @@ func (this S3Backend) Ls(path string) (files []os.FileInfo, err error) {
FTime: 0, FTime: 0,
}) })
} }
if this.timeout > 0 && time.Since(start) > this.timeout {
return false
}
return aws.BoolValue(objs.IsTruncated) return aws.BoolValue(objs.IsTruncated)
}, },
) )