From 2616c078b530de8bc3b6c34d28143cea467c6634 Mon Sep 17 00:00:00 2001
From: MickaelK <mickael@kerjean.me>
Date: Tue, 10 Jun 2025 21:42:17 +1000
Subject: [PATCH] feature (s3): add timeout option for very large buckets

---
 server/plugin/plg_backend_s3/index.go | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/server/plugin/plg_backend_s3/index.go b/server/plugin/plg_backend_s3/index.go
index 54da3495..44eb8c54 100644
--- a/server/plugin/plg_backend_s3/index.go
+++ b/server/plugin/plg_backend_s3/index.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strconv"
 	"sync"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -31,6 +32,7 @@ type S3Backend struct {
 	params     map[string]string
 	Context    context.Context
 	threadSize int
+	timeout    time.Duration
 }
 
 func init() {
@@ -78,6 +80,10 @@ func (this S3Backend) Init(params map[string]string, app *App) (IBackend, error)
 	if params["endpoint"] != "" {
 		config.Endpoint = aws.String(params["endpoint"])
 	}
+	var timeout time.Duration
+	if secs, err := strconv.Atoi(params["timeout"]); err == nil {
+		timeout = time.Duration(secs) * time.Second
+	}
 	threadSize, err := strconv.Atoi(params["number_thread"])
 	if err != nil {
 		threadSize = 50
@@ -90,6 +96,7 @@ func (this S3Backend) Init(params map[string]string, app *App) (IBackend, error)
 		client:     s3.New(session.New(config)),
 		Context:    app.Context,
 		threadSize: threadSize,
+		timeout:    timeout,
 	}
 	return backend, nil
 }
@@ -118,7 +125,7 @@ func (this S3Backend) LoginForm() Form {
 				Placeholder: "Advanced",
 				Target: []string{
 					"s3_region", "s3_endpoint", "s3_role_arn", "s3_session_token",
-					"s3_path", "s3_encryption_key", "s3_number_thread",
+					"s3_path", "s3_encryption_key", "s3_number_thread", "s3_timeout",
 				},
 			},
 			FormElement{
@@ -163,6 +170,12 @@ func (this S3Backend) LoginForm() Form {
 				Type:        "text",
 				Placeholder: "Num. Thread",
 			},
+			FormElement{
+				Id:          "s3_timeout",
+				Name:        "timeout",
+				Type:        "number",
+				Placeholder: "List Object Timeout",
+			},
 		},
 	}
 }
@@ -197,6 +210,7 @@ func (this S3Backend) Ls(path string) (files []os.FileInfo, err error) {
 		return files, nil
 	}
 	client := s3.New(this.createSession(p.bucket))
+	start := time.Now()
 	err = client.ListObjectsV2PagesWithContext(
 		this.Context,
 		&s3.ListObjectsV2Input{
@@ -232,6 +246,9 @@ func (this S3Backend) Ls(path string) (files []os.FileInfo, err error) {
 					FTime: 0,
 				})
 			}
+			if this.timeout > 0 && time.Since(start) > this.timeout {
+				return false
+			}
 			return aws.BoolValue(objs.IsTruncated)
 		},
 	)