From 136afbeac261cf8901975ef9f37b42b602baee3d Mon Sep 17 00:00:00 2001
From: Jay Thomason <email@jaythomason.com>
Date: Tue, 25 May 2021 05:00:23 -0700
Subject: [PATCH] improve (s3): support for prefixes with > 1k objects (#395)

Previously filestash would only show the first 1000 objects in a given
s3 prefix.

We solve this by paginating the ListObjectsV2 responses and iterating
over all pages instead of just the first one.

Tested by manually connecting to a private s3 bucket with multiple
thousands of objects under a single prefix and verifying that all
objects are visible in the UI.
---
 server/plugin/plg_backend_s3/index.go | 50 +++++++++++++--------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/server/plugin/plg_backend_s3/index.go b/server/plugin/plg_backend_s3/index.go
index fbf2aa39..36c848f0 100644
--- a/server/plugin/plg_backend_s3/index.go
+++ b/server/plugin/plg_backend_s3/index.go
@@ -144,32 +144,32 @@ func (s S3Backend) Ls(path string) (files []os.FileInfo, err error) {
 	}
 	client := s3.New(s.createSession(p.bucket))
 
-	objs, errTmp := client.ListObjectsV2(&s3.ListObjectsV2Input{
-		Bucket:    aws.String(p.bucket),
-		Prefix:    aws.String(p.path),
-		Delimiter: aws.String("/"),
-	})
-	if errTmp != nil {
-		err = errTmp
-		return
-	}
-	for i, object := range objs.Contents {
-		if i == 0 && *object.Key == p.path {
-			continue
-		}
-		files = append(files, &File{
-			FName: filepath.Base(*object.Key),
-			FType: "file",
-			FTime: object.LastModified.Unix(),
-			FSize: *object.Size,
+	err = client.ListObjectsV2Pages(
+		&s3.ListObjectsV2Input{
+			Bucket:    aws.String(p.bucket),
+			Prefix:    aws.String(p.path),
+			Delimiter: aws.String("/"),
+		},
+		func(objs *s3.ListObjectsV2Output, lastPage bool) bool {
+			for i, object := range objs.Contents {
+				if i == 0 && *object.Key == p.path {
+					continue
+				}
+				files = append(files, &File{
+					FName: filepath.Base(*object.Key),
+					FType: "file",
+					FTime: object.LastModified.Unix(),
+					FSize: *object.Size,
+				})
+			}
+			for _, object := range objs.CommonPrefixes {
+				files = append(files, &File{
+					FName: filepath.Base(*object.Prefix),
+					FType: "directory",
+				})
+			}
+			return true
 		})
-	}
-	for _, object := range objs.CommonPrefixes {
-		files = append(files, &File{
-			FName: filepath.Base(*object.Prefix),
-			FType: "directory",
-		})
-	}
 
 	return files, err
 }