From 2f74a00879d476b0b2fb5dcc83ca70c45dedecf1 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 23 Apr 2019 18:55:19 +1000 Subject: [PATCH] fix (thumbnail): memory leak found using ab --- server/plugin/plg_image_light/index.go | 8 ++++-- server/plugin/plg_image_light/lib/raw.go | 2 +- server/plugin/plg_image_light/lib/resizer.c | 15 +++++----- server/plugin/plg_image_light/lib/resizer.go | 29 ++++++++++++-------- server/plugin/plg_image_light/lib/resizer.h | 4 +-- 5 files changed, 33 insertions(+), 25 deletions(-) diff --git a/server/plugin/plg_image_light/index.go b/server/plugin/plg_image_light/index.go index 2c4b6884..0947a30d 100644 --- a/server/plugin/plg_image_light/index.go +++ b/server/plugin/plg_image_light/index.go @@ -140,7 +140,8 @@ func Init(conf *Configuration) { ///////////////////////// // Specify transformation transform := &lib.Transform{ - Temporary: GetAbsolutePath(ImageCachePath + "image_" + QuickString(10)), + Input: GetAbsolutePath(ImageCachePath + "imagein_" + QuickString(10)), + Output: GetAbsolutePath(ImageCachePath + "imageout_" + QuickString(10)), Size: thumb_size(), Crop: true, Quality: thumb_quality(), @@ -163,7 +164,7 @@ func Init(conf *Configuration) { ///////////////////////////// // Insert file in the fs // => lower RAM usage while processing - file, err := os.OpenFile(transform.Temporary, os.O_WRONLY|os.O_CREATE, os.ModePerm) + file, err := os.OpenFile(transform.Input, os.O_WRONLY|os.O_CREATE, os.ModePerm) if err != nil { return reader, ErrFilesystemError } @@ -171,7 +172,8 @@ func Init(conf *Configuration) { file.Close() reader.Close() defer func() { - os.Remove(transform.Temporary) + os.Remove(transform.Input) + os.Remove(transform.Output) }() ///////////////////////// diff --git a/server/plugin/plg_image_light/lib/raw.go b/server/plugin/plg_image_light/lib/raw.go index 2991c676..f81a08fa 100644 --- a/server/plugin/plg_image_light/lib/raw.go +++ b/server/plugin/plg_image_light/lib/raw.go @@ -46,7 +46,7 @@ func IsRaw(mType string) bool { } func ExtractPreview(t *Transform) error { - filename := C.CString(t.Temporary) + filename := C.CString(t.Input) err := C.raw_process(filename, C.int(t.Size)) if err == LIBRAW_MEMORY_ERROR { // libraw acts weird sometimes and I couldn't diff --git a/server/plugin/plg_image_light/lib/resizer.c b/server/plugin/plg_image_light/lib/resizer.c index 8306bf13..b8d83948 100644 --- a/server/plugin/plg_image_light/lib/resizer.c +++ b/server/plugin/plg_image_light/lib/resizer.c @@ -1,17 +1,16 @@ #include #include -int resizer_init(const int ncpu, const int cache_max, const int cache_mem){ +int resizer_init(const int ncpu){ if(VIPS_INIT("filestash")){ return 1; } - vips_concurrency_set(100); - vips_cache_set_max(cache_max); - vips_cache_set_max_mem(cache_mem); + vips_concurrency_set(1); + vips_cache_set_max(0); return 0; } -int resizer_process(const char *filename, void **buf, size_t *len, int size, int crop, int quality, int exif){ +int resizer_process(const char *input, const char *output, int size, int crop, int quality, int exif){ VipsImage *img; int err; @@ -22,7 +21,7 @@ int resizer_process(const char *filename, void **buf, size_t *len, int size, int if(crop == VIPS_INTERESTING_CENTRE){ // Generate a thumbnails: a square picture crop in the center - err = vips_thumbnail(filename, &img, size, + err = vips_thumbnail(input, &img, size, "size", VIPS_SIZE_BOTH, "auto_rotate", TRUE, "crop", VIPS_INTERESTING_CENTRE, @@ -30,7 +29,7 @@ int resizer_process(const char *filename, void **buf, size_t *len, int size, int ); }else{ // normal resize of an image with libvips - err = vips_thumbnail(filename, &img, size, + err = vips_thumbnail(input, &img, size, "size", VIPS_SIZE_DOWN, "auto_rotate", TRUE, "crop", VIPS_INTERESTING_NONE, @@ -41,7 +40,7 @@ int resizer_process(const char *filename, void **buf, size_t *len, int size, int return err; } - err = vips_jpegsave_buffer(img, buf, len, "Q", quality, "strip", exif, NULL); + err = vips_jpegsave(img, output, NULL); g_object_unref(img); return err; } diff --git a/server/plugin/plg_image_light/lib/resizer.go b/server/plugin/plg_image_light/lib/resizer.go index f6a8d178..4d27b83c 100644 --- a/server/plugin/plg_image_light/lib/resizer.go +++ b/server/plugin/plg_image_light/lib/resizer.go @@ -8,14 +8,20 @@ import "C" import ( . "github.com/mickael-kerjean/filestash/server/common" "io" + "os" "runtime" "unsafe" + "sync" ) -var LIBVIPS_INSTALLED = false +var ( + LIBVIPS_INSTALLED = false + VIPS_LOCK = &sync.Mutex{} +) type Transform struct { - Temporary string + Input string + Output string Size int Crop bool Quality int @@ -23,7 +29,7 @@ type Transform struct { } func init() { - if C.resizer_init(C.int(runtime.NumCPU()), 50, 1024) != 0 { + if C.resizer_init(C.int(runtime.NumCPU())) != 0 { Log.Warning("Can't load libvips") return } @@ -34,17 +40,18 @@ func CreateThumbnail(t *Transform) (io.ReadCloser, error) { if LIBVIPS_INSTALLED == false { return nil, NewError("Libvips not installed", 501) } - filename := C.CString(t.Temporary) - defer C.free(unsafe.Pointer(filename)) - var buffer unsafe.Pointer - len := C.size_t(0) + filenameInput := C.CString(t.Input) + defer C.free(unsafe.Pointer(filenameInput)) - if C.resizer_process(filename, &buffer, &len, C.int(t.Size), boolToCInt(t.Crop), C.int(t.Quality), boolToCInt(t.Exif)) != 0 { + filenameOutput := C.CString(t.Output) + defer C.free(unsafe.Pointer(filenameOutput)) + + VIPS_LOCK.Lock() + if C.resizer_process(filenameInput, filenameOutput, C.int(t.Size), boolToCInt(t.Crop), C.int(t.Quality), boolToCInt(t.Exif)) != 0 { return nil, NewError("", 500) } - buf := C.GoBytes(buffer, C.int(len)) - C.g_free(C.gpointer(buffer)) - return NewReadCloserFromBytes(buf), nil + VIPS_LOCK.Unlock() + return os.OpenFile(t.Output, os.O_RDONLY, os.ModePerm) } func boolToCInt(val bool) C.int { diff --git a/server/plugin/plg_image_light/lib/resizer.h b/server/plugin/plg_image_light/lib/resizer.h index ad8d0f43..6452dbdf 100644 --- a/server/plugin/plg_image_light/lib/resizer.h +++ b/server/plugin/plg_image_light/lib/resizer.h @@ -1,6 +1,6 @@ #include #include -int resizer_init(const int ncpu, const int cache_max, const int cache_mem); +int resizer_init(const int ncpu); -int resizer_process(const char *filename, void **buf, size_t *len, int size, int crop, int quality, int exif); +int resizer_process(const char *input, const char *output, int size, int crop, int quality, int exif);