chore (maintenance): image thumbnail

the issue of plg_image_c is its reliance on shared libraries like libraw
which depends on lcms2 and libheif which depends on a whole bunch of
other stuff. That make releasing a fat binary that just work of
Filestash tricky

if someone ever read this, I would gladly get help to integrate the
whole build of those things within CI without prolonging the build time
by 2 hours.

server/plugin/index.go

@@ -31,12 +31,11 @@ import (
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_handler_console"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_handler_mcp"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_image_ascii"
-	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_image_c"
-	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_image_transcode"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_search_stateless"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_security_scanner"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_security_svg"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_starter_http"
+	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_thumbnail_c"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_video_transcoder"
 )
 

server/plugin/plg_thumbnail_c/image_gif.c

@@ -0,0 +1,204 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <gif_lib.h>
+#include <webp/encode.h>
+#include <unistd.h>
+#include "utils.h"
+#include "image_gif_vendor.h"
+
+int DGifSlurp2(GifFileType *GifFile);
+int DGifCloseFile2(GifFileType *GifFile, int *ErrorCode);
+
+int gif_to_webp(int inputDesc, int outputDesc, int targetSize) {
+#ifdef HAS_DEBUG
+  clock_t t;
+  t = clock();
+#endif
+  int status = 0;
+  int error;
+  if (targetSize < 0) targetSize = -targetSize;
+
+  // STEP1: setup gif
+  GifFileType* gif;
+  uint8_t* gif_rgba;
+  uint8_t* scaled_rgba;
+  if((gif = DGifOpenFileHandle(inputDesc, &error)) == NULL) {
+    status = 1;
+    goto CLEANUP_AND_ABORT;
+  }
+  DEBUG("after gif opened");
+  if (DGifSlurp2(gif) != GIF_OK) {
+    status = 1;
+    goto CLEANUP_AND_ABORT_A;
+  }
+  int width = gif->SWidth;
+  int height = gif->SHeight;
+  int scale_factor = (width > targetSize) ? width / targetSize : 1;
+  int thumb_width = width / scale_factor;
+  int thumb_height = height / scale_factor;
+  DEBUG("after gif ready");
+
+  // STEP2: convert frame to RGBA
+  if (gif->ImageCount == 0) {
+	status = 1;
+    goto CLEANUP_AND_ABORT_A;
+  } else if (!(gif_rgba = (uint8_t*)malloc(width * height * 4))) {
+    status = 1;
+    goto CLEANUP_AND_ABORT_A;
+  }
+  GifColorType* colorMapEntry;
+  ColorMapObject* colorMap = (gif->Image.ColorMap) ? gif->Image.ColorMap : gif->SColorMap;
+  SavedImage* firstFrame = &gif->SavedImages[0];
+  GifByteType* gifBytes = firstFrame->RasterBits;
+  for (int i = 0; i < gif->SWidth * gif->SHeight; ++i) {
+    colorMapEntry = &colorMap->Colors[gifBytes[i]];
+    gif_rgba[i * 4 + 0] = colorMapEntry->Red;
+    gif_rgba[i * 4 + 1] = colorMapEntry->Green;
+    gif_rgba[i * 4 + 2] = colorMapEntry->Blue;
+    gif_rgba[i * 4 + 3] = 0xFF;
+  }
+  DEBUG("after gif rgba convert");
+
+  // STEP3: scale the image
+  if (!(scaled_rgba = (uint8_t*)malloc(thumb_width * thumb_height * 4))) {
+    free(gif_rgba);
+    status = 1;
+    goto CLEANUP_AND_ABORT_A;
+  }
+  int x, y, srcIndex, destIndex;
+  for (int i = 0; i < thumb_height; ++i) {
+    for (int j = 0; j < thumb_width; ++j) {
+      x = j * width / thumb_width;
+      y = i * height / thumb_height;
+      srcIndex = (y * width + x) * 4;
+      destIndex = (i * thumb_width + j) * 4;
+      memcpy(&scaled_rgba[destIndex], &gif_rgba[srcIndex], 4);
+    }
+  }
+  free(gif_rgba);
+  DEBUG("after image scaled");
+
+  // STEP4: write image as webp
+  uint8_t* webp_output_data;
+  size_t webp_output_size = WebPEncodeRGBA(scaled_rgba, thumb_width, thumb_height, thumb_width * 4, 75, &webp_output_data);
+  free(scaled_rgba);
+  if (webp_output_size == 0) {
+    status = 1;
+    goto CLEANUP_AND_ABORT_A;
+  }
+  if (write(outputDesc, webp_output_data, webp_output_size) != webp_output_size) {
+    status = 1;
+    ERROR("unexpected number of bytes written");
+  }
+  WebPFree(webp_output_data);
+  DEBUG("after webp written");
+
+ CLEANUP_AND_ABORT_A:
+  DGifCloseFile2(gif, &error);
+
+ CLEANUP_AND_ABORT:
+  return status;
+}
+
+// adapted from https://android.googlesource.com/platform/external/giflib/+/dc07290edccc2c3fc4062da835306f809cea1fdc/dgif_lib.c
+// we got rid of unecessary stuff for our use case and reduce the processing
+// to the first frame only which isn't possible using stock libgif functions
+int DGifSlurp2(GifFileType *GifFile) {
+  clock_t t = clock();
+  size_t ImageSize;
+  GifRecordType RecordType;
+  SavedImage *sp;
+  GifByteType *ExtData;
+  int ExtFunction;
+  GifFile->ExtensionBlocks = NULL;
+  GifFile->ExtensionBlockCount = 0;
+  do {
+    if (DGifGetRecordType(GifFile, &RecordType) == GIF_ERROR) {
+      return GIF_ERROR;
+    }
+
+    if (RecordType == IMAGE_DESC_RECORD_TYPE) {
+      if (DGifGetImageDesc(GifFile) == GIF_ERROR) {
+        return GIF_ERROR;
+      }
+
+      sp = &GifFile->SavedImages[GifFile->ImageCount - 1];
+      if (sp->ImageDesc.Width < 0 && sp->ImageDesc.Height < 0 && sp->ImageDesc.Width > (INT_MAX / sp->ImageDesc.Height)) {
+        return GIF_ERROR;
+      }
+      ImageSize = sp->ImageDesc.Width * sp->ImageDesc.Height;
+      if (ImageSize > (SIZE_MAX / sizeof(GifPixelType))) {
+        return GIF_ERROR;
+      }
+      sp->RasterBits = (unsigned char *)reallocarray(NULL, ImageSize, sizeof(GifPixelType));
+      if (sp->RasterBits == NULL) {
+        return GIF_ERROR;
+      }
+      if (DGifGetLine(GifFile, sp->RasterBits, ImageSize) == GIF_ERROR) {
+        return GIF_ERROR;
+      }
+      return GIF_OK;
+    } else if (RecordType == EXTENSION_RECORD_TYPE) {
+      if (DGifGetExtension(GifFile, &ExtFunction, &ExtData) == GIF_ERROR) {
+        return GIF_ERROR;
+      }
+      while (ExtData != NULL) {
+        if (DGifGetExtensionNext(GifFile, &ExtData) == GIF_ERROR) {
+          return GIF_ERROR;
+        }
+      }
+    }
+  } while (RecordType != TERMINATE_RECORD_TYPE);
+
+  return GIF_OK;
+}
+
+
+// adapted from: https://android.googlesource.com/platform/external/giflib/+/dc07290edccc2c3fc4062da835306f809cea1fdc/dgif_lib.c#626
+// as we don't want libgif to manage the lifecycle of the file descriptor, in our case
+// this is the responsibility of the downstream program, that's why we've recopied it here
+// a commented the fclose call
+int DGifCloseFile2(GifFileType *GifFile, int *ErrorCode)
+{
+  GifFilePrivateType *Private;
+  if (GifFile == NULL || GifFile->Private == NULL) {
+    return GIF_ERROR;
+  }
+  if (GifFile->Image.ColorMap) {
+    GifFreeMapObject(GifFile->Image.ColorMap);
+    GifFile->Image.ColorMap = NULL;
+  }
+  if (GifFile->SColorMap) {
+    GifFreeMapObject(GifFile->SColorMap);
+    GifFile->SColorMap = NULL;
+  }
+  if (GifFile->SavedImages) {
+    GifFreeSavedImages(GifFile);
+    GifFile->SavedImages = NULL;
+  }
+  GifFreeExtensions(&GifFile->ExtensionBlockCount, &GifFile->ExtensionBlocks);
+  Private = (GifFilePrivateType *) GifFile->Private;
+  if (!IS_READABLE(Private)) {
+    if (ErrorCode != NULL) {
+      *ErrorCode = D_GIF_ERR_NOT_READABLE;
+    }
+	free((char *)GifFile->Private);
+	free(GifFile);
+    return GIF_ERROR;
+  }
+  if (Private->File /*&& (fclose(Private->File) != 0)*/) {
+	if (ErrorCode != NULL) {
+      *ErrorCode = D_GIF_ERR_CLOSE_FAILED;
+    }
+	free((char *)GifFile->Private);
+	free(GifFile);
+    return GIF_ERROR;
+  }
+  free((char *)GifFile->Private);
+  free(GifFile);
+  if (ErrorCode != NULL) {
+	*ErrorCode = D_GIF_SUCCEEDED;
+  }
+  return GIF_OK;
+}

server/plugin/plg_thumbnail_c/image_gif.go

@@ -0,0 +1,10 @@
+package plg_image_c
+
+// #include "image_gif.h"
+// #cgo LDFLAGS: -l:libgif.a -l:libwebp.a
+import "C"
+
+func gif(input uintptr, output uintptr, size int) {
+	C.gif_to_webp(C.int(input), C.int(output), C.int(size))
+	return
+}

server/plugin/plg_thumbnail_c/image_gif.h

@@ -0,0 +1,4 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+int gif_to_webp(int inputDesc, int outputDesc, int targetSize);

server/plugin/plg_thumbnail_c/image_gif_vendor.h

@@ -0,0 +1,8 @@
+#define FILE_STATE_READ     0x08
+#define IS_READABLE(Private)    (Private->FileState & FILE_STATE_READ)
+#define INT_MAX 2147483647
+
+typedef struct GifFilePrivateType {
+  GifWord FileState;
+  FILE *File;
+} GifFilePrivateType;

server/plugin/plg_thumbnail_c/image_jpeg.c

@@ -0,0 +1,128 @@
+#include <stdio.h>
+#include <jpeglib.h>
+#include <setjmp.h>
+#include <stdlib.h>
+#include "utils.h"
+
+#define JPEG_QUALITY 50
+
+typedef struct filestash_jpeg_error_mgr {
+  struct jpeg_error_mgr pub;
+  jmp_buf jmp;
+} *filestash_jpeg_error_ptr;
+
+void filestash_jpeg_error_exit (j_common_ptr cinfo);
+
+int jpeg_to_jpeg(int inputDesc, int outputDesc, int targetSize) {
+#ifdef HAS_DEBUG
+  clock_t t;
+  t = clock();
+#endif
+  int status = 0;
+  FILE* input = fdopen(inputDesc, "rb");
+  FILE* output = fdopen(outputDesc, "wb");
+  if (!input || !output) {
+    return 1;
+  }
+
+  struct jpeg_decompress_struct jpeg_config_input;
+  struct jpeg_compress_struct jpeg_config_output;
+  struct filestash_jpeg_error_mgr jerr;
+
+  jpeg_config_input.err = jpeg_std_error(&jerr.pub);
+  jpeg_config_output.err = jpeg_std_error(&jerr.pub);
+  jpeg_config_input.dct_method = JDCT_IFAST;
+  jpeg_config_input.do_fancy_upsampling = FALSE;
+  jpeg_config_input.two_pass_quantize = FALSE;
+  jpeg_config_input.dither_mode = JDITHER_ORDERED;
+
+  jpeg_create_decompress(&jpeg_config_input);
+  jpeg_create_compress(&jpeg_config_output);
+  jpeg_stdio_src(&jpeg_config_input, input);
+  jpeg_stdio_dest(&jpeg_config_output, output);
+
+  jerr.pub.error_exit = filestash_jpeg_error_exit;
+  if (setjmp(jerr.jmp)) {
+    ERROR("exception");
+    goto CLEANUP_AND_ABORT;
+  }
+
+  DEBUG("after constructor decompress");
+  if(jpeg_read_header(&jpeg_config_input, TRUE) != JPEG_HEADER_OK) {
+    status = 1;
+    ERROR("not a jpeg");
+    goto CLEANUP_AND_ABORT;
+  }
+  DEBUG("after header read");
+  jpeg_config_input.dct_method = JDCT_IFAST;
+  jpeg_config_input.do_fancy_upsampling = FALSE;
+  jpeg_config_input.two_pass_quantize = FALSE;
+  jpeg_config_input.dither_mode = JDITHER_ORDERED;
+  jpeg_calc_output_dimensions(&jpeg_config_input);
+
+  int image_min_size = min(jpeg_config_input.output_width, jpeg_config_input.output_height);
+  jpeg_config_input.scale_num = 1;
+  jpeg_config_input.scale_denom = 1;
+  int targetSizeAbs = abs(targetSize);
+  if (image_min_size / 8 >= targetSizeAbs) {
+    jpeg_config_input.scale_num = 1;
+    jpeg_config_input.scale_denom = 8;
+  } else if (image_min_size * 2 / 8 >= targetSizeAbs) {
+    jpeg_config_input.scale_num = 1;
+    jpeg_config_input.scale_denom = 4;
+  } else if (image_min_size * 3 / 8 >= targetSizeAbs) {
+    jpeg_config_input.scale_num = 3;
+    jpeg_config_input.scale_denom = 8;
+  } else if (image_min_size * 4 / 8 >= targetSizeAbs) {
+    jpeg_config_input.scale_num = 4;
+    jpeg_config_input.scale_denom = 8;
+  } else if (image_min_size * 5 / 8 >= targetSizeAbs) {
+    jpeg_config_input.scale_num = 5;
+    jpeg_config_input.scale_denom = 8;
+  } else if (image_min_size * 6 / 8 >= targetSizeAbs) {
+    jpeg_config_input.scale_num = 6;
+    jpeg_config_input.scale_denom = 8;
+  } else if (image_min_size * 7 / 8 >= targetSizeAbs) {
+    jpeg_config_input.scale_num = 7;
+    jpeg_config_input.scale_denom = 8;
+  }
+
+  DEBUG("start decompress");
+  if(jpeg_start_decompress(&jpeg_config_input) == FALSE) {
+    ERROR("jpeg_start_decompress");
+    status = 1;
+    goto CLEANUP_AND_ABORT;
+  }
+  DEBUG("processing image setup");
+  int jpeg_row_stride = jpeg_config_input.output_width * jpeg_config_input.output_components;
+  jpeg_config_output.image_width = jpeg_config_input.output_width;
+  jpeg_config_output.image_height = jpeg_config_input.output_height;
+  jpeg_config_output.input_components = jpeg_config_input.num_components;
+  jpeg_config_output.in_color_space = jpeg_config_input.out_color_space;
+  jpeg_set_defaults(&jpeg_config_output);
+  jpeg_set_quality(&jpeg_config_output, JPEG_QUALITY, TRUE);
+  jpeg_start_compress(&jpeg_config_output, TRUE);
+  JSAMPARRAY buffer = jpeg_config_input.mem->alloc_sarray((j_common_ptr) &jpeg_config_input, JPOOL_IMAGE, jpeg_row_stride, 1);
+
+  DEBUG("processing image");
+  while (jpeg_config_output.next_scanline < jpeg_config_output.image_height) {
+    jpeg_read_scanlines(&jpeg_config_input, buffer, 1);
+    jpeg_write_scanlines(&jpeg_config_output, buffer, 1);
+  }
+
+  DEBUG("end decompress");
+  jpeg_finish_decompress(&jpeg_config_input);
+  DEBUG("finish decompress");
+  jpeg_finish_compress(&jpeg_config_output);
+
+ CLEANUP_AND_ABORT:
+  jpeg_destroy_decompress(&jpeg_config_input);
+  jpeg_destroy_compress(&jpeg_config_output);
+  DEBUG("final");
+  return status;
+}
+
+void filestash_jpeg_error_exit (j_common_ptr cinfo) {
+  filestash_jpeg_error_ptr filestash_err = (filestash_jpeg_error_ptr) cinfo->err;
+  longjmp(filestash_err->jmp, 1);
+}

server/plugin/plg_thumbnail_c/image_jpeg.h

@@ -0,0 +1 @@
+int jpeg_to_jpeg(int input, int output, int targetSize);

server/plugin/plg_thumbnail_c/image_jpeg_freebsd.go

@@ -0,0 +1,11 @@
+package plg_image_c
+
+// #include "image_jpeg.h"
+// #cgo LDFLAGS: -L /usr/local/lib -L /usr/lib -L /lib -l:libjpeg.a
+// #cgo CFLAGS: -I /usr/local/include
+import "C"
+
+func jpeg(input uintptr, output uintptr, size int) {
+	C.jpeg_to_jpeg(C.int(input), C.int(output), C.int(size))
+	return
+}

server/plugin/plg_thumbnail_c/image_jpeg_linux.go

@@ -0,0 +1,10 @@
+package plg_image_c
+
+// #include "image_jpeg.h"
+// #cgo LDFLAGS: -l:libjpeg.a
+import "C"
+
+func jpeg(input uintptr, output uintptr, size int) {
+	C.jpeg_to_jpeg(C.int(input), C.int(output), C.int(size))
+	return
+}

server/plugin/plg_thumbnail_c/image_png.c

@@ -0,0 +1,132 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <png.h>
+#include <webp/encode.h>
+#include "utils.h"
+
+void png_read_error(png_structp png_ptr, png_const_charp error_msg) {
+  longjmp(png_jmpbuf(png_ptr), 1);
+}
+
+void png_read_warning(png_structp png_ptr, png_const_charp warning_msg) {
+  longjmp(png_jmpbuf(png_ptr), 1);
+}
+
+int png_to_webp(int inputDesc, int outputDesc, int targetSize) {
+#ifdef HAS_DEBUG
+  clock_t t;
+  t = clock();
+#endif
+  if (targetSize < 0 ) {
+    targetSize = -targetSize;
+  }
+  int status = 0;
+  FILE* input = fdopen(inputDesc, "rb");
+  FILE* output = fdopen(outputDesc, "wb");
+  if (!input || !output) {
+    return 1;
+  }
+
+  // STEP1: setup png
+  png_structp png_ptr = NULL;
+  png_infop info_ptr = NULL;
+  if(!(png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, png_read_error, png_read_warning))) {
+    status = 1;
+    goto CLEANUP_AND_ABORT;
+  }
+  if (!(info_ptr = png_create_info_struct(png_ptr))) {
+    status = 1;
+    goto CLEANUP_AND_ABORT_A;
+  }
+  if (setjmp(png_jmpbuf(png_ptr))) {
+    status = 1;
+    goto CLEANUP_AND_ABORT_B;
+  }
+  png_init_io(png_ptr, input);
+  png_read_info(png_ptr, info_ptr);
+  png_set_strip_alpha(png_ptr);
+  png_uint_32 width = png_get_image_width(png_ptr, info_ptr);
+  png_uint_32 height = png_get_image_height(png_ptr, info_ptr);
+  png_byte color_type = png_get_color_type(png_ptr, info_ptr);
+  png_byte bit_depth = png_get_bit_depth(png_ptr, info_ptr);
+  if (color_type == PNG_COLOR_TYPE_PALETTE) {
+    png_set_palette_to_rgb(png_ptr);
+  }
+  if (color_type == PNG_COLOR_TYPE_GRAY) {
+    png_set_expand_gray_1_2_4_to_8(png_ptr);
+  }
+  if (color_type & PNG_COLOR_MASK_ALPHA) {
+    png_set_strip_alpha(png_ptr);
+  }
+  png_read_update_info(png_ptr, info_ptr);
+  DEBUG("after png construct");
+
+  // STEP2: process the image
+  int scale_factor = height > targetSize ? height / targetSize : 1;
+  png_uint_32 thumb_width = width / scale_factor;
+  png_uint_32 thumb_height = height / scale_factor;
+
+  if (thumb_width == 0 || thumb_height == 0) {
+    ERROR("0 dimensions");
+    status = 1;
+    goto CLEANUP_AND_ABORT_B;
+  }
+  uint8_t* webp_image_data = (uint8_t*)malloc(thumb_width * thumb_height * 3);
+  if (!webp_image_data) {
+    ERROR("malloc error");
+    status = 1;
+    goto CLEANUP_AND_ABORT_B;
+  }
+  png_bytep row = (png_bytep)malloc(png_get_rowbytes(png_ptr, info_ptr));
+  if (!row) {
+    ERROR("malloc error");
+    status = 1;
+    goto CLEANUP_AND_ABORT_B;
+  }
+  DEBUG("after png malloc");
+  for (png_uint_32 y = 0; y < height; y++) {
+    png_read_row(png_ptr, row, NULL);
+    if (y % scale_factor == 0 && (y / scale_factor < thumb_height)) {
+      for (png_uint_32 x = 0; x < width; x += scale_factor) {
+        if (x / scale_factor < thumb_width) {
+          png_uint_32 thumb_x = x / scale_factor;
+          png_uint_32 thumb_y = y / scale_factor;
+          memcpy(webp_image_data + (thumb_y * thumb_width + thumb_x) * 3, row + x * 3, 3);
+        }
+      }
+    }
+  }
+  DEBUG("after png process");
+  free(row);
+  png_destroy_read_struct(&png_ptr, &info_ptr, NULL);
+  DEBUG("after png cleanup");
+
+  // STEP3: save as webp
+  uint8_t* webp_output_data = NULL;
+  size_t webp_output_size = WebPEncodeRGB(webp_image_data, thumb_width, thumb_height, thumb_width * 3, 75, &webp_output_data);
+  free(webp_image_data);
+  DEBUG("after webp init");
+  if (webp_output_data == NULL) {
+    status = 1;
+    goto CLEANUP_AND_ABORT_B;
+  } else if (webp_output_size == 0) {
+    status = 1;
+    goto CLEANUP_AND_ABORT_C;
+  }
+  fwrite(webp_output_data, webp_output_size, 1, output);
+  fflush(output);
+  DEBUG("after webp written");
+
+ CLEANUP_AND_ABORT_C:
+  if (webp_output_data != NULL) WebPFree(webp_output_data);
+
+ CLEANUP_AND_ABORT_B:
+  if (info_ptr != NULL) png_free_data(png_ptr, info_ptr, PNG_FREE_ALL, -1);
+
+ CLEANUP_AND_ABORT_A:
+  if (png_ptr != NULL) png_destroy_read_struct(&png_ptr, (info_ptr != NULL) ? &info_ptr : NULL, NULL);
+
+ CLEANUP_AND_ABORT:
+  return status;
+}

server/plugin/plg_thumbnail_c/image_png.h

@@ -0,0 +1,6 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+int png_to_png(int input, int output, int targetSize);
+
+int png_to_webp(int input, int output, int targetSize);

server/plugin/plg_thumbnail_c/image_png_freebsd.go

@@ -0,0 +1,11 @@
+package plg_image_c
+
+// #include "image_png.h"
+// #cgo LDFLAGS: -L /usr/local/lib -L /usr/lib -L /lib -l:libsharpyuv.a -l:libpng.a -l:libz.a -l:libwebp.a -l:libpthread.a -fopenmp
+// #cgo CFLAGS: -I /usr/local/include
+import "C"
+
+func png(input uintptr, output uintptr, size int) {
+	C.png_to_webp(C.int(input), C.int(output), C.int(size))
+	return
+}

server/plugin/plg_thumbnail_c/image_png_linux.go

@@ -0,0 +1,10 @@
+package plg_image_c
+
+// #include "image_png.h"
+// #cgo LDFLAGS: -l:libpng.a -l:libz.a -l:libwebp.a -fopenmp -lm
+import "C"
+
+func png(input uintptr, output uintptr, size int) {
+	C.png_to_webp(C.int(input), C.int(output), C.int(size))
+	return
+}

server/plugin/plg_thumbnail_c/image_webp.c

@@ -0,0 +1,106 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <webp/decode.h>
+#include <webp/encode.h>
+#include "utils.h"
+
+#define WEBP_QUALITY 75
+#define INITIAL_BUFFER_SIZE 1024*64 // 128kB
+#define MAX_BUFFER_SIZE 1024*1024*2 // 2MB
+
+int webp_to_webp(int inputDesc, int outputDesc, int targetSize) {
+#ifdef HAS_DEBUG
+  clock_t t;
+  t = clock();
+#endif
+  if (targetSize < 0) {
+    targetSize = -targetSize;
+  }
+  int status = 0;
+  FILE* input = fdopen(inputDesc, "rb");
+  FILE* output = fdopen(outputDesc, "wb");
+  if (!input || !output) {
+    ERROR("setup");
+    return 1;
+  }
+
+  // STEP1: setup everything
+  size_t data_size = 0;
+  size_t buffer_size = INITIAL_BUFFER_SIZE;
+  uint8_t* data = (uint8_t*)malloc(buffer_size);
+  if (!data) {
+    ERROR("malloc");
+    return 1;
+  }
+  size_t bytes_read;
+  while ((bytes_read = fread(data + data_size, 1, buffer_size - data_size, input)) > 0) {
+    data_size += bytes_read;
+    if (buffer_size - data_size == 0) {
+      DEBUG("realloc");
+      if (buffer_size >= MAX_BUFFER_SIZE) {
+        free(data);
+        ERROR("abort");
+        return 1;
+      }
+      buffer_size *= 2;
+      if (buffer_size > MAX_BUFFER_SIZE) buffer_size = MAX_BUFFER_SIZE;
+      uint8_t* new_data = (uint8_t*)realloc(data, buffer_size);
+      if (!new_data) {
+        free(data);
+        ERROR("realloc");
+        return 1;
+      }
+      data = new_data;
+    }
+  }
+
+  // STEP2: decode
+  int width, height, scale_factor;
+  if (!WebPGetInfo(data, data_size, &width, &height)) {
+    free(data);
+    ERROR("init");
+    return 1;
+  }
+  DEBUG("init");
+  WebPDecoderConfig config;
+  if (!WebPInitDecoderConfig(&config)) {
+    free(data);
+    ERROR("config");
+    return 1;
+  }
+  scale_factor = (height > targetSize) ? height / targetSize : 1;
+  config.options.use_scaling = 1;
+  config.options.scaled_width = width / scale_factor;
+  config.options.scaled_height = height / scale_factor;
+  config.output.colorspace = MODE_rgbA;
+  DEBUG("config");
+  if (WebPDecode(data, data_size, &config) != VP8_STATUS_OK) {
+    WebPFreeDecBuffer(&config.output);
+    free(data);
+    ERROR("decode");
+    return 1;
+  }
+  free(data);
+  DEBUG("decode");
+
+  // STEP3: encode
+  size_t output_size = 0;
+  uint8_t* output_data = NULL;
+  output_size = WebPEncodeRGBA(
+      config.output.u.RGBA.rgba, config.options.scaled_width,
+      config.options.scaled_height, config.output.u.RGBA.stride,
+      WEBP_QUALITY, &output_data
+  );
+  if (output_data == NULL) {
+    WebPFreeDecBuffer(&config.output);
+    ERROR("encode");
+    return 1;
+  }
+  DEBUG("encode");
+  fwrite(output_data, output_size, 1, output);
+  fflush(output);
+  WebPFree(output_data);
+  WebPFreeDecBuffer(&config.output);
+  DEBUG("done");
+  return status;
+}

server/plugin/plg_thumbnail_c/image_webp.go

@@ -0,0 +1,10 @@
+package plg_image_c
+
+// #include "image_webp.h"
+// #cgo LDFLAGS: -l:libwebp.a
+import "C"
+
+func webp(input uintptr, output uintptr, size int) {
+	C.webp_to_webp(C.int(input), C.int(output), C.int(size))
+	return
+}

server/plugin/plg_thumbnail_c/image_webp.h

@@ -0,0 +1 @@
+int webp_to_webp(int inputDesc, int outputDesc, int targetSize);

server/plugin/plg_thumbnail_c/index.go

@@ -0,0 +1,95 @@
+package plg_image_c
+
+import (
+	"io"
+	"net/http"
+	"os"
+
+	. "github.com/mickael-kerjean/filestash/server/common"
+)
+
+/*
+ * All the transcoders are reponsible for:
+ * 1. create thumbnails if needed
+ * 2. transcode various files if needed
+ *
+ * Under the hood, our transcoders are C programs that takes 3 arguments:
+ * 1/2. the input/output file descriptors. We use file descriptors to communicate from go -> C -> go
+ * 3. the target size. by convention those program handles:
+ *    - positive size: when we want to transcode a file with best effort in regards to quality and
+ *      not lose metadata, typically when this will be open in an image viewer from which we might have
+ *      frontend code to extract exif/xmp metadata, ...
+ *    - negative size: when we want transcode to be done as quickly as possible, typically when we want
+ *      to create a thumbnail and don't care/need anything else than speed
+ */
+
+func init() {
+	Hooks.Register.Thumbnailer("image/jpeg", &transcoder{runner(jpeg), "image/jpeg", -200})
+	Hooks.Register.Thumbnailer("image/png", &transcoder{runner(png), "image/webp", -200})
+	Hooks.Register.Thumbnailer("image/gif", &transcoder{runner(gif), "image/webp", -300})
+	Hooks.Register.Thumbnailer("image/webp", &transcoder{runner(webp), "image/webp", -200})
+}
+
+type transcoder struct {
+	fn   func(input io.ReadCloser, size int) (io.ReadCloser, error)
+	mime string
+	size int
+}
+
+func (this transcoder) Generate(reader io.ReadCloser, ctx *App, res *http.ResponseWriter, req *http.Request) (io.ReadCloser, error) {
+	thumb, err := this.fn(reader, this.size)
+	if err == nil && this.mime != "" {
+		(*res).Header().Set("Content-Type", this.mime)
+	}
+	return thumb, err
+}
+
+/*
+ * uuuh, what is this stuff you might rightly wonder? Trying to send a go stream to C isn't obvious,
+ * but if you try to stream from C back to go in the same time, this is what you endup with.
+ * To my knowledge using file descriptor is the best way we can do that if we don't make the assumption
+ * that everything fits in memory.
+ */
+func runner(fn func(uintptr, uintptr, int)) func(io.ReadCloser, int) (io.ReadCloser, error) {
+	return func(inputGo io.ReadCloser, size int) (io.ReadCloser, error) {
+		inputC, tmpw, err := os.Pipe()
+		logErrors(err, "plg_image_c::pipe")
+		if err != nil {
+			return nil, err
+		}
+		outputGo, outputC, err := os.Pipe()
+		logErrors(err, "plg_image_c::pipe")
+		if err != nil {
+			tmpw.Close()
+			return nil, err
+		}
+
+		go func() {
+			fn(inputC.Fd(), outputC.Fd(), size) // <-- all this code so we can do that
+			logErrors(inputC.Close(), "plg_image_c::inputC")
+			logErrors(inputGo.Close(), "plg_image_c::inputGo")
+			logErrors(outputC.Close(), "plg_image_c::outputC")
+		}()
+		go func() {
+			io.Copy(tmpw, inputGo)
+			logErrors(tmpw.Close(), "plg_image_c::tmpw")
+		}()
+		return outputGo, nil
+	}
+}
+
+func logErrors(err error, msg string) {
+	if err == nil {
+		return
+	}
+	Log.Debug(msg + ": " + err.Error())
+}
+
+func contains(s []string, str string) bool {
+	for _, v := range s {
+		if v == str {
+			return true
+		}
+	}
+	return false
+}

server/plugin/plg_thumbnail_c/utils.h

@@ -0,0 +1,11 @@
+#define HAS_DEBUG 0
+#include <time.h>
+#if HAS_DEBUG == 1
+#define DEBUG(r) (fprintf(stderr,  "[DEBUG::('" r "')(%.2Fms)]", ((double)clock() - t)/CLOCKS_PER_SEC * 1000))
+#else
+#define DEBUG(r) ((void)0)
+#endif
+
+#define ERROR(r) (fprintf(stderr, "[ERROR:('" r "')]"))
+
+#define min(a, b) (a > b ? b : a)