feature (transcoding): live transcoding

client/components/bundle.js

@@ -1,6 +1,7 @@
 // taken from https://reacttraining.com/react-router/web/guides/code-splitting
 import React from 'react';
 import Path from 'path';
+import load from "little-loader";
 
 export class Bundle extends React.Component {
     state = { mod: null };
@@ -17,26 +18,42 @@ export class Bundle extends React.Component {
         this.setState({
             mod: null
         });
-        props.loader
-            .then((_mod) => {
-                this.setState({
-                    mod: function(mod){
-                        if(mod['default']){
-                            return mod.default;
-                        }else if(mod['__esModule'] === true){                        
-                            return mod[props.symbol] ? mod[props.symbol] : null;
-                        }else{
-                            return mod;
-                        }
-                    }(_mod)
-                })
-            })
-            .catch((err) => {
-                console.warn(err)
-            })
+        Promise.all(
+            [props.loader].concat(
+                (props.overrides || []).map((src) => loadAsPromise(src))
+            )
+        ).then((m) => {
+            const _mod = m[0];
+            this.setState({
+                mod: function(mod){
+                    if(mod['default']){
+                        return mod.default;
+                    }else if(mod['__esModule'] === true){
+                        return mod[props.symbol] ? mod[props.symbol] : null;
+                    }else{
+                        return mod;
+                    }
+                }(_mod)
+            });
+        }).catch((err) => {
+            console.warn(err)
+        });
     }
 
     render() {
         return this.state.mod ? this.props.children(this.state.mod) : null
     }
 }
+
+
+function loadAsPromise(src){
+    return new Promise((done, error) => {
+        load(src, function(err){
+            if(err){
+                error(err);
+                return;
+            }
+            done();
+        })
+    });
+}

client/index.js

@@ -55,6 +55,8 @@ window.onerror = function (msg, url, lineNo, colNo, error) {
     Log.report(msg, url, lineNo, colNo, error)
 }
 
+window.overrides = {}; // server generated frontend overrides
+
 if ("serviceWorker" in navigator) {
     window.addEventListener("load", function() {
         navigator.serviceWorker.register("/sw_cache.js").catch(function(err){

client/pages/viewerpage.js

@@ -9,7 +9,7 @@ import { debounce, opener, notify } from '../helpers/';
 import { FileDownloader, ImageViewer, PDFViewer, FormViewer } from './viewerpage/';
 
 const VideoPlayer = (props) => (
-    <Bundle loader={import(/* webpackChunkName: "video" */"../pages/viewerpage/videoplayer")} symbol="VideoPlayer">
+    <Bundle loader={import(/* webpackChunkName: "video" */"../pages/viewerpage/videoplayer")} symbol="VideoPlayer" overrides={["/overrides/video-transcoder.js"]} >
       {(Comp) => <Comp {...props}/>}
     </Bundle>
 );

client/pages/viewerpage/videoplayer.js

@@ -13,30 +13,29 @@ import './videoplayer.scss';
 export class VideoPlayer extends React.Component {
     constructor(props){
         super(props);
+        if(!window.overrides["video-map-sources"]){
+            window.overrides["video-map-sources"] = function(s){ return s; };
+        }
     }
 
     componentDidMount(){
         this.player = videojs(this.refs.$video, {
-            fluid: true,
             controls: true,
-            aspectRatio: "16:9",
-            sources: [{
+            sources: window.overrides["video-map-sources"]([{
                 src: this.props.data,
                 type: getMimeType(this.props.data)
-            }]
+            }])
         });
     }
 
     componentWillReceiveProps(nextProps){
         if(this.props.data === nextProps.data){
             this.player = videojs(this.refs.$video, {
-                fluid: true,
                 controls: true,
-                aspectRatio: "16:9",
-                sources: [{
+                sources: window.overrides["video-map-sources"]([{
                     src: nextProps.data,
                     type: getMimeType(this.props.data)
-                }]
+                }])
             });
         }
     }
@@ -54,7 +53,7 @@ export class VideoPlayer extends React.Component {
               <div className="video_container">
                 <ReactCSSTransitionGroup transitionName="video" transitionAppear={true} transitionLeave={false} transitionEnter={true} transitionEnterTimeout={300} transitionAppearTimeout={300}>
                   <div key={this.props.data} data-vjs-player>
-                    <video ref="$video" className="video-js vjs-fluid vjs-default-skin vjs-big-play-centered" style={{boxShadow: 'rgba(0, 0, 0, 0.14) 0px 4px 5px 0px, rgba(0, 0, 0, 0.12) 0px 1px 10px 0px, rgba(0, 0, 0, 0.2) 0px 2px 4px -1px'}}></video>
+                    <video ref="$video" className="video-js vjs-fill vjs-default-skin vjs-big-play-centered" style={{boxShadow: 'rgba(0, 0, 0, 0.14) 0px 4px 5px 0px, rgba(0, 0, 0, 0.12) 0px 1px 10px 0px, rgba(0, 0, 0, 0.2) 0px 2px 4px -1px'}}></video>
                   </div>
                 </ReactCSSTransitionGroup>
                 <Pager path={this.props.path} />

client/pages/viewerpage/videoplayer.scss

@@ -31,9 +31,10 @@
 /* SKIN FOR VIDEOJS: */
 .video-js{
     font-size: 14px; /* Chromium bug */
+    max-height: 500px;
 
     .vjs-control-bar{
-        background: rgba(255,255,255,0.15);
+        background: rgba(255,255,255,0.2);
     }
     .vjs-load-progress div{
         background: var(--primary);
@@ -43,7 +44,7 @@
         border-color: var(--emphasis-primary)!important;
         border-width: 2px;
         margin-top: -40px;
-        border-radius: 10px;
+        border-radius: 8px;
         &:before{ font-size: 45px; color: var(--bg-color); }
     }
 }

config/mime.json

@@ -72,6 +72,7 @@
 	"mef": "image/x-mamiya-mef",
 	"mid": "audio/midi",
 	"midi": "application/x-midi",
+	"mkv": "video/x-matroska",
 	"mml": "text/mathml",
 	"mng": "video/x-mng",
 	"mos": "image/x-aptus-mos",

docker/Dockerfile

@@ -14,7 +14,7 @@ RUN apt-get update > /dev/null && \
     apt-get purge -y --auto-remove gnupg && \
     #################
     # Optional dependencies
-    apt-get install -y curl tor emacs zip poppler-utils > /dev/null && \
+    apt-get install -y curl tor emacs ffmpeg zip poppler-utils > /dev/null && \
     # org-mode: html export
     curl https://raw.githubusercontent.com/mickael-kerjean/filestash/master/server/.assets/emacs/htmlize.el > /usr/share/emacs/site-lisp/htmlize.el && \
     # org-mode: markdown export

package.json

@@ -14,6 +14,7 @@
   "dependencies": {
     "aes-js": "git+https://github.com/mickael-kerjean/aes-js.git",
     "bcryptjs": "^2.4.3",
+    "little-loader": "^0.2.0",
     "react-selectable": "git+https://github.com/mickael-kerjean/react-selectable.git"
   },
   "devDependencies": {
@@ -45,7 +46,6 @@
     "react-router": "^4.1.1",
     "react-router-dom": "^4.1.1",
     "react-sticky": "^6.0.2",
-    "requirejs": "^2.3.5",
     "rxjs": "^5.4.0",
     "sass-loader": "^6.0.6",
     "sass-variable-loader": "^0.1.2",

server/common/plugin.go

@@ -36,11 +36,11 @@ func (this Get) ProcessFileContentBeforeSend() []func(io.ReadCloser, *App, *http
 	return process_file_content_before_send
 }
 
-var http_endpoint []func(*mux.Router) error
-func (this Register) HttpEndpoint(fn func(*mux.Router) error) {
+var http_endpoint []func(*mux.Router, *App) error
+func (this Register) HttpEndpoint(fn func(*mux.Router, *App) error) {
 	http_endpoint = append(http_endpoint, fn)
 }
-func (this Get) HttpEndpoint() []func(*mux.Router) error {
+func (this Get) HttpEndpoint() []func(*mux.Router, *App) error {
 	return http_endpoint
 }
 
@@ -51,3 +51,21 @@ func (this Register) Starter(fn func(*mux.Router)) {
 func (this Get) Starter() []func(*mux.Router) {
 	return starter_process
 }
+
+
+/*
+ * UI Overrides
+ * They are the means by which server plugin change the frontend behaviors.
+ */
+var overrides []string
+func (this Register) FrontendOverrides(url string) {
+	overrides = append(overrides, url)
+}
+func (this Get) FrontendOverrides() []string {
+	return overrides
+}
+
+const OverrideVideoSourceMapper = "/overrides/video-transcoder.js"
+func init() {
+	Hooks.Register.FrontendOverrides(OverrideVideoSourceMapper)
+}

server/main.go

@@ -101,7 +101,13 @@ func Init(a *App) {
 	middlewares = []Middleware{ IndexHeaders }
 	r.HandleFunc("/about",       NewMiddlewareChain(AboutHandler,                                    middlewares, *a)).Methods("GET")
 	for _, obj := range Hooks.Get.HttpEndpoint() {
-		obj(r)
+		obj(r, a)
+	}
+	for _, obj := range Hooks.Get.FrontendOverrides() {
+		r.HandleFunc(obj, func(res http.ResponseWriter, req *http.Request) {
+			res.WriteHeader(http.StatusOK)
+			res.Write([]byte("/* FRONTOFFICE OVERRIDES */"))
+		})
 	}
 	r.HandleFunc("/robots.txt", func(res http.ResponseWriter, req *http.Request) {
 		res.Write([]byte(""))

server/plugin/index.go

@@ -3,6 +3,7 @@ package plugin
 import (
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_starter_tunnel"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_starter_tor"
+	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_video_transcoder"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_image_light"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_backend_backblaze"
 	_ "github.com/mickael-kerjean/filestash/server/plugin/plg_backend_dav"

server/plugin/plg_handler_console/index.go

@@ -38,7 +38,7 @@ var console_enable = func() bool {
 
 func init() {
 	console_enable()
-	Hooks.Register.HttpEndpoint(func(r *mux.Router) error {
+	Hooks.Register.HttpEndpoint(func(r *mux.Router, _ *App) error {
 		if console_enable() == false {
 			return nil
 		}

server/plugin/plg_security_scanner/index.go

@@ -41,7 +41,7 @@ func init() {
 	}
 	billionsOfLol = bytes.NewBuffer(b)
 
-	Hooks.Register.HttpEndpoint(func(r *mux.Router) error{
+	Hooks.Register.HttpEndpoint(func(r *mux.Router, _ *App) error{
 		// DEFAULT
 		r.HandleFunc("/index.php", WelcomePackHandle)
 		r.PathPrefix("/html/").Handler(http.HandlerFunc(WelcomePackHandle))

server/plugin/plg_video_transcoder/index.go

@@ -0,0 +1,218 @@
+package plg_video_transcoder
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"github.com/gorilla/mux"
+	. "github.com/mickael-kerjean/filestash/server/common"
+	. "github.com/mickael-kerjean/filestash/server/middleware"
+	"io"
+	"math"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	HLS_SEGMENT_LENGTH = 10
+	CLEAR_CACHE_AFTER = 12
+	VideoCachePath = "data/cache/video/"
+)
+
+func init(){
+	ffmpegIsInstalled := false
+	ffprobeIsInstalled := false
+	if _, err := exec.LookPath("ffmpeg"); err == nil {
+		ffmpegIsInstalled = true
+	}
+	if _, err := exec.LookPath("ffprobe"); err == nil {
+		ffprobeIsInstalled = true
+	}
+	plugin_enable := func() bool {
+		return Config.Get("features.video.enable_transcoder").Schema(func(f *FormElement) *FormElement {
+			if f == nil {
+				f = &FormElement{}
+			}
+			f.Name = "enable_transcoder"
+			f.Type = "boolean"
+			f.Description = "Enable/Disable on demand video transcoding. The transcoder"
+			f.Default = true
+			if ffmpegIsInstalled == false || ffprobeIsInstalled == false {
+				f.Default = false
+			}
+			return f
+		}).Bool()
+	}
+	if plugin_enable() == false {
+		return
+	} else if ffmpegIsInstalled == false {
+		Log.Warning("[plugin video transcoder] ffmpeg needs to be installed")
+		return
+	} else if ffprobeIsInstalled == false {
+		Log.Warning("[plugin video transcoder] ffprobe needs to be installed")
+		return
+	}
+
+	cachePath := filepath.Join(GetCurrentDir(), VideoCachePath)
+	os.RemoveAll(cachePath)
+	os.MkdirAll(cachePath, os.ModePerm)
+
+	Hooks.Register.ProcessFileContentBeforeSend(hls_playlist)
+	Hooks.Register.HttpEndpoint(func(r *mux.Router, app *App) error {
+		r.PathPrefix("/hls/hls_{segment}.ts").Handler(NewMiddlewareChain(
+			hls_transcode,
+			[]Middleware{ SecureHeaders },
+			*app,
+		)).Methods("GET")
+		return nil
+	})
+
+	Hooks.Register.HttpEndpoint(func(r *mux.Router, app *App) error {
+		r.HandleFunc(OverrideVideoSourceMapper, func(res http.ResponseWriter, req *http.Request) {
+			res.Header().Set("Content-Type", "application/javascript")
+			res.Write([]byte(`
+                window.overrides["video-map-sources"] = function(sources){
+                    return sources.map(function(source){
+                        source.src = source.src + "&transcode=hls";
+                        source.type = "application/x-mpegURL";
+                        return source;
+                    })
+                }
+`))
+		})
+		return nil
+	})
+}
+
+func hls_playlist(reader io.ReadCloser, ctx *App, res *http.ResponseWriter, req *http.Request) (io.ReadCloser, error) {
+	query := req.URL.Query()
+	if query.Get("transcode") != "hls" {
+		return reader, nil
+	}
+	path := query.Get("path")
+	if strings.HasPrefix(GetMimeType(path), "video/") == false {
+		return reader, nil
+	}
+
+	cacheName := "vid_" + GenerateID(ctx) + "_" + QuickHash(path, 10) + ".dat"
+	cachePath := filepath.Join(
+		GetCurrentDir(),
+		VideoCachePath,
+		cacheName,
+	)
+	f, err := os.OpenFile(cachePath, os.O_CREATE | os.O_RDWR, os.ModePerm)
+	if err != nil {
+		Log.Stdout("ERR %+v", err)
+		return reader, err
+	}
+	io.Copy(f, reader)
+	reader.Close()
+	f.Close()
+	time.AfterFunc(CLEAR_CACHE_AFTER * time.Hour, func() { os.Remove(cachePath) })
+
+	p, err := ffprobe(cachePath)
+	if err != nil {
+		return reader, err
+	}
+
+	var response string
+	var i int
+	response =  "#EXTM3U\n"
+	response += "#EXT-X-VERSION:3\n"
+	response += "#EXT-X-MEDIA-SEQUENCE:0\n"
+	response += "#EXT-X-ALLOW-CACHE:YES\n"
+	response += fmt.Sprintf("#EXT-X-TARGETDURATION:%d\n", HLS_SEGMENT_LENGTH)
+	for i=0; i< int(p.Format.Duration) / HLS_SEGMENT_LENGTH; i++ {
+		response += fmt.Sprintf("#EXTINF:%d.0000, nodesc\n", HLS_SEGMENT_LENGTH)
+		response += fmt.Sprintf("/hls/hls_%d.ts?path=%s\n", i, cacheName)
+	}
+	if md := math.Mod(p.Format.Duration, HLS_SEGMENT_LENGTH); md > 0 {
+		response += fmt.Sprintf("#EXTINF:%.4f, nodesc\n", md)
+		response += fmt.Sprintf("/hls/hls_%d.ts?path=%s\n", i, cacheName)
+	}
+	response += "#EXT-X-ENDLIST\n"
+	return NewReadCloserFromBytes([]byte(response)), nil
+}
+
+func hls_transcode(ctx App, res http.ResponseWriter, req *http.Request) {
+	segmentNumber, err := strconv.Atoi(mux.Vars(req)["segment"])
+	if err != nil {
+		Log.Info("[plugin hls] invalid segment request '%s'", mux.Vars(req)["segment"])
+		res.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	startTime := segmentNumber * HLS_SEGMENT_LENGTH
+	cachePath := filepath.Join(
+		GetCurrentDir(),
+		VideoCachePath,
+		req.URL.Query().Get("path"),
+	)
+	if _, err := os.Stat(cachePath); os.IsNotExist(err) {
+		Log.Info("[plugin hls]: invalid video")
+		res.WriteHeader(http.StatusServiceUnavailable)
+		return
+	}
+
+	cmd := exec.Command("ffmpeg", []string{
+		"-timelimit", "30",
+		"-ss", fmt.Sprintf("%d.00", startTime),
+		"-i", cachePath,
+		"-t", fmt.Sprintf("%d.00", HLS_SEGMENT_LENGTH),
+		"-vf", fmt.Sprintf("scale=-2:%d", 720),
+		"-vcodec", "libx264",
+		"-preset", "veryfast",
+		"-acodec", "aac",
+		"-pix_fmt", "yuv420p",
+		"-x264opts:0", "subme=0:me_range=4:rc_lookahead=10:me=dia:no_chroma_me:8x8dct=0:partitions=none",
+		"-force_key_frames", fmt.Sprintf("expr:gte(t,n_forced*%d.000)", HLS_SEGMENT_LENGTH),
+		"-f", "ssegment",
+		"-segment_time", fmt.Sprintf("%d.00", HLS_SEGMENT_LENGTH),
+		"-segment_start_number", fmt.Sprintf("%d", segmentNumber),
+		"-initial_offset", fmt.Sprintf("%d.00", startTime),
+		"-vsync", "2",
+		"pipe:out%03d.ts",
+	}...)
+
+	var str bytes.Buffer
+	cmd.Stdout = res
+	cmd.Stderr = &str
+	_ = cmd.Run()
+}
+
+type FFProbeData struct {
+	Format struct {
+		Duration float64 `json:"duration,string"`
+		BitRate int `json:"bit_rate,string"`
+	} `json: "format"`
+	Streams []struct {
+		CodecType string `json:"codec_type"`
+		CodecName string `json:"codec_name"`
+		PixelFormat string `json:"pix_fmt"`
+	} `json:"streams"`
+}
+
+func ffprobe(videoPath string) (FFProbeData, error) {
+	var stream bytes.Buffer
+	var probe FFProbeData
+
+	cmd := exec.Command(
+		"ffprobe", strings.Split(fmt.Sprintf(
+			"-v quiet -print_format json -show_format -show_streams %s",
+			videoPath,
+		), " ")...
+	)
+	cmd.Stdout = &stream
+	if err := cmd.Run(); err != nil {
+		return probe, nil
+	}
+	cmd.Run()
+	if err := json.Unmarshal([]byte(stream.String()), &probe); err != nil {
+		return probe, err
+	}
+	return probe, nil
+}

webpack.config.js

@@ -6,7 +6,6 @@ const UglifyJSPlugin = require("uglifyjs-webpack-plugin");
 const CopyWebpackPlugin = require('copy-webpack-plugin');
 const CompressionPlugin = require('compression-webpack-plugin');
 
-
 let config = {
     entry: {
         app: path.join(__dirname, 'client', 'index.js')