Backup_Repos/filestash
1
0
Fork 0
mirror of https://github.com/mickael-kerjean/filestash synced 2025-12-06 08:22:24 +01:00
Code Issues Projects Releases Wiki Activity

fix (build): transform the build to push for static program

Browse source
This commit is contained in:
Mickael Kerjean 2019-05-01 11:35:27 +10:00
parent f8cc366165
commit 5f7de7826a
17 changed files with 181 additions and 231 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

163
.drone.yml
View file

@ -15,33 +15,36 @@ steps:
image: alpine:latest
depends_on: [ clone ]
commands:
- mkdir -p ./dist/data/
- cp -R config dist/data/config
- mkdir -p ./dist/data/state/
- cp -R config dist/data/state/config
- name: build_go
image: golang:1.11-stretch
depends_on: [ clone ]
- name: build_frontend
image: node:8-alpine
depends_on: [ build_prepare ]
commands:
- apk add make git > /dev/null
- npm install --silent
- make build_frontend
- name: build_backend
image: golang:1.12-stretch
depends_on: [ build_prepare ]
environment:
CGO_LDFLAGS_ALLOW: '-fopenmp'
commands:
- apt-get update > /dev/null && apt-get install -y libglib2.0-dev curl make > /dev/null
- make build_init
- find server/plugin/plg_* -type f -name '*.a' -exec mv {} /usr/local/lib/ \;
- go get -t ./server/...
- find server/plugin/plg_* -type f -name "install.sh" -exec {} \;
- go build -ldflags "-X github.com/mickael-kerjean/filestash/server/common.BUILD_NUMBER=`date -u +%Y%m%d`" -o dist/filestash server/main.go
- make build_backend
- timeout 1 ./dist/filestash || true
- name: build_js
image: node:8-alpine
depends_on: [ clone ]
commands:
- npm install > /dev/null 2>&1
- NODE_ENV=production npm run build
- name: build_finalise
image: alpine:latest
depends_on: [ clone ]
image: debian:latest
depends_on: [ build_go, build_js ]
commands:
- cp -R dist /tmp/filestash
- tar -C /tmp/ -zcvf filestash.tar.gz ./filestash
- tar -C /tmp/ -zcf filestash_linux-amd64.tar.gz ./filestash
- name: test_prepare
image: alpine:latest
@ -50,28 +53,28 @@ steps:
PASSWORD:
from_secret: GIT_PASSWORD
commands:
- apk --no-cache add git > /dev/null 2>&1
- apk add git > /dev/null
- cd ../ && git clone "https://readonly:$PASSWORD@git.kerjean.me/filestash/test.git"
- chmod -R 777 ./test/
- chmod -R 777 ./filestash/dist/data
- name: test_go
image: golang:1.11-alpine
depends_on: [ test_prepare ]
commands:
- apk --no-cache add git gcc libc-dev poppler-utils > /dev/null 2>&1
- cp ../test/assets/* /tmp/
- go get -t ../test/unit_go/...
- go test -v --tags "fts5" ../test/unit_go/...
- name: test_js
- name: test_frontend
image: node:8-alpine
depends_on: [ test_prepare ]
commands:
- cd ../test/unit_js
- npm install > /dev/null 2>&1
- npm install --silent
- npm test
- name: test_backend
image: golang:1.12-alpine
depends_on: [ test_prepare ]
commands:
- apk add git gcc libc-dev poppler-utils > /dev/null
- cp ../test/assets/* /tmp/
- go get -t ../test/unit_go/...
- go test -v --tags "fts5" ../test/unit_go/...
- name: test_e2e
image: machines/puppeteer
depends_on: [ build_prepare, build_js, build_go ]
@ -80,10 +83,10 @@ steps:
APP_URL: http://127.0.0.1:8334
CI: true
commands:
- ./dist/filestash &
- ./dist/filestash > /dev/null &
- cd ../test/e2e
- npm install > /dev/null 2>&1
- node servers/webdav.js &
- npm install > /dev/null
- node servers/webdav.js > /dev/null &
- npm test
- name: release_artifact
@ -98,54 +101,56 @@ steps:
from_secret: SSH_USERNAME
password:
from_secret: SSH_PASSWORD
source: filestash.tar.gz
source: filestash_linux-amd64.tar.gz
target: /app/pages/data/projects/filestash/downloads/latest/
# - name: release_docker
# image: debian:latest
# depends_on: [ release_artifact ]
# when:
# branch: master
# settings:
# dockerfile: ./docker/prod/Dockerfile
# repo: machines/filestash
# username:
# from_secret: DOCKER_USERNAME
# password:
# from_secret: DOCKER_PASSWORD
# tags: latest
- name: release_docker
image: plugins/docker
depends_on: [ release_artifact ]
when:
branch: master
settings:
dockerfile: ./docker/Dockerfile
repo: machines/filestash
username:
from_secret: DOCKER_USERNAME
password:
from_secret: DOCKER_PASSWORD
tags: latest
# - name: deploy
# image: appleboy/drone-ssh
# depends_on: [ release_docker ]
# when:
# branch: master
# settings:
# command_timeout: 10m
# host: hal.kerjean.me
# user:
# from_secret: SSH_USERNAME
# password:
# from_secret: SSH_PASSWORD
# port: 22
# script:
# - cd /app/filestash/
# - docker-compose pull
# - docker-compose up -d --force-recreate --build
# - docker image prune -f
- name: deploy
image: appleboy/drone-ssh
depends_on: [ release_docker ]
when:
branch: master
settings:
command_timeout: 10m
host: hal.kerjean.me
user:
from_secret: SSH_USERNAME
password:
from_secret: SSH_PASSWORD
port: 22
script:
- cd /app/filestash/
- docker-compose pull
- docker-compose up -d --force-recreate --build
- docker image prune -f
# - name: report
# image: plugins/irc
# depends_on: [ deploy ]
# settings:
# prefix: build
# nick: blipblip
# channel: filestash
# host: chat.freenode.net
# port: 6667
# template: >
# {{#success build.status}}
# Build success: https://github.com/mickael-kerjean/filestash/commit/{{build.commit}}
# {{else}}
# Build failure: er}} https://github.com/mickael-kerjean/filestash/commit/{{build.commit}}
# {{/success}}
- name: report
image: plugins/irc
depends_on: [ deploy ]
when:
branch: master
settings:
prefix: build
nick: blipblip
channel: filestash
host: chat.freenode.net
port: 6667
template: >
{{#success build.status}}
Build success: https://github.com/mickael-kerjean/filestash/commit/{{build.commit}}
{{else}}
Build failure: er}} https://github.com/mickael-kerjean/filestash/commit/{{build.commit}}
{{/success}}

15
Makefile
View file

@ -1,20 +1,11 @@
all:
make build_backend
build_init:
find server/plugin/plg_* -type f -name "install.sh" -exec {} \;
build_frontend:
NODE_ENV=production npm run build
build_backend:
PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/ CGO_CFLAGS_ALLOW='-fopenmp' go build --tags "fts5" -ldflags "-X github.com/mickael-kerjean/filestash/server/common.BUILD_NUMBER=`date -u +%Y%m%d`" -o dist/filestash server/main.go
build_backend_reload:
gin --bin dist/filestash-reload --path ./server/ --port 8333 --appPort 8334 run server/main.go
build_plugins:
go build -buildmode=plugin -o ./dist/data/plugin/image.so server/plugin/plg_image_light/index.go
go build -buildmode=plugin -o ./dist/data/plugin/backend_dav.so server/plugin/plg_backend_dav/index.go
go build -buildmode=plugin -o ./dist/data/plugin/backend_ldap.so server/plugin/plg_backend_ldap/index.go
go build -buildmode=plugin -o ./dist/data/plugin/backend_mysql.so server/plugin/plg_backend_mysql/index.go
go build -buildmode=plugin -o dist/data/plugin/backend_backblaze.so server/plugin/plg_backend_backblaze/index.go
go build -buildmode=plugin -o dist/data/plugin/security_scanner.so server/plugin/plg_security_scanner/index.go
go build -buildmode=plugin -o dist/data/plugin/security_svg.so server/plugin/plg_security_svg/index.go

1
client/pages/filespage/share.scss
View file

@ -7,6 +7,7 @@
font-weight: 100;
.component_icon{
float: right;
width: 24px;
}
}
.copy {

1
config/emacs.el
View file

@ -34,7 +34,6 @@
"\\lstset{columns=flexible, breaklines=true}"
"\\definecolor{doctitle}{RGB}{51,51,51}"
"\\usepackage{lmodern}"
"\\usepackage{helvet}"
"\\usepackage{xcolor}"
"\\renewcommand{\\familydefault}{\\sfdefault}"
"\\usepackage{sectsty}"

49
docker/Dockerfile Normal file
View file

@ -0,0 +1,49 @@
FROM debian:stable-slim
MAINTAINER mickael@kerjean.me
RUN echo "deb http://debian.mirrors.ovh.net/debian stretch main" > /etc/apt/sources.list && \
apt-get update > /dev/null && \
#################
# Install
apt-get install -y libglib2.0-0 curl > /dev/null && \
curl -s https://downloads.filestash.app/latest/filestash_linux-amd64.tar.gz > /tmp/filestash.tar.gz && \
mkdir /app/ && \
tar zxf /tmp/filestash.tar.gz -C /tmp/ && \
mv /tmp/filestash/* /app/ && \
rm -rf /tmp/filestash* && \
#################
# Optional dependencies
apt-get install -y curl emacs zip poppler-utils > /dev/null&& \
# Minimal latex dependencies for org-mode
cd ~/ && apt-get install -y wget perl > /dev/null && \
curl -sL "https://yihui.name/gh/tinytex/tools/install-unx.sh" | sh && \
mv ~/.TinyTeX /usr/share/tinytex && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install wasy && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install ulem && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install marvosym && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install wasysym && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install xcolor && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install listings && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install parskip && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install float && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install wrapfig && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install sectsty && \
ln -s /usr/share/tinytex/bin/x86_64-linux/pdflatex /usr/local/bin/pdflatex && \
apt-get purge -y --auto-remove perl wget && \
# Cleanup
find /usr/share/ -name 'doc' | xargs rm -rf && \
find /usr/share/emacs -name '*.pbm' | xargs rm -f && \
find /usr/share/emacs -name '*.png' | xargs rm -f && \
find /usr/share/emacs -name '*.xpm' | xargs rm -f && \
#################
# Finalise the image
useradd filestash && \
chown -R filestash:filestash /app/ && \
rm -rf /var/lib/apt/lists/* && \
rm -rf /tmp/*
EXPOSE 8334
VOLUME ["/app/data/"]
WORKDIR "/app"
USER filestash
CMD ["/app/filestash"]

0
docker/prod/docker-compose.yml → docker/docker-compose.yml
View file

85
docker/prod/Dockerfile
View file

@ -1,85 +0,0 @@
FROM alpine:latest
MAINTAINER mickael@kerjean.me
ENV GOPATH /tmp/go
ENV CGO_LDFLAGS_ALLOW '-fopenmp'
RUN mkdir -p $GOPATH/src/github.com/mickael-kerjean/ && \
#################
# Dependencies
apk --no-cache --virtual .build-deps add make gcc g++ curl nodejs git npm python2 binutils-gold && \
apk --no-cache --virtual .go add go --repository http://dl-3.alpinelinux.org/alpine/edge/community && \
mkdir /tmp/deps && \
cd /tmp/deps && \
# libvips #######
cd /tmp/deps && \
curl -L -X GET https://github.com/libvips/libvips/releases/download/v8.7.0/vips-8.7.0.tar.gz > libvips.tar.gz && \
tar -zxf libvips.tar.gz && \
cd vips-8.7.0/ && \
apk --no-cache add libexif-dev tiff-dev jpeg-dev libjpeg-turbo-dev libpng-dev librsvg-dev giflib-dev glib-dev fftw-dev glib-dev libc-dev expat-dev orc-dev && \
./configure && \
make -j 6 && \
make install && \
# libraw ########
cd /tmp/deps && \
curl -X GET https://www.libraw.org/data/LibRaw-0.19.0.tar.gz > libraw.tar.gz && \
tar -zxf libraw.tar.gz && \
cd LibRaw-0.19.0/ && \
./configure && \
make -j 6 && \
make install && \
#################
# Prepare Build
cd $GOPATH/src/github.com/mickael-kerjean && \
git clone --depth 1 https://github.com/mickael-kerjean/filestash && \
cd filestash && \
mkdir -p ./dist/data/state/ && \
mv config ./dist/data/state/ && \
#################
# Compile Frontend
npm install && \
npm rebuild node-sass && \
NODE_ENV=production npm run build && \
#################
# Compile Backend
cd $GOPATH/src/github.com/mickael-kerjean/filestash/server && go get && cd ../ && \
go build --tags "fts5" -ldflags "-X github.com/mickael-kerjean/filestash/server/common.BUILD_NUMBER=`date -u +%Y%m%d`" -o ./dist/filestash ./server/main.go && \
#################
# Compile Plugins
mkdir -p ./dist/data/plugin && \
go get -v ./server/plugin/... > /dev/null 2>&1 || true && \
make build_plugins && \
#################
# External dependencies
apk --no-cache add curl emacs texlive zip poppler-utils && \
curl https://raw.githubusercontent.com/mickael-kerjean/filestash_latex/master/wrapfig.sty > /usr/share/texmf-dist/tex/latex/base/wrapfig.sty && \
curl https://raw.githubusercontent.com/mickael-kerjean/filestash_latex/master/capt-of.sty > /usr/share/texmf-dist/tex/latex/base/capt-of.sty && \
curl https://raw.githubusercontent.com/mickael-kerjean/filestash_latex/master/sectsty.sty > /usr/share/texmf-dist/tex/latex/base/sectsty.sty && \
texhash && \
apk --no-cache del curl && \
# put emacs on a diet program
find /usr/share/emacs -name '*.pbm' | xargs rm && \
find /usr/share/emacs -name '*.png' | xargs rm && \
find /usr/share/emacs -name '*.xpm' | xargs rm && \
# put latex on a diet program
rm -rf /usr/share/texmf-dist/doc && \
#################
# Finalise the build
cd $GOPATH/src/github.com/mickael-kerjean/filestash/ && \
apk --no-cache add ca-certificates && \
mv dist /app && \
cd /app && \
rm -rf $GOPATH && \
rm -rf /tmp/* && \
apk del .build-deps && \
apk del .go && \
#################
# Create machine user
addgroup -S filestash && adduser -S -g filestash filestash && \
chown -R filestash:filestash /app/
EXPOSE 8334
VOLUME ["/app/data/"]
WORKDIR "/app"
USER filestash
CMD ["/app/filestash"]

4
server/plugin/plg_backend_ldap/index.go
View file

@ -1,4 +1,4 @@
package main
package plg_backend_ldap
/*
* Introduction
@ -26,7 +26,7 @@ import (
var LDAPCache AppCache
func Init(config *Configuration) {
func init() {
Backend.Register("ldap", LDAP{})
LDAPCache = NewAppCache(2, 1)
LDAPCache.OnEvict(func(key string, value interface{}) {

41
server/plugin/plg_image_heavy/index.go
View file

@ -1,41 +0,0 @@
package main
import (
"bytes"
. "github.com/mickael-kerjean/filestash/server/common"
"github.com/nfnt/resize"
"image/jpeg"
"io"
"net/http"
)
func Init(config *Configuration) {
plugin_enable := config.Get("feature.image.enable").Default(true).Bool()
plugin_thumbsize := uint(config.Get("feature.image.thumbnail_size").Default(300).Int())
Hooks.Register.ProcessFileContentBeforeSend(func(reader io.ReadCloser, ctx *App, res *http.ResponseWriter, req *http.Request) (io.ReadCloser, error){
if plugin_enable == false {
return reader, nil
}
query := req.URL.Query()
mType := GetMimeType(query.Get("path"))
if mType != "image/jpeg" {
return reader, nil
} else if query.Get("thumbnail") != "true" {
return reader, nil
}
(*res).Header().Set("Cache-Control", "max-age=3600")
img, err := jpeg.Decode(reader)
if err != nil {
return reader, nil
}
reader.Close()
img = resize.Resize(plugin_thumbsize, 0, img, resize.Lanczos3)
out := bytes.NewBufferString("")
jpeg.Encode(out, img, &jpeg.Options{50})
return NewReadCloserFromReader(out), nil
})
}

2
server/plugin/plg_image_light/deps/README.md
View file

@ -10,7 +10,7 @@ To test the libraries are working fine:
```
# libtranscode:
gcc -Wall -c src/libtranscode_test.c
gcc -o main_transcode.bin libtranscode_test.o -lm -lgomp -llcms2 -lstdc++ -L. -l:libtranscode.a
gcc -o main_transcode.bin libtranscode_test.o -lm -lpthread -L. -l:libtranscode.a
# libresize:
gcc -Wall -c src/libresize_test.c `pkg-config --cflags glib-2.0`

6
server/plugin/plg_image_light/deps/create_libresize.sh
View file

@ -72,3 +72,9 @@ ar x /usr/lib/x86_64-linux-gnu/libpoppler.a
ar rcs libresize.a *.o
rm *.o *.ao
################################################
# Stage 4: Gather all the related headers
#cd /usr/include/
#tar zcf /tmp/libresize-headers.tar.gz .
#scp /tmp/libresize-headers.tar.gz mickael@hal.kerjean.me:/home/app/pages/data/projects/filestash/downloads/upload/libresize-linux-headers.tar.gz

25
server/plugin/plg_image_light/deps/create_libtranscode.sh
View file

@ -14,8 +14,22 @@ export PATH=~/.local/bin:$PATH
################################################
# Stage 1: Get libraw and its dependencies
INITIAL_PATH=`pwd`
apt install -y libraw-dev
cd /tmp/
# libgomp and libstdc++
apt-get install -y libgcc-6-dev
# libjpeg
apt-get install -y libjpeg-dev
# liblcms2
curl -L -O https://downloads.sourceforge.net/project/lcms/lcms/2.9/lcms2-2.9.tar.gz
tar zxf lcms2-2.9.tar.gz
cd lcms2-2.9
./configure --enable-static --without-zlib --without-threads
make -j 8
make install
cd $INITIAL_PATH
################################################
# Stage 2: Create our own library as a static build
gcc -Wall -c src/libtranscode.c
@ -24,6 +38,17 @@ gcc -Wall -c src/libtranscode.c
# Stage 3: Gather and assemble all the bits and pieces together
ar x /usr/lib/x86_64-linux-gnu/libraw.a
ar x /usr/lib/x86_64-linux-gnu/libjpeg.a
ar x /usr/lib/gcc/x86_64-linux-gnu/6/libstdc++.a
ar x /usr/local/lib/liblcms2.a
ar x /usr/lib/gcc/x86_64-linux-gnu/6/libgomp.a
ar x /usr/lib/x86_64-linux-gnu/libpthread.a
ar rcs libtranscode.a *.o
rm *.o
#scp libtranscode.a mickael@hal.kerjean.me:/home/app/pages/data/projects/filestash/downloads/upload/libtranscode-linux-x86-64.a
################################################
# Stage 4: Gather all the related headers
#cd /usr/include/
#tar zcf /tmp/libtranscode-headers.tar.gz .
#scp /tmp/libtranscode-headers.tar.gz mickael@hal.kerjean.me:/home/app/pages/data/projects/filestash/downloads/upload/libtranscode-linux-headers.tar.gz

2
server/plugin/plg_image_light/deps/src/libresize.h
View file

@ -1,4 +1,4 @@
#include <stdio.h>
#include <stdlib.h>
#include <vips/vips.h>
int image_resize(const char *filename, void **buf, size_t *len, int size, int crop, int quality, int exif);

3
server/plugin/plg_image_light/deps/src/libtranscode.c
View file

@ -20,7 +20,6 @@ int image_transcode_compute(const char* filename, int min_width) {
//////////////////////
// use thumbnail if available
/*
if(libraw_unpack_thumb(raw) == 0){
has_thumbnail = TRUE;
if(raw->thumbnail.twidth > min_width && raw->thumbnail.tformat == LIBRAW_THUMBNAIL_JPEG){
@ -29,8 +28,6 @@ int image_transcode_compute(const char* filename, int min_width) {
return err;
}
}
*/
fprintf(stdout, "HERE\n");
fflush(stdout);
//////////////////////

7
server/plugin/plg_image_light/install.sh
View file

@ -4,5 +4,8 @@ set -e
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
cd "$(dirname "$0")"/deps
echo `pwd`
curl https://downloads.filestash.app/upload/libresize-linux-x86-64.a > libresize.a
curl https://downloads.filestash.app/upload/libtranscode-linux-x86-64.a > libtranscode.a
curl -s https://downloads.filestash.app/upload/libresize-linux-x86-64.a > libresize.a
curl -Ls https://downloads.filestash.app/upload/libresize-linux-headers.tar.gz | tar zxf - -C /usr/local/include/
curl -s https://downloads.filestash.app/upload/libtranscode-linux-x86-64.a > libtranscode.a
curl -Ls https://downloads.filestash.app/upload/libtranscode-linux-headers.tar.gz | tar zxf - -C /usr/local/include/

6
server/plugin/plg_image_light/lib_resize.go
View file

@ -1,7 +1,7 @@
package plg_image_light
// #cgo pkg-config:glib-2.0
// #cgo CFLAGS: -I./deps/src
// #cgo LDFLAGS: -lm -lgmodule-2.0 -lgobject-2.0 -lglib-2.0 -ldl -L./deps -l:libresize.a
// #cgo LDFLAGS: -lm -lgmodule-2.0 -lgobject-2.0 -lglib-2.0 -ldl -L./deps -l:libresize.a
// #include "libresize.h"
import "C"
@ -18,7 +18,7 @@ var VIPS_LOCK = semaphore.NewWeighted(int64(10))
func CreateThumbnail(t *Transform) (io.ReadCloser, error) {
VIPS_LOCK.Acquire(context.Background(), 1)
defer VIPS_LOCK.Release(1)
filename := C.CString(t.Input)
defer C.free(unsafe.Pointer(filename))
var buffer unsafe.Pointer
@ -27,7 +27,7 @@ func CreateThumbnail(t *Transform) (io.ReadCloser, error) {
return nil, NewError("", 500)
}
buf := C.GoBytes(buffer, C.int(len))
C.g_free(C.gpointer(buffer))
C.g_free(C.gpointer(buffer))
return NewReadCloserFromBytes(buf), nil
}

2
server/plugin/plg_image_light/lib_transcode.go
View file

@ -1,6 +1,6 @@
package plg_image_light
// #cgo CFLAGS: -I./deps/src
// #cgo LDFLAGS: -lm -lgomp -llcms2 -lstdc++ -L./deps -l:libtranscode.a
// #cgo LDFLAGS: -lm -lpthread -L./deps -l:libtranscode.a
// #include "libtranscode.h"
import "C"