fix (build): transform the build to push for static program

This commit is contained in:
Mickael Kerjean 2019-05-01 11:35:27 +10:00
parent f8cc366165
commit 5f7de7826a
17 changed files with 181 additions and 231 deletions

View file

@ -15,33 +15,36 @@ steps:
image: alpine:latest image: alpine:latest
depends_on: [ clone ] depends_on: [ clone ]
commands: commands:
- mkdir -p ./dist/data/ - mkdir -p ./dist/data/state/
- cp -R config dist/data/config - cp -R config dist/data/state/config
- name: build_go - name: build_frontend
image: golang:1.11-stretch image: node:8-alpine
depends_on: [ clone ] depends_on: [ build_prepare ]
commands:
- apk add make git > /dev/null
- npm install --silent
- make build_frontend
- name: build_backend
image: golang:1.12-stretch
depends_on: [ build_prepare ]
environment: environment:
CGO_LDFLAGS_ALLOW: '-fopenmp' CGO_LDFLAGS_ALLOW: '-fopenmp'
commands: commands:
- apt-get update > /dev/null && apt-get install -y libglib2.0-dev curl make > /dev/null
- make build_init
- find server/plugin/plg_* -type f -name '*.a' -exec mv {} /usr/local/lib/ \;
- go get -t ./server/... - go get -t ./server/...
- find server/plugin/plg_* -type f -name "install.sh" -exec {} \; - make build_backend
- go build -ldflags "-X github.com/mickael-kerjean/filestash/server/common.BUILD_NUMBER=`date -u +%Y%m%d`" -o dist/filestash server/main.go
- timeout 1 ./dist/filestash || true - timeout 1 ./dist/filestash || true
- name: build_js
image: node:8-alpine
depends_on: [ clone ]
commands:
- npm install > /dev/null 2>&1
- NODE_ENV=production npm run build
- name: build_finalise - name: build_finalise
image: alpine:latest image: debian:latest
depends_on: [ clone ] depends_on: [ build_go, build_js ]
commands: commands:
- cp -R dist /tmp/filestash - cp -R dist /tmp/filestash
- tar -C /tmp/ -zcvf filestash.tar.gz ./filestash - tar -C /tmp/ -zcf filestash_linux-amd64.tar.gz ./filestash
- name: test_prepare - name: test_prepare
image: alpine:latest image: alpine:latest
@ -50,28 +53,28 @@ steps:
PASSWORD: PASSWORD:
from_secret: GIT_PASSWORD from_secret: GIT_PASSWORD
commands: commands:
- apk --no-cache add git > /dev/null 2>&1 - apk add git > /dev/null
- cd ../ && git clone "https://readonly:$PASSWORD@git.kerjean.me/filestash/test.git" - cd ../ && git clone "https://readonly:$PASSWORD@git.kerjean.me/filestash/test.git"
- chmod -R 777 ./test/ - chmod -R 777 ./test/
- chmod -R 777 ./filestash/dist/data - chmod -R 777 ./filestash/dist/data
- name: test_go - name: test_frontend
image: golang:1.11-alpine
depends_on: [ test_prepare ]
commands:
- apk --no-cache add git gcc libc-dev poppler-utils > /dev/null 2>&1
- cp ../test/assets/* /tmp/
- go get -t ../test/unit_go/...
- go test -v --tags "fts5" ../test/unit_go/...
- name: test_js
image: node:8-alpine image: node:8-alpine
depends_on: [ test_prepare ] depends_on: [ test_prepare ]
commands: commands:
- cd ../test/unit_js - cd ../test/unit_js
- npm install > /dev/null 2>&1 - npm install --silent
- npm test - npm test
- name: test_backend
image: golang:1.12-alpine
depends_on: [ test_prepare ]
commands:
- apk add git gcc libc-dev poppler-utils > /dev/null
- cp ../test/assets/* /tmp/
- go get -t ../test/unit_go/...
- go test -v --tags "fts5" ../test/unit_go/...
- name: test_e2e - name: test_e2e
image: machines/puppeteer image: machines/puppeteer
depends_on: [ build_prepare, build_js, build_go ] depends_on: [ build_prepare, build_js, build_go ]
@ -80,10 +83,10 @@ steps:
APP_URL: http://127.0.0.1:8334 APP_URL: http://127.0.0.1:8334
CI: true CI: true
commands: commands:
- ./dist/filestash & - ./dist/filestash > /dev/null &
- cd ../test/e2e - cd ../test/e2e
- npm install > /dev/null 2>&1 - npm install > /dev/null
- node servers/webdav.js & - node servers/webdav.js > /dev/null &
- npm test - npm test
- name: release_artifact - name: release_artifact
@ -98,54 +101,56 @@ steps:
from_secret: SSH_USERNAME from_secret: SSH_USERNAME
password: password:
from_secret: SSH_PASSWORD from_secret: SSH_PASSWORD
source: filestash.tar.gz source: filestash_linux-amd64.tar.gz
target: /app/pages/data/projects/filestash/downloads/latest/ target: /app/pages/data/projects/filestash/downloads/latest/
# - name: release_docker - name: release_docker
# image: debian:latest image: plugins/docker
# depends_on: [ release_artifact ] depends_on: [ release_artifact ]
# when: when:
# branch: master branch: master
# settings: settings:
# dockerfile: ./docker/prod/Dockerfile dockerfile: ./docker/Dockerfile
# repo: machines/filestash repo: machines/filestash
# username: username:
# from_secret: DOCKER_USERNAME from_secret: DOCKER_USERNAME
# password: password:
# from_secret: DOCKER_PASSWORD from_secret: DOCKER_PASSWORD
# tags: latest tags: latest
# - name: deploy - name: deploy
# image: appleboy/drone-ssh image: appleboy/drone-ssh
# depends_on: [ release_docker ] depends_on: [ release_docker ]
# when: when:
# branch: master branch: master
# settings: settings:
# command_timeout: 10m command_timeout: 10m
# host: hal.kerjean.me host: hal.kerjean.me
# user: user:
# from_secret: SSH_USERNAME from_secret: SSH_USERNAME
# password: password:
# from_secret: SSH_PASSWORD from_secret: SSH_PASSWORD
# port: 22 port: 22
# script: script:
# - cd /app/filestash/ - cd /app/filestash/
# - docker-compose pull - docker-compose pull
# - docker-compose up -d --force-recreate --build - docker-compose up -d --force-recreate --build
# - docker image prune -f - docker image prune -f
# - name: report - name: report
# image: plugins/irc image: plugins/irc
# depends_on: [ deploy ] depends_on: [ deploy ]
# settings: when:
# prefix: build branch: master
# nick: blipblip settings:
# channel: filestash prefix: build
# host: chat.freenode.net nick: blipblip
# port: 6667 channel: filestash
# template: > host: chat.freenode.net
# {{#success build.status}} port: 6667
# Build success: https://github.com/mickael-kerjean/filestash/commit/{{build.commit}} template: >
# {{else}} {{#success build.status}}
# Build failure: er}} https://github.com/mickael-kerjean/filestash/commit/{{build.commit}} Build success: https://github.com/mickael-kerjean/filestash/commit/{{build.commit}}
# {{/success}} {{else}}
Build failure: er}} https://github.com/mickael-kerjean/filestash/commit/{{build.commit}}
{{/success}}

View file

@ -1,20 +1,11 @@
all: all:
make build_backend make build_backend
build_init:
find server/plugin/plg_* -type f -name "install.sh" -exec {} \;
build_frontend: build_frontend:
NODE_ENV=production npm run build NODE_ENV=production npm run build
build_backend: build_backend:
PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/ CGO_CFLAGS_ALLOW='-fopenmp' go build --tags "fts5" -ldflags "-X github.com/mickael-kerjean/filestash/server/common.BUILD_NUMBER=`date -u +%Y%m%d`" -o dist/filestash server/main.go PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/ CGO_CFLAGS_ALLOW='-fopenmp' go build --tags "fts5" -ldflags "-X github.com/mickael-kerjean/filestash/server/common.BUILD_NUMBER=`date -u +%Y%m%d`" -o dist/filestash server/main.go
build_backend_reload:
gin --bin dist/filestash-reload --path ./server/ --port 8333 --appPort 8334 run server/main.go
build_plugins:
go build -buildmode=plugin -o ./dist/data/plugin/image.so server/plugin/plg_image_light/index.go
go build -buildmode=plugin -o ./dist/data/plugin/backend_dav.so server/plugin/plg_backend_dav/index.go
go build -buildmode=plugin -o ./dist/data/plugin/backend_ldap.so server/plugin/plg_backend_ldap/index.go
go build -buildmode=plugin -o ./dist/data/plugin/backend_mysql.so server/plugin/plg_backend_mysql/index.go
go build -buildmode=plugin -o dist/data/plugin/backend_backblaze.so server/plugin/plg_backend_backblaze/index.go
go build -buildmode=plugin -o dist/data/plugin/security_scanner.so server/plugin/plg_security_scanner/index.go
go build -buildmode=plugin -o dist/data/plugin/security_svg.so server/plugin/plg_security_svg/index.go

View file

@ -7,6 +7,7 @@
font-weight: 100; font-weight: 100;
.component_icon{ .component_icon{
float: right; float: right;
width: 24px;
} }
} }
.copy { .copy {

View file

@ -34,7 +34,6 @@
"\\lstset{columns=flexible, breaklines=true}" "\\lstset{columns=flexible, breaklines=true}"
"\\definecolor{doctitle}{RGB}{51,51,51}" "\\definecolor{doctitle}{RGB}{51,51,51}"
"\\usepackage{lmodern}" "\\usepackage{lmodern}"
"\\usepackage{helvet}"
"\\usepackage{xcolor}" "\\usepackage{xcolor}"
"\\renewcommand{\\familydefault}{\\sfdefault}" "\\renewcommand{\\familydefault}{\\sfdefault}"
"\\usepackage{sectsty}" "\\usepackage{sectsty}"

49
docker/Dockerfile Normal file
View file

@ -0,0 +1,49 @@
FROM debian:stable-slim
MAINTAINER mickael@kerjean.me
RUN echo "deb http://debian.mirrors.ovh.net/debian stretch main" > /etc/apt/sources.list && \
apt-get update > /dev/null && \
#################
# Install
apt-get install -y libglib2.0-0 curl > /dev/null && \
curl -s https://downloads.filestash.app/latest/filestash_linux-amd64.tar.gz > /tmp/filestash.tar.gz && \
mkdir /app/ && \
tar zxf /tmp/filestash.tar.gz -C /tmp/ && \
mv /tmp/filestash/* /app/ && \
rm -rf /tmp/filestash* && \
#################
# Optional dependencies
apt-get install -y curl emacs zip poppler-utils > /dev/null&& \
# Minimal latex dependencies for org-mode
cd ~/ && apt-get install -y wget perl > /dev/null && \
curl -sL "https://yihui.name/gh/tinytex/tools/install-unx.sh" | sh && \
mv ~/.TinyTeX /usr/share/tinytex && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install wasy && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install ulem && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install marvosym && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install wasysym && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install xcolor && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install listings && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install parskip && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install float && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install wrapfig && \
/usr/share/tinytex/bin/x86_64-linux/tlmgr install sectsty && \
ln -s /usr/share/tinytex/bin/x86_64-linux/pdflatex /usr/local/bin/pdflatex && \
apt-get purge -y --auto-remove perl wget && \
# Cleanup
find /usr/share/ -name 'doc' | xargs rm -rf && \
find /usr/share/emacs -name '*.pbm' | xargs rm -f && \
find /usr/share/emacs -name '*.png' | xargs rm -f && \
find /usr/share/emacs -name '*.xpm' | xargs rm -f && \
#################
# Finalise the image
useradd filestash && \
chown -R filestash:filestash /app/ && \
rm -rf /var/lib/apt/lists/* && \
rm -rf /tmp/*
EXPOSE 8334
VOLUME ["/app/data/"]
WORKDIR "/app"
USER filestash
CMD ["/app/filestash"]

View file

@ -1,85 +0,0 @@
FROM alpine:latest
MAINTAINER mickael@kerjean.me
ENV GOPATH /tmp/go
ENV CGO_LDFLAGS_ALLOW '-fopenmp'
RUN mkdir -p $GOPATH/src/github.com/mickael-kerjean/ && \
#################
# Dependencies
apk --no-cache --virtual .build-deps add make gcc g++ curl nodejs git npm python2 binutils-gold && \
apk --no-cache --virtual .go add go --repository http://dl-3.alpinelinux.org/alpine/edge/community && \
mkdir /tmp/deps && \
cd /tmp/deps && \
# libvips #######
cd /tmp/deps && \
curl -L -X GET https://github.com/libvips/libvips/releases/download/v8.7.0/vips-8.7.0.tar.gz > libvips.tar.gz && \
tar -zxf libvips.tar.gz && \
cd vips-8.7.0/ && \
apk --no-cache add libexif-dev tiff-dev jpeg-dev libjpeg-turbo-dev libpng-dev librsvg-dev giflib-dev glib-dev fftw-dev glib-dev libc-dev expat-dev orc-dev && \
./configure && \
make -j 6 && \
make install && \
# libraw ########
cd /tmp/deps && \
curl -X GET https://www.libraw.org/data/LibRaw-0.19.0.tar.gz > libraw.tar.gz && \
tar -zxf libraw.tar.gz && \
cd LibRaw-0.19.0/ && \
./configure && \
make -j 6 && \
make install && \
#################
# Prepare Build
cd $GOPATH/src/github.com/mickael-kerjean && \
git clone --depth 1 https://github.com/mickael-kerjean/filestash && \
cd filestash && \
mkdir -p ./dist/data/state/ && \
mv config ./dist/data/state/ && \
#################
# Compile Frontend
npm install && \
npm rebuild node-sass && \
NODE_ENV=production npm run build && \
#################
# Compile Backend
cd $GOPATH/src/github.com/mickael-kerjean/filestash/server && go get && cd ../ && \
go build --tags "fts5" -ldflags "-X github.com/mickael-kerjean/filestash/server/common.BUILD_NUMBER=`date -u +%Y%m%d`" -o ./dist/filestash ./server/main.go && \
#################
# Compile Plugins
mkdir -p ./dist/data/plugin && \
go get -v ./server/plugin/... > /dev/null 2>&1 || true && \
make build_plugins && \
#################
# External dependencies
apk --no-cache add curl emacs texlive zip poppler-utils && \
curl https://raw.githubusercontent.com/mickael-kerjean/filestash_latex/master/wrapfig.sty > /usr/share/texmf-dist/tex/latex/base/wrapfig.sty && \
curl https://raw.githubusercontent.com/mickael-kerjean/filestash_latex/master/capt-of.sty > /usr/share/texmf-dist/tex/latex/base/capt-of.sty && \
curl https://raw.githubusercontent.com/mickael-kerjean/filestash_latex/master/sectsty.sty > /usr/share/texmf-dist/tex/latex/base/sectsty.sty && \
texhash && \
apk --no-cache del curl && \
# put emacs on a diet program
find /usr/share/emacs -name '*.pbm' | xargs rm && \
find /usr/share/emacs -name '*.png' | xargs rm && \
find /usr/share/emacs -name '*.xpm' | xargs rm && \
# put latex on a diet program
rm -rf /usr/share/texmf-dist/doc && \
#################
# Finalise the build
cd $GOPATH/src/github.com/mickael-kerjean/filestash/ && \
apk --no-cache add ca-certificates && \
mv dist /app && \
cd /app && \
rm -rf $GOPATH && \
rm -rf /tmp/* && \
apk del .build-deps && \
apk del .go && \
#################
# Create machine user
addgroup -S filestash && adduser -S -g filestash filestash && \
chown -R filestash:filestash /app/
EXPOSE 8334
VOLUME ["/app/data/"]
WORKDIR "/app"
USER filestash
CMD ["/app/filestash"]

View file

@ -1,4 +1,4 @@
package main package plg_backend_ldap
/* /*
* Introduction * Introduction
@ -26,7 +26,7 @@ import (
var LDAPCache AppCache var LDAPCache AppCache
func Init(config *Configuration) { func init() {
Backend.Register("ldap", LDAP{}) Backend.Register("ldap", LDAP{})
LDAPCache = NewAppCache(2, 1) LDAPCache = NewAppCache(2, 1)
LDAPCache.OnEvict(func(key string, value interface{}) { LDAPCache.OnEvict(func(key string, value interface{}) {

View file

@ -1,41 +0,0 @@
package main
import (
"bytes"
. "github.com/mickael-kerjean/filestash/server/common"
"github.com/nfnt/resize"
"image/jpeg"
"io"
"net/http"
)
func Init(config *Configuration) {
plugin_enable := config.Get("feature.image.enable").Default(true).Bool()
plugin_thumbsize := uint(config.Get("feature.image.thumbnail_size").Default(300).Int())
Hooks.Register.ProcessFileContentBeforeSend(func(reader io.ReadCloser, ctx *App, res *http.ResponseWriter, req *http.Request) (io.ReadCloser, error){
if plugin_enable == false {
return reader, nil
}
query := req.URL.Query()
mType := GetMimeType(query.Get("path"))
if mType != "image/jpeg" {
return reader, nil
} else if query.Get("thumbnail") != "true" {
return reader, nil
}
(*res).Header().Set("Cache-Control", "max-age=3600")
img, err := jpeg.Decode(reader)
if err != nil {
return reader, nil
}
reader.Close()
img = resize.Resize(plugin_thumbsize, 0, img, resize.Lanczos3)
out := bytes.NewBufferString("")
jpeg.Encode(out, img, &jpeg.Options{50})
return NewReadCloserFromReader(out), nil
})
}

View file

@ -10,7 +10,7 @@ To test the libraries are working fine:
``` ```
# libtranscode: # libtranscode:
gcc -Wall -c src/libtranscode_test.c gcc -Wall -c src/libtranscode_test.c
gcc -o main_transcode.bin libtranscode_test.o -lm -lgomp -llcms2 -lstdc++ -L. -l:libtranscode.a gcc -o main_transcode.bin libtranscode_test.o -lm -lpthread -L. -l:libtranscode.a
# libresize: # libresize:
gcc -Wall -c src/libresize_test.c `pkg-config --cflags glib-2.0` gcc -Wall -c src/libresize_test.c `pkg-config --cflags glib-2.0`

View file

@ -72,3 +72,9 @@ ar x /usr/lib/x86_64-linux-gnu/libpoppler.a
ar rcs libresize.a *.o ar rcs libresize.a *.o
rm *.o *.ao rm *.o *.ao
################################################
# Stage 4: Gather all the related headers
#cd /usr/include/
#tar zcf /tmp/libresize-headers.tar.gz .
#scp /tmp/libresize-headers.tar.gz mickael@hal.kerjean.me:/home/app/pages/data/projects/filestash/downloads/upload/libresize-linux-headers.tar.gz

View file

@ -14,8 +14,22 @@ export PATH=~/.local/bin:$PATH
################################################ ################################################
# Stage 1: Get libraw and its dependencies # Stage 1: Get libraw and its dependencies
INITIAL_PATH=`pwd`
apt install -y libraw-dev apt install -y libraw-dev
cd /tmp/
# libgomp and libstdc++
apt-get install -y libgcc-6-dev
# libjpeg
apt-get install -y libjpeg-dev
# liblcms2
curl -L -O https://downloads.sourceforge.net/project/lcms/lcms/2.9/lcms2-2.9.tar.gz
tar zxf lcms2-2.9.tar.gz
cd lcms2-2.9
./configure --enable-static --without-zlib --without-threads
make -j 8
make install
cd $INITIAL_PATH
################################################ ################################################
# Stage 2: Create our own library as a static build # Stage 2: Create our own library as a static build
gcc -Wall -c src/libtranscode.c gcc -Wall -c src/libtranscode.c
@ -24,6 +38,17 @@ gcc -Wall -c src/libtranscode.c
# Stage 3: Gather and assemble all the bits and pieces together # Stage 3: Gather and assemble all the bits and pieces together
ar x /usr/lib/x86_64-linux-gnu/libraw.a ar x /usr/lib/x86_64-linux-gnu/libraw.a
ar x /usr/lib/x86_64-linux-gnu/libjpeg.a ar x /usr/lib/x86_64-linux-gnu/libjpeg.a
ar x /usr/lib/gcc/x86_64-linux-gnu/6/libstdc++.a
ar x /usr/local/lib/liblcms2.a
ar x /usr/lib/gcc/x86_64-linux-gnu/6/libgomp.a
ar x /usr/lib/x86_64-linux-gnu/libpthread.a
ar rcs libtranscode.a *.o ar rcs libtranscode.a *.o
rm *.o rm *.o
#scp libtranscode.a mickael@hal.kerjean.me:/home/app/pages/data/projects/filestash/downloads/upload/libtranscode-linux-x86-64.a
################################################
# Stage 4: Gather all the related headers
#cd /usr/include/
#tar zcf /tmp/libtranscode-headers.tar.gz .
#scp /tmp/libtranscode-headers.tar.gz mickael@hal.kerjean.me:/home/app/pages/data/projects/filestash/downloads/upload/libtranscode-linux-headers.tar.gz

View file

@ -1,4 +1,4 @@
#include <stdio.h> #include <stdlib.h>
#include <vips/vips.h> #include <vips/vips.h>
int image_resize(const char *filename, void **buf, size_t *len, int size, int crop, int quality, int exif); int image_resize(const char *filename, void **buf, size_t *len, int size, int crop, int quality, int exif);

View file

@ -20,7 +20,6 @@ int image_transcode_compute(const char* filename, int min_width) {
////////////////////// //////////////////////
// use thumbnail if available // use thumbnail if available
/*
if(libraw_unpack_thumb(raw) == 0){ if(libraw_unpack_thumb(raw) == 0){
has_thumbnail = TRUE; has_thumbnail = TRUE;
if(raw->thumbnail.twidth > min_width && raw->thumbnail.tformat == LIBRAW_THUMBNAIL_JPEG){ if(raw->thumbnail.twidth > min_width && raw->thumbnail.tformat == LIBRAW_THUMBNAIL_JPEG){
@ -29,8 +28,6 @@ int image_transcode_compute(const char* filename, int min_width) {
return err; return err;
} }
} }
*/
fprintf(stdout, "HERE\n");
fflush(stdout); fflush(stdout);
////////////////////// //////////////////////

View file

@ -4,5 +4,8 @@ set -e
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
cd "$(dirname "$0")"/deps cd "$(dirname "$0")"/deps
echo `pwd` echo `pwd`
curl https://downloads.filestash.app/upload/libresize-linux-x86-64.a > libresize.a curl -s https://downloads.filestash.app/upload/libresize-linux-x86-64.a > libresize.a
curl https://downloads.filestash.app/upload/libtranscode-linux-x86-64.a > libtranscode.a curl -Ls https://downloads.filestash.app/upload/libresize-linux-headers.tar.gz | tar zxf - -C /usr/local/include/
curl -s https://downloads.filestash.app/upload/libtranscode-linux-x86-64.a > libtranscode.a
curl -Ls https://downloads.filestash.app/upload/libtranscode-linux-headers.tar.gz | tar zxf - -C /usr/local/include/

View file

@ -1,6 +1,6 @@
package plg_image_light package plg_image_light
// #cgo CFLAGS: -I./deps/src // #cgo CFLAGS: -I./deps/src
// #cgo LDFLAGS: -lm -lgomp -llcms2 -lstdc++ -L./deps -l:libtranscode.a // #cgo LDFLAGS: -lm -lpthread -L./deps -l:libtranscode.a
// #include "libtranscode.h" // #include "libtranscode.h"
import "C" import "C"