Docker+Build optimizations (#1833)

* Docker CI builds: half the size, less than half the build time
* Add an "Official Build" Designator
* Fix .git constantly invalidating build cache, use distro ffmpeg
* Fix official build detection, add some compiler image docs

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>

.github/workflows/build.yml

@@ -53,9 +53,11 @@ jobs:
         key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/go.sum') }}
 
     - name: Start build container
+      env:
+        official-build: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/develop') || (github.event_name == 'release' && github.ref != 'refs/tags/latest_develop') }}
       run: |
         mkdir -p .go-cache
-        docker run -d --name build --mount type=bind,source="$(pwd)",target=/stash,consistency=delegated --mount type=bind,source="$(pwd)/.go-cache",target=/root/.cache/go-build,consistency=delegated -w /stash $COMPILER_IMAGE tail -f /dev/null
+        docker run -d --name build --mount type=bind,source="$(pwd)",target=/stash,consistency=delegated --mount type=bind,source="$(pwd)/.go-cache",target=/root/.cache/go-build,consistency=delegated --env OFFICIAL_BUILD=${{ env.official-build }} -w /stash $COMPILER_IMAGE tail -f /dev/null
 
     - name: Pre-install
       run: docker exec -t build /bin/bash -c "make pre-ui"

Makefile

@@ -41,8 +41,13 @@ ifndef STASH_VERSION
 	$(eval STASH_VERSION := $(shell git describe --tags --exclude latest_develop))
 endif
 
+ifndef OFFICIAL_BUILD
+    $(eval OFFICIAL_BUILD := false)
+endif
+
 build: pre-build
 	$(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/pkg/api.version=$(STASH_VERSION)' -X 'github.com/stashapp/stash/pkg/api.buildstamp=$(BUILD_DATE)' -X 'github.com/stashapp/stash/pkg/api.githash=$(GITHASH)')
+	$(eval LDFLAGS := $(LDFLAGS) -X 'github.com/stashapp/stash/pkg/api.officialBuild=$(OFFICIAL_BUILD)')
 	go build $(OUTPUT) -mod=vendor -v -tags "sqlite_omit_load_extension osusergo netgo" $(GO_BUILD_FLAGS) -ldflags "$(LDFLAGS) $(EXTRA_LDFLAGS)"
 
 # strips debug symbols from the release build
@@ -195,5 +200,5 @@ validate-backend: lint it
 
 # locally builds and tags a 'stash/build' docker image
 .PHONY: docker-build
-docker-build:
+docker-build: pre-build
-	docker build -t stash/build -f docker/build/x86_64/Dockerfile .
+	docker build --build-arg GITHASH=$(GITHASH) --build-arg STASH_VERSION=$(STASH_VERSION) -t stash/build -f docker/build/x86_64/Dockerfile .

docker/build/x86_64/Dockerfile

@@ -1,30 +1,23 @@
-# This dockerfile must be built from the top-level stash directory
+# This dockerfile should be built with `make docker-build` from the stash root.
-# ie from top-level stash:
-# docker build -t stash/build -f docker/build/x86_64/Dockerfile .
 
 # Build Frontend
 FROM node:alpine as frontend
-RUN apk add --no-cache make git
+RUN apk add --no-cache make
 ## cache node_modules separately
 COPY ./ui/v2.5/package.json ./ui/v2.5/yarn.lock /stash/ui/v2.5/
 WORKDIR /stash
 RUN yarn --cwd ui/v2.5 install --frozen-lockfile.
 COPY Makefile /stash/
-COPY ./.git /stash/.git
 COPY ./graphql /stash/graphql/
 COPY ./ui /stash/ui/
 RUN make generate-frontend
+ARG GITHASH
+ARG STASH_VERSION
 RUN BUILD_DATE=$(date +"%Y-%m-%d %H:%M:%S") make ui
 
 # Build Backend
 FROM golang:1.17-alpine as backend
-RUN apk add --no-cache xz make alpine-sdk
+RUN apk add --no-cache make alpine-sdk
-## install ffmpeg
-WORKDIR /
-RUN wget -O /ffmpeg.tar.xz https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \
-    tar xf /ffmpeg.tar.xz && \
-    rm ffmpeg.tar.xz && \
-    mv /ffmpeg*/ /ffmpeg/
 WORKDIR /stash
 COPY ./go* ./*.go Makefile gqlgen.yml .gqlgenc.yml /stash/
 COPY ./scripts /stash/scripts/
@@ -32,12 +25,14 @@ COPY ./vendor /stash/vendor/
 COPY ./pkg /stash/pkg/
 COPY --from=frontend /stash /stash/
 RUN make generate-backend
+ARG GITHASH
+ARG STASH_VERSION
 RUN make build
 
 # Final Runnable Image
 FROM alpine:latest
-RUN apk add --no-cache ca-certificates vips-tools
+RUN apk add --no-cache ca-certificates vips-tools ffmpeg
-COPY --from=backend /stash/stash /ffmpeg/ffmpeg /ffmpeg/ffprobe /usr/bin/
+COPY --from=backend /stash/stash /usr/bin/
 ENV STASH_CONFIG_FILE=/root/.stash/config.yml
 EXPOSE 9999
 ENTRYPOINT ["stash"]

docker/ci/x86_64/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=$BUILDPLATFORM ubuntu:20.04  AS prep
+FROM --platform=$BUILDPLATFORM alpine:latest AS binary
 ARG TARGETPLATFORM
 WORKDIR /
 COPY stash-*  /
@@ -8,15 +8,11 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm/v6" ];   then BIN=stash-pi; \
     elif [ "$TARGETPLATFORM" = "linux/amd64" ];  then BIN=stash-linux; \
     fi; \
     mv $BIN /stash
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-pip && pip3 install cloudscraper
-FROM ubuntu:20.04 as app
-run apt update && apt install -y python3 python-is-python3 python3-requests python3-requests-toolbelt python3-lxml python3-mechanicalsoup ffmpeg libvips-tools && rm -rf /var/lib/apt/lists/*
-COPY --from=prep /stash /usr/bin/
-COPY --from=prep /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages
 
+FROM --platform=$BUILDPLATFORM alpine:latest AS app
+COPY --from=binary /stash /usr/bin/
+RUN apk add --no-cache ca-certificates python3 py3-requests py3-requests-toolbelt py3-lxml py3-pip ffmpeg vips-tools && pip install mechanicalsoup cloudscraper
 ENV STASH_CONFIG_FILE=/root/.stash/config.yml
 
 EXPOSE 9999
 CMD ["stash"]
-

docker/compiler/Dockerfile

@@ -48,10 +48,20 @@ RUN mkdir -p /root/.ssh; \
     ssh-keyscan github.com > /root/.ssh/known_hosts;
 
 # Notes for self:
+
+# To test locally:
+# make generate
+# make ui
+# cd docker/compiler
+# make build
+# docker run -it -v /PATH_TO_STASH:/go/stash stashapp/compiler:latest /bin/bash
+# cd stash
+# make cross-compile-all
+# # binaries will show up in /dist
+
 # Windows:
 # GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++  go build -ldflags "-extldflags '-static'" -tags extended
 
-
 # Darwin
 # CC=o64-clang CXX=o64-clang++ GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 go build -tags extended
 # env goreleaser --config=goreleaser-extended.yml --skip-publish --skip-validate --rm-dist --release-notes=temp/0.48-relnotes-ready.md

pkg/api/server.go

@@ -34,6 +34,7 @@ import (
 var version string
 var buildstamp string
 var githash string
+var officialBuild string
 
 func Start(uiBox embed.FS, loginUIBox embed.FS) {
 	initialiseImages()
@@ -255,12 +256,21 @@ func Start(uiBox embed.FS, loginUIBox embed.FS) {
 
 func printVersion() {
 	versionString := githash
+	if IsOfficialBuild() {
+		versionString += " - Official Build"
+	} else {
+		versionString += " - Unofficial Build"
+	}
 	if version != "" {
 		versionString = version + " (" + versionString + ")"
 	}
 	fmt.Printf("stash version: %s - %s\n", versionString, buildstamp)
 }
 
+func IsOfficialBuild() bool {
+	return officialBuild == "true"
+}
+
 func GetVersion() (string, string, string) {
 	return version, githash, buildstamp
 }