From a4e25f32ea8e8f59fcda4fb40670e4f6450f157f Mon Sep 17 00:00:00 2001 From: NodudeWasTaken <75137537+NodudeWasTaken@users.noreply.github.com> Date: Mon, 24 Jun 2024 05:33:27 +0200 Subject: [PATCH] Add apple encoder and fix extra_hw_frames bug (#4986) * Fixes format in full hw encoding to nv12 for cuda, vaapi and qsv now * Remove extra_hw_frames * Add apple transcoder support * Up the duration to discover decoding errors * yuv420p is not supported on intel --------- Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com> --- pkg/ffmpeg/codec_hardware.go | 162 +++++++++++++++++++++++++-------- pkg/ffmpeg/ffmpeg.go | 14 +++ pkg/ffmpeg/ffmpeg_test.go | 75 +++++++++++++++ pkg/ffmpeg/filter.go | 31 +++---- pkg/ffmpeg/stream_segmented.go | 2 +- pkg/ffmpeg/stream_transcode.go | 4 +- 6 files changed, 227 insertions(+), 61 deletions(-) create mode 100644 pkg/ffmpeg/ffmpeg_test.go diff --git a/pkg/ffmpeg/codec_hardware.go b/pkg/ffmpeg/codec_hardware.go index 4c39cb3b9..e4797a84a 100644 --- a/pkg/ffmpeg/codec_hardware.go +++ b/pkg/ffmpeg/codec_hardware.go @@ -4,7 +4,9 @@ import ( "bytes" "context" "fmt" + "math" "regexp" + "strconv" "strings" "github.com/stashapp/stash/pkg/logger" @@ -25,7 +27,7 @@ var ( VideoCodecVVPX VideoCodec = "vp8_vaapi" ) -const minHeight int = 256 +const minHeight int = 480 // Tests all (given) hardware codec's func (f *FFMpeg) InitHWSupport(ctx context.Context) { @@ -38,17 +40,19 @@ func (f *FFMpeg) InitHWSupport(ctx context.Context) { VideoCodecR264, VideoCodecIVP9, VideoCodecVVP9, + VideoCodecM264, } { var args Args args = append(args, "-hide_banner") args = args.LogLevel(LogLevelWarning) args = f.hwDeviceInit(args, codec, false) args = args.Format("lavfi") - args = args.Input(fmt.Sprintf("color=c=red:s=%dx%d", 1280, 720)) + vFile := &models.VideoFile{Width: 1280, Height: 720} + args = args.Input(fmt.Sprintf("color=c=red:s=%dx%d", vFile.Width, vFile.Height)) args = args.Duration(0.1) // Test scaling - videoFilter := f.hwMaxResFilter(codec, 1280, 720, minHeight, false) + videoFilter := f.hwMaxResFilter(codec, vFile, minHeight, false) args = append(args, CodecInit(codec)...) args = args.VideoFilter(videoFilter) @@ -93,9 +97,9 @@ func (f *FFMpeg) hwCanFullHWTranscode(ctx context.Context, codec VideoCodec, vf args = args.XError() args = f.hwDeviceInit(args, codec, true) args = args.Input(vf.Path) - args = args.Duration(0.1) + args = args.Duration(1) - videoFilter := f.hwMaxResFilter(codec, vf.Width, vf.Height, reqHeight, true) + videoFilter := f.hwMaxResFilter(codec, vf, reqHeight, true) args = append(args, CodecInit(codec)...) args = args.VideoFilter(videoFilter) @@ -128,12 +132,12 @@ func (f *FFMpeg) hwDeviceInit(args Args, toCodec VideoCodec, fullhw bool) Args { args = append(args, "-hwaccel_device") args = append(args, "0") if fullhw { + args = append(args, "-threads") + args = append(args, "1") args = append(args, "-hwaccel") args = append(args, "cuda") args = append(args, "-hwaccel_output_format") args = append(args, "cuda") - args = append(args, "-extra_hw_frames") - args = append(args, "5") } case VideoCodecV264, VideoCodecVVP9: @@ -158,6 +162,16 @@ func (f *FFMpeg) hwDeviceInit(args Args, toCodec VideoCodec, fullhw bool) Args { args = append(args, "-filter_hw_device") args = append(args, "hw") } + case VideoCodecM264: + if fullhw { + args = append(args, "-hwaccel") + args = append(args, "videotoolbox") + args = append(args, "-hwaccel_output_format") + args = append(args, "videotoolbox_vld") + } else { + args = append(args, "-init_hw_device") + args = append(args, "videotoolbox=vt") + } } return args @@ -175,7 +189,7 @@ func (f *FFMpeg) hwFilterInit(toCodec VideoCodec, fullhw bool) VideoFilter { } case VideoCodecN264: if !fullhw { - videoFilter = videoFilter.Append("format=nv12") + videoFilter = videoFilter.Append("format=yuv420p") videoFilter = videoFilter.Append("hwupload_cuda") } case VideoCodecI264, @@ -184,80 +198,146 @@ func (f *FFMpeg) hwFilterInit(toCodec VideoCodec, fullhw bool) VideoFilter { videoFilter = videoFilter.Append("hwupload=extra_hw_frames=64") videoFilter = videoFilter.Append("format=qsv") } + case VideoCodecM264: + if !fullhw { + videoFilter = videoFilter.Append("format=nv12") + videoFilter = videoFilter.Append("hwupload") + } } return videoFilter } -var scaler_re = regexp.MustCompile(`scale=(?P[-\d]+:[-\d]+)`) +var scaler_re = regexp.MustCompile(`scale=(?P([-\d]+):([-\d]+))`) -func templateReplaceScale(input string, template string, match []int, minusonehack bool) string { +func templateReplaceScale(input string, template string, match []int, vf *models.VideoFile, minusonehack bool) string { result := []byte{} - res := string(scaler_re.ExpandString(result, template, input, match)) - - // BUG: [scale_qsv]: Size values less than -1 are not acceptable. - // Fix: Replace all instances of -2 with -1 in a scale operation if minusonehack { - res = strings.ReplaceAll(res, "-2", "-1") + // Parse width and height + w, err := strconv.Atoi(input[match[4]:match[5]]) + if err != nil { + logger.Error("failed to parse width") + return input + } + h, err := strconv.Atoi(input[match[6]:match[7]]) + if err != nil { + logger.Error("failed to parse height") + return input + } + + // Calculate ratio + ratio := float64(vf.Width) / float64(vf.Height) + if w < 0 { + w = int(math.Round(float64(h) * ratio)) + } else if h < 0 { + h = int(math.Round(float64(w) / ratio)) + } + + // Fix not divisible by 2 errors + if w%2 != 0 { + w++ + } + if h%2 != 0 { + h++ + } + + template = strings.ReplaceAll(template, "$value", fmt.Sprintf("%d:%d", w, h)) } + res := string(scaler_re.ExpandString(result, template, input, match)) + matchStart := match[0] matchEnd := match[1] return input[0:matchStart] + res + input[matchEnd:] } -// Replace video filter scaling with hardware scaling for full hardware transcoding -func (f *FFMpeg) hwCodecFilter(args VideoFilter, codec VideoCodec, fullhw bool) VideoFilter { +// Replace video filter scaling with hardware scaling for full hardware transcoding (also fixes the format) +func (f *FFMpeg) hwCodecFilter(args VideoFilter, codec VideoCodec, vf *models.VideoFile, fullhw bool) VideoFilter { sargs := string(args) match := scaler_re.FindStringSubmatchIndex(sargs) if match == nil { - return args + return f.hwApplyFullHWFilter(args, codec, fullhw) } + return f.hwApplyScaleTemplate(sargs, codec, match, vf, fullhw) +} + +// Apply format switching if applicable +func (f *FFMpeg) hwApplyFullHWFilter(args VideoFilter, codec VideoCodec, fullhw bool) VideoFilter { switch codec { case VideoCodecN264: - template := "scale_cuda=$value" - // In 10bit inputs you might get an error like "10 bit encode not supported" - if fullhw && f.version.major >= 5 { - template += ":format=nv12" + if fullhw && f.version.Gteq(FFMpegVersion{major: 5}) { // Added in FFMpeg 5 + args = args.Append("scale_cuda=format=yuv420p") + } + case VideoCodecV264, VideoCodecVVP9: + if fullhw && f.version.Gteq(FFMpegVersion{major: 3, minor: 1}) { // Added in FFMpeg 3.1 + args = args.Append("scale_vaapi=format=nv12") + } + case VideoCodecI264, VideoCodecIVP9: + if fullhw && f.version.Gteq(FFMpegVersion{major: 3, minor: 3}) { // Added in FFMpeg 3.3 + args = args.Append("scale_qsv=format=nv12") } - args = VideoFilter(templateReplaceScale(sargs, template, match, false)) - case VideoCodecV264, - VideoCodecVVP9: - template := "scale_vaapi=$value" - args = VideoFilter(templateReplaceScale(sargs, template, match, false)) - case VideoCodecI264, - VideoCodecIVP9: - template := "scale_qsv=$value" - args = VideoFilter(templateReplaceScale(sargs, template, match, true)) } return args } +// Switch scaler +func (f *FFMpeg) hwApplyScaleTemplate(sargs string, codec VideoCodec, match []int, vf *models.VideoFile, fullhw bool) VideoFilter { + var template string + + switch codec { + case VideoCodecN264: + template = "scale_cuda=$value" + if fullhw && f.version.Gteq(FFMpegVersion{major: 5}) { // Added in FFMpeg 5 + template += ":format=yuv420p" + } + case VideoCodecV264, VideoCodecVVP9: + template = "scale_vaapi=$value" + if fullhw && f.version.Gteq(FFMpegVersion{major: 3, minor: 1}) { // Added in FFMpeg 3.1 + template += ":format=nv12" + } + case VideoCodecI264, VideoCodecIVP9: + template = "scale_qsv=$value" + if fullhw && f.version.Gteq(FFMpegVersion{major: 3, minor: 3}) { // Added in FFMpeg 3.3 + template += ":format=nv12" + } + case VideoCodecM264: + template = "scale_vt=$value" + default: + return VideoFilter(sargs) + } + + // BUG: [scale_qsv]: Size values less than -1 are not acceptable. + isIntel := codec == VideoCodecI264 || codec == VideoCodecIVP9 + // BUG: scale_vt doesn't call ff_scale_adjust_dimensions, thus cant accept negative size values + isApple := codec == VideoCodecM264 + return VideoFilter(templateReplaceScale(sargs, template, match, vf, isIntel || isApple)) +} + // Returns the max resolution for a given codec, or a default -func (f *FFMpeg) hwCodecMaxRes(codec VideoCodec, dW int, dH int) (int, int) { +func (f *FFMpeg) hwCodecMaxRes(codec VideoCodec) (int, int) { switch codec { case VideoCodecN264, VideoCodecI264: return 4096, 4096 } - return dW, dH + return 0, 0 } // Return a maxres filter -func (f *FFMpeg) hwMaxResFilter(toCodec VideoCodec, width int, height int, reqHeight int, fullhw bool) VideoFilter { - if width == 0 || height == 0 { +func (f *FFMpeg) hwMaxResFilter(toCodec VideoCodec, vf *models.VideoFile, reqHeight int, fullhw bool) VideoFilter { + if vf.Width == 0 || vf.Height == 0 { return "" } videoFilter := f.hwFilterInit(toCodec, fullhw) - maxWidth, maxHeight := f.hwCodecMaxRes(toCodec, width, height) - videoFilter = videoFilter.ScaleMaxLM(width, height, reqHeight, maxWidth, maxHeight) - return f.hwCodecFilter(videoFilter, toCodec, fullhw) + maxWidth, maxHeight := f.hwCodecMaxRes(toCodec) + videoFilter = videoFilter.ScaleMaxLM(vf.Width, vf.Height, reqHeight, maxWidth, maxHeight) + return f.hwCodecFilter(videoFilter, toCodec, vf, fullhw) } // Return if a hardware accelerated for HLS is available @@ -267,7 +347,8 @@ func (f *FFMpeg) hwCodecHLSCompatible() *VideoCodec { case VideoCodecN264, VideoCodecI264, VideoCodecV264, - VideoCodecR264: + VideoCodecR264, + VideoCodecM264: // Note that the Apple encoder sucks at startup, thus HLS quality is crap return &element } } @@ -279,7 +360,8 @@ func (f *FFMpeg) hwCodecMP4Compatible() *VideoCodec { for _, element := range f.hwCodecSupport { switch element { case VideoCodecN264, - VideoCodecI264: + VideoCodecI264, + VideoCodecM264: return &element } } diff --git a/pkg/ffmpeg/ffmpeg.go b/pkg/ffmpeg/ffmpeg.go index e929cc7f8..5ee98a873 100644 --- a/pkg/ffmpeg/ffmpeg.go +++ b/pkg/ffmpeg/ffmpeg.go @@ -195,6 +195,20 @@ type FFMpegVersion struct { patch int } +// Gteq returns true if the version is greater than or equal to the other version. +func (v FFMpegVersion) Gteq(other FFMpegVersion) bool { + if v.major > other.major { + return true + } + if v.major == other.major && v.minor > other.minor { + return true + } + if v.major == other.major && v.minor == other.minor && v.patch >= other.patch { + return true + } + return false +} + // FFMpeg provides an interface to ffmpeg. type FFMpeg struct { ffmpeg string diff --git a/pkg/ffmpeg/ffmpeg_test.go b/pkg/ffmpeg/ffmpeg_test.go new file mode 100644 index 000000000..3e9151ed9 --- /dev/null +++ b/pkg/ffmpeg/ffmpeg_test.go @@ -0,0 +1,75 @@ +// Package ffmpeg provides a wrapper around the ffmpeg and ffprobe executables. +package ffmpeg + +import "testing" + +func TestFFMpegVersion_GreaterThan(t *testing.T) { + tests := []struct { + name string + this FFMpegVersion + other FFMpegVersion + want bool + }{ + { + "major greater, minor equal, patch equal", + FFMpegVersion{2, 0, 0}, + FFMpegVersion{1, 0, 0}, + true, + }, + { + "major greater, minor less, patch less", + FFMpegVersion{2, 1, 1}, + FFMpegVersion{1, 0, 0}, + true, + }, + { + "major equal, minor greater, patch equal", + FFMpegVersion{1, 1, 0}, + FFMpegVersion{1, 0, 0}, + true, + }, + { + "major equal, minor equal, patch greater", + FFMpegVersion{1, 0, 1}, + FFMpegVersion{1, 0, 0}, + true, + }, + { + "major equal, minor equal, patch equal", + FFMpegVersion{1, 0, 0}, + FFMpegVersion{1, 0, 0}, + true, + }, + { + "major less, minor equal, patch equal", + FFMpegVersion{1, 0, 0}, + FFMpegVersion{2, 0, 0}, + false, + }, + { + "major equal, minor less, patch equal", + FFMpegVersion{1, 0, 0}, + FFMpegVersion{1, 1, 0}, + false, + }, + { + "major equal, minor equal, patch less", + FFMpegVersion{1, 0, 0}, + FFMpegVersion{1, 0, 1}, + false, + }, + { + "major less, minor less, patch less", + FFMpegVersion{1, 0, 0}, + FFMpegVersion{2, 1, 1}, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.this.Gteq(tt.other); got != tt.want { + t.Errorf("FFMpegVersion.GreaterThan() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/ffmpeg/filter.go b/pkg/ffmpeg/filter.go index 52be57c9c..dd6ecc106 100644 --- a/pkg/ffmpeg/filter.go +++ b/pkg/ffmpeg/filter.go @@ -59,33 +59,28 @@ func (f VideoFilter) ScaleMax(inputWidth, inputHeight, maxSize int) VideoFilter return f.ScaleDimensions(maxSize, -2) } -// ScaleMaxLM returns a VideoFilter scaling to maxSize with respect to a max size. +// ScaleMaxLM scales an image to fit within specified maximum dimensions while maintaining its aspect ratio. func (f VideoFilter) ScaleMaxLM(width int, height int, reqHeight int, maxWidth int, maxHeight int) VideoFilter { - // calculate the aspect ratio of the current resolution - aspectRatio := width / height + if maxWidth == 0 || maxHeight == 0 { + return f.ScaleMax(width, height, reqHeight) + } - // find the max height + aspectRatio := float64(width) / float64(height) desiredHeight := reqHeight if desiredHeight == 0 { desiredHeight = height } + desiredWidth := int(float64(desiredHeight) * aspectRatio) - // calculate the desired width based on the desired height and the aspect ratio - desiredWidth := int(desiredHeight * aspectRatio) - - // check which dimension to scale based on the maximum resolution - if desiredHeight > maxHeight || desiredWidth > maxWidth { - if desiredHeight-maxHeight > desiredWidth-maxWidth { - // scale the height down to the maximum height - return f.ScaleDimensions(-2, maxHeight) - } else { - // scale the width down to the maximum width - return f.ScaleDimensions(maxWidth, -2) - } + if desiredHeight <= maxHeight && desiredWidth <= maxWidth { + return f.ScaleMax(width, height, reqHeight) } - // the current resolution can be scaled to the desired height without exceeding the maximum resolution - return f.ScaleMax(width, height, reqHeight) + if float64(desiredHeight-maxHeight) > float64(desiredWidth-maxWidth) { + return f.ScaleDimensions(-2, maxHeight) + } else { + return f.ScaleDimensions(maxWidth, -2) + } } // Fps returns a VideoFilter setting the frames per second. diff --git a/pkg/ffmpeg/stream_segmented.go b/pkg/ffmpeg/stream_segmented.go index 1058fb8eb..56ef392f1 100644 --- a/pkg/ffmpeg/stream_segmented.go +++ b/pkg/ffmpeg/stream_segmented.go @@ -342,7 +342,7 @@ func (s *runningStream) makeStreamArgs(sm *StreamManager, segment int) Args { videoOnly := ProbeAudioCodec(s.vf.AudioCodec) == MissingUnsupported - videoFilter := sm.encoder.hwMaxResFilter(codec, s.vf.Width, s.vf.Height, s.maxTranscodeSize, fullhw) + videoFilter := sm.encoder.hwMaxResFilter(codec, s.vf, s.maxTranscodeSize, fullhw) args = append(args, s.streamType.Args(codec, segment, videoFilter, videoOnly, s.outputDir)...) diff --git a/pkg/ffmpeg/stream_transcode.go b/pkg/ffmpeg/stream_transcode.go index ce56e0795..714652470 100644 --- a/pkg/ffmpeg/stream_transcode.go +++ b/pkg/ffmpeg/stream_transcode.go @@ -60,7 +60,7 @@ func CodecInit(codec VideoCodec) (args Args) { ) case VideoCodecM264: args = append(args, - "-prio_speed", "1", + "-realtime", "1", ) case VideoCodecO264: args = append(args, @@ -198,7 +198,7 @@ func (o TranscodeOptions) makeStreamArgs(sm *StreamManager) Args { videoOnly := ProbeAudioCodec(o.VideoFile.AudioCodec) == MissingUnsupported - videoFilter := sm.encoder.hwMaxResFilter(codec, o.VideoFile.Width, o.VideoFile.Height, maxTranscodeSize, fullhw) + videoFilter := sm.encoder.hwMaxResFilter(codec, o.VideoFile, maxTranscodeSize, fullhw) args = append(args, o.StreamType.Args(codec, videoFilter, videoOnly)...)