path: root/libraries/libopenshot/limit-hwaccel.patch

From e74d71f5451c6f2be9c546d61fac412f2d55f851 Mon Sep 17 00:00:00 2001
From: "FeRD (Frank Dana)" <ferdnyc@gmail.com>
Date: Mon, 10 Feb 2020 01:50:31 -0500
Subject: [PATCH] FFmpegReader/Writer: limit hwaccel to FFmpeg 3.4+

Add a new #define HAVE_HW_ACCEL, which is only set on FFmpeg 3.4+,
and use that to restrict the use of hw-accel features, leaving
IS_FFMPEG_3_2 to determine only whether code is compatible with
FFmpeg 3.2+.
---
 include/FFmpegReader.h    |  7 ++--
 include/FFmpegUtilities.h |  4 +++
 src/FFmpegReader.cpp      | 41 +++++++++++++----------
 src/FFmpegWriter.cpp      | 70 +++++++++++++++++++++------------------
 4 files changed, 68 insertions(+), 54 deletions(-)

diff --git a/include/FFmpegReader.h b/include/FFmpegReader.h
index 9faa86a3..cc782cd5 100644
--- a/include/FFmpegReader.h
+++ b/include/FFmpegReader.h
@@ -98,7 +98,7 @@ namespace openshot {
 		AVFormatContext *pFormatCtx;
 		int i, videoStream, audioStream;
 		AVCodecContext *pCodecCtx, *aCodecCtx;
-#if (LIBAVFORMAT_VERSION_MAJOR >= 57)
+#if HAVE_HW_ACCEL
 		AVBufferRef *hw_device_ctx = NULL; //PM
 #endif
 		AVStream *pStream, *aStream;
@@ -147,12 +147,11 @@ namespace openshot {
 		int64_t current_video_frame;    // can't reliably use PTS of video to determine this
 
 		int hw_de_supported = 0;    // Is set by FFmpegReader
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 		AVPixelFormat hw_de_av_pix_fmt = AV_PIX_FMT_NONE;
 		AVHWDeviceType hw_de_av_device_type = AV_HWDEVICE_TYPE_NONE;
-#endif
-
 		int IsHardwareDecodeSupported(int codecid);
+#endif
 
 		/// Check for the correct frames per second value by scanning the 1st few seconds of video packets.
 		void CheckFPS();
diff --git a/include/FFmpegUtilities.h b/include/FFmpegUtilities.h
index c673305e..62d64df1 100644
--- a/include/FFmpegUtilities.h
+++ b/include/FFmpegUtilities.h
@@ -40,6 +40,10 @@
 	#ifndef IS_FFMPEG_3_2
 	#define IS_FFMPEG_3_2 (LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 64, 101))
 	#endif
+	
+	#ifndef HAVE_HW_ACCEL
+	#define HAVE_HW_ACCEL (LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 107, 100))
+	#endif
 
 	// Include the FFmpeg headers
 	extern "C" {
diff --git a/src/FFmpegReader.cpp b/src/FFmpegReader.cpp
index 9234ecfe..b548fa8f 100644
--- a/src/FFmpegReader.cpp
+++ b/src/FFmpegReader.cpp
@@ -35,13 +35,13 @@
 
 #define ENABLE_VAAPI 0
 
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 #pragma message "You are compiling with experimental hardware decode"
 #else
 #pragma message "You are compiling only with software decode"
 #endif
 
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 #define MAX_SUPPORTED_WIDTH 1950
 #define MAX_SUPPORTED_HEIGHT 1100
 
@@ -71,14 +71,14 @@ typedef struct VAAPIDecodeContext {
 	 enum AVPixelFormat    surface_format;
 	 int                   surface_count;
  } VAAPIDecodeContext;
-#endif
-#endif
+#endif // ENABLE_VAAPI
+#endif // HAVE_HW_ACCEL
 
 
 using namespace openshot;
 
 int hw_de_on = 0;
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 	AVPixelFormat hw_de_av_pix_fmt_global = AV_PIX_FMT_NONE;
 	AVHWDeviceType hw_de_av_device_type_global = AV_HWDEVICE_TYPE_NONE;
 #endif
@@ -153,7 +153,7 @@ bool AudioLocation::is_near(AudioLocation location, int samples_per_frame, int64
 	return false;
 }
 
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 
 // Get hardware pix format
 static enum AVPixelFormat get_hw_dec_format(AVCodecContext *ctx, const enum AVPixelFormat *pix_fmts)
@@ -234,7 +234,7 @@ int FFmpegReader::IsHardwareDecodeSupported(int codecid)
 	}
 	return ret;
 }
-#endif
+#endif // HAVE_HW_ACCEL
 
 void FFmpegReader::Open() {
 	// Open reader if not already open
@@ -287,7 +287,7 @@ void FFmpegReader::Open() {
 			// If hw accel is selected but hardware cannot handle repeat with software decoding
 			do {
 				pCodecCtx = AV_GET_CODEC_CONTEXT(pStream, pCodec);
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 				if (hw_de_on && (retry_decode_open==2)) {
 					// Up to here no decision is made if hardware or software decode
 					hw_de_supported = IsHardwareDecodeSupported(pCodecCtx->codec_id);
@@ -304,7 +304,7 @@ void FFmpegReader::Open() {
 
 				// Init options
 				av_dict_set(&opts, "strict", "experimental", 0);
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 				if (hw_de_on && hw_de_supported) {
 					// Open Hardware Acceleration
 					int i_decoder_hw = 0;
@@ -433,13 +433,13 @@ void FFmpegReader::Open() {
 						  throw InvalidCodec("Hardware device create failed.", path);
 					}
 				}
-#endif
+#endif // HAVE_HW_ACCEL
 
 				// Open video codec
 				if (avcodec_open2(pCodecCtx, pCodec, &opts) < 0)
 					throw InvalidCodec("A video codec was found, but could not be opened.", path);
 
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 				if (hw_de_on && hw_de_supported) {
 					AVHWFramesConstraints *constraints = NULL;
 					void *hwconfig = NULL;
@@ -449,7 +449,7 @@ void FFmpegReader::Open() {
 #if ENABLE_VAAPI
 					((AVVAAPIHWConfig *)hwconfig)->config_id = ((VAAPIDecodeContext *)(pCodecCtx->priv_data))->va_config;
 					constraints = av_hwdevice_get_hwframe_constraints(hw_device_ctx,hwconfig);
-#endif
+#endif // ENABLE_VAAPI
 					if (constraints) {
 						if (pCodecCtx->coded_width < constraints->min_width  	||
 								pCodecCtx->coded_height < constraints->min_height ||
@@ -506,7 +506,7 @@ void FFmpegReader::Open() {
 				}
 #else
 				retry_decode_open = 0;
-#endif
+#endif // HAVE_HW_ACCEL
 			} while (retry_decode_open); // retry_decode_open
 			// Free options
 			av_dict_free(&opts);
@@ -592,14 +592,14 @@ void FFmpegReader::Close() {
 		if (info.has_video) {
 			avcodec_flush_buffers(pCodecCtx);
 			AV_FREE_CONTEXT(pCodecCtx);
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 			if (hw_de_on) {
 				if (hw_device_ctx) {
 					av_buffer_unref(&hw_device_ctx);
 					hw_device_ctx = NULL;
 				}
 			}
-#endif
+#endif // HAVE_HW_ACCEL
 		}
 		if (info.has_audio) {
 			avcodec_flush_buffers(aCodecCtx);
@@ -1100,19 +1100,22 @@ bool FFmpegReader::GetAVFrame() {
 
 		ret = avcodec_send_packet(pCodecCtx, packet);
 
+	#if HAVE_HW_ACCEL
 		// Get the format from the variables set in get_hw_dec_format
 		hw_de_av_pix_fmt = hw_de_av_pix_fmt_global;
 		hw_de_av_device_type = hw_de_av_device_type_global;
-
+	#endif // HAVE_HW_ACCEL
 		if (ret < 0 || ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) {
 			ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (Packet not sent)");
 		}
 		else {
 			AVFrame *next_frame2;
+	#if HAVE_HW_ACCEL
 			if (hw_de_on && hw_de_supported) {
 				next_frame2 = AV_ALLOCATE_FRAME();
 			}
 			else
+	#endif // HAVE_HW_ACCEL
 			{
 				next_frame2 = next_frame;
 			}
@@ -1125,6 +1128,7 @@ bool FFmpegReader::GetAVFrame() {
 				if (ret != 0) {
 					ZmqLogger::Instance()->AppendDebugMethod("FFmpegReader::GetAVFrame (invalid return frame received)");
 				}
+	#if HAVE_HW_ACCEL
 				if (hw_de_on && hw_de_supported) {
 					int err;
 					if (next_frame2->format == hw_de_av_pix_fmt) {
@@ -1138,6 +1142,7 @@ bool FFmpegReader::GetAVFrame() {
 					}
 				}
 				else
+	#endif // HAVE_HW_ACCEL
 				{	// No hardware acceleration used -> no copy from GPU memory needed
 					next_frame = next_frame2;
 				}
@@ -1151,9 +1156,11 @@ bool FFmpegReader::GetAVFrame() {
 												(AVPixelFormat)(pStream->codecpar->format), info.width, info.height);
 				}
 			}
+	#if HAVE_HW_ACCEL
 			if (hw_de_on && hw_de_supported) {
 				AV_FREE_FRAME(&next_frame2);
 			}
+	#endif // HAVE_HW_ACCEL
 		}
 #else
 		avcodec_decode_video2(pCodecCtx, next_frame, &frameFinished, packet);
@@ -1169,7 +1176,7 @@ bool FFmpegReader::GetAVFrame() {
 			av_picture_copy((AVPicture *) pFrame, (AVPicture *) next_frame, pCodecCtx->pix_fmt, info.width,
 							info.height);
 		}
-#endif
+#endif // IS_FFMPEG_3_2
 	}
 
 	// deallocate the frame
diff --git a/src/FFmpegWriter.cpp b/src/FFmpegWriter.cpp
index 245bd9bd..8d060d77 100644
--- a/src/FFmpegWriter.cpp
+++ b/src/FFmpegWriter.cpp
@@ -35,7 +35,7 @@
 
 using namespace openshot;
 
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 #pragma message "You are compiling with experimental hardware encode"
 #else
 #pragma message "You are compiling only with software encode"
@@ -44,7 +44,7 @@ using namespace openshot;
 // Multiplexer parameters temporary storage
 AVDictionary *mux_dict = NULL;
 
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 int hw_en_on = 1;					// Is set in UI
 int hw_en_supported = 0;	// Is set by FFmpegWriter
 AVPixelFormat hw_en_av_pix_fmt = AV_PIX_FMT_NONE;
@@ -81,7 +81,7 @@ static int set_hwframe_ctx(AVCodecContext *ctx, AVBufferRef *hw_device_ctx, int6
 	av_buffer_unref(&hw_frames_ref);
 	return err;
 }
-#endif
+#endif // HAVE_HW_ACCEL
 
 FFmpegWriter::FFmpegWriter(std::string path) :
 		path(path), fmt(NULL), oc(NULL), audio_st(NULL), video_st(NULL), audio_pts(0), video_pts(0), samples(NULL),
@@ -171,7 +171,7 @@ void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction f
 	if (codec.length() > 0) {
 		AVCodec *new_codec;
 		// Check if the codec selected is a hardware accelerated codec
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 #if defined(__linux__)
 		if (strstr(codec.c_str(), "_vaapi") != NULL) {
 			new_codec = avcodec_find_encoder_by_name(codec.c_str());
@@ -225,7 +225,7 @@ void FFmpegWriter::SetVideoOptions(bool has_video, std::string codec, Fraction f
 	#endif //__linux__
 #else // not ffmpeg 3
 		new_codec = avcodec_find_encoder_by_name(codec.c_str());
-#endif //IS_FFMPEG_3_2
+#endif // HAVE_HW_ACCEL
 		if (new_codec == NULL)
 			throw InvalidCodec("A valid video codec could not be found for this file.", path);
 		else {
@@ -392,11 +392,11 @@ void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string va
 			// This might be better in an extra methods as more options
 			// and way to set quality are possible
 		#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 39, 101)
-			#if IS_FFMPEG_3_2
+			#if HAVE_HW_ACCEL
 				if (hw_en_on) {
 					av_opt_set_int(c->priv_data, "qp", std::min(std::stoi(value),63), 0); // 0-63
 				} else
-			#endif
+			#endif // HAVE_HW_ACCEL
 				{
 				switch (c->codec_id) {
 					#if (LIBAVCODEC_VERSION_MAJOR >= 58)
@@ -442,7 +442,7 @@ void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string va
 			// This might be better in an extra methods as more options
 			// and way to set quality are possible
 #if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(55, 39, 101)
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 			if (hw_en_on) {
 				double mbs = 15000000.0;
 				if (info.video_bit_rate > 0) {
@@ -455,7 +455,7 @@ void FFmpegWriter::SetOption(StreamType stream, std::string name, std::string va
 				}
 				c->bit_rate = (int)(mbs);
 			} else
-#endif
+#endif // HAVE_HW_ACCEL
 			{
 				switch (c->codec_id) {
 #if (LIBAVCODEC_VERSION_MAJOR >= 58)
@@ -955,16 +955,14 @@ void FFmpegWriter::flush_encoders() {
 // Close the video codec
 void FFmpegWriter::close_video(AVFormatContext *oc, AVStream *st)
 {
-#if IS_FFMPEG_3_2
-	//  #if defined(__linux__)
-		if (hw_en_on && hw_en_supported) {
-			if (hw_device_ctx) {
-				av_buffer_unref(&hw_device_ctx);
-				hw_device_ctx = NULL;
-			}
+#if HAVE_HW_ACCEL
+	if (hw_en_on && hw_en_supported) {
+		if (hw_device_ctx) {
+			av_buffer_unref(&hw_device_ctx);
+			hw_device_ctx = NULL;
 		}
-	//  #endif
-#endif
+	}
+#endif // HAVE_HW_ACCEL
 }
 
 // Close the audio codec
@@ -1342,7 +1340,7 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
 	// Set number of threads equal to number of processors (not to exceed 16)
 	video_codec->thread_count = std::min(FF_NUM_PROCESSORS, 16);
 
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 	if (hw_en_on && hw_en_supported) {
 		//char *dev_hw = NULL;
 		char adapter[256];
@@ -1385,7 +1383,7 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
 				throw InvalidCodec("Could not create hwdevice", path);
 		}
 	}
-#endif
+#endif // HAVE_HW_ACCEL
 
 	/* find the video encoder */
 	codec = avcodec_find_encoder_by_name(info.vcodec.c_str());
@@ -1402,7 +1400,7 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
 	AVDictionary *opts = NULL;
 	av_dict_set(&opts, "strict", "experimental", 0);
 
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 	if (hw_en_on && hw_en_supported) {
 		video_codec->pix_fmt   = hw_en_av_pix_fmt;
 
@@ -1451,7 +1449,7 @@ void FFmpegWriter::open_video(AVFormatContext *oc, AVStream *st) {
 					"width", info.width, "height", info.height, av_err2str(err), -1);
 		}
 	}
-#endif
+#endif // HAVE_HW_ACCEL
 
 	/* open the codec */
 	if (avcodec_open2(video_codec, codec, &opts) < 0)
@@ -1900,14 +1898,17 @@ void FFmpegWriter::process_video_packet(std::shared_ptr<Frame> frame) {
 		frame_source = allocate_avframe(PIX_FMT_RGBA, source_image_width, source_image_height, &bytes_source, (uint8_t *) pixels);
 #if IS_FFMPEG_3_2
 		AVFrame *frame_final;
+	#if HAVE_HW_ACCEL
 		if (hw_en_on && hw_en_supported) {
 			frame_final = allocate_avframe(AV_PIX_FMT_NV12, info.width, info.height, &bytes_final, NULL);
-		} else {
+		} else
+	#endif // HAVE_HW_ACCEL
+		{
 			frame_final = allocate_avframe((AVPixelFormat)(video_st->codecpar->format), info.width, info.height, &bytes_final, NULL);
 		}
 #else
 		AVFrame *frame_final = allocate_avframe(video_codec->pix_fmt, info.width, info.height, &bytes_final, NULL);
-#endif
+#endif // IS_FFMPEG_3_2
 
 		// Fill with data
 		AV_COPY_PICTURE_DATA(frame_source, (uint8_t *) pixels, PIX_FMT_RGBA, source_image_width, source_image_height);
@@ -1977,7 +1978,7 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame *fra
 
 		// Assign the initial AVFrame PTS from the frame counter
 		frame_final->pts = write_video_count;
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 		if (hw_en_on && hw_en_supported) {
 			if (!(hw_frame = av_frame_alloc())) {
 				fprintf(stderr, "Error code: av_hwframe_alloc\n");
@@ -1994,7 +1995,7 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame *fra
 			}
 			av_frame_copy_props(hw_frame, frame_final);
 		}
-#endif
+#endif // HAVE_HW_ACCEL
 		/* encode the image */
 		int got_packet_ptr = 0;
 		int error_code = 0;
@@ -2003,9 +2004,12 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame *fra
 		int frameFinished = 0;
 		int ret;
 
+	#if HAVE_HW_ACCEL
 		if (hw_en_on && hw_en_supported) {
 			ret = avcodec_send_frame(video_codec, hw_frame); //hw_frame!!!
-		} else {
+		} else
+	#endif // HAVE_HW_ACCEL
+		{
 			ret = avcodec_send_frame(video_codec, frame_final);
 		}
 		error_code = ret;
@@ -2062,8 +2066,8 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame *fra
 			// got data back (so encode this frame)
 			got_packet_ptr = 1;
 		}
-#endif
-#endif
+#endif // LIBAVFORMAT_VERSION_MAJOR >= 54
+#endif // IS_FFMPEG_3_2
 
 		/* if zero size, it means the image was buffered */
 		if (error_code == 0 && got_packet_ptr) {
@@ -2095,14 +2099,14 @@ bool FFmpegWriter::write_video_packet(std::shared_ptr<Frame> frame, AVFrame *fra
 
 		// Deallocate packet
 		AV_FREE_PACKET(&pkt);
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 		if (hw_en_on && hw_en_supported) {
 			if (hw_frame) {
 				av_frame_free(&hw_frame);
 				hw_frame = NULL;
 			}
 		}
-#endif
+#endif // HAVE_HW_ACCEL
 	}
 
 	// Success
@@ -2125,11 +2129,11 @@ void FFmpegWriter::InitScalers(int source_width, int source_height) {
 	// Init software rescalers vector (many of them, one for each thread)
 	for (int x = 0; x < num_of_rescalers; x++) {
 		// Init the software scaler from FFMpeg
-#if IS_FFMPEG_3_2
+#if HAVE_HW_ACCEL
 		if (hw_en_on && hw_en_supported) {
 			img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA, info.width, info.height, AV_PIX_FMT_NV12, scale_mode, NULL, NULL, NULL);
 		} else
-#endif
+#endif // HAVE_HW_ACCEL
 		{
 			img_convert_ctx = sws_getContext(source_width, source_height, PIX_FMT_RGBA, info.width, info.height, AV_GET_CODEC_PIXEL_FORMAT(video_st, video_st->codec), scale_mode,
 											 NULL, NULL, NULL);