$OpenBSD: patch-lib_codec_ffmpeg_c,v 1.5 2019/02/18 10:05:39 ajacoutot Exp $

Update for newer FFmpeg API.

Index: lib/codec_ffmpeg.c
--- lib/codec_ffmpeg.c.orig
+++ lib/codec_ffmpeg.c
@@ -23,11 +23,7 @@
 #include "codec_ffmpeg.h"
 #include "iaxclient_lib.h"
 
-#ifdef WIN32
 #include "libavcodec/avcodec.h"
-#else
-#include <ffmpeg/avcodec.h>
-#endif
 
 struct slice_header_t
 {
@@ -63,30 +59,30 @@ struct decoder_ctx
 
 static struct slice_set_t * g_slice_set = 0;
 
-static enum CodecID map_iaxc_codec_to_avcodec(int format)
+static enum AVCodecID map_iaxc_codec_to_avcodec(int format)
 {
 	switch (format)
 	{
 	case IAXC_FORMAT_H261:
-		return CODEC_ID_H261;
+		return AV_CODEC_ID_H261;
 
 	case IAXC_FORMAT_H263:
-		return CODEC_ID_H263;
+		return AV_CODEC_ID_H263;
 
 	case IAXC_FORMAT_H263_PLUS:
-		return CODEC_ID_H263P;
+		return AV_CODEC_ID_H263P;
 
 	case IAXC_FORMAT_MPEG4:
-		return CODEC_ID_MPEG4;
+		return AV_CODEC_ID_MPEG4;
 
 	case IAXC_FORMAT_H264:
-		return CODEC_ID_H264;
+		return AV_CODEC_ID_H264;
 
 	case IAXC_FORMAT_THEORA:
-		return CODEC_ID_THEORA;
+		return AV_CODEC_ID_THEORA;
 
 	default:
-		return CODEC_ID_NONE;
+		return AV_CODEC_ID_NONE;
 	}
 }
 
@@ -165,10 +161,14 @@ static int pass_frame_to_decoder(AVCodecContext * avct
 {
 	int bytes_decoded;
 	int got_picture;
+	AVPacket pkt;
 
-	bytes_decoded = avcodec_decode_video(avctx, picture, &got_picture,
-			in, inlen);
+	pkt.data = in;
+	pkt.size = inlen;
 
+	bytes_decoded = avcodec_decode_video2(avctx, picture, &got_picture,
+			&pkt);
+
 	if ( bytes_decoded != inlen )
 	{
 		fprintf(stderr,
@@ -309,10 +309,14 @@ static int decode_rtp_slice(struct iaxc_video_codec * 
 	{
 		int bytes_decoded;
 		int got_picture;
+		AVPacket pkt;
 
-		bytes_decoded = avcodec_decode_video(d->avctx, d->picture,
-				&got_picture, (unsigned char *)in, inlen);
+		pkt.data = (unsigned char *)in;
+		pkt.size = inlen;
 
+		bytes_decoded = avcodec_decode_video2(d->avctx, d->picture,
+				&got_picture, &pkt);
+
 		if ( bytes_decoded < 0 )
 		{
 			fprintf(stderr,
@@ -374,7 +378,7 @@ static int encode(struct iaxc_video_codec *c,
 	struct encoder_ctx *e = (struct encoder_ctx *) c->encstate;
 	int encoded_size;
 
-	avcodec_get_frame_defaults(e->picture);
+	av_frame_unref(e->picture);
 
 	e->picture->data[0] = (unsigned char *)in;
 	e->picture->data[1] = (unsigned char *)in
@@ -441,6 +445,7 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 {
 	struct encoder_ctx *e;
 	struct decoder_ctx *d;
+	AVDictionary *opts = NULL;
 	AVCodec *codec;
 	int ff_enc_id, ff_dec_id;
 	char *name;
@@ -454,7 +459,6 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 		return NULL;
 	}
 
-	avcodec_init();
 	avcodec_register_all();
 
 	c->format = format;
@@ -475,10 +479,10 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 	if (!c->encstate)
 		goto bail;
 	e = c->encstate;
-	e->avctx = avcodec_alloc_context();
+	e->avctx = avcodec_alloc_context3(NULL);
 	if (!e->avctx)
 		goto bail;
-	e->picture = avcodec_alloc_frame();
+	e->picture = av_frame_alloc();
 	if (!e->picture)
 		goto bail;
 	/* The idea here is that the encoded frame that will land in this
@@ -496,7 +500,7 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 	if (!c->decstate)
 		goto bail;
 	d = c->decstate;
-	d->avctx = avcodec_alloc_context();
+	d->avctx = avcodec_alloc_context3(NULL);
 	if (!d->avctx)
 		goto bail;
 	d->picture = avcodec_alloc_frame();
@@ -521,20 +525,20 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 
 	/* This determines how often i-frames are sent */
 	e->avctx->gop_size = framerate * 3;
-	e->avctx->pix_fmt = PIX_FMT_YUV420P;
+	e->avctx->pix_fmt = AV_PIX_FMT_YUV420P;
 	e->avctx->has_b_frames = 0;
 
-	e->avctx->mb_qmin = e->avctx->qmin = 10;
-	e->avctx->mb_qmax = e->avctx->qmax = 10;
-
+#if 0
 	e->avctx->lmin = 2 * FF_QP2LAMBDA;
 	e->avctx->lmax = 10 * FF_QP2LAMBDA;
+#endif
+
 	e->avctx->global_quality = FF_QP2LAMBDA * 2;
 	e->avctx->qblur = 0.5;
 	e->avctx->global_quality = 10;
 
-	e->avctx->flags |= CODEC_FLAG_PSNR;
-	e->avctx->flags |= CODEC_FLAG_QSCALE;
+	e->avctx->flags |= AV_CODEC_FLAG_PSNR;
+	e->avctx->flags |= AV_CODEC_FLAG_QSCALE;
 
 	e->avctx->mb_decision = FF_MB_DECISION_SIMPLE;
 
@@ -555,37 +559,38 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 	case IAXC_FORMAT_H263:
 		/* TODO: H263 only works with specific resolutions. */
 		name = "H.263";
-		e->avctx->flags |= CODEC_FLAG_AC_PRED;
+		e->avctx->flags |= AV_CODEC_FLAG_AC_PRED;
 		if (fragsize)
 		{
 			c->decode = decode_rtp_slice;
 			e->avctx->rtp_payload_size = fragsize;
 			e->avctx->flags |=
-				CODEC_FLAG_TRUNCATED | CODEC_FLAG2_STRICT_GOP;
+				AV_CODEC_FLAG_TRUNCATED;
+			av_dict_set(&opts, "strict_gop", "1", 0);
 			e->avctx->rtp_callback = encode_rtp_callback;
-			d->avctx->flags |= CODEC_FLAG_TRUNCATED;
+			d->avctx->flags |= AV_CODEC_FLAG_TRUNCATED;
 		}
 		break;
 
 	case IAXC_FORMAT_H263_PLUS:
-		/* Although the encoder is CODEC_ID_H263P, the decoder
+		/* Although the encoder is AV_CODEC_ID_H263P, the decoder
 		 * is the regular h.263, so we handle this special case
 		 * here.
 		 */
-		ff_dec_id = CODEC_ID_H263;
+		ff_dec_id = AV_CODEC_ID_H263;
 		name = "H.263+";
-		e->avctx->flags |= CODEC_FLAG_AC_PRED;
+		e->avctx->flags |= AV_CODEC_FLAG_AC_PRED;
 		if (fragsize)
 		{
 			c->decode = decode_rtp_slice;
 			e->avctx->rtp_payload_size = fragsize;
 			e->avctx->flags |=
-				CODEC_FLAG_TRUNCATED |
-				CODEC_FLAG_H263P_SLICE_STRUCT |
-				CODEC_FLAG2_STRICT_GOP |
-				CODEC_FLAG2_LOCAL_HEADER;
+				AV_CODEC_FLAG_TRUNCATED |
+				AV_CODEC_FLAG2_LOCAL_HEADER;
+			av_dict_set(&opts, "strict_gop", "1", 0);
+			av_dict_set(&opts, "structured_slices", "1", 0);
 			e->avctx->rtp_callback = encode_rtp_callback;
-			d->avctx->flags |= CODEC_FLAG_TRUNCATED;
+			d->avctx->flags |= AV_CODEC_FLAG_TRUNCATED;
 		}
 		break;
 
@@ -595,12 +600,11 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 		e->avctx->rtp_payload_size = fragsize;
 		e->avctx->rtp_callback = encode_rtp_callback;
 		e->avctx->flags |=
-			CODEC_FLAG_TRUNCATED |
-			CODEC_FLAG_H263P_SLICE_STRUCT |
-			CODEC_FLAG2_STRICT_GOP |
-			CODEC_FLAG2_LOCAL_HEADER;
-
-		d->avctx->flags |= CODEC_FLAG_TRUNCATED;
+			AV_CODEC_FLAG_TRUNCATED |
+			AV_CODEC_FLAG2_LOCAL_HEADER;
+		av_dict_set(&opts, "strict_gop", "1", 0);
+		av_dict_set(&opts, "structured_slices", "1", 0);
+		d->avctx->flags |= AV_CODEC_FLAG_TRUNCATED;
 		break;
 
 	case IAXC_FORMAT_H264:
@@ -611,35 +615,35 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 		 */
 
 		/* Headers are not repeated */
-		/* e->avctx->flags |= CODEC_FLAG_GLOBAL_HEADER; */
+		/* e->avctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; */
 
 		/* Slower, less blocky */
-		/* e->avctx->flags |= CODEC_FLAG_LOOP_FILTER; */
+		/* e->avctx->flags |= AV_CODEC_FLAG_LOOP_FILTER; */
 
-		e->avctx->flags |= CODEC_FLAG_PASS1;
-		/* e->avctx->flags |= CODEC_FLAG_PASS2; */
+		e->avctx->flags |= AV_CODEC_FLAG_PASS1;
+		/* e->avctx->flags |= AV_CODEC_FLAG_PASS2; */
 
 		/* Compute psnr values at encode-time (avctx->error[]) */
-		/* e->avctx->flags |= CODEC_FLAG_PSNR; */
+		/* e->avctx->flags |= AV_CODEC_FLAG_PSNR; */
 
-		/* e->avctx->flags2 |= CODEC_FLAG2_8X8DCT; */
+		/* e->avctx->flags2 |= AV_CODEC_FLAG2_8X8DCT; */
 
 		/* Access Unit Delimiters */
-		e->avctx->flags2 |= CODEC_FLAG2_AUD;
+		av_dict_set(&opts, "aud", "1", 0);
 
 		/* Allow b-frames to be used as reference */
-		/* e->avctx->flags2 |= CODEC_FLAG2_BPYRAMID; */
+		/* e->avctx->flags2 |= AV_CODEC_FLAG2_BPYRAMID; */
 
 		/* b-frame rate distortion optimization */
-		/* e->avctx->flags2 |= CODEC_FLAG2_BRDO; */
+		/* e->avctx->flags2 |= AV_CODEC_FLAG2_BRDO; */
 
-		/* e->avctx->flags2 |= CODEC_FLAG2_FASTPSKIP; */
+		/* e->avctx->flags2 |= AV_CODEC_FLAG2_FASTPSKIP; */
 
 		/* Multiple references per partition */
-		/* e->avctx->flags2 |= CODEC_FLAG2_MIXED_REFS; */
+		/* e->avctx->flags2 |= AV_CODEC_FLAG2_MIXED_REFS; */
 
 		/* Weighted biprediction for b-frames */
-		/* e->avctx->flags2 |= CODEC_FLAG2_WPRED; */
+		/* e->avctx->flags2 |= AV_CODEC_FLAG2_WPRED; */
 
 		/*
 		 * Decoder flags
@@ -686,7 +690,7 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 		goto bail;
 	}
 
-	if (avcodec_open(e->avctx, codec))
+	if (avcodec_open2(e->avctx, codec, &opts))
 	{
 		iaxci_usermsg(IAXC_TEXT_TYPE_ERROR,
 			     "codec_ffmpeg: cannot open encoder %s\n", name);
@@ -701,7 +705,7 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 			     ff_dec_id);
 		goto bail;
 	}
-	if (avcodec_open(d->avctx, codec))
+	if (avcodec_open2(d->avctx, codec, NULL))
 	{
 		iaxci_usermsg(IAXC_TEXT_TYPE_ERROR,
 			     "codec_ffmpeg: cannot open decoder %s\n", name);
@@ -709,8 +713,8 @@ struct iaxc_video_codec *codec_video_ffmpeg_new(int fo
 	}
 
 	{
-		enum PixelFormat fmts[] = { PIX_FMT_YUV420P, -1 };
-		if (d->avctx->get_format(d->avctx, fmts) != PIX_FMT_YUV420P)
+		enum AVPixelFormat fmts[] = { AV_PIX_FMT_YUV420P, -1 };
+		if (d->avctx->get_format(d->avctx, fmts) != AV_PIX_FMT_YUV420P)
 		{
 			iaxci_usermsg(IAXC_TEXT_TYPE_ERROR,
 					"codec_ffmpeg: cannot set decode format to YUV420P\n");
@@ -728,17 +732,16 @@ bail:
 int codec_video_ffmpeg_check_codec(int format)
 {
 	AVCodec *codec;
-	enum CodecID codec_id;
+	enum AVCodecID codec_id;
 
 	/* These functions are idempotent, so it is okay that we
 	 * may call them elsewhere at a different time.
 	 */
-	avcodec_init();
 	avcodec_register_all();
 
 	codec_id = map_iaxc_codec_to_avcodec(format);
 
-	if (codec_id == CODEC_ID_NONE)
+	if (codec_id == AV_CODEC_ID_NONE)
 		return 0;
 
 	codec = avcodec_find_encoder(codec_id);
