20#include <freerdp/config.h>
22#include <winpr/assert.h>
23#include <winpr/wlog.h>
24#include <winpr/stream.h>
25#include <freerdp/log.h>
27#include <freerdp/codec/video.h>
28#include <freerdp/codec/h264.h>
30#define VTAG FREERDP_TAG("codec.video")
32#if defined(WITH_SWSCALE)
35static BOOL freerdp_video_fill_plane_info(wLog* log, BYTE* data[4],
int lineSize[4],
36 FREERDP_VIDEO_FORMAT format, UINT32 width, UINT32 height,
39#include "image_ffmpeg.h"
42#if defined(WITH_VIDEO_FFMPEG) && !defined(WITH_SWSCALE_LOADING)
43#define WITH_MJPEG_DECODER
44#include <libavcodec/avcodec.h>
47struct s_FREERDP_VIDEO_CONTEXT
51 struct SwsContext* sws;
53#if defined(WITH_MJPEG_DECODER)
54 AVCodecContext* mjpegDecoder;
55 AVPacket* mjpegPacket;
70static enum AVPixelFormat video_format_to_av(wLog* log, FREERDP_VIDEO_FORMAT format)
75 case FREERDP_VIDEO_FORMAT_MJPEG:
76 case FREERDP_VIDEO_FORMAT_H264:
77 return AV_PIX_FMT_NONE;
80 case FREERDP_VIDEO_FORMAT_YUV420P:
81 return AV_PIX_FMT_YUV420P;
82 case FREERDP_VIDEO_FORMAT_NV12:
83 return AV_PIX_FMT_NV12;
86 case FREERDP_VIDEO_FORMAT_YUYV422:
87 return AV_PIX_FMT_YUYV422;
90 case FREERDP_VIDEO_FORMAT_RGB24:
91 return AV_PIX_FMT_RGB24;
92 case FREERDP_VIDEO_FORMAT_RGB32:
93 return AV_PIX_FMT_RGB32;
95 case FREERDP_VIDEO_FORMAT_NONE:
97 WLog_Print(log, WLOG_WARN,
"Could not map FREERDP_VIDEO_FORMAT %s [0x%08" PRIx32
"]",
98 freerdp_video_format_string(format), format);
99 return AV_PIX_FMT_NONE;
103BOOL freerdp_video_available(
void)
105 return freerdp_swscale_available() && freerdp_avutil_available();
108FREERDP_VIDEO_CONTEXT* freerdp_video_context_new(UINT32 width, UINT32 height)
110 FREERDP_VIDEO_CONTEXT* context =
nullptr;
112 wLog* log = WLog_Get(VTAG);
113 if (!freerdp_video_available())
115 WLog_Print(log, WLOG_ERROR,
"Video codecs not available - FFmpeg not loaded");
119 context = (FREERDP_VIDEO_CONTEXT*)calloc(1,
sizeof(FREERDP_VIDEO_CONTEXT));
124 context->width = width;
125 context->height = height;
127#if defined(WITH_MJPEG_DECODER)
129 const AVCodec* codec = avcodec_find_decoder(AV_CODEC_ID_MJPEG);
132 WLog_Print(context->log, WLOG_ERROR,
"avcodec_find_decoder failed to find MJPEG codec");
136 context->mjpegDecoder = avcodec_alloc_context3(codec);
137 if (!context->mjpegDecoder)
139 WLog_Print(context->log, WLOG_ERROR,
"avcodec_alloc_context3 failed");
143 context->mjpegDecoder->width = (int)width;
144 context->mjpegDecoder->height = (int)height;
146 context->mjpegDecoder->err_recognition |= AV_EF_EXPLODE;
148 if (avcodec_open2(context->mjpegDecoder, codec,
nullptr) < 0)
150 WLog_Print(context->log, WLOG_ERROR,
"avcodec_open2 failed");
154 context->mjpegPacket = av_packet_alloc();
155 if (!context->mjpegPacket)
157 WLog_Print(context->log, WLOG_ERROR,
"av_packet_alloc failed");
161 context->mjpegFrame = av_frame_alloc();
162 if (!context->mjpegFrame)
164 WLog_Print(context->log, WLOG_ERROR,
"av_frame_alloc failed");
171#if defined(WITH_MJPEG_DECODER)
173 freerdp_video_context_free(context);
178void freerdp_video_context_free(FREERDP_VIDEO_CONTEXT* context)
185 freerdp_sws_freeContext(context->sws);
186 context->sws =
nullptr;
189#if defined(WITH_MJPEG_DECODER)
190 if (context->mjpegFrame)
191 av_frame_free(&context->mjpegFrame);
193 if (context->mjpegPacket)
195 context->mjpegPacket->data =
nullptr;
196 context->mjpegPacket->size = 0;
197 av_packet_free(&context->mjpegPacket);
200 if (context->mjpegDecoder)
201 avcodec_free_context(&context->mjpegDecoder);
206 h264_context_free(context->h264);
207 context->h264 =
nullptr;
213static UINT32 video_get_h264_bitrate(wLog* log, UINT32 height)
220 { 1080, 2700 }, { 720, 1250 }, { 480, 700 }, { 360, 400 },
221 { 240, 170 }, { 180, 140 }, { 0, 100 },
223 const size_t nBitrates = ARRAYSIZE(bitrates);
225 for (
size_t i = 0; i < nBitrates; i++)
227 if (height >= bitrates[i].height)
229 UINT32 bitrate = bitrates[i].bitrate;
230 WLog_Print(log, WLOG_DEBUG,
"Auto-calculated H.264 bitrate: %u kbps", bitrate);
231 return bitrate * 1000;
239BOOL freerdp_video_context_reconfigure(FREERDP_VIDEO_CONTEXT* context, UINT32 width, UINT32 height,
240 UINT32 framerate, UINT32 bitrate, UINT32 usageType)
242 WINPR_ASSERT(context);
244 if (width == 0 || height == 0)
246 WLog_Print(context->log, WLOG_ERROR,
"Invalid dimensions: %ux%u", width, height);
251 bitrate = video_get_h264_bitrate(context->log, height);
255 context->h264 = h264_context_new(TRUE);
258 WLog_Print(context->log, WLOG_ERROR,
"h264_context_new failed");
263 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_USAGETYPE, usageType))
266 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_FRAMERATE, framerate))
269 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_BITRATE, bitrate))
272 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_RATECONTROL,
273 H264_RATECONTROL_CQP))
276 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_QP, 26))
279 if (!h264_context_set_option(context->h264, H264_CONTEXT_OPTION_HW_ACCEL, FALSE))
282 if (!context->h264Configured || (context->width != width) || (context->height != height))
284 if (!h264_context_reset(context->h264, width, height))
286 WLog_Print(context->log, WLOG_ERROR,
"h264_context_reset failed");
291 context->h264Framerate = framerate;
292 context->h264Bitrate = bitrate;
293 context->h264UsageType = usageType;
294 context->h264Configured = TRUE;
295 context->width = width;
296 context->height = height;
303 h264_context_free(context->h264);
304 context->h264 =
nullptr;
309static BOOL freerdp_video_decode_mjpeg(FREERDP_VIDEO_CONTEXT* context,
const BYTE* srcData,
310 size_t srcSize, BYTE* dstData[4],
int dstLineSize[4],
311 enum AVPixelFormat* dstFormat);
313static BOOL freerdp_video_convert_to_yuv(FREERDP_VIDEO_CONTEXT* context,
const BYTE* srcData[4],
314 const int srcLineSize[4],
enum AVPixelFormat srcFormat,
315 BYTE* dstData[3],
const int dstLineSize[3],
316 FREERDP_VIDEO_FORMAT dstFormat, UINT32 width,
319BOOL freerdp_video_sample_convert(FREERDP_VIDEO_CONTEXT* context, FREERDP_VIDEO_FORMAT srcFormat,
320 const void* srcSampleData,
size_t srcSampleLength,
321 FREERDP_VIDEO_FORMAT dstFormat,
wStream* output)
323 WINPR_ASSERT(context);
324 WINPR_ASSERT(srcSampleData);
325 WINPR_ASSERT(output);
327 if (srcFormat == dstFormat)
329 if (!Stream_EnsureRemainingCapacity(output, srcSampleLength))
331 Stream_Write(output, srcSampleData, srcSampleLength);
335 if (!freerdp_video_available())
337 WLog_Print(context->log, WLOG_ERROR,
"Video codecs not available");
341 if (srcSampleLength == 0)
343 WLog_Print(context->log, WLOG_ERROR,
"Invalid source sample length: 0");
347 if (!freerdp_video_conversion_supported(srcFormat, dstFormat))
349 WLog_Print(context->log, WLOG_ERROR,
"Conversion from format %u to %u not supported",
350 srcFormat, dstFormat);
354 const BOOL srcCompressed =
355 (srcFormat == FREERDP_VIDEO_FORMAT_MJPEG || srcFormat == FREERDP_VIDEO_FORMAT_H264);
356 const BOOL dstCompressed =
357 (dstFormat == FREERDP_VIDEO_FORMAT_MJPEG || dstFormat == FREERDP_VIDEO_FORMAT_H264);
359 BYTE* intermediate_data[4] = WINPR_C_ARRAY_INIT;
360 int intermediate_linesize[4] = WINPR_C_ARRAY_INIT;
361 enum AVPixelFormat intermediate_format = AV_PIX_FMT_NONE;
365 if (srcFormat == FREERDP_VIDEO_FORMAT_MJPEG)
367 if (!freerdp_video_decode_mjpeg(context, (
const BYTE*)srcSampleData, srcSampleLength,
368 intermediate_data, intermediate_linesize,
369 &intermediate_format))
371 WLog_Print(context->log, WLOG_ERROR,
"MJPEG decoding failed");
375 else if (srcFormat == FREERDP_VIDEO_FORMAT_H264)
377 WLog_Print(context->log, WLOG_ERROR,
"H264 decoding not supported");
383 intermediate_format = video_format_to_av(context->log, srcFormat);
386 if (dstFormat == FREERDP_VIDEO_FORMAT_H264)
388 if (!context->h264Configured)
390 WLog_Print(context->log, WLOG_ERROR,
"H264 encoder not configured");
394 const UINT32 hwAccel = h264_context_get_option(context->h264, H264_CONTEXT_OPTION_HW_ACCEL);
395 const FREERDP_VIDEO_FORMAT yuvFormat =
396 hwAccel ? FREERDP_VIDEO_FORMAT_NV12 : FREERDP_VIDEO_FORMAT_YUV420P;
398 BYTE* yuvData[3] = WINPR_C_ARRAY_INIT;
399 UINT32 yuvStrides[3] = WINPR_C_ARRAY_INIT;
401 if (h264_get_yuv_buffer(context->h264, 0, context->width, context->height, yuvData,
404 WLog_Print(context->log, WLOG_ERROR,
"h264_get_yuv_buffer failed");
408 int yuvLineSizes[4] = { (int)yuvStrides[0], (
int)yuvStrides[1], (int)yuvStrides[2], 0 };
412 const BYTE* cIntermediateData[4] = { intermediate_data[0], intermediate_data[1],
413 intermediate_data[2], intermediate_data[3] };
414 if (!freerdp_video_convert_to_yuv(context, cIntermediateData, intermediate_linesize,
415 intermediate_format, yuvData, yuvLineSizes, yuvFormat,
416 context->width, context->height))
418 WLog_Print(context->log, WLOG_ERROR,
"YUV conversion failed");
424 BYTE* srcPlanes[4] = WINPR_C_ARRAY_INIT;
425 int srcStrides[4] = WINPR_C_ARRAY_INIT;
427 if (!freerdp_video_fill_plane_info(context->log, srcPlanes, srcStrides, srcFormat,
428 context->width, context->height,
429 (
const BYTE*)srcSampleData))
431 WLog_Print(context->log, WLOG_ERROR,
"Failed to fill plane info");
435 const BYTE* cSrcPlanes[4] = { srcPlanes[0], srcPlanes[1], srcPlanes[2], srcPlanes[3] };
436 if (!freerdp_video_convert_to_yuv(
437 context, cSrcPlanes, srcStrides, video_format_to_av(context->log, srcFormat),
438 yuvData, yuvLineSizes, yuvFormat, context->width, context->height))
440 WLog_Print(context->log, WLOG_ERROR,
"YUV conversion failed");
445 BYTE* h264Data =
nullptr;
448 if (h264_compress(context->h264, &h264Data, &h264Size) < 0)
450 WLog_Print(context->log, WLOG_ERROR,
"H264 compression failed");
454 if (!Stream_EnsureRemainingCapacity(output, h264Size))
456 WLog_Print(context->log, WLOG_ERROR,
"Failed to ensure stream capacity");
460 Stream_Write(output, h264Data, h264Size);
463 else if (dstCompressed)
465 WLog_Print(context->log, WLOG_ERROR,
"Only H264 encoding is supported");
469 WLog_Print(context->log, WLOG_ERROR,
"Raw format output not yet implemented");
473static BOOL freerdp_video_decode_mjpeg(FREERDP_VIDEO_CONTEXT* context,
const BYTE* srcData,
474 size_t srcSize, BYTE* dstData[4],
int dstLineSize[4],
475 enum AVPixelFormat* dstFormat)
477#if defined(WITH_MJPEG_DECODER)
478 WINPR_ASSERT(context);
479 WINPR_ASSERT(srcData);
480 WINPR_ASSERT(dstData);
481 WINPR_ASSERT(dstLineSize);
482 WINPR_ASSERT(dstFormat);
484 if (!context->mjpegDecoder)
486 WLog_Print(context->log, WLOG_ERROR,
"MJPEG decoder not initialized");
490 context->mjpegPacket->data = WINPR_CAST_CONST_PTR_AWAY(srcData, uint8_t*);
491 WINPR_ASSERT(srcSize <= INT32_MAX);
492 context->mjpegPacket->size = (int)srcSize;
494 if (avcodec_send_packet(context->mjpegDecoder, context->mjpegPacket) < 0)
496 WLog_Print(context->log, WLOG_ERROR,
"avcodec_send_packet failed");
500 if (avcodec_receive_frame(context->mjpegDecoder, context->mjpegFrame) < 0)
502 WLog_Print(context->log, WLOG_ERROR,
"avcodec_receive_frame failed");
507 for (
size_t i = 0; i < 4; i++)
509 dstData[i] = context->mjpegFrame->data[i];
510 dstLineSize[i] = context->mjpegFrame->linesize[i];
514 *dstFormat = context->mjpegDecoder->pix_fmt;
518 WINPR_UNUSED(context);
519 WINPR_UNUSED(srcData);
520 WINPR_UNUSED(srcSize);
521 WINPR_UNUSED(dstData);
522 WINPR_UNUSED(dstLineSize);
523 WINPR_UNUSED(dstFormat);
524 WLog_Print(context->log, WLOG_ERROR,
525 "MJPEG decoder not available (requires direct FFmpeg linking)");
530static BOOL freerdp_video_convert_to_yuv(FREERDP_VIDEO_CONTEXT* context,
const BYTE* srcData[4],
531 const int srcLineSize[4],
enum AVPixelFormat srcFormat,
532 BYTE* dstData[3],
const int dstLineSize[3],
533 FREERDP_VIDEO_FORMAT dstFormat, UINT32 width,
536 WINPR_ASSERT(srcData);
537 WINPR_ASSERT(srcLineSize);
538 WINPR_ASSERT(dstData);
539 WINPR_ASSERT(dstLineSize);
541 if (!freerdp_swscale_available())
543 WLog_Print(context->log, WLOG_ERROR,
544 "swscale not available - install FFmpeg to enable video processing");
548 enum AVPixelFormat srcPixFmt = srcFormat;
549 enum AVPixelFormat dstPixFmt = video_format_to_av(context->log, dstFormat);
551 if (srcPixFmt == AV_PIX_FMT_NONE || dstPixFmt == AV_PIX_FMT_NONE)
553 WLog_Print(context->log, WLOG_ERROR,
"Unsupported pixel format");
558 struct SwsContext* sws =
nullptr;
559 BOOL needFree = FALSE;
561 if (context && context->sws)
567 sws = freerdp_sws_getContext((
int)width, (
int)height, srcPixFmt, (
int)width, (
int)height,
568 dstPixFmt, 0,
nullptr,
nullptr,
nullptr);
571 WLog_Print(context->log, WLOG_ERROR,
"sws_getContext failed");
582 const BYTE* cSrcData[4] = { srcData[0], srcData[1], srcData[2], srcData[3] };
585 uint8_t* localDstData[4] = { dstData[0], dstData[1], dstData[2],
nullptr };
586 int localDstLineSize[4] = { dstLineSize[0], dstLineSize[1], dstLineSize[2], 0 };
588 const int result = freerdp_sws_scale(sws, cSrcData, srcLineSize, 0, (
int)height, localDstData,
593 freerdp_sws_freeContext(sws);
599static BOOL freerdp_video_fill_plane_info(wLog* log, BYTE* data[4],
int lineSize[4],
600 FREERDP_VIDEO_FORMAT format, UINT32 width, UINT32 height,
604 WINPR_ASSERT(lineSize);
606 enum AVPixelFormat pixFmt = video_format_to_av(log, format);
607 if (pixFmt == AV_PIX_FMT_NONE)
609 WLog_Print(log, WLOG_ERROR,
"Unsupported pixel format");
613 if (!freerdp_avutil_available())
615 WLog_Print(log, WLOG_ERROR,
"avutil not available");
619 if (freerdp_av_image_fill_linesizes(lineSize, pixFmt, (
int)width) < 0)
621 WLog_Print(log, WLOG_ERROR,
"av_image_fill_linesizes failed");
625 if (freerdp_av_image_fill_pointers(data, pixFmt, (
int)height,
626 WINPR_CAST_CONST_PTR_AWAY(buffer, BYTE*), lineSize) < 0)
628 WLog_Print(log, WLOG_ERROR,
"av_image_fill_pointers failed");
635BOOL freerdp_video_conversion_supported(FREERDP_VIDEO_FORMAT srcFormat,
636 FREERDP_VIDEO_FORMAT dstFormat)
638 if (srcFormat == dstFormat)
641 if (!freerdp_video_available())
644 if (srcFormat == FREERDP_VIDEO_FORMAT_MJPEG)
646#if !defined(WITH_MJPEG_DECODER)
650 else if (srcFormat == FREERDP_VIDEO_FORMAT_H264)
655 if (dstFormat == FREERDP_VIDEO_FORMAT_MJPEG)
659 else if (dstFormat == FREERDP_VIDEO_FORMAT_H264)
661#if !defined(WITH_OPENH264) && !defined(WITH_VIDEO_FFMPEG) && !defined(WITH_MEDIA_FOUNDATION) && \
662 !defined(WITH_MEDIACODEC)
674BOOL freerdp_video_available(
void)
679FREERDP_VIDEO_CONTEXT* freerdp_video_context_new(WINPR_ATTR_UNUSED UINT32 width,
680 WINPR_ATTR_UNUSED UINT32 height)
683 WINPR_UNUSED(height);
684 return calloc(1,
sizeof(
char));
687void freerdp_video_context_free(FREERDP_VIDEO_CONTEXT* context)
692BOOL freerdp_video_conversion_supported(FREERDP_VIDEO_FORMAT srcFormat,
693 FREERDP_VIDEO_FORMAT dstFormat)
695 if (srcFormat == dstFormat)
700BOOL freerdp_video_context_reconfigure(WINPR_ATTR_UNUSED FREERDP_VIDEO_CONTEXT* context,
701 WINPR_ATTR_UNUSED UINT32 width,
702 WINPR_ATTR_UNUSED UINT32 height,
703 WINPR_ATTR_UNUSED UINT32 framerate,
704 WINPR_ATTR_UNUSED UINT32 bitrate,
705 WINPR_ATTR_UNUSED UINT32 usageType)
710BOOL freerdp_video_sample_convert(WINPR_ATTR_UNUSED FREERDP_VIDEO_CONTEXT* context,
711 FREERDP_VIDEO_FORMAT srcFormat,
const void* srcSampleData,
712 size_t srcSampleLength, FREERDP_VIDEO_FORMAT dstFormat,
715 if (srcFormat == dstFormat)
717 if (!Stream_EnsureRemainingCapacity(output, srcSampleLength))
720 Stream_Write(output, srcSampleData, srcSampleLength);
729const char* freerdp_video_format_string(UINT32 format)
737 EVCASE(FREERDP_VIDEO_FORMAT_NONE);
738 EVCASE(FREERDP_VIDEO_FORMAT_MJPEG);
739 EVCASE(FREERDP_VIDEO_FORMAT_H264);
740 EVCASE(FREERDP_VIDEO_FORMAT_YUV420P);
741 EVCASE(FREERDP_VIDEO_FORMAT_NV12);
742 EVCASE(FREERDP_VIDEO_FORMAT_YUYV422);
743 EVCASE(FREERDP_VIDEO_FORMAT_RGB24);
744 EVCASE(FREERDP_VIDEO_FORMAT_RGB32);
746 return "FREERDP_VIDEO_FORMAT_UNKNOWN";