35 #include <TargetConditionals.h>
37 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
38 # define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
40 #ifndef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
41 # define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder CFSTR("EnableHardwareAcceleratedVideoDecoder")
44 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
48 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12
59 CVPixelBufferRelease(
ref->pixbuf);
101 if (
ref->hw_frames_ctx) {
141 #define AV_W8(p, v) *(p) = (v)
149 int vt_extradata_size = 6 + 2 +
h->ps.sps->data_size + 3 +
h->ps.pps->data_size;
157 AV_W8(p + 1,
h->ps.sps->data[1]);
158 AV_W8(p + 2,
h->ps.sps->data[2]);
159 AV_W8(p + 3,
h->ps.sps->data[3]);
162 AV_WB16(p + 6,
h->ps.sps->data_size);
163 memcpy(p + 8,
h->ps.sps->data,
h->ps.sps->data_size);
164 p += 8 +
h->ps.sps->data_size;
166 AV_WB16(p + 1,
h->ps.pps->data_size);
167 memcpy(p + 3,
h->ps.pps->data,
h->ps.pps->data_size);
169 p += 3 +
h->ps.pps->data_size;
170 av_assert0(p - vt_extradata == vt_extradata_size);
175 memcpy(vtctx->
sps,
h->ps.sps->data + 1, 3);
177 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
185 int i, num_vps = 0, num_sps = 0, num_pps = 0;
194 int vt_extradata_size = 23 + 3 + 3 + 3;
197 #define COUNT_SIZE_PS(T, t) \
198 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
199 if (h->ps.t##ps_list[i]) { \
200 const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \
201 vt_extradata_size += 2 + lps->data_size; \
210 vt_extradata =
av_malloc(vt_extradata_size);
254 else if (
pps->entropy_coding_sync_enabled_flag &&
pps->tiles_enabled_flag)
256 else if (
pps->entropy_coding_sync_enabled_flag)
258 else if (
pps->tiles_enabled_flag)
262 AV_W8(p + 15, 0xfc | parallelismType);
268 AV_W8(p + 16,
sps->chroma_format_idc | 0xfc);
274 AV_W8(p + 17, (
sps->bit_depth - 8) | 0xfc);
280 AV_W8(p + 18, (
sps->bit_depth_chroma - 8) | 0xfc);
291 AV_W8(p + 21, 0 << 6 |
292 sps->max_sub_layers << 3 |
293 sps->temporal_id_nesting_flag << 2 |
301 #define APPEND_PS(T, t) \
308 HEVC_NAL_##T##PS & 0x3f); \
310 AV_WB16(p + 1, num_##t##ps); \
312 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
313 if (h->ps.t##ps_list[i]) { \
314 const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \
316 AV_WB16(p, lps->data_size); \
318 memcpy(p + 2, lps->data, lps->data_size); \
319 p += 2 + lps->data_size; \
327 av_assert0(p - vt_extradata == vt_extradata_size);
329 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
341 if (
h->is_avc == 1) {
358 memcpy(vtctx->
sps,
h->ps.sps->data + 1, 3);
412 CVPixelBufferRelease(vtctx->
frame);
418 #if CONFIG_VIDEOTOOLBOX
436 CVPixelBufferRef pixbuf = (CVPixelBufferRef)vtctx->
frame;
437 OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
439 int width = CVPixelBufferGetWidth(pixbuf);
440 int height = CVPixelBufferGetHeight(pixbuf);
454 CVPixelBufferRelease(
ref->pixbuf);
490 if (!
ref->hw_frames_ctx)
496 static void videotoolbox_write_mp4_descr_length(
PutByteContext *pb,
int length)
501 for (
i = 3;
i >= 0;
i--) {
502 b = (length >> (
i * 7)) & 0x7F;
506 bytestream2_put_byteu(pb,
b);
510 static CFDataRef videotoolbox_esds_extradata_create(
AVCodecContext *avctx)
524 bytestream2_put_byteu(&pb, 0);
528 bytestream2_put_byteu(&pb, 0x03);
529 videotoolbox_write_mp4_descr_length(&pb, full_size);
531 bytestream2_put_byteu(&pb, 0);
534 bytestream2_put_byteu(&pb, 0x04);
535 videotoolbox_write_mp4_descr_length(&pb, config_size);
536 bytestream2_put_byteu(&pb, 32);
537 bytestream2_put_byteu(&pb, 0x11);
543 bytestream2_put_byteu(&pb, 0x05);
549 bytestream2_put_byteu(&pb, 0x06);
550 bytestream2_put_byteu(&pb, 0x01);
551 bytestream2_put_byteu(&pb, 0x02);
555 data = CFDataCreate(kCFAllocatorDefault, rw_extradata,
s);
561 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
566 CMBlockBufferRef block_buf;
567 CMSampleBufferRef sample_buf;
572 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,
583 status = CMSampleBufferCreate(kCFAllocatorDefault,
598 CFRelease(block_buf);
603 static void videotoolbox_decoder_callback(
void *opaque,
604 void *sourceFrameRefCon,
606 VTDecodeInfoFlags
flags,
607 CVImageBufferRef image_buffer,
614 CVPixelBufferRelease(vtctx->
frame);
620 "vt decoder cb: output image buffer is null: %i\n", status);
624 vtctx->
frame = CVPixelBufferRetain(image_buffer);
627 static OSStatus videotoolbox_session_decode_frame(
AVCodecContext *avctx)
630 CMSampleBufferRef sample_buf;
634 sample_buf = videotoolbox_sample_buffer_create(videotoolbox->
cm_fmt_desc,
641 status = VTDecompressionSessionDecodeFrame(videotoolbox->
session,
647 status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->
session);
649 CFRelease(sample_buf);
654 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType
codec_type,
655 CFDictionaryRef decoder_spec,
659 CMFormatDescriptionRef cm_fmt_desc;
662 status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
675 static CFDictionaryRef videotoolbox_buffer_attributes_create(
int width,
679 CFMutableDictionaryRef buffer_attributes;
680 CFMutableDictionaryRef io_surface_properties;
681 CFNumberRef cv_pix_fmt;
685 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
width);
686 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
height);
687 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
pix_fmt);
689 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
691 &kCFTypeDictionaryKeyCallBacks,
692 &kCFTypeDictionaryValueCallBacks);
693 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
695 &kCFTypeDictionaryKeyCallBacks,
696 &kCFTypeDictionaryValueCallBacks);
699 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
700 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
701 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey,
w);
702 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey,
h);
704 CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
706 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
709 CFRelease(io_surface_properties);
710 CFRelease(cv_pix_fmt);
714 return buffer_attributes;
717 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType
codec_type,
720 CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
722 &kCFTypeDictionaryKeyCallBacks,
723 &kCFTypeDictionaryValueCallBacks);
725 CFDictionarySetValue(config_info,
731 CFMutableDictionaryRef avc_info;
734 avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
736 &kCFTypeDictionaryKeyCallBacks,
737 &kCFTypeDictionaryValueCallBacks);
740 case kCMVideoCodecType_MPEG4Video :
742 data = videotoolbox_esds_extradata_create(avctx);
744 CFDictionarySetValue(avc_info, CFSTR(
"esds"),
data);
746 case kCMVideoCodecType_H264 :
749 CFDictionarySetValue(avc_info, CFSTR(
"avcC"),
data);
754 CFDictionarySetValue(avc_info, CFSTR(
"hvcC"),
data);
760 CFDictionarySetValue(config_info,
761 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
775 VTDecompressionOutputCallbackRecord decoder_cb;
776 CFDictionaryRef decoder_spec;
777 CFDictionaryRef buf_attr;
807 decoder_spec = videotoolbox_decoder_config_create(videotoolbox->
cm_codec_type, avctx);
820 CFRelease(decoder_spec);
826 buf_attr = videotoolbox_buffer_attributes_create(avctx->
width,
830 decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
833 status = VTDecompressionSessionCreate(
NULL,
841 CFRelease(decoder_spec);
846 case kVTVideoDecoderNotAvailableNowErr:
849 case kVTVideoDecoderUnsupportedDataFormatErr:
852 case kVTCouldNotFindVideoDecoderErr:
855 case kVTVideoDecoderMalfunctionErr:
858 case kVTVideoDecoderBadDataErr:
881 VTDecompressionSessionInvalidate(videotoolbox->
session);
882 CFRelease(videotoolbox->
session);
887 static const char *videotoolbox_error_string(OSStatus status)
890 case kVTVideoDecoderBadDataErr:
892 case kVTVideoDecoderMalfunctionErr:
893 return "decoder malfunction";
894 case kVTInvalidSessionErr:
895 return "invalid session";
909 videotoolbox_stop(avctx);
910 if (videotoolbox_start(avctx) != 0) {
918 status = videotoolbox_session_decode_frame(avctx);
919 if (status != noErr) {
920 if (status == kVTVideoDecoderMalfunctionErr || status == kVTInvalidSessionErr)
922 av_log(avctx,
AV_LOG_ERROR,
"Failed to decode frame (%s, %d)\n", videotoolbox_error_string(status), (
int)status);
931 return videotoolbox_buffer_create(avctx,
frame);
939 int ret = videotoolbox_common_end_frame(avctx,
frame);
973 h->output_frame->crop_right = 0;
974 h->output_frame->crop_left = 0;
975 h->output_frame->crop_top = 0;
976 h->output_frame->crop_bottom = 0;
978 int ret = videotoolbox_common_end_frame(avctx,
frame);
1004 return videotoolbox_common_end_frame(avctx,
frame);
1016 videotoolbox_stop(avctx);
1051 "Either hw_frames_ctx or hw_device_ctx must be set.\n");
1072 hw_frames->
sw_format = videotoolbox_best_pixel_format(avctx);
1096 "Failed to map underlying FFmpeg pixel format %s (%s range) to "
1097 "a VideoToolbox format!\n",
1098 attempted_format ? attempted_format->
name :
"<unknown>",
1104 err = videotoolbox_start(avctx);
1123 frames_ctx->
sw_format = videotoolbox_best_pixel_format(avctx);
1129 .
name =
"h263_videotoolbox",
1134 .start_frame = videotoolbox_mpeg_start_frame,
1135 .decode_slice = videotoolbox_mpeg_decode_slice,
1136 .end_frame = videotoolbox_mpeg_end_frame,
1137 .frame_params = videotoolbox_frame_params,
1138 .init = videotoolbox_common_init,
1144 .
name =
"hevc_videotoolbox",
1149 .start_frame = videotoolbox_hevc_start_frame,
1150 .decode_slice = videotoolbox_hevc_decode_slice,
1151 .decode_params = videotoolbox_hevc_decode_params,
1152 .end_frame = videotoolbox_hevc_end_frame,
1153 .frame_params = videotoolbox_frame_params,
1154 .init = videotoolbox_common_init,
1160 .
name =
"h264_videotoolbox",
1168 .end_frame = videotoolbox_h264_end_frame,
1169 .frame_params = videotoolbox_frame_params,
1170 .init = videotoolbox_common_init,
1176 .
name =
"mpeg1_videotoolbox",
1181 .start_frame = videotoolbox_mpeg_start_frame,
1182 .decode_slice = videotoolbox_mpeg_decode_slice,
1183 .end_frame = videotoolbox_mpeg_end_frame,
1184 .frame_params = videotoolbox_frame_params,
1185 .init = videotoolbox_common_init,
1191 .
name =
"mpeg2_videotoolbox",
1196 .start_frame = videotoolbox_mpeg_start_frame,
1197 .decode_slice = videotoolbox_mpeg_decode_slice,
1198 .end_frame = videotoolbox_mpeg_end_frame,
1199 .frame_params = videotoolbox_frame_params,
1200 .init = videotoolbox_common_init,
1206 .
name =
"mpeg4_videotoolbox",
1211 .start_frame = videotoolbox_mpeg_start_frame,
1212 .decode_slice = videotoolbox_mpeg_decode_slice,
1213 .end_frame = videotoolbox_mpeg_end_frame,
1214 .frame_params = videotoolbox_frame_params,
1215 .init = videotoolbox_common_init,
1229 if (cv_pix_fmt_type == 0) {
1230 cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1240 return av_videotoolbox_alloc_context_with_pix_fmt(
AV_PIX_FMT_NONE,
false);
1255 return videotoolbox_start(avctx);
1261 videotoolbox_stop(avctx);
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Convenience header that includes libavutil's core.
#define bytestream2_put_ne16
static av_always_inline void bytestream2_init_writer(PutByteContext *p, uint8_t *buf, int buf_size)
static av_always_inline int bytestream2_size_p(PutByteContext *p)
#define bytestream2_put_ne32
#define bytestream2_put_ne24
static av_always_inline unsigned int bytestream2_put_buffer(PutByteContext *p, const uint8_t *src, unsigned int size)
#define flags(name, subs,...)
static int FUNC() pps(CodedBitstreamContext *ctx, RWContext *rw, H264RawPPS *current)
static int FUNC() sps(CodedBitstreamContext *ctx, RWContext *rw, H264RawSPS *current)
static int FUNC() vps(CodedBitstreamContext *ctx, RWContext *rw, H265RawVPS *current)
int ff_attach_decode_data(AVFrame *frame)
static enum AVPixelFormat pix_fmt
@ AV_CODEC_ID_MPEG2VIDEO
preferred ID for MPEG-1/2 video decoding
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
AVBufferRef * av_buffer_create(uint8_t *data, buffer_size_t size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
AVBufferRef * av_buffer_ref(AVBufferRef *buf)
Create a new reference to an AVBuffer.
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
#define AVERROR_EXTERNAL
Generic error in an external library.
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
void av_frame_unref(AVFrame *frame)
Unreference all the buffers referenced by frame and reset the frame fields.
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
#define AV_LOG_VERBOSE
Detailed information.
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
void * av_fast_realloc(void *ptr, unsigned int *size, size_t min_size)
Reallocate the given buffer if it is not large enough, otherwise do nothing.
H.264 / AVC / MPEG-4 part10 codec.
const AVHWAccel ff_h263_videotoolbox_hwaccel
const AVHWAccel ff_mpeg4_videotoolbox_hwaccel
const AVHWAccel ff_hevc_videotoolbox_hwaccel
const AVHWAccel ff_mpeg2_videotoolbox_hwaccel
const AVHWAccel ff_mpeg1_videotoolbox_hwaccel
const AVHWAccel ff_h264_videotoolbox_hwaccel
int av_hwframe_ctx_init(AVBufferRef *ref)
Finalize the context before use.
AVBufferRef * av_hwframe_ctx_alloc(AVBufferRef *device_ref_in)
Allocate an AVHWFramesContext tied to a given device context.
enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt)
Convert a VideoToolbox (actually CoreVideo) format to AVPixelFormat.
uint32_t av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, bool full_range)
Same as av_map_videotoolbox_format_from_pixfmt function, but can map and return full range pixel form...
An API-specific header for AV_HWDEVICE_TYPE_VIDEOTOOLBOX.
const char * av_color_range_name(enum AVColorRange range)
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
@ AVCOL_RANGE_JPEG
Full range content.
AVPixelFormat
Pixel format.
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
@ AV_PIX_FMT_VIDEOTOOLBOX
hardware decoding through Videotoolbox
enum AVMediaType codec_type
A reference to a data buffer.
uint8_t * data
The data buffer.
main external API structure.
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
int width
picture width / height.
enum AVColorRange color_range
MPEG vs JPEG YUV range.
AVBufferRef * hw_frames_ctx
A reference to the AVHWFramesContext describing the input (for encoding) or output (decoding) frames.
void * hwaccel_context
Hardware accelerator context.
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
AVBufferRef * hw_device_ctx
A reference to the AVHWDeviceContext describing the device which will be used by a hardware encoder/d...
int coded_width
Bitstream width / height, may be different from width/height e.g.
struct AVCodecInternal * internal
Private context used for internal data.
void * hwaccel_priv_data
hwaccel-specific private data
int depth
Number of bits in the component.
This structure describes decoded (raw) audio or video data.
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
AVBufferRef * hw_frames_ctx
For hwaccel-format frames, this should be a reference to the AVHWFramesContext describing the frame.
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
AVBufferRef * private_ref
AVBufferRef for internal use by a single libav* library.
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
const char * name
Name of the hardware accelerated codec.
This struct describes a set or pool of "hardware" frames (i.e.
enum AVPixelFormat format
The pixel format identifying the underlying HW surface type.
AVBufferRef * device_ref
A reference to the parent AVHWDeviceContext.
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
int width
The allocated dimensions of the frames in this pool.
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
AVComponentDescriptor comp[4]
Parameters that describe how pixels are packed.
This struct holds all the information that needs to be passed between the caller and libavcodec for i...
int cm_codec_type
CoreMedia codec type that Videotoolbox will use to create the decompression session.
CMVideoFormatDescriptionRef cm_fmt_desc
CoreMedia Format Description that Videotoolbox will use to create the decompression session.
VTDecompressionOutputCallback output_callback
The output callback that must be passed to the session.
OSType cv_pix_fmt_type
CVPixelBuffer Format Type that Videotoolbox will use for decoded frames.
VTDecompressionSessionRef session
Videotoolbox decompression session object.
This struct stores per-frame lavc-internal data and is attached to it via private_ref.
int(* post_process)(void *logctx, AVFrame *frame)
The callback to perform some delayed processing on the frame right before it is returned to the calle...
uint8_t frame_only_constraint_flag
uint8_t progressive_source_flag
uint8_t interlaced_source_flag
uint8_t profile_compatibility_flag[32]
uint8_t non_packed_constraint_flag
struct AVBufferRef * cached_hw_frames_ctx
struct AVVideotoolboxContext * vt_ctx
AVBufferRef * hw_frames_ctx
int min_spatial_segmentation_idc
static int ref[MAX_W *MAX_W]