FFmpeg  4.4.5
videotoolboxenc.c
Go to the documentation of this file.
1 /*
2  * copyright (c) 2015 Rick Kern <kernrj@gmail.com>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <VideoToolbox/VideoToolbox.h>
22 #include <CoreVideo/CoreVideo.h>
23 #include <CoreMedia/CoreMedia.h>
24 #include <TargetConditionals.h>
25 #include <Availability.h>
26 #include "avcodec.h"
27 #include "libavutil/opt.h"
28 #include "libavutil/avassert.h"
29 #include "libavutil/avstring.h"
30 #include "libavcodec/avcodec.h"
31 #include "libavutil/pixdesc.h"
32 #include "internal.h"
33 #include <pthread.h>
34 #include "atsc_a53.h"
35 #include "h264.h"
36 #include "h264_sei.h"
37 #include <dlfcn.h>
38 
39 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
40 enum { kCMVideoCodecType_HEVC = 'hvc1' };
41 #endif
42 
43 #if !HAVE_KCMVIDEOCODECTYPE_HEVCWITHALPHA
45 #endif
46 
47 #if !HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
50 #endif
51 
52 #ifndef TARGET_CPU_ARM64
53 # define TARGET_CPU_ARM64 0
54 #endif
55 
56 typedef OSStatus (*getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc,
57  size_t parameterSetIndex,
58  const uint8_t **parameterSetPointerOut,
59  size_t *parameterSetSizeOut,
60  size_t *parameterSetCountOut,
61  int *NALUnitHeaderLengthOut);
62 
63 //These symbols may not be present
64 static struct{
68 
72 
94 
97 
100 
103 
106 
107 #define GET_SYM(symbol, defaultVal) \
108 do{ \
109  CFStringRef* handle = (CFStringRef*)dlsym(RTLD_DEFAULT, #symbol); \
110  if(!handle) \
111  compat_keys.symbol = CFSTR(defaultVal); \
112  else \
113  compat_keys.symbol = *handle; \
114 }while(0)
115 
117 
118 static void loadVTEncSymbols(){
119  compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex =
120  (getParameterSetAtIndex)dlsym(
121  RTLD_DEFAULT,
122  "CMVideoFormatDescriptionGetHEVCParameterSetAtIndex"
123  );
124 
128 
132 
133  GET_SYM(kVTProfileLevel_H264_Baseline_4_0, "H264_Baseline_4_0");
134  GET_SYM(kVTProfileLevel_H264_Baseline_4_2, "H264_Baseline_4_2");
135  GET_SYM(kVTProfileLevel_H264_Baseline_5_0, "H264_Baseline_5_0");
136  GET_SYM(kVTProfileLevel_H264_Baseline_5_1, "H264_Baseline_5_1");
137  GET_SYM(kVTProfileLevel_H264_Baseline_5_2, "H264_Baseline_5_2");
138  GET_SYM(kVTProfileLevel_H264_Baseline_AutoLevel, "H264_Baseline_AutoLevel");
139  GET_SYM(kVTProfileLevel_H264_Main_4_2, "H264_Main_4_2");
140  GET_SYM(kVTProfileLevel_H264_Main_5_1, "H264_Main_5_1");
141  GET_SYM(kVTProfileLevel_H264_Main_5_2, "H264_Main_5_2");
142  GET_SYM(kVTProfileLevel_H264_Main_AutoLevel, "H264_Main_AutoLevel");
143  GET_SYM(kVTProfileLevel_H264_High_3_0, "H264_High_3_0");
144  GET_SYM(kVTProfileLevel_H264_High_3_1, "H264_High_3_1");
145  GET_SYM(kVTProfileLevel_H264_High_3_2, "H264_High_3_2");
146  GET_SYM(kVTProfileLevel_H264_High_4_0, "H264_High_4_0");
147  GET_SYM(kVTProfileLevel_H264_High_4_1, "H264_High_4_1");
148  GET_SYM(kVTProfileLevel_H264_High_4_2, "H264_High_4_2");
149  GET_SYM(kVTProfileLevel_H264_High_5_1, "H264_High_5_1");
150  GET_SYM(kVTProfileLevel_H264_High_5_2, "H264_High_5_2");
151  GET_SYM(kVTProfileLevel_H264_High_AutoLevel, "H264_High_AutoLevel");
152  GET_SYM(kVTProfileLevel_H264_Extended_5_0, "H264_Extended_5_0");
153  GET_SYM(kVTProfileLevel_H264_Extended_AutoLevel, "H264_Extended_AutoLevel");
154 
155  GET_SYM(kVTProfileLevel_HEVC_Main_AutoLevel, "HEVC_Main_AutoLevel");
156  GET_SYM(kVTProfileLevel_HEVC_Main10_AutoLevel, "HEVC_Main10_AutoLevel");
157 
160  "TargetQualityForAlpha");
161 
163  "EnableHardwareAcceleratedVideoEncoder");
165  "RequireHardwareAcceleratedVideoEncoder");
166 }
167 
168 typedef enum VT_H264Profile {
176 
177 typedef enum VTH264Entropy{
180  VT_CABAC
182 
183 typedef enum VT_HEVCProfile {
189 
190 static const uint8_t start_code[] = { 0, 0, 0, 1 };
191 
192 typedef struct ExtraSEI {
193  void *data;
194  size_t size;
195 } ExtraSEI;
196 
197 typedef struct BufNode {
198  CMSampleBufferRef cm_buffer;
200  struct BufNode* next;
201  int error;
202 } BufNode;
203 
204 typedef struct VTEncContext {
205  AVClass *class;
206  enum AVCodecID codec_id;
207  VTCompressionSessionRef session;
208  CFStringRef ycbcr_matrix;
209  CFStringRef color_primaries;
210  CFStringRef transfer_function;
212 
215 
217 
220 
223 
226 
233 
237 
238  bool flushing;
241 
242  /* can't be bool type since AVOption will access it as int */
243  int a53_cc;
244 } VTEncContext;
245 
246 static int vtenc_populate_extradata(AVCodecContext *avctx,
247  CMVideoCodecType codec_type,
248  CFStringRef profile_level,
249  CFNumberRef gamma_level,
250  CFDictionaryRef enc_info,
251  CFDictionaryRef pixel_buffer_info);
252 
253 /**
254  * NULL-safe release of *refPtr, and sets value to NULL.
255  */
256 static void vt_release_num(CFNumberRef* refPtr){
257  if (!*refPtr) {
258  return;
259  }
260 
261  CFRelease(*refPtr);
262  *refPtr = NULL;
263 }
264 
265 static void set_async_error(VTEncContext *vtctx, int err)
266 {
267  BufNode *info;
268 
269  pthread_mutex_lock(&vtctx->lock);
270 
271  vtctx->async_error = err;
272 
273  info = vtctx->q_head;
274  vtctx->q_head = vtctx->q_tail = NULL;
275 
276  while (info) {
277  BufNode *next = info->next;
278  CFRelease(info->cm_buffer);
279  av_free(info);
280  info = next;
281  }
282 
283  pthread_mutex_unlock(&vtctx->lock);
284 }
285 
286 static void clear_frame_queue(VTEncContext *vtctx)
287 {
288  set_async_error(vtctx, 0);
289 }
290 
291 static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf, ExtraSEI **sei)
292 {
293  BufNode *info;
294 
295  pthread_mutex_lock(&vtctx->lock);
296 
297  if (vtctx->async_error) {
298  pthread_mutex_unlock(&vtctx->lock);
299  return vtctx->async_error;
300  }
301 
302  if (vtctx->flushing && vtctx->frame_ct_in == vtctx->frame_ct_out) {
303  *buf = NULL;
304 
305  pthread_mutex_unlock(&vtctx->lock);
306  return 0;
307  }
308 
309  while (!vtctx->q_head && !vtctx->async_error && wait && !vtctx->flushing) {
310  pthread_cond_wait(&vtctx->cv_sample_sent, &vtctx->lock);
311  }
312 
313  if (!vtctx->q_head) {
314  pthread_mutex_unlock(&vtctx->lock);
315  *buf = NULL;
316  return 0;
317  }
318 
319  info = vtctx->q_head;
320  vtctx->q_head = vtctx->q_head->next;
321  if (!vtctx->q_head) {
322  vtctx->q_tail = NULL;
323  }
324 
325  vtctx->frame_ct_out++;
326  pthread_mutex_unlock(&vtctx->lock);
327 
328  *buf = info->cm_buffer;
329  if (sei && *buf) {
330  *sei = info->sei;
331  } else if (info->sei) {
332  if (info->sei->data) av_free(info->sei->data);
333  av_free(info->sei);
334  }
335  av_free(info);
336 
337 
338  return 0;
339 }
340 
341 static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer, ExtraSEI *sei)
342 {
343  BufNode *info = av_malloc(sizeof(BufNode));
344  if (!info) {
345  set_async_error(vtctx, AVERROR(ENOMEM));
346  return;
347  }
348 
349  CFRetain(buffer);
350  info->cm_buffer = buffer;
351  info->sei = sei;
352  info->next = NULL;
353 
354  pthread_mutex_lock(&vtctx->lock);
355 
356  if (!vtctx->q_head) {
357  vtctx->q_head = info;
358  } else {
359  vtctx->q_tail->next = info;
360  }
361 
362  vtctx->q_tail = info;
363 
365  pthread_mutex_unlock(&vtctx->lock);
366 }
367 
368 static int count_nalus(size_t length_code_size,
369  CMSampleBufferRef sample_buffer,
370  int *count)
371 {
372  size_t offset = 0;
373  int status;
374  int nalu_ct = 0;
375  uint8_t size_buf[4];
376  size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
377  CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
378 
379  if (length_code_size > 4)
380  return AVERROR_INVALIDDATA;
381 
382  while (offset < src_size) {
383  size_t curr_src_len;
384  size_t box_len = 0;
385  size_t i;
386 
387  status = CMBlockBufferCopyDataBytes(block,
388  offset,
389  length_code_size,
390  size_buf);
391 
392  for (i = 0; i < length_code_size; i++) {
393  box_len <<= 8;
394  box_len |= size_buf[i];
395  }
396 
397  curr_src_len = box_len + length_code_size;
398  offset += curr_src_len;
399 
400  nalu_ct++;
401  }
402 
403  *count = nalu_ct;
404  return 0;
405 }
406 
407 static CMVideoCodecType get_cm_codec_type(enum AVCodecID id,
408  enum AVPixelFormat fmt,
409  double alpha_quality)
410 {
411  switch (id) {
412  case AV_CODEC_ID_H264: return kCMVideoCodecType_H264;
413  case AV_CODEC_ID_HEVC:
414  if (fmt == AV_PIX_FMT_BGRA && alpha_quality > 0.0) {
416  }
417  return kCMVideoCodecType_HEVC;
418  default: return 0;
419  }
420 }
421 
422 /**
423  * Get the parameter sets from a CMSampleBufferRef.
424  * @param dst If *dst isn't NULL, the parameters are copied into existing
425  * memory. *dst_size must be set accordingly when *dst != NULL.
426  * If *dst is NULL, it will be allocated.
427  * In all cases, *dst_size is set to the number of bytes used starting
428  * at *dst.
429  */
430 static int get_params_size(
431  AVCodecContext *avctx,
432  CMVideoFormatDescriptionRef vid_fmt,
433  size_t *size)
434 {
435  VTEncContext *vtctx = avctx->priv_data;
436  size_t total_size = 0;
437  size_t ps_count;
438  int is_count_bad = 0;
439  size_t i;
440  int status;
441  status = vtctx->get_param_set_func(vid_fmt,
442  0,
443  NULL,
444  NULL,
445  &ps_count,
446  NULL);
447  if (status) {
448  is_count_bad = 1;
449  ps_count = 0;
450  status = 0;
451  }
452 
453  for (i = 0; i < ps_count || is_count_bad; i++) {
454  const uint8_t *ps;
455  size_t ps_size;
456  status = vtctx->get_param_set_func(vid_fmt,
457  i,
458  &ps,
459  &ps_size,
460  NULL,
461  NULL);
462  if (status) {
463  /*
464  * When ps_count is invalid, status != 0 ends the loop normally
465  * unless we didn't get any parameter sets.
466  */
467  if (i > 0 && is_count_bad) status = 0;
468 
469  break;
470  }
471 
472  total_size += ps_size + sizeof(start_code);
473  }
474 
475  if (status) {
476  av_log(avctx, AV_LOG_ERROR, "Error getting parameter set sizes: %d\n", status);
477  return AVERROR_EXTERNAL;
478  }
479 
480  *size = total_size;
481  return 0;
482 }
483 
484 static int copy_param_sets(
485  AVCodecContext *avctx,
486  CMVideoFormatDescriptionRef vid_fmt,
487  uint8_t *dst,
488  size_t dst_size)
489 {
490  VTEncContext *vtctx = avctx->priv_data;
491  size_t ps_count;
492  int is_count_bad = 0;
493  int status;
494  size_t offset = 0;
495  size_t i;
496 
497  status = vtctx->get_param_set_func(vid_fmt,
498  0,
499  NULL,
500  NULL,
501  &ps_count,
502  NULL);
503  if (status) {
504  is_count_bad = 1;
505  ps_count = 0;
506  status = 0;
507  }
508 
509 
510  for (i = 0; i < ps_count || is_count_bad; i++) {
511  const uint8_t *ps;
512  size_t ps_size;
513  size_t next_offset;
514 
515  status = vtctx->get_param_set_func(vid_fmt,
516  i,
517  &ps,
518  &ps_size,
519  NULL,
520  NULL);
521  if (status) {
522  if (i > 0 && is_count_bad) status = 0;
523 
524  break;
525  }
526 
527  next_offset = offset + sizeof(start_code) + ps_size;
528  if (dst_size < next_offset) {
529  av_log(avctx, AV_LOG_ERROR, "Error: buffer too small for parameter sets.\n");
531  }
532 
533  memcpy(dst + offset, start_code, sizeof(start_code));
534  offset += sizeof(start_code);
535 
536  memcpy(dst + offset, ps, ps_size);
537  offset = next_offset;
538  }
539 
540  if (status) {
541  av_log(avctx, AV_LOG_ERROR, "Error getting parameter set data: %d\n", status);
542  return AVERROR_EXTERNAL;
543  }
544 
545  return 0;
546 }
547 
548 static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer)
549 {
550  CMVideoFormatDescriptionRef vid_fmt;
551  size_t total_size;
552  int status;
553 
554  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
555  if (!vid_fmt) {
556  av_log(avctx, AV_LOG_ERROR, "No video format.\n");
557  return AVERROR_EXTERNAL;
558  }
559 
560  status = get_params_size(avctx, vid_fmt, &total_size);
561  if (status) {
562  av_log(avctx, AV_LOG_ERROR, "Could not get parameter sets.\n");
563  return status;
564  }
565 
566  avctx->extradata = av_mallocz(total_size + AV_INPUT_BUFFER_PADDING_SIZE);
567  if (!avctx->extradata) {
568  return AVERROR(ENOMEM);
569  }
570  avctx->extradata_size = total_size;
571 
572  status = copy_param_sets(avctx, vid_fmt, avctx->extradata, total_size);
573 
574  if (status) {
575  av_log(avctx, AV_LOG_ERROR, "Could not copy param sets.\n");
576  return status;
577  }
578 
579  return 0;
580 }
581 
583  void *ctx,
584  void *sourceFrameCtx,
585  OSStatus status,
586  VTEncodeInfoFlags flags,
587  CMSampleBufferRef sample_buffer)
588 {
589  AVCodecContext *avctx = ctx;
590  VTEncContext *vtctx = avctx->priv_data;
591  ExtraSEI *sei = sourceFrameCtx;
592 
593  if (vtctx->async_error) {
594  return;
595  }
596 
597  if (status) {
598  av_log(avctx, AV_LOG_ERROR, "Error encoding frame: %d\n", (int)status);
600  return;
601  }
602 
603  if (!sample_buffer) {
604  return;
605  }
606 
607  if (!avctx->extradata && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)) {
608  int set_status = set_extradata(avctx, sample_buffer);
609  if (set_status) {
610  set_async_error(vtctx, set_status);
611  return;
612  }
613  }
614 
615  vtenc_q_push(vtctx, sample_buffer, sei);
616 }
617 
619  AVCodecContext *avctx,
620  CMSampleBufferRef sample_buffer,
621  size_t *size)
622 {
623  VTEncContext *vtctx = avctx->priv_data;
624  CMVideoFormatDescriptionRef vid_fmt;
625  int isize;
626  int status;
627 
628  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
629  if (!vid_fmt) {
630  av_log(avctx, AV_LOG_ERROR, "Error getting buffer format description.\n");
631  return AVERROR_EXTERNAL;
632  }
633 
634  status = vtctx->get_param_set_func(vid_fmt,
635  0,
636  NULL,
637  NULL,
638  NULL,
639  &isize);
640  if (status) {
641  av_log(avctx, AV_LOG_ERROR, "Error getting length code size: %d\n", status);
642  return AVERROR_EXTERNAL;
643  }
644 
645  *size = isize;
646  return 0;
647 }
648 
649 /*
650  * Returns true on success.
651  *
652  * If profile_level_val is NULL and this method returns true, don't specify the
653  * profile/level to the encoder.
654  */
656  CFStringRef *profile_level_val)
657 {
658  VTEncContext *vtctx = avctx->priv_data;
659  int64_t profile = vtctx->profile;
660 
661  if (profile == H264_PROF_AUTO && vtctx->level) {
662  //Need to pick a profile if level is not auto-selected.
664  }
665 
666  *profile_level_val = NULL;
667 
668  switch (profile) {
669  case H264_PROF_AUTO:
670  return true;
671 
672  case H264_PROF_BASELINE:
673  switch (vtctx->level) {
674  case 0: *profile_level_val =
675  compat_keys.kVTProfileLevel_H264_Baseline_AutoLevel; break;
676  case 13: *profile_level_val = kVTProfileLevel_H264_Baseline_1_3; break;
677  case 30: *profile_level_val = kVTProfileLevel_H264_Baseline_3_0; break;
678  case 31: *profile_level_val = kVTProfileLevel_H264_Baseline_3_1; break;
679  case 32: *profile_level_val = kVTProfileLevel_H264_Baseline_3_2; break;
680  case 40: *profile_level_val =
681  compat_keys.kVTProfileLevel_H264_Baseline_4_0; break;
682  case 41: *profile_level_val = kVTProfileLevel_H264_Baseline_4_1; break;
683  case 42: *profile_level_val =
684  compat_keys.kVTProfileLevel_H264_Baseline_4_2; break;
685  case 50: *profile_level_val =
686  compat_keys.kVTProfileLevel_H264_Baseline_5_0; break;
687  case 51: *profile_level_val =
688  compat_keys.kVTProfileLevel_H264_Baseline_5_1; break;
689  case 52: *profile_level_val =
690  compat_keys.kVTProfileLevel_H264_Baseline_5_2; break;
691  }
692  break;
693 
694  case H264_PROF_MAIN:
695  switch (vtctx->level) {
696  case 0: *profile_level_val =
697  compat_keys.kVTProfileLevel_H264_Main_AutoLevel; break;
698  case 30: *profile_level_val = kVTProfileLevel_H264_Main_3_0; break;
699  case 31: *profile_level_val = kVTProfileLevel_H264_Main_3_1; break;
700  case 32: *profile_level_val = kVTProfileLevel_H264_Main_3_2; break;
701  case 40: *profile_level_val = kVTProfileLevel_H264_Main_4_0; break;
702  case 41: *profile_level_val = kVTProfileLevel_H264_Main_4_1; break;
703  case 42: *profile_level_val =
704  compat_keys.kVTProfileLevel_H264_Main_4_2; break;
705  case 50: *profile_level_val = kVTProfileLevel_H264_Main_5_0; break;
706  case 51: *profile_level_val =
707  compat_keys.kVTProfileLevel_H264_Main_5_1; break;
708  case 52: *profile_level_val =
709  compat_keys.kVTProfileLevel_H264_Main_5_2; break;
710  }
711  break;
712 
713  case H264_PROF_HIGH:
714  switch (vtctx->level) {
715  case 0: *profile_level_val =
716  compat_keys.kVTProfileLevel_H264_High_AutoLevel; break;
717  case 30: *profile_level_val =
718  compat_keys.kVTProfileLevel_H264_High_3_0; break;
719  case 31: *profile_level_val =
720  compat_keys.kVTProfileLevel_H264_High_3_1; break;
721  case 32: *profile_level_val =
722  compat_keys.kVTProfileLevel_H264_High_3_2; break;
723  case 40: *profile_level_val =
724  compat_keys.kVTProfileLevel_H264_High_4_0; break;
725  case 41: *profile_level_val =
726  compat_keys.kVTProfileLevel_H264_High_4_1; break;
727  case 42: *profile_level_val =
728  compat_keys.kVTProfileLevel_H264_High_4_2; break;
729  case 50: *profile_level_val = kVTProfileLevel_H264_High_5_0; break;
730  case 51: *profile_level_val =
731  compat_keys.kVTProfileLevel_H264_High_5_1; break;
732  case 52: *profile_level_val =
733  compat_keys.kVTProfileLevel_H264_High_5_2; break;
734  }
735  break;
736  case H264_PROF_EXTENDED:
737  switch (vtctx->level) {
738  case 0: *profile_level_val =
739  compat_keys.kVTProfileLevel_H264_Extended_AutoLevel; break;
740  case 50: *profile_level_val =
741  compat_keys.kVTProfileLevel_H264_Extended_5_0; break;
742  }
743  break;
744  }
745 
746  if (!*profile_level_val) {
747  av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
748  return false;
749  }
750 
751  return true;
752 }
753 
754 /*
755  * Returns true on success.
756  *
757  * If profile_level_val is NULL and this method returns true, don't specify the
758  * profile/level to the encoder.
759  */
761  CFStringRef *profile_level_val)
762 {
763  VTEncContext *vtctx = avctx->priv_data;
764  int64_t profile = vtctx->profile;
765 
766  *profile_level_val = NULL;
767 
768  switch (profile) {
769  case HEVC_PROF_AUTO:
770  return true;
771  case HEVC_PROF_MAIN:
772  *profile_level_val =
773  compat_keys.kVTProfileLevel_HEVC_Main_AutoLevel;
774  break;
775  case HEVC_PROF_MAIN10:
776  *profile_level_val =
777  compat_keys.kVTProfileLevel_HEVC_Main10_AutoLevel;
778  break;
779  }
780 
781  if (!*profile_level_val) {
782  av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
783  return false;
784  }
785 
786  return true;
787 }
788 
790  enum AVPixelFormat fmt,
791  enum AVColorRange range,
792  int* av_pixel_format,
793  int* range_guessed)
794 {
795  if (range_guessed) *range_guessed = range != AVCOL_RANGE_MPEG &&
796  range != AVCOL_RANGE_JPEG;
797 
798  //MPEG range is used when no range is set
799  if (fmt == AV_PIX_FMT_NV12) {
800  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
801  kCVPixelFormatType_420YpCbCr8BiPlanarFullRange :
802  kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
803  } else if (fmt == AV_PIX_FMT_YUV420P) {
804  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
805  kCVPixelFormatType_420YpCbCr8PlanarFullRange :
806  kCVPixelFormatType_420YpCbCr8Planar;
807  } else if (fmt == AV_PIX_FMT_BGRA) {
808  *av_pixel_format = kCVPixelFormatType_32BGRA;
809  } else if (fmt == AV_PIX_FMT_P010LE) {
810  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
813  } else {
814  return AVERROR(EINVAL);
815  }
816 
817  return 0;
818 }
819 
820 static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict) {
821  VTEncContext *vtctx = avctx->priv_data;
822 
823  if (vtctx->color_primaries) {
824  CFDictionarySetValue(dict,
825  kCVImageBufferColorPrimariesKey,
826  vtctx->color_primaries);
827  }
828 
829  if (vtctx->transfer_function) {
830  CFDictionarySetValue(dict,
831  kCVImageBufferTransferFunctionKey,
832  vtctx->transfer_function);
833  }
834 
835  if (vtctx->ycbcr_matrix) {
836  CFDictionarySetValue(dict,
837  kCVImageBufferYCbCrMatrixKey,
838  vtctx->ycbcr_matrix);
839  }
840 }
841 
843  CFMutableDictionaryRef* dict)
844 {
845  CFNumberRef cv_color_format_num = NULL;
846  CFNumberRef width_num = NULL;
847  CFNumberRef height_num = NULL;
848  CFMutableDictionaryRef pixel_buffer_info = NULL;
849  int cv_color_format;
850  int status = get_cv_pixel_format(avctx,
851  avctx->pix_fmt,
852  avctx->color_range,
853  &cv_color_format,
854  NULL);
855  if (status) return status;
856 
857  pixel_buffer_info = CFDictionaryCreateMutable(
858  kCFAllocatorDefault,
859  20,
860  &kCFCopyStringDictionaryKeyCallBacks,
861  &kCFTypeDictionaryValueCallBacks);
862 
863  if (!pixel_buffer_info) goto pbinfo_nomem;
864 
865  cv_color_format_num = CFNumberCreate(kCFAllocatorDefault,
866  kCFNumberSInt32Type,
867  &cv_color_format);
868  if (!cv_color_format_num) goto pbinfo_nomem;
869 
870  CFDictionarySetValue(pixel_buffer_info,
871  kCVPixelBufferPixelFormatTypeKey,
872  cv_color_format_num);
873  vt_release_num(&cv_color_format_num);
874 
875  width_num = CFNumberCreate(kCFAllocatorDefault,
876  kCFNumberSInt32Type,
877  &avctx->width);
878  if (!width_num) return AVERROR(ENOMEM);
879 
880  CFDictionarySetValue(pixel_buffer_info,
881  kCVPixelBufferWidthKey,
882  width_num);
883  vt_release_num(&width_num);
884 
885  height_num = CFNumberCreate(kCFAllocatorDefault,
886  kCFNumberSInt32Type,
887  &avctx->height);
888  if (!height_num) goto pbinfo_nomem;
889 
890  CFDictionarySetValue(pixel_buffer_info,
891  kCVPixelBufferHeightKey,
892  height_num);
893  vt_release_num(&height_num);
894 
895  add_color_attr(avctx, pixel_buffer_info);
896 
897  *dict = pixel_buffer_info;
898  return 0;
899 
900 pbinfo_nomem:
901  vt_release_num(&cv_color_format_num);
902  vt_release_num(&width_num);
903  vt_release_num(&height_num);
904  if (pixel_buffer_info) CFRelease(pixel_buffer_info);
905 
906  return AVERROR(ENOMEM);
907 }
908 
910  CFStringRef *primaries)
911 {
912  enum AVColorPrimaries pri = avctx->color_primaries;
913  switch (pri) {
915  *primaries = NULL;
916  break;
917 
918  case AVCOL_PRI_BT470BG:
919  *primaries = kCVImageBufferColorPrimaries_EBU_3213;
920  break;
921 
922  case AVCOL_PRI_SMPTE170M:
923  *primaries = kCVImageBufferColorPrimaries_SMPTE_C;
924  break;
925 
926  case AVCOL_PRI_BT709:
927  *primaries = kCVImageBufferColorPrimaries_ITU_R_709_2;
928  break;
929 
930  case AVCOL_PRI_BT2020:
931  *primaries = compat_keys.kCVImageBufferColorPrimaries_ITU_R_2020;
932  break;
933 
934  default:
935  av_log(avctx, AV_LOG_ERROR, "Color primaries %s is not supported.\n", av_color_primaries_name(pri));
936  *primaries = NULL;
937  return -1;
938  }
939 
940  return 0;
941 }
942 
944  CFStringRef *transfer_fnc,
945  CFNumberRef *gamma_level)
946 {
947  enum AVColorTransferCharacteristic trc = avctx->color_trc;
948  Float32 gamma;
949  *gamma_level = NULL;
950 
951  switch (trc) {
953  *transfer_fnc = NULL;
954  break;
955 
956  case AVCOL_TRC_BT709:
957  *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_709_2;
958  break;
959 
960  case AVCOL_TRC_SMPTE240M:
961  *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_240M_1995;
962  break;
963 
964 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_2084_PQ
965  case AVCOL_TRC_SMPTE2084:
966  *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
967  break;
968 #endif
969 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_LINEAR
970  case AVCOL_TRC_LINEAR:
971  *transfer_fnc = kCVImageBufferTransferFunction_Linear;
972  break;
973 #endif
974 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG
976  *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_2100_HLG;
977  break;
978 #endif
979 
980  case AVCOL_TRC_GAMMA22:
981  gamma = 2.2;
982  *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
983  *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
984  break;
985 
986  case AVCOL_TRC_GAMMA28:
987  gamma = 2.8;
988  *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
989  *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
990  break;
991 
992  case AVCOL_TRC_BT2020_10:
993  case AVCOL_TRC_BT2020_12:
994  *transfer_fnc = compat_keys.kCVImageBufferTransferFunction_ITU_R_2020;
995  break;
996 
997  default:
998  *transfer_fnc = NULL;
999  av_log(avctx, AV_LOG_ERROR, "Transfer function %s is not supported.\n", av_color_transfer_name(trc));
1000  return -1;
1001  }
1002 
1003  return 0;
1004 }
1005 
1006 static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix) {
1007  switch(avctx->colorspace) {
1008  case AVCOL_SPC_BT709:
1009  *matrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2;
1010  break;
1011 
1012  case AVCOL_SPC_UNSPECIFIED:
1013  *matrix = NULL;
1014  break;
1015 
1016  case AVCOL_SPC_BT470BG:
1017  case AVCOL_SPC_SMPTE170M:
1018  *matrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
1019  break;
1020 
1021  case AVCOL_SPC_SMPTE240M:
1022  *matrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
1023  break;
1024 
1025  case AVCOL_SPC_BT2020_NCL:
1026  *matrix = compat_keys.kCVImageBufferYCbCrMatrix_ITU_R_2020;
1027  break;
1028 
1029  default:
1030  av_log(avctx, AV_LOG_ERROR, "Color space %s is not supported.\n", av_color_space_name(avctx->colorspace));
1031  return -1;
1032  }
1033 
1034  return 0;
1035 }
1036 
1037 // constant quality only on Macs with Apple Silicon
1038 static bool vtenc_qscale_enabled(void)
1039 {
1040  return TARGET_OS_OSX && TARGET_CPU_ARM64;
1041 }
1042 
1044  CMVideoCodecType codec_type,
1045  CFStringRef profile_level,
1046  CFNumberRef gamma_level,
1047  CFDictionaryRef enc_info,
1048  CFDictionaryRef pixel_buffer_info,
1049  VTCompressionSessionRef *session)
1050 {
1051  VTEncContext *vtctx = avctx->priv_data;
1052  SInt32 bit_rate = avctx->bit_rate;
1053  SInt32 max_rate = avctx->rc_max_rate;
1054  Float32 quality = avctx->global_quality / FF_QP2LAMBDA;
1055  CFNumberRef bit_rate_num;
1056  CFNumberRef quality_num;
1057  CFNumberRef bytes_per_second;
1058  CFNumberRef one_second;
1059  CFArrayRef data_rate_limits;
1060  int64_t bytes_per_second_value = 0;
1061  int64_t one_second_value = 0;
1062  void *nums[2];
1063 
1064  int status = VTCompressionSessionCreate(kCFAllocatorDefault,
1065  avctx->width,
1066  avctx->height,
1067  codec_type,
1068  enc_info,
1069  pixel_buffer_info,
1070  kCFAllocatorDefault,
1072  avctx,
1073  session);
1074 
1075  if (status || !vtctx->session) {
1076  av_log(avctx, AV_LOG_ERROR, "Error: cannot create compression session: %d\n", status);
1077 
1078 #if !TARGET_OS_IPHONE
1079  if (!vtctx->allow_sw) {
1080  av_log(avctx, AV_LOG_ERROR, "Try -allow_sw 1. The hardware encoder may be busy, or not supported.\n");
1081  }
1082 #endif
1083 
1084  return AVERROR_EXTERNAL;
1085  }
1086 
1087  if (avctx->flags & AV_CODEC_FLAG_QSCALE && !vtenc_qscale_enabled()) {
1088  av_log(avctx, AV_LOG_ERROR, "Error: -q:v qscale not available for encoder. Use -b:v bitrate instead.\n");
1089  return AVERROR_EXTERNAL;
1090  }
1091 
1092  if (avctx->flags & AV_CODEC_FLAG_QSCALE) {
1093  quality = quality >= 100 ? 1.0 : quality / 100;
1094  quality_num = CFNumberCreate(kCFAllocatorDefault,
1095  kCFNumberFloat32Type,
1096  &quality);
1097  if (!quality_num) return AVERROR(ENOMEM);
1098 
1099  status = VTSessionSetProperty(vtctx->session,
1100  kVTCompressionPropertyKey_Quality,
1101  quality_num);
1102  CFRelease(quality_num);
1103  } else {
1104  bit_rate_num = CFNumberCreate(kCFAllocatorDefault,
1105  kCFNumberSInt32Type,
1106  &bit_rate);
1107  if (!bit_rate_num) return AVERROR(ENOMEM);
1108 
1109  status = VTSessionSetProperty(vtctx->session,
1110  kVTCompressionPropertyKey_AverageBitRate,
1111  bit_rate_num);
1112  CFRelease(bit_rate_num);
1113  }
1114 
1115  if (status) {
1116  av_log(avctx, AV_LOG_ERROR, "Error setting bitrate property: %d\n", status);
1117  return AVERROR_EXTERNAL;
1118  }
1119 
1120  if (vtctx->codec_id == AV_CODEC_ID_H264 && max_rate > 0) {
1121  // kVTCompressionPropertyKey_DataRateLimits is not available for HEVC
1122  bytes_per_second_value = max_rate >> 3;
1123  bytes_per_second = CFNumberCreate(kCFAllocatorDefault,
1124  kCFNumberSInt64Type,
1125  &bytes_per_second_value);
1126  if (!bytes_per_second) {
1127  return AVERROR(ENOMEM);
1128  }
1129  one_second_value = 1;
1130  one_second = CFNumberCreate(kCFAllocatorDefault,
1131  kCFNumberSInt64Type,
1132  &one_second_value);
1133  if (!one_second) {
1134  CFRelease(bytes_per_second);
1135  return AVERROR(ENOMEM);
1136  }
1137  nums[0] = (void *)bytes_per_second;
1138  nums[1] = (void *)one_second;
1139  data_rate_limits = CFArrayCreate(kCFAllocatorDefault,
1140  (const void **)nums,
1141  2,
1142  &kCFTypeArrayCallBacks);
1143 
1144  if (!data_rate_limits) {
1145  CFRelease(bytes_per_second);
1146  CFRelease(one_second);
1147  return AVERROR(ENOMEM);
1148  }
1149  status = VTSessionSetProperty(vtctx->session,
1150  kVTCompressionPropertyKey_DataRateLimits,
1151  data_rate_limits);
1152 
1153  CFRelease(bytes_per_second);
1154  CFRelease(one_second);
1155  CFRelease(data_rate_limits);
1156 
1157  if (status) {
1158  av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status);
1159  return AVERROR_EXTERNAL;
1160  }
1161  }
1162 
1163  if (vtctx->codec_id == AV_CODEC_ID_HEVC) {
1164  if (avctx->pix_fmt == AV_PIX_FMT_BGRA && vtctx->alpha_quality > 0.0) {
1165  CFNumberRef alpha_quality_num = CFNumberCreate(kCFAllocatorDefault,
1166  kCFNumberDoubleType,
1167  &vtctx->alpha_quality);
1168  if (!alpha_quality_num) return AVERROR(ENOMEM);
1169 
1170  status = VTSessionSetProperty(vtctx->session,
1171  compat_keys.kVTCompressionPropertyKey_TargetQualityForAlpha,
1172  alpha_quality_num);
1173  CFRelease(alpha_quality_num);
1174  }
1175  }
1176 
1177  if (profile_level) {
1178  status = VTSessionSetProperty(vtctx->session,
1179  kVTCompressionPropertyKey_ProfileLevel,
1180  profile_level);
1181  if (status) {
1182  av_log(avctx, AV_LOG_ERROR, "Error setting profile/level property: %d. Output will be encoded using a supported profile/level combination.\n", status);
1183  }
1184  }
1185 
1186  if (avctx->gop_size > 0) {
1187  CFNumberRef interval = CFNumberCreate(kCFAllocatorDefault,
1188  kCFNumberIntType,
1189  &avctx->gop_size);
1190  if (!interval) {
1191  return AVERROR(ENOMEM);
1192  }
1193 
1194  status = VTSessionSetProperty(vtctx->session,
1195  kVTCompressionPropertyKey_MaxKeyFrameInterval,
1196  interval);
1197  CFRelease(interval);
1198 
1199  if (status) {
1200  av_log(avctx, AV_LOG_ERROR, "Error setting 'max key-frame interval' property: %d\n", status);
1201  return AVERROR_EXTERNAL;
1202  }
1203  }
1204 
1205  if (vtctx->frames_before) {
1206  status = VTSessionSetProperty(vtctx->session,
1207  kVTCompressionPropertyKey_MoreFramesBeforeStart,
1208  kCFBooleanTrue);
1209 
1210  if (status == kVTPropertyNotSupportedErr) {
1211  av_log(avctx, AV_LOG_WARNING, "frames_before property is not supported on this device. Ignoring.\n");
1212  } else if (status) {
1213  av_log(avctx, AV_LOG_ERROR, "Error setting frames_before property: %d\n", status);
1214  }
1215  }
1216 
1217  if (vtctx->frames_after) {
1218  status = VTSessionSetProperty(vtctx->session,
1219  kVTCompressionPropertyKey_MoreFramesAfterEnd,
1220  kCFBooleanTrue);
1221 
1222  if (status == kVTPropertyNotSupportedErr) {
1223  av_log(avctx, AV_LOG_WARNING, "frames_after property is not supported on this device. Ignoring.\n");
1224  } else if (status) {
1225  av_log(avctx, AV_LOG_ERROR, "Error setting frames_after property: %d\n", status);
1226  }
1227  }
1228 
1229  if (avctx->sample_aspect_ratio.num != 0) {
1230  CFNumberRef num;
1231  CFNumberRef den;
1232  CFMutableDictionaryRef par;
1233  AVRational *avpar = &avctx->sample_aspect_ratio;
1234 
1235  av_reduce(&avpar->num, &avpar->den,
1236  avpar->num, avpar->den,
1237  0xFFFFFFFF);
1238 
1239  num = CFNumberCreate(kCFAllocatorDefault,
1240  kCFNumberIntType,
1241  &avpar->num);
1242 
1243  den = CFNumberCreate(kCFAllocatorDefault,
1244  kCFNumberIntType,
1245  &avpar->den);
1246 
1247 
1248 
1249  par = CFDictionaryCreateMutable(kCFAllocatorDefault,
1250  2,
1251  &kCFCopyStringDictionaryKeyCallBacks,
1252  &kCFTypeDictionaryValueCallBacks);
1253 
1254  if (!par || !num || !den) {
1255  if (par) CFRelease(par);
1256  if (num) CFRelease(num);
1257  if (den) CFRelease(den);
1258 
1259  return AVERROR(ENOMEM);
1260  }
1261 
1262  CFDictionarySetValue(
1263  par,
1264  kCMFormatDescriptionKey_PixelAspectRatioHorizontalSpacing,
1265  num);
1266 
1267  CFDictionarySetValue(
1268  par,
1269  kCMFormatDescriptionKey_PixelAspectRatioVerticalSpacing,
1270  den);
1271 
1272  status = VTSessionSetProperty(vtctx->session,
1273  kVTCompressionPropertyKey_PixelAspectRatio,
1274  par);
1275 
1276  CFRelease(par);
1277  CFRelease(num);
1278  CFRelease(den);
1279 
1280  if (status) {
1281  av_log(avctx,
1282  AV_LOG_ERROR,
1283  "Error setting pixel aspect ratio to %d:%d: %d.\n",
1284  avctx->sample_aspect_ratio.num,
1285  avctx->sample_aspect_ratio.den,
1286  status);
1287 
1288  return AVERROR_EXTERNAL;
1289  }
1290  }
1291 
1292 
1293  if (vtctx->transfer_function) {
1294  status = VTSessionSetProperty(vtctx->session,
1295  kVTCompressionPropertyKey_TransferFunction,
1296  vtctx->transfer_function);
1297 
1298  if (status) {
1299  av_log(avctx, AV_LOG_WARNING, "Could not set transfer function: %d\n", status);
1300  }
1301  }
1302 
1303 
1304  if (vtctx->ycbcr_matrix) {
1305  status = VTSessionSetProperty(vtctx->session,
1306  kVTCompressionPropertyKey_YCbCrMatrix,
1307  vtctx->ycbcr_matrix);
1308 
1309  if (status) {
1310  av_log(avctx, AV_LOG_WARNING, "Could not set ycbcr matrix: %d\n", status);
1311  }
1312  }
1313 
1314 
1315  if (vtctx->color_primaries) {
1316  status = VTSessionSetProperty(vtctx->session,
1317  kVTCompressionPropertyKey_ColorPrimaries,
1318  vtctx->color_primaries);
1319 
1320  if (status) {
1321  av_log(avctx, AV_LOG_WARNING, "Could not set color primaries: %d\n", status);
1322  }
1323  }
1324 
1325  if (gamma_level) {
1326  status = VTSessionSetProperty(vtctx->session,
1327  kCVImageBufferGammaLevelKey,
1328  gamma_level);
1329 
1330  if (status) {
1331  av_log(avctx, AV_LOG_WARNING, "Could not set gamma level: %d\n", status);
1332  }
1333  }
1334 
1335  if (!vtctx->has_b_frames) {
1336  status = VTSessionSetProperty(vtctx->session,
1337  kVTCompressionPropertyKey_AllowFrameReordering,
1338  kCFBooleanFalse);
1339 
1340  if (status) {
1341  av_log(avctx, AV_LOG_ERROR, "Error setting 'allow frame reordering' property: %d\n", status);
1342  return AVERROR_EXTERNAL;
1343  }
1344  }
1345 
1346  if (vtctx->entropy != VT_ENTROPY_NOT_SET) {
1347  CFStringRef entropy = vtctx->entropy == VT_CABAC ?
1348  compat_keys.kVTH264EntropyMode_CABAC:
1349  compat_keys.kVTH264EntropyMode_CAVLC;
1350 
1351  status = VTSessionSetProperty(vtctx->session,
1352  compat_keys.kVTCompressionPropertyKey_H264EntropyMode,
1353  entropy);
1354 
1355  if (status) {
1356  av_log(avctx, AV_LOG_ERROR, "Error setting entropy property: %d\n", status);
1357  }
1358  }
1359 
1360  if (vtctx->realtime) {
1361  status = VTSessionSetProperty(vtctx->session,
1362  compat_keys.kVTCompressionPropertyKey_RealTime,
1363  kCFBooleanTrue);
1364 
1365  if (status) {
1366  av_log(avctx, AV_LOG_ERROR, "Error setting realtime property: %d\n", status);
1367  }
1368  }
1369 
1370  status = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
1371  if (status) {
1372  av_log(avctx, AV_LOG_ERROR, "Error: cannot prepare encoder: %d\n", status);
1373  return AVERROR_EXTERNAL;
1374  }
1375 
1376  return 0;
1377 }
1378 
1380 {
1381  CFMutableDictionaryRef enc_info;
1382  CFMutableDictionaryRef pixel_buffer_info;
1383  CMVideoCodecType codec_type;
1384  VTEncContext *vtctx = avctx->priv_data;
1385  CFStringRef profile_level;
1386  CFNumberRef gamma_level = NULL;
1387  int status;
1388 
1389  codec_type = get_cm_codec_type(avctx->codec_id, avctx->pix_fmt, vtctx->alpha_quality);
1390  if (!codec_type) {
1391  av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n", avctx->codec_id);
1392  return AVERROR(EINVAL);
1393  }
1394 
1395  vtctx->codec_id = avctx->codec_id;
1396  avctx->max_b_frames = 16;
1397 
1398  if (vtctx->codec_id == AV_CODEC_ID_H264) {
1399  vtctx->get_param_set_func = CMVideoFormatDescriptionGetH264ParameterSetAtIndex;
1400 
1401  vtctx->has_b_frames = avctx->max_b_frames > 0;
1402  if(vtctx->has_b_frames && vtctx->profile == H264_PROF_BASELINE){
1403  av_log(avctx, AV_LOG_WARNING, "Cannot use B-frames with baseline profile. Output will not contain B-frames.\n");
1404  vtctx->has_b_frames = 0;
1405  }
1406 
1407  if (vtctx->entropy == VT_CABAC && vtctx->profile == H264_PROF_BASELINE) {
1408  av_log(avctx, AV_LOG_WARNING, "CABAC entropy requires 'main' or 'high' profile, but baseline was requested. Encode will not use CABAC entropy.\n");
1409  vtctx->entropy = VT_ENTROPY_NOT_SET;
1410  }
1411 
1412  if (!get_vt_h264_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1413  } else {
1414  vtctx->get_param_set_func = compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex;
1415  if (!vtctx->get_param_set_func) return AVERROR(EINVAL);
1416  if (!get_vt_hevc_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1417  // HEVC has b-byramid
1418  vtctx->has_b_frames = avctx->max_b_frames > 0 ? 2 : 0;
1419  }
1420 
1421  enc_info = CFDictionaryCreateMutable(
1422  kCFAllocatorDefault,
1423  20,
1424  &kCFCopyStringDictionaryKeyCallBacks,
1425  &kCFTypeDictionaryValueCallBacks
1426  );
1427 
1428  if (!enc_info) return AVERROR(ENOMEM);
1429 
1430 #if !TARGET_OS_IPHONE
1431  if(vtctx->require_sw) {
1432  CFDictionarySetValue(enc_info,
1433  compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
1434  kCFBooleanFalse);
1435  } else if (!vtctx->allow_sw) {
1436  CFDictionarySetValue(enc_info,
1437  compat_keys.kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
1438  kCFBooleanTrue);
1439  } else {
1440  CFDictionarySetValue(enc_info,
1441  compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
1442  kCFBooleanTrue);
1443  }
1444 #endif
1445 
1446  if (avctx->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX) {
1447  status = create_cv_pixel_buffer_info(avctx, &pixel_buffer_info);
1448  if (status)
1449  goto init_cleanup;
1450  } else {
1451  pixel_buffer_info = NULL;
1452  }
1453 
1454  vtctx->dts_delta = vtctx->has_b_frames ? -1 : 0;
1455 
1456  get_cv_transfer_function(avctx, &vtctx->transfer_function, &gamma_level);
1457  get_cv_ycbcr_matrix(avctx, &vtctx->ycbcr_matrix);
1458  get_cv_color_primaries(avctx, &vtctx->color_primaries);
1459 
1460 
1461  if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
1462  status = vtenc_populate_extradata(avctx,
1463  codec_type,
1464  profile_level,
1465  gamma_level,
1466  enc_info,
1467  pixel_buffer_info);
1468  if (status)
1469  goto init_cleanup;
1470  }
1471 
1472  status = vtenc_create_encoder(avctx,
1473  codec_type,
1474  profile_level,
1475  gamma_level,
1476  enc_info,
1477  pixel_buffer_info,
1478  &vtctx->session);
1479 
1480 init_cleanup:
1481  if (gamma_level)
1482  CFRelease(gamma_level);
1483 
1484  if (pixel_buffer_info)
1485  CFRelease(pixel_buffer_info);
1486 
1487  CFRelease(enc_info);
1488 
1489  return status;
1490 }
1491 
1493 {
1494  VTEncContext *vtctx = avctx->priv_data;
1495  CFBooleanRef has_b_frames_cfbool;
1496  int status;
1497 
1499 
1500  pthread_mutex_init(&vtctx->lock, NULL);
1502 
1503  vtctx->session = NULL;
1504  status = vtenc_configure_encoder(avctx);
1505  if (status) return status;
1506 
1507  status = VTSessionCopyProperty(vtctx->session,
1508  kVTCompressionPropertyKey_AllowFrameReordering,
1509  kCFAllocatorDefault,
1510  &has_b_frames_cfbool);
1511 
1512  if (!status && has_b_frames_cfbool) {
1513  //Some devices don't output B-frames for main profile, even if requested.
1514  // HEVC has b-pyramid
1515  vtctx->has_b_frames = (CFBooleanGetValue(has_b_frames_cfbool) && avctx->codec_id == AV_CODEC_ID_HEVC) ? 2 : 1;
1516  CFRelease(has_b_frames_cfbool);
1517  }
1518  avctx->has_b_frames = vtctx->has_b_frames;
1519 
1520  return 0;
1521 }
1522 
1523 static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
1524 {
1525  CFArrayRef attachments;
1526  CFDictionaryRef attachment;
1527  CFBooleanRef not_sync;
1528  CFIndex len;
1529 
1530  attachments = CMSampleBufferGetSampleAttachmentsArray(buffer, false);
1531  len = !attachments ? 0 : CFArrayGetCount(attachments);
1532 
1533  if (!len) {
1534  *is_key_frame = true;
1535  return;
1536  }
1537 
1538  attachment = CFArrayGetValueAtIndex(attachments, 0);
1539 
1540  if (CFDictionaryGetValueIfPresent(attachment,
1541  kCMSampleAttachmentKey_NotSync,
1542  (const void **)&not_sync))
1543  {
1544  *is_key_frame = !CFBooleanGetValue(not_sync);
1545  } else {
1546  *is_key_frame = true;
1547  }
1548 }
1549 
1550 static int is_post_sei_nal_type(int nal_type){
1551  return nal_type != H264_NAL_SEI &&
1552  nal_type != H264_NAL_SPS &&
1553  nal_type != H264_NAL_PPS &&
1554  nal_type != H264_NAL_AUD;
1555 }
1556 
1557 /*
1558  * Finds the sei message start/size of type find_sei_type.
1559  * If more than one of that type exists, the last one is returned.
1560  */
1561 static int find_sei_end(AVCodecContext *avctx,
1562  uint8_t *nal_data,
1563  size_t nal_size,
1564  uint8_t **sei_end)
1565 {
1566  int nal_type;
1567  size_t sei_payload_size = 0;
1568  int sei_payload_type = 0;
1569  *sei_end = NULL;
1570  uint8_t *nal_start = nal_data;
1571 
1572  if (!nal_size)
1573  return 0;
1574 
1575  nal_type = *nal_data & 0x1F;
1576  if (nal_type != H264_NAL_SEI)
1577  return 0;
1578 
1579  nal_data++;
1580  nal_size--;
1581 
1582  if (nal_data[nal_size - 1] == 0x80)
1583  nal_size--;
1584 
1585  while (nal_size > 0 && *nal_data > 0) {
1586  do{
1587  sei_payload_type += *nal_data;
1588  nal_data++;
1589  nal_size--;
1590  } while (nal_size > 0 && *nal_data == 0xFF);
1591 
1592  if (!nal_size) {
1593  av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing type.\n");
1594  return AVERROR_INVALIDDATA;
1595  }
1596 
1597  do{
1598  sei_payload_size += *nal_data;
1599  nal_data++;
1600  nal_size--;
1601  } while (nal_size > 0 && *nal_data == 0xFF);
1602 
1603  if (nal_size < sei_payload_size) {
1604  av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing size.\n");
1605  return AVERROR_INVALIDDATA;
1606  }
1607 
1608  nal_data += sei_payload_size;
1609  nal_size -= sei_payload_size;
1610  }
1611 
1612  *sei_end = nal_data;
1613 
1614  return nal_data - nal_start + 1;
1615 }
1616 
1617 /**
1618  * Copies the data inserting emulation prevention bytes as needed.
1619  * Existing data in the destination can be taken into account by providing
1620  * dst with a dst_offset > 0.
1621  *
1622  * @return The number of bytes copied on success. On failure, the negative of
1623  * the number of bytes needed to copy src is returned.
1624  */
1625 static int copy_emulation_prev(const uint8_t *src,
1626  size_t src_size,
1627  uint8_t *dst,
1628  ssize_t dst_offset,
1629  size_t dst_size)
1630 {
1631  int zeros = 0;
1632  int wrote_bytes;
1633  uint8_t* dst_start;
1634  uint8_t* dst_end = dst + dst_size;
1635  const uint8_t* src_end = src + src_size;
1636  int start_at = dst_offset > 2 ? dst_offset - 2 : 0;
1637  int i;
1638  for (i = start_at; i < dst_offset && i < dst_size; i++) {
1639  if (!dst[i])
1640  zeros++;
1641  else
1642  zeros = 0;
1643  }
1644 
1645  dst += dst_offset;
1646  dst_start = dst;
1647  for (; src < src_end; src++, dst++) {
1648  if (zeros == 2) {
1649  int insert_ep3_byte = *src <= 3;
1650  if (insert_ep3_byte) {
1651  if (dst < dst_end)
1652  *dst = 3;
1653  dst++;
1654  }
1655 
1656  zeros = 0;
1657  }
1658 
1659  if (dst < dst_end)
1660  *dst = *src;
1661 
1662  if (!*src)
1663  zeros++;
1664  else
1665  zeros = 0;
1666  }
1667 
1668  wrote_bytes = dst - dst_start;
1669 
1670  if (dst > dst_end)
1671  return -wrote_bytes;
1672 
1673  return wrote_bytes;
1674 }
1675 
1676 static int write_sei(const ExtraSEI *sei,
1677  int sei_type,
1678  uint8_t *dst,
1679  size_t dst_size)
1680 {
1681  uint8_t *sei_start = dst;
1682  size_t remaining_sei_size = sei->size;
1683  size_t remaining_dst_size = dst_size;
1684  int header_bytes;
1685  int bytes_written;
1686  ssize_t offset;
1687 
1688  if (!remaining_dst_size)
1689  return AVERROR_BUFFER_TOO_SMALL;
1690 
1691  while (sei_type && remaining_dst_size != 0) {
1692  int sei_byte = sei_type > 255 ? 255 : sei_type;
1693  *dst = sei_byte;
1694 
1695  sei_type -= sei_byte;
1696  dst++;
1697  remaining_dst_size--;
1698  }
1699 
1700  if (!dst_size)
1701  return AVERROR_BUFFER_TOO_SMALL;
1702 
1703  while (remaining_sei_size && remaining_dst_size != 0) {
1704  int size_byte = remaining_sei_size > 255 ? 255 : remaining_sei_size;
1705  *dst = size_byte;
1706 
1707  remaining_sei_size -= size_byte;
1708  dst++;
1709  remaining_dst_size--;
1710  }
1711 
1712  if (remaining_dst_size < sei->size)
1713  return AVERROR_BUFFER_TOO_SMALL;
1714 
1715  header_bytes = dst - sei_start;
1716 
1717  offset = header_bytes;
1718  bytes_written = copy_emulation_prev(sei->data,
1719  sei->size,
1720  sei_start,
1721  offset,
1722  dst_size);
1723  if (bytes_written < 0)
1724  return AVERROR_BUFFER_TOO_SMALL;
1725 
1726  bytes_written += header_bytes;
1727  return bytes_written;
1728 }
1729 
1730 /**
1731  * Copies NAL units and replaces length codes with
1732  * H.264 Annex B start codes. On failure, the contents of
1733  * dst_data may have been modified.
1734  *
1735  * @param length_code_size Byte length of each length code
1736  * @param sample_buffer NAL units prefixed with length codes.
1737  * @param sei Optional A53 closed captions SEI data.
1738  * @param dst_data Must be zeroed before calling this function.
1739  * Contains the copied NAL units prefixed with
1740  * start codes when the function returns
1741  * successfully.
1742  * @param dst_size Length of dst_data
1743  * @return 0 on success
1744  * AVERROR_INVALIDDATA if length_code_size is invalid
1745  * AVERROR_BUFFER_TOO_SMALL if dst_data is too small
1746  * or if a length_code in src_data specifies data beyond
1747  * the end of its buffer.
1748  */
1750  AVCodecContext *avctx,
1751  size_t length_code_size,
1752  CMSampleBufferRef sample_buffer,
1753  ExtraSEI *sei,
1754  uint8_t *dst_data,
1755  size_t dst_size)
1756 {
1757  size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1758  size_t remaining_src_size = src_size;
1759  size_t remaining_dst_size = dst_size;
1760  size_t src_offset = 0;
1761  int wrote_sei = 0;
1762  int status;
1763  uint8_t size_buf[4];
1764  uint8_t nal_type;
1765  CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
1766 
1767  if (length_code_size > 4) {
1768  return AVERROR_INVALIDDATA;
1769  }
1770 
1771  while (remaining_src_size > 0) {
1772  size_t curr_src_len;
1773  size_t curr_dst_len;
1774  size_t box_len = 0;
1775  size_t i;
1776 
1777  uint8_t *dst_box;
1778 
1779  status = CMBlockBufferCopyDataBytes(block,
1780  src_offset,
1781  length_code_size,
1782  size_buf);
1783  if (status) {
1784  av_log(avctx, AV_LOG_ERROR, "Cannot copy length: %d\n", status);
1785  return AVERROR_EXTERNAL;
1786  }
1787 
1788  status = CMBlockBufferCopyDataBytes(block,
1789  src_offset + length_code_size,
1790  1,
1791  &nal_type);
1792 
1793  if (status) {
1794  av_log(avctx, AV_LOG_ERROR, "Cannot copy type: %d\n", status);
1795  return AVERROR_EXTERNAL;
1796  }
1797 
1798  nal_type &= 0x1F;
1799 
1800  for (i = 0; i < length_code_size; i++) {
1801  box_len <<= 8;
1802  box_len |= size_buf[i];
1803  }
1804 
1805  if (sei && !wrote_sei && is_post_sei_nal_type(nal_type)) {
1806  //No SEI NAL unit - insert.
1807  int wrote_bytes;
1808 
1809  memcpy(dst_data, start_code, sizeof(start_code));
1810  dst_data += sizeof(start_code);
1811  remaining_dst_size -= sizeof(start_code);
1812 
1813  *dst_data = H264_NAL_SEI;
1814  dst_data++;
1815  remaining_dst_size--;
1816 
1817  wrote_bytes = write_sei(sei,
1819  dst_data,
1820  remaining_dst_size);
1821 
1822  if (wrote_bytes < 0)
1823  return wrote_bytes;
1824 
1825  remaining_dst_size -= wrote_bytes;
1826  dst_data += wrote_bytes;
1827 
1828  if (remaining_dst_size <= 0)
1829  return AVERROR_BUFFER_TOO_SMALL;
1830 
1831  *dst_data = 0x80;
1832 
1833  dst_data++;
1834  remaining_dst_size--;
1835 
1836  wrote_sei = 1;
1837  }
1838 
1839  curr_src_len = box_len + length_code_size;
1840  curr_dst_len = box_len + sizeof(start_code);
1841 
1842  if (remaining_src_size < curr_src_len) {
1843  return AVERROR_BUFFER_TOO_SMALL;
1844  }
1845 
1846  if (remaining_dst_size < curr_dst_len) {
1847  return AVERROR_BUFFER_TOO_SMALL;
1848  }
1849 
1850  dst_box = dst_data + sizeof(start_code);
1851 
1852  memcpy(dst_data, start_code, sizeof(start_code));
1853  status = CMBlockBufferCopyDataBytes(block,
1854  src_offset + length_code_size,
1855  box_len,
1856  dst_box);
1857 
1858  if (status) {
1859  av_log(avctx, AV_LOG_ERROR, "Cannot copy data: %d\n", status);
1860  return AVERROR_EXTERNAL;
1861  }
1862 
1863  if (sei && !wrote_sei && nal_type == H264_NAL_SEI) {
1864  //Found SEI NAL unit - append.
1865  int wrote_bytes;
1866  int old_sei_length;
1867  int extra_bytes;
1868  uint8_t *new_sei;
1869  old_sei_length = find_sei_end(avctx, dst_box, box_len, &new_sei);
1870  if (old_sei_length < 0)
1871  return status;
1872 
1873  wrote_bytes = write_sei(sei,
1875  new_sei,
1876  remaining_dst_size - old_sei_length);
1877  if (wrote_bytes < 0)
1878  return wrote_bytes;
1879 
1880  if (new_sei + wrote_bytes >= dst_data + remaining_dst_size)
1881  return AVERROR_BUFFER_TOO_SMALL;
1882 
1883  new_sei[wrote_bytes++] = 0x80;
1884  extra_bytes = wrote_bytes - (dst_box + box_len - new_sei);
1885 
1886  dst_data += extra_bytes;
1887  remaining_dst_size -= extra_bytes;
1888 
1889  wrote_sei = 1;
1890  }
1891 
1892  src_offset += curr_src_len;
1893  dst_data += curr_dst_len;
1894 
1895  remaining_src_size -= curr_src_len;
1896  remaining_dst_size -= curr_dst_len;
1897  }
1898 
1899  return 0;
1900 }
1901 
1902 /**
1903  * Returns a sufficient number of bytes to contain the sei data.
1904  * It may be greater than the minimum required.
1905  */
1906 static int get_sei_msg_bytes(const ExtraSEI* sei, int type){
1907  int copied_size;
1908  if (sei->size == 0)
1909  return 0;
1910 
1911  copied_size = -copy_emulation_prev(sei->data,
1912  sei->size,
1913  NULL,
1914  0,
1915  0);
1916 
1917  if ((sei->size % 255) == 0) //may result in an extra byte
1918  copied_size++;
1919 
1920  return copied_size + sei->size / 255 + 1 + type / 255 + 1;
1921 }
1922 
1924  AVCodecContext *avctx,
1925  CMSampleBufferRef sample_buffer,
1926  AVPacket *pkt,
1927  ExtraSEI *sei)
1928 {
1929  VTEncContext *vtctx = avctx->priv_data;
1930 
1931  int status;
1932  bool is_key_frame;
1933  bool add_header;
1934  size_t length_code_size;
1935  size_t header_size = 0;
1936  size_t in_buf_size;
1937  size_t out_buf_size;
1938  size_t sei_nalu_size = 0;
1939  int64_t dts_delta;
1940  int64_t time_base_num;
1941  int nalu_count;
1942  CMTime pts;
1943  CMTime dts;
1944  CMVideoFormatDescriptionRef vid_fmt;
1945 
1946 
1947  vtenc_get_frame_info(sample_buffer, &is_key_frame);
1948  status = get_length_code_size(avctx, sample_buffer, &length_code_size);
1949  if (status) return status;
1950 
1951  add_header = is_key_frame && !(avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER);
1952 
1953  if (add_header) {
1954  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
1955  if (!vid_fmt) {
1956  av_log(avctx, AV_LOG_ERROR, "Cannot get format description.\n");
1957  return AVERROR_EXTERNAL;
1958  }
1959 
1960  int status = get_params_size(avctx, vid_fmt, &header_size);
1961  if (status) return status;
1962  }
1963 
1964  status = count_nalus(length_code_size, sample_buffer, &nalu_count);
1965  if(status)
1966  return status;
1967 
1968  if (sei) {
1969  size_t msg_size = get_sei_msg_bytes(sei,
1971 
1972  sei_nalu_size = sizeof(start_code) + 1 + msg_size + 1;
1973  }
1974 
1975  in_buf_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1976  out_buf_size = header_size +
1977  in_buf_size +
1978  sei_nalu_size +
1979  nalu_count * ((int)sizeof(start_code) - (int)length_code_size);
1980 
1981  status = ff_alloc_packet2(avctx, pkt, out_buf_size, out_buf_size);
1982  if (status < 0)
1983  return status;
1984 
1985  if (add_header) {
1986  status = copy_param_sets(avctx, vid_fmt, pkt->data, out_buf_size);
1987  if(status) return status;
1988  }
1989 
1990  status = copy_replace_length_codes(
1991  avctx,
1992  length_code_size,
1993  sample_buffer,
1994  sei,
1995  pkt->data + header_size,
1996  pkt->size - header_size
1997  );
1998 
1999  if (status) {
2000  av_log(avctx, AV_LOG_ERROR, "Error copying packet data: %d\n", status);
2001  return status;
2002  }
2003 
2004  if (is_key_frame) {
2006  }
2007 
2008  pts = CMSampleBufferGetPresentationTimeStamp(sample_buffer);
2009  dts = CMSampleBufferGetDecodeTimeStamp (sample_buffer);
2010 
2011  if (CMTIME_IS_INVALID(dts)) {
2012  if (!vtctx->has_b_frames) {
2013  dts = pts;
2014  } else {
2015  av_log(avctx, AV_LOG_ERROR, "DTS is invalid.\n");
2016  return AVERROR_EXTERNAL;
2017  }
2018  }
2019 
2020  dts_delta = vtctx->dts_delta >= 0 ? vtctx->dts_delta : 0;
2021  time_base_num = avctx->time_base.num;
2022  pkt->pts = pts.value / time_base_num;
2023  pkt->dts = dts.value / time_base_num - dts_delta;
2024  pkt->size = out_buf_size;
2025 
2026  return 0;
2027 }
2028 
2029 /*
2030  * contiguous_buf_size is 0 if not contiguous, and the size of the buffer
2031  * containing all planes if so.
2032  */
2034  AVCodecContext *avctx,
2035  const AVFrame *frame,
2036  int *color,
2037  int *plane_count,
2038  size_t *widths,
2039  size_t *heights,
2040  size_t *strides,
2041  size_t *contiguous_buf_size)
2042 {
2043  VTEncContext *vtctx = avctx->priv_data;
2044  int av_format = frame->format;
2045  int av_color_range = frame->color_range;
2046  int i;
2047  int range_guessed;
2048  int status;
2049 
2050  status = get_cv_pixel_format(avctx, av_format, av_color_range, color, &range_guessed);
2051  if (status) {
2052  av_log(avctx,
2053  AV_LOG_ERROR,
2054  "Could not get pixel format for color format '%s' range '%s'.\n",
2055  av_get_pix_fmt_name(av_format),
2056  av_color_range > AVCOL_RANGE_UNSPECIFIED &&
2057  av_color_range < AVCOL_RANGE_NB ?
2058  av_color_range_name(av_color_range) :
2059  "Unknown");
2060 
2061  return AVERROR(EINVAL);
2062  }
2063 
2064  if (range_guessed) {
2065  if (!vtctx->warned_color_range) {
2066  vtctx->warned_color_range = true;
2067  av_log(avctx,
2069  "Color range not set for %s. Using MPEG range.\n",
2070  av_get_pix_fmt_name(av_format));
2071  }
2072  }
2073 
2074  switch (av_format) {
2075  case AV_PIX_FMT_NV12:
2076  *plane_count = 2;
2077 
2078  widths [0] = avctx->width;
2079  heights[0] = avctx->height;
2080  strides[0] = frame ? frame->linesize[0] : avctx->width;
2081 
2082  widths [1] = (avctx->width + 1) / 2;
2083  heights[1] = (avctx->height + 1) / 2;
2084  strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) & -2;
2085  break;
2086 
2087  case AV_PIX_FMT_YUV420P:
2088  *plane_count = 3;
2089 
2090  widths [0] = avctx->width;
2091  heights[0] = avctx->height;
2092  strides[0] = frame ? frame->linesize[0] : avctx->width;
2093 
2094  widths [1] = (avctx->width + 1) / 2;
2095  heights[1] = (avctx->height + 1) / 2;
2096  strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) / 2;
2097 
2098  widths [2] = (avctx->width + 1) / 2;
2099  heights[2] = (avctx->height + 1) / 2;
2100  strides[2] = frame ? frame->linesize[2] : (avctx->width + 1) / 2;
2101  break;
2102 
2103  case AV_PIX_FMT_BGRA:
2104  *plane_count = 1;
2105 
2106  widths [0] = avctx->width;
2107  heights[0] = avctx->height;
2108  strides[0] = frame ? frame->linesize[0] : avctx->width * 4;
2109  break;
2110 
2111  case AV_PIX_FMT_P010LE:
2112  *plane_count = 2;
2113  widths[0] = avctx->width;
2114  heights[0] = avctx->height;
2115  strides[0] = frame ? frame->linesize[0] : (avctx->width * 2 + 63) & -64;
2116 
2117  widths[1] = (avctx->width + 1) / 2;
2118  heights[1] = (avctx->height + 1) / 2;
2119  strides[1] = frame ? frame->linesize[1] : ((avctx->width + 1) / 2 + 63) & -64;
2120  break;
2121 
2122  default:
2123  av_log(
2124  avctx,
2125  AV_LOG_ERROR,
2126  "Could not get frame format info for color %d range %d.\n",
2127  av_format,
2128  av_color_range);
2129 
2130  return AVERROR(EINVAL);
2131  }
2132 
2133  *contiguous_buf_size = 0;
2134  for (i = 0; i < *plane_count; i++) {
2135  if (i < *plane_count - 1 &&
2136  frame->data[i] + strides[i] * heights[i] != frame->data[i + 1]) {
2137  *contiguous_buf_size = 0;
2138  break;
2139  }
2140 
2141  *contiguous_buf_size += strides[i] * heights[i];
2142  }
2143 
2144  return 0;
2145 }
2146 
2147 //Not used on OSX - frame is never copied.
2149  const AVFrame *frame,
2150  CVPixelBufferRef cv_img,
2151  const size_t *plane_strides,
2152  const size_t *plane_rows)
2153 {
2154  int i, j;
2155  size_t plane_count;
2156  int status;
2157  int rows;
2158  int src_stride;
2159  int dst_stride;
2160  uint8_t *src_addr;
2161  uint8_t *dst_addr;
2162  size_t copy_bytes;
2163 
2164  status = CVPixelBufferLockBaseAddress(cv_img, 0);
2165  if (status) {
2166  av_log(
2167  avctx,
2168  AV_LOG_ERROR,
2169  "Error: Could not lock base address of CVPixelBuffer: %d.\n",
2170  status
2171  );
2172  }
2173 
2174  if (CVPixelBufferIsPlanar(cv_img)) {
2175  plane_count = CVPixelBufferGetPlaneCount(cv_img);
2176  for (i = 0; frame->data[i]; i++) {
2177  if (i == plane_count) {
2178  CVPixelBufferUnlockBaseAddress(cv_img, 0);
2179  av_log(avctx,
2180  AV_LOG_ERROR,
2181  "Error: different number of planes in AVFrame and CVPixelBuffer.\n"
2182  );
2183 
2184  return AVERROR_EXTERNAL;
2185  }
2186 
2187  dst_addr = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cv_img, i);
2188  src_addr = (uint8_t*)frame->data[i];
2189  dst_stride = CVPixelBufferGetBytesPerRowOfPlane(cv_img, i);
2190  src_stride = plane_strides[i];
2191  rows = plane_rows[i];
2192 
2193  if (dst_stride == src_stride) {
2194  memcpy(dst_addr, src_addr, src_stride * rows);
2195  } else {
2196  copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
2197 
2198  for (j = 0; j < rows; j++) {
2199  memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
2200  }
2201  }
2202  }
2203  } else {
2204  if (frame->data[1]) {
2205  CVPixelBufferUnlockBaseAddress(cv_img, 0);
2206  av_log(avctx,
2207  AV_LOG_ERROR,
2208  "Error: different number of planes in AVFrame and non-planar CVPixelBuffer.\n"
2209  );
2210 
2211  return AVERROR_EXTERNAL;
2212  }
2213 
2214  dst_addr = (uint8_t*)CVPixelBufferGetBaseAddress(cv_img);
2215  src_addr = (uint8_t*)frame->data[0];
2216  dst_stride = CVPixelBufferGetBytesPerRow(cv_img);
2217  src_stride = plane_strides[0];
2218  rows = plane_rows[0];
2219 
2220  if (dst_stride == src_stride) {
2221  memcpy(dst_addr, src_addr, src_stride * rows);
2222  } else {
2223  copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
2224 
2225  for (j = 0; j < rows; j++) {
2226  memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
2227  }
2228  }
2229  }
2230 
2231  status = CVPixelBufferUnlockBaseAddress(cv_img, 0);
2232  if (status) {
2233  av_log(avctx, AV_LOG_ERROR, "Error: Could not unlock CVPixelBuffer base address: %d.\n", status);
2234  return AVERROR_EXTERNAL;
2235  }
2236 
2237  return 0;
2238 }
2239 
2241  const AVFrame *frame,
2242  CVPixelBufferRef *cv_img)
2243 {
2244  int plane_count;
2245  int color;
2246  size_t widths [AV_NUM_DATA_POINTERS];
2247  size_t heights[AV_NUM_DATA_POINTERS];
2248  size_t strides[AV_NUM_DATA_POINTERS];
2249  int status;
2250  size_t contiguous_buf_size;
2251  CVPixelBufferPoolRef pix_buf_pool;
2252  VTEncContext* vtctx = avctx->priv_data;
2253 
2254  if (avctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX) {
2256 
2257  *cv_img = (CVPixelBufferRef)frame->data[3];
2258  av_assert0(*cv_img);
2259 
2260  CFRetain(*cv_img);
2261  return 0;
2262  }
2263 
2264  memset(widths, 0, sizeof(widths));
2265  memset(heights, 0, sizeof(heights));
2266  memset(strides, 0, sizeof(strides));
2267 
2268  status = get_cv_pixel_info(
2269  avctx,
2270  frame,
2271  &color,
2272  &plane_count,
2273  widths,
2274  heights,
2275  strides,
2276  &contiguous_buf_size
2277  );
2278 
2279  if (status) {
2280  av_log(
2281  avctx,
2282  AV_LOG_ERROR,
2283  "Error: Cannot convert format %d color_range %d: %d\n",
2284  frame->format,
2285  frame->color_range,
2286  status
2287  );
2288 
2289  return AVERROR_EXTERNAL;
2290  }
2291 
2292  pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2293  if (!pix_buf_pool) {
2294  /* On iOS, the VT session is invalidated when the APP switches from
2295  * foreground to background and vice versa. Fetch the actual error code
2296  * of the VT session to detect that case and restart the VT session
2297  * accordingly. */
2298  OSStatus vtstatus;
2299 
2300  vtstatus = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
2301  if (vtstatus == kVTInvalidSessionErr) {
2302  CFRelease(vtctx->session);
2303  vtctx->session = NULL;
2304  status = vtenc_configure_encoder(avctx);
2305  if (status == 0)
2306  pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2307  }
2308  if (!pix_buf_pool) {
2309  av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
2310  return AVERROR_EXTERNAL;
2311  }
2312  else
2313  av_log(avctx, AV_LOG_WARNING, "VT session restarted because of a "
2314  "kVTInvalidSessionErr error.\n");
2315  }
2316 
2317  status = CVPixelBufferPoolCreatePixelBuffer(NULL,
2318  pix_buf_pool,
2319  cv_img);
2320 
2321 
2322  if (status) {
2323  av_log(avctx, AV_LOG_ERROR, "Could not create pixel buffer from pool: %d.\n", status);
2324  return AVERROR_EXTERNAL;
2325  }
2326 
2327  status = copy_avframe_to_pixel_buffer(avctx, frame, *cv_img, strides, heights);
2328  if (status) {
2329  CFRelease(*cv_img);
2330  *cv_img = NULL;
2331  return status;
2332  }
2333 
2334  return 0;
2335 }
2336 
2338  CFDictionaryRef* dict_out)
2339 {
2340  CFDictionaryRef dict = NULL;
2341  if (frame->pict_type == AV_PICTURE_TYPE_I) {
2342  const void *keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame };
2343  const void *vals[] = { kCFBooleanTrue };
2344 
2345  dict = CFDictionaryCreate(NULL, keys, vals, 1, NULL, NULL);
2346  if(!dict) return AVERROR(ENOMEM);
2347  }
2348 
2349  *dict_out = dict;
2350  return 0;
2351 }
2352 
2354  VTEncContext *vtctx,
2355  const AVFrame *frame)
2356 {
2357  CMTime time;
2358  CFDictionaryRef frame_dict;
2359  CVPixelBufferRef cv_img = NULL;
2360  AVFrameSideData *side_data = NULL;
2361  ExtraSEI *sei = NULL;
2362  int status = create_cv_pixel_buffer(avctx, frame, &cv_img);
2363 
2364  if (status) return status;
2365 
2366  status = create_encoder_dict_h264(frame, &frame_dict);
2367  if (status) {
2368  CFRelease(cv_img);
2369  return status;
2370  }
2371 
2373  if (vtctx->a53_cc && side_data && side_data->size) {
2374  sei = av_mallocz(sizeof(*sei));
2375  if (!sei) {
2376  av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
2377  } else {
2378  int ret = ff_alloc_a53_sei(frame, 0, &sei->data, &sei->size);
2379  if (ret < 0) {
2380  av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
2381  av_free(sei);
2382  sei = NULL;
2383  }
2384  }
2385  }
2386 
2387  time = CMTimeMake(frame->pts * avctx->time_base.num, avctx->time_base.den);
2388  status = VTCompressionSessionEncodeFrame(
2389  vtctx->session,
2390  cv_img,
2391  time,
2392  kCMTimeInvalid,
2393  frame_dict,
2394  sei,
2395  NULL
2396  );
2397 
2398  if (frame_dict) CFRelease(frame_dict);
2399  CFRelease(cv_img);
2400 
2401  if (status) {
2402  av_log(avctx, AV_LOG_ERROR, "Error: cannot encode frame: %d\n", status);
2403  return AVERROR_EXTERNAL;
2404  }
2405 
2406  return 0;
2407 }
2408 
2410  AVCodecContext *avctx,
2411  AVPacket *pkt,
2412  const AVFrame *frame,
2413  int *got_packet)
2414 {
2415  VTEncContext *vtctx = avctx->priv_data;
2416  bool get_frame;
2417  int status;
2418  CMSampleBufferRef buf = NULL;
2419  ExtraSEI *sei = NULL;
2420 
2421  if (frame) {
2422  status = vtenc_send_frame(avctx, vtctx, frame);
2423 
2424  if (status) {
2425  status = AVERROR_EXTERNAL;
2426  goto end_nopkt;
2427  }
2428 
2429  if (vtctx->frame_ct_in == 0) {
2430  vtctx->first_pts = frame->pts;
2431  } else if(vtctx->frame_ct_in == vtctx->has_b_frames) {
2432  vtctx->dts_delta = frame->pts - vtctx->first_pts;
2433  }
2434 
2435  vtctx->frame_ct_in++;
2436  } else if(!vtctx->flushing) {
2437  vtctx->flushing = true;
2438 
2439  status = VTCompressionSessionCompleteFrames(vtctx->session,
2440  kCMTimeIndefinite);
2441 
2442  if (status) {
2443  av_log(avctx, AV_LOG_ERROR, "Error flushing frames: %d\n", status);
2444  status = AVERROR_EXTERNAL;
2445  goto end_nopkt;
2446  }
2447  }
2448 
2449  *got_packet = 0;
2450  get_frame = vtctx->dts_delta >= 0 || !frame;
2451  if (!get_frame) {
2452  status = 0;
2453  goto end_nopkt;
2454  }
2455 
2456  status = vtenc_q_pop(vtctx, !frame, &buf, &sei);
2457  if (status) goto end_nopkt;
2458  if (!buf) goto end_nopkt;
2459 
2460  status = vtenc_cm_to_avpacket(avctx, buf, pkt, sei);
2461  if (sei) {
2462  if (sei->data) av_free(sei->data);
2463  av_free(sei);
2464  }
2465  CFRelease(buf);
2466  if (status) goto end_nopkt;
2467 
2468  *got_packet = 1;
2469  return 0;
2470 
2471 end_nopkt:
2473  return status;
2474 }
2475 
2477  CMVideoCodecType codec_type,
2478  CFStringRef profile_level,
2479  CFNumberRef gamma_level,
2480  CFDictionaryRef enc_info,
2481  CFDictionaryRef pixel_buffer_info)
2482 {
2483  VTEncContext *vtctx = avctx->priv_data;
2484  int status;
2485  CVPixelBufferPoolRef pool = NULL;
2486  CVPixelBufferRef pix_buf = NULL;
2487  CMTime time;
2488  CMSampleBufferRef buf = NULL;
2489 
2490  status = vtenc_create_encoder(avctx,
2491  codec_type,
2492  profile_level,
2493  gamma_level,
2494  enc_info,
2495  pixel_buffer_info,
2496  &vtctx->session);
2497  if (status)
2498  goto pe_cleanup;
2499 
2500  pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2501  if(!pool){
2502  av_log(avctx, AV_LOG_ERROR, "Error getting pixel buffer pool.\n");
2503  goto pe_cleanup;
2504  }
2505 
2506  status = CVPixelBufferPoolCreatePixelBuffer(NULL,
2507  pool,
2508  &pix_buf);
2509 
2510  if(status != kCVReturnSuccess){
2511  av_log(avctx, AV_LOG_ERROR, "Error creating frame from pool: %d\n", status);
2512  goto pe_cleanup;
2513  }
2514 
2515  time = CMTimeMake(0, avctx->time_base.den);
2516  status = VTCompressionSessionEncodeFrame(vtctx->session,
2517  pix_buf,
2518  time,
2519  kCMTimeInvalid,
2520  NULL,
2521  NULL,
2522  NULL);
2523 
2524  if (status) {
2525  av_log(avctx,
2526  AV_LOG_ERROR,
2527  "Error sending frame for extradata: %d\n",
2528  status);
2529 
2530  goto pe_cleanup;
2531  }
2532 
2533  //Populates extradata - output frames are flushed and param sets are available.
2534  status = VTCompressionSessionCompleteFrames(vtctx->session,
2535  kCMTimeIndefinite);
2536 
2537  if (status)
2538  goto pe_cleanup;
2539 
2540  status = vtenc_q_pop(vtctx, 0, &buf, NULL);
2541  if (status) {
2542  av_log(avctx, AV_LOG_ERROR, "popping: %d\n", status);
2543  goto pe_cleanup;
2544  }
2545 
2546  CFRelease(buf);
2547 
2548 
2549 
2550 pe_cleanup:
2551  if(vtctx->session)
2552  CFRelease(vtctx->session);
2553 
2554  vtctx->session = NULL;
2555  vtctx->frame_ct_out = 0;
2556 
2557  av_assert0(status != 0 || (avctx->extradata && avctx->extradata_size > 0));
2558 
2559  return status;
2560 }
2561 
2563 {
2564  VTEncContext *vtctx = avctx->priv_data;
2565 
2566  if(!vtctx->session) {
2568  pthread_mutex_destroy(&vtctx->lock);
2569  return 0;
2570  }
2571 
2572  VTCompressionSessionCompleteFrames(vtctx->session,
2573  kCMTimeIndefinite);
2574  clear_frame_queue(vtctx);
2576  pthread_mutex_destroy(&vtctx->lock);
2577  CFRelease(vtctx->session);
2578  vtctx->session = NULL;
2579 
2580  if (vtctx->color_primaries) {
2581  CFRelease(vtctx->color_primaries);
2582  vtctx->color_primaries = NULL;
2583  }
2584 
2585  if (vtctx->transfer_function) {
2586  CFRelease(vtctx->transfer_function);
2587  vtctx->transfer_function = NULL;
2588  }
2589 
2590  if (vtctx->ycbcr_matrix) {
2591  CFRelease(vtctx->ycbcr_matrix);
2592  vtctx->ycbcr_matrix = NULL;
2593  }
2594 
2595  return 0;
2596 }
2597 
2598 static const enum AVPixelFormat avc_pix_fmts[] = {
2603 };
2604 
2605 static const enum AVPixelFormat hevc_pix_fmts[] = {
2612 };
2613 
2614 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
2615 #define COMMON_OPTIONS \
2616  { "allow_sw", "Allow software encoding", OFFSET(allow_sw), AV_OPT_TYPE_BOOL, \
2617  { .i64 = 0 }, 0, 1, VE }, \
2618  { "require_sw", "Require software encoding", OFFSET(require_sw), AV_OPT_TYPE_BOOL, \
2619  { .i64 = 0 }, 0, 1, VE }, \
2620  { "realtime", "Hint that encoding should happen in real-time if not faster (e.g. capturing from camera).", \
2621  OFFSET(realtime), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \
2622  { "frames_before", "Other frames will come before the frames in this session. This helps smooth concatenation issues.", \
2623  OFFSET(frames_before), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \
2624  { "frames_after", "Other frames will come after the frames in this session. This helps smooth concatenation issues.", \
2625  OFFSET(frames_after), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
2626 
2627 #define OFFSET(x) offsetof(VTEncContext, x)
2628 static const AVOption h264_options[] = {
2629  { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = H264_PROF_AUTO }, H264_PROF_AUTO, H264_PROF_COUNT, VE, "profile" },
2630  { "baseline", "Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_BASELINE }, INT_MIN, INT_MAX, VE, "profile" },
2631  { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" },
2632  { "high", "High Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_HIGH }, INT_MIN, INT_MAX, VE, "profile" },
2633  { "extended", "Extend Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_EXTENDED }, INT_MIN, INT_MAX, VE, "profile" },
2634 
2635  { "level", "Level", OFFSET(level), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 52, VE, "level" },
2636  { "1.3", "Level 1.3, only available with Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = 13 }, INT_MIN, INT_MAX, VE, "level" },
2637  { "3.0", "Level 3.0", 0, AV_OPT_TYPE_CONST, { .i64 = 30 }, INT_MIN, INT_MAX, VE, "level" },
2638  { "3.1", "Level 3.1", 0, AV_OPT_TYPE_CONST, { .i64 = 31 }, INT_MIN, INT_MAX, VE, "level" },
2639  { "3.2", "Level 3.2", 0, AV_OPT_TYPE_CONST, { .i64 = 32 }, INT_MIN, INT_MAX, VE, "level" },
2640  { "4.0", "Level 4.0", 0, AV_OPT_TYPE_CONST, { .i64 = 40 }, INT_MIN, INT_MAX, VE, "level" },
2641  { "4.1", "Level 4.1", 0, AV_OPT_TYPE_CONST, { .i64 = 41 }, INT_MIN, INT_MAX, VE, "level" },
2642  { "4.2", "Level 4.2", 0, AV_OPT_TYPE_CONST, { .i64 = 42 }, INT_MIN, INT_MAX, VE, "level" },
2643  { "5.0", "Level 5.0", 0, AV_OPT_TYPE_CONST, { .i64 = 50 }, INT_MIN, INT_MAX, VE, "level" },
2644  { "5.1", "Level 5.1", 0, AV_OPT_TYPE_CONST, { .i64 = 51 }, INT_MIN, INT_MAX, VE, "level" },
2645  { "5.2", "Level 5.2", 0, AV_OPT_TYPE_CONST, { .i64 = 52 }, INT_MIN, INT_MAX, VE, "level" },
2646 
2647  { "coder", "Entropy coding", OFFSET(entropy), AV_OPT_TYPE_INT, { .i64 = VT_ENTROPY_NOT_SET }, VT_ENTROPY_NOT_SET, VT_CABAC, VE, "coder" },
2648  { "cavlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
2649  { "vlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
2650  { "cabac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
2651  { "ac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
2652 
2653  { "a53cc", "Use A53 Closed Captions (if available)", OFFSET(a53_cc), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, VE },
2654 
2656  { NULL },
2657 };
2658 
2660  .class_name = "h264_videotoolbox",
2661  .item_name = av_default_item_name,
2662  .option = h264_options,
2663  .version = LIBAVUTIL_VERSION_INT,
2664 };
2665 
2667  .name = "h264_videotoolbox",
2668  .long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.264 Encoder"),
2669  .type = AVMEDIA_TYPE_VIDEO,
2670  .id = AV_CODEC_ID_H264,
2671  .priv_data_size = sizeof(VTEncContext),
2673  .init = vtenc_init,
2674  .encode2 = vtenc_frame,
2675  .close = vtenc_close,
2676  .capabilities = AV_CODEC_CAP_DELAY,
2677  .priv_class = &h264_videotoolbox_class,
2678  .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE |
2680 };
2681 
2682 static const AVOption hevc_options[] = {
2683  { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = HEVC_PROF_AUTO }, HEVC_PROF_AUTO, HEVC_PROF_COUNT, VE, "profile" },
2684  { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" },
2685  { "main10", "Main10 Profile", 0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN10 }, INT_MIN, INT_MAX, VE, "profile" },
2686 
2687  { "alpha_quality", "Compression quality for the alpha channel", OFFSET(alpha_quality), AV_OPT_TYPE_DOUBLE, { .dbl = 0.0 }, 0.0, 1.0, VE },
2688 
2690  { NULL },
2691 };
2692 
2694  .class_name = "hevc_videotoolbox",
2695  .item_name = av_default_item_name,
2696  .option = hevc_options,
2697  .version = LIBAVUTIL_VERSION_INT,
2698 };
2699 
2701  .name = "hevc_videotoolbox",
2702  .long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.265 Encoder"),
2703  .type = AVMEDIA_TYPE_VIDEO,
2704  .id = AV_CODEC_ID_HEVC,
2705  .priv_data_size = sizeof(VTEncContext),
2707  .init = vtenc_init,
2708  .encode2 = vtenc_frame,
2709  .close = vtenc_close,
2710  .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE,
2711  .priv_class = &hevc_videotoolbox_class,
2712  .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE |
2714  .wrapper_name = "videotoolbox",
2715 };
int ff_alloc_a53_sei(const AVFrame *frame, size_t prefix_len, void **data, size_t *sei_size)
Check AVFrame for A53 side data and allocate and fill SEI message with A53 info.
Definition: atsc_a53.c:25
#define av_cold
Definition: attributes.h:88
uint8_t
simple assert() macros that are a bit more flexible than ISO C assert().
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
Libavcodec external API header.
static av_cold int init(AVCodecContext *avctx)
Definition: avrndec.c:31
#define flags(name, subs,...)
Definition: cbs_av1.c:572
static int FUNC() sei(CodedBitstreamContext *ctx, RWContext *rw, H264RawSEI *current)
#define NULL
Definition: coverity.c:32
long long int64_t
Definition: coverity.c:34
static AVFrame * frame
int ff_alloc_packet2(AVCodecContext *avctx, AVPacket *avpkt, int64_t size, int64_t min_size)
Check AVPacket size and/or allocate data.
Definition: encode.c:33
int
#define pthread_mutex_lock(a)
Definition: ffprobe.c:63
#define pthread_mutex_unlock(a)
Definition: ffprobe.c:67
#define AV_NUM_DATA_POINTERS
Definition: frame.h:319
@ AV_OPT_TYPE_CONST
Definition: opt.h:234
@ AV_OPT_TYPE_INT
Definition: opt.h:225
@ AV_OPT_TYPE_DOUBLE
Definition: opt.h:227
@ AV_OPT_TYPE_BOOL
Definition: opt.h:242
#define AV_CODEC_CAP_DELAY
Encoder or decoder requires flushing with NULL input at the end in order to give the complete and cor...
Definition: codec.h:77
#define AV_CODEC_FLAG_QSCALE
Use fixed qscale.
Definition: avcodec.h:275
#define AV_CODEC_FLAG_GLOBAL_HEADER
Place global headers in extradata instead of every keyframe.
Definition: avcodec.h:329
AVCodecID
Identify the syntax and semantics of the bitstream.
Definition: codec_id.h:46
#define AV_CODEC_CAP_HARDWARE
Codec is backed by a hardware implementation.
Definition: codec.h:157
@ AV_CODEC_ID_H264
Definition: codec_id.h:76
@ AV_CODEC_ID_HEVC
Definition: codec_id.h:223
#define AV_INPUT_BUFFER_PADDING_SIZE
Required number of additionally allocated bytes at the end of the input bitstream for decoding.
Definition: avcodec.h:215
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
Definition: avpacket.c:634
#define AV_PKT_FLAG_KEY
The packet contains a keyframe.
Definition: packet.h:410
#define FF_QP2LAMBDA
factor to convert from H.263 QP to lambda
Definition: avutil.h:227
#define AVERROR_BUFFER_TOO_SMALL
Buffer too small.
Definition: error.h:51
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:57
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:59
#define AVERROR(e)
Definition: error.h:43
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
Definition: frame.c:738
@ AV_FRAME_DATA_A53_CC
ATSC A53 Part 4 Closed Captions.
Definition: frame.h:58
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:235
int av_reduce(int *dst_num, int *dst_den, int64_t num, int64_t den, int64_t max)
Reduce a fraction.
Definition: rational.c:35
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:237
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:274
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
H.264 common definitions.
@ H264_NAL_PPS
Definition: h264.h:42
@ H264_NAL_SPS
Definition: h264.h:41
@ H264_NAL_SEI
Definition: h264.h:40
@ H264_NAL_AUD
Definition: h264.h:43
cl_device_type type
int i
Definition: input.c:407
#define FF_CODEC_CAP_INIT_THREADSAFE
The codec does not modify any global variables in the init function, allowing to call the init functi...
Definition: internal.h:41
#define FF_CODEC_CAP_INIT_CLEANUP
The codec allows calling the close function for deallocation even if the init function returned a fai...
Definition: internal.h:49
common internal API header
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:117
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:309
static const uint32_t color[16+AV_CLASS_CATEGORY_NB]
Definition: log.c:92
AVOptions.
static av_always_inline int pthread_cond_signal(pthread_cond_t *cond)
Definition: os2threads.h:152
static av_always_inline int pthread_cond_destroy(pthread_cond_t *cond)
Definition: os2threads.h:144
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
#define PTHREAD_ONCE_INIT
Definition: os2threads.h:71
static av_always_inline int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr)
Definition: os2threads.h:133
static av_always_inline int pthread_once(pthread_once_t *once_control, void(*init_routine)(void))
Definition: os2threads.h:210
_fmutex pthread_mutex_t
Definition: os2threads.h:53
static av_always_inline int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
Definition: os2threads.h:192
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
const char * av_color_transfer_name(enum AVColorTransferCharacteristic transfer)
Definition: pixdesc.c:2940
const char * av_color_range_name(enum AVColorRange range)
Definition: pixdesc.c:2901
const char * av_color_space_name(enum AVColorSpace space)
Definition: pixdesc.c:2961
const char * av_color_primaries_name(enum AVColorPrimaries primaries)
Definition: pixdesc.c:2919
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2489
AVColorRange
Visual content value range.
Definition: pixfmt.h:551
@ AVCOL_RANGE_MPEG
Narrow or limited range content.
Definition: pixfmt.h:569
@ AVCOL_RANGE_UNSPECIFIED
Definition: pixfmt.h:552
@ AVCOL_RANGE_NB
Not part of ABI.
Definition: pixfmt.h:587
@ AVCOL_RANGE_JPEG
Full range content.
Definition: pixfmt.h:586
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:65
@ AV_PIX_FMT_P010LE
like NV12, with 10bpp per component, data in the high bits, zeros in the low bits,...
Definition: pixfmt.h:284
@ AV_PIX_FMT_VIDEOTOOLBOX
hardware decoding through Videotoolbox
Definition: pixfmt.h:282
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
@ AV_PIX_FMT_BGRA
packed BGRA 8:8:8:8, 32bpp, BGRABGRA...
Definition: pixfmt.h:95
AVColorPrimaries
Chromaticity coordinates of the source primaries.
Definition: pixfmt.h:458
@ AVCOL_PRI_BT470BG
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM
Definition: pixfmt.h:465
@ AVCOL_PRI_BT709
also ITU-R BT1361 / IEC 61966-2-4 / SMPTE RP177 Annex B
Definition: pixfmt.h:460
@ AVCOL_PRI_UNSPECIFIED
Definition: pixfmt.h:461
@ AVCOL_PRI_SMPTE170M
also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC
Definition: pixfmt.h:466
@ AVCOL_PRI_BT2020
ITU-R BT2020.
Definition: pixfmt.h:469
AVColorTransferCharacteristic
Color Transfer Characteristic.
Definition: pixfmt.h:483
@ AVCOL_TRC_SMPTE2084
SMPTE ST 2084 for 10-, 12-, 14- and 16-bit systems.
Definition: pixfmt.h:500
@ AVCOL_TRC_GAMMA22
also ITU-R BT470M / ITU-R BT1700 625 PAL & SECAM
Definition: pixfmt.h:488
@ AVCOL_TRC_SMPTE240M
Definition: pixfmt.h:491
@ AVCOL_TRC_LINEAR
"Linear transfer characteristics"
Definition: pixfmt.h:492
@ AVCOL_TRC_GAMMA28
also ITU-R BT470BG
Definition: pixfmt.h:489
@ AVCOL_TRC_ARIB_STD_B67
ARIB STD-B67, known as "Hybrid log-gamma".
Definition: pixfmt.h:504
@ AVCOL_TRC_BT2020_12
ITU-R BT2020 for 12-bit system.
Definition: pixfmt.h:499
@ AVCOL_TRC_BT2020_10
ITU-R BT2020 for 10-bit system.
Definition: pixfmt.h:498
@ AVCOL_TRC_UNSPECIFIED
Definition: pixfmt.h:486
@ AVCOL_TRC_BT709
also ITU-R BT1361
Definition: pixfmt.h:485
@ AVCOL_SPC_BT709
also ITU-R BT1361 / IEC 61966-2-4 xvYCC709 / SMPTE RP177 Annex B
Definition: pixfmt.h:514
@ AVCOL_SPC_BT470BG
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM / IEC 61966-2-4 xvYCC601
Definition: pixfmt.h:518
@ AVCOL_SPC_BT2020_NCL
ITU-R BT2020 non-constant luminance system.
Definition: pixfmt.h:523
@ AVCOL_SPC_UNSPECIFIED
Definition: pixfmt.h:515
@ AVCOL_SPC_SMPTE170M
also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC
Definition: pixfmt.h:519
@ AVCOL_SPC_SMPTE240M
functionally identical to above
Definition: pixfmt.h:520
mfxU16 profile
Definition: qsvenc.c:45
enum AVMediaType codec_type
Definition: rtp.c:37
static char buffer[20]
Definition: seek.c:32
@ SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35
Definition: sei.h:34
Describe the class of an AVClass context structure.
Definition: log.h:67
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
main external API structure.
Definition: avcodec.h:536
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:746
int width
picture width / height.
Definition: avcodec.h:709
int global_quality
Global quality for codecs which cannot change it per frame.
Definition: avcodec.h:602
enum AVColorRange color_range
MPEG vs JPEG YUV range.
Definition: avcodec.h:1171
enum AVColorPrimaries color_primaries
Chromaticity coordinates of the source primaries.
Definition: avcodec.h:1150
int max_b_frames
maximum number of B-frames between non-B-frames Note: The output will be delayed by max_b_frames+1 re...
Definition: avcodec.h:796
AVRational sample_aspect_ratio
sample aspect ratio (0 if unknown) That is the width of a pixel divided by the height of the pixel.
Definition: avcodec.h:915
int has_b_frames
Size of the frame reordering buffer in the decoder.
Definition: avcodec.h:826
int64_t bit_rate
the average bitrate
Definition: avcodec.h:586
enum AVColorSpace colorspace
YUV colorspace type.
Definition: avcodec.h:1164
int gop_size
the number of pictures in a group of pictures, or 0 for intra_only
Definition: avcodec.h:731
int64_t rc_max_rate
maximum bitrate
Definition: avcodec.h:1416
enum AVColorTransferCharacteristic color_trc
Color Transfer Characteristic.
Definition: avcodec.h:1157
AVRational time_base
This is the fundamental unit of time (in seconds) in terms of which frame timestamps are represented.
Definition: avcodec.h:659
int flags
AV_CODEC_FLAG_*.
Definition: avcodec.h:616
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
Definition: avcodec.h:637
enum AVCodecID codec_id
Definition: avcodec.h:546
int extradata_size
Definition: avcodec.h:638
void * priv_data
Definition: avcodec.h:563
AVCodec.
Definition: codec.h:197
const char * name
Name of the codec implementation.
Definition: codec.h:204
Structure to hold side data for an AVFrame.
Definition: frame.h:220
This structure describes decoded (raw) audio or video data.
Definition: frame.h:318
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:411
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:332
enum AVColorRange color_range
MPEG vs JPEG YUV range.
Definition: frame.h:562
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:349
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:391
enum AVPictureType pict_type
Picture type of the frame.
Definition: frame.h:401
AVOption.
Definition: opt.h:248
This structure stores compressed data.
Definition: packet.h:346
int flags
A combination of AV_PKT_FLAG values.
Definition: packet.h:375
int size
Definition: packet.h:370
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: packet.h:362
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed.
Definition: packet.h:368
uint8_t * data
Definition: packet.h:369
Rational number (pair of numerator and denominator).
Definition: rational.h:58
int num
Numerator.
Definition: rational.h:59
int den
Denominator.
Definition: rational.h:60
CMSampleBufferRef cm_buffer
struct BufNode * next
ExtraSEI * sei
int64_t frames_before
pthread_cond_t cv_sample_sent
CFStringRef transfer_function
int64_t frame_ct_out
BufNode * q_tail
double alpha_quality
int64_t frames_after
getParameterSetAtIndex get_param_set_func
CFStringRef ycbcr_matrix
enum AVCodecID codec_id
int64_t require_sw
pthread_mutex_t lock
BufNode * q_head
VTCompressionSessionRef session
CFStringRef color_primaries
int64_t frame_ct_in
uint8_t level
Definition: svq3.c:206
#define av_free(p)
#define av_malloc(s)
#define av_log(a,...)
#define src
Definition: vp8dsp.c:255
static int16_t block[64]
Definition: dct.c:116
AVPacket * pkt
Definition: movenc.c:59
AVFormatContext * ctx
Definition: movenc.c:48
static int64_t pts
int size
static int get_frame(AVFilterContext *ctx, int is_second)
Definition: vf_nnedi.c:666
static const uint8_t offset[127][2]
Definition: vf_spp.c:107
static int write_sei(const ExtraSEI *sei, int sei_type, uint8_t *dst, size_t dst_size)
static const AVClass h264_videotoolbox_class
CFStringRef kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder
static av_cold int vtenc_init(AVCodecContext *avctx)
CFStringRef kVTProfileLevel_H264_High_4_0
static int create_encoder_dict_h264(const AVFrame *frame, CFDictionaryRef *dict_out)
CFStringRef kVTProfileLevel_H264_Main_5_2
@ kCMVideoCodecType_HEVC
static int vtenc_create_encoder(AVCodecContext *avctx, CMVideoCodecType codec_type, CFStringRef profile_level, CFNumberRef gamma_level, CFDictionaryRef enc_info, CFDictionaryRef pixel_buffer_info, VTCompressionSessionRef *session)
static int create_cv_pixel_buffer_info(AVCodecContext *avctx, CFMutableDictionaryRef *dict)
static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer)
static CMVideoCodecType get_cm_codec_type(enum AVCodecID id, enum AVPixelFormat fmt, double alpha_quality)
CFStringRef kCVImageBufferYCbCrMatrix_ITU_R_2020
static int is_post_sei_nal_type(int nal_type)
CFStringRef kVTProfileLevel_H264_High_3_0
CFStringRef kVTProfileLevel_H264_Main_5_1
CFStringRef kVTProfileLevel_H264_Baseline_4_2
static void vtenc_output_callback(void *ctx, void *sourceFrameCtx, OSStatus status, VTEncodeInfoFlags flags, CMSampleBufferRef sample_buffer)
CFStringRef kVTProfileLevel_H264_High_4_1
static void set_async_error(VTEncContext *vtctx, int err)
static av_cold int vtenc_frame(AVCodecContext *avctx, AVPacket *pkt, const AVFrame *frame, int *got_packet)
getParameterSetAtIndex CMVideoFormatDescriptionGetHEVCParameterSetAtIndex
CFStringRef kVTCompressionPropertyKey_RealTime
#define VE
CFStringRef kVTProfileLevel_H264_Baseline_5_1
VT_HEVCProfile
@ HEVC_PROF_AUTO
@ HEVC_PROF_MAIN
@ HEVC_PROF_MAIN10
@ HEVC_PROF_COUNT
CFStringRef kVTProfileLevel_H264_High_3_1
#define COMMON_OPTIONS
static int copy_emulation_prev(const uint8_t *src, size_t src_size, uint8_t *dst, ssize_t dst_offset, size_t dst_size)
Copies the data inserting emulation prevention bytes as needed.
@ kCMVideoCodecType_HEVCWithAlpha
static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf, ExtraSEI **sei)
static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict)
static void clear_frame_queue(VTEncContext *vtctx)
CFStringRef kVTH264EntropyMode_CAVLC
static pthread_once_t once_ctrl
@ kCVPixelFormatType_420YpCbCr10BiPlanarFullRange
static bool get_vt_h264_profile_level(AVCodecContext *avctx, CFStringRef *profile_level_val)
static int vtenc_send_frame(AVCodecContext *avctx, VTEncContext *vtctx, const AVFrame *frame)
static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx, const AVFrame *frame, CVPixelBufferRef cv_img, const size_t *plane_strides, const size_t *plane_rows)
static const AVOption h264_options[]
CFStringRef kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder
static int get_cv_transfer_function(AVCodecContext *avctx, CFStringRef *transfer_fnc, CFNumberRef *gamma_level)
CFStringRef kVTProfileLevel_H264_Baseline_5_2
static void vt_release_num(CFNumberRef *refPtr)
NULL-safe release of *refPtr, and sets value to NULL.
static const AVOption hevc_options[]
static enum AVPixelFormat hevc_pix_fmts[]
static bool get_vt_hevc_profile_level(AVCodecContext *avctx, CFStringRef *profile_level_val)
static enum AVPixelFormat avc_pix_fmts[]
static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
static int vtenc_cm_to_avpacket(AVCodecContext *avctx, CMSampleBufferRef sample_buffer, AVPacket *pkt, ExtraSEI *sei)
CFStringRef kVTProfileLevel_H264_High_4_2
CFStringRef kVTProfileLevel_H264_High_3_2
VT_H264Profile
@ H264_PROF_EXTENDED
@ H264_PROF_COUNT
@ H264_PROF_AUTO
@ H264_PROF_HIGH
@ H264_PROF_BASELINE
@ H264_PROF_MAIN
OSStatus(* getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc, size_t parameterSetIndex, const uint8_t **parameterSetPointerOut, size_t *parameterSetSizeOut, size_t *parameterSetCountOut, int *NALUnitHeaderLengthOut)
static int get_cv_color_primaries(AVCodecContext *avctx, CFStringRef *primaries)
static struct @163 compat_keys
static int find_sei_end(AVCodecContext *avctx, uint8_t *nal_data, size_t nal_size, uint8_t **sei_end)
CFStringRef kVTProfileLevel_H264_Baseline_4_0
CFStringRef kVTProfileLevel_H264_Extended_AutoLevel
CFStringRef kCVImageBufferColorPrimaries_ITU_R_2020
VTH264Entropy
@ VT_CAVLC
@ VT_ENTROPY_NOT_SET
@ VT_CABAC
static const uint8_t start_code[]
AVCodec ff_h264_videotoolbox_encoder
static int copy_param_sets(AVCodecContext *avctx, CMVideoFormatDescriptionRef vid_fmt, uint8_t *dst, size_t dst_size)
static bool vtenc_qscale_enabled(void)
CFStringRef kVTProfileLevel_HEVC_Main_AutoLevel
CFStringRef kCVImageBufferTransferFunction_ITU_R_2020
static void loadVTEncSymbols()
static int count_nalus(size_t length_code_size, CMSampleBufferRef sample_buffer, int *count)
CFStringRef kVTProfileLevel_H264_High_AutoLevel
static int get_length_code_size(AVCodecContext *avctx, CMSampleBufferRef sample_buffer, size_t *size)
static av_cold int vtenc_close(AVCodecContext *avctx)
CFStringRef kVTCompressionPropertyKey_H264EntropyMode
static int vtenc_populate_extradata(AVCodecContext *avctx, CMVideoCodecType codec_type, CFStringRef profile_level, CFNumberRef gamma_level, CFDictionaryRef enc_info, CFDictionaryRef pixel_buffer_info)
CFStringRef kVTH264EntropyMode_CABAC
CFStringRef kVTProfileLevel_H264_Main_AutoLevel
static const AVClass hevc_videotoolbox_class
static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer, ExtraSEI *sei)
AVCodec ff_hevc_videotoolbox_encoder
#define OFFSET(x)
CFStringRef kVTCompressionPropertyKey_TargetQualityForAlpha
#define TARGET_CPU_ARM64
static int copy_replace_length_codes(AVCodecContext *avctx, size_t length_code_size, CMSampleBufferRef sample_buffer, ExtraSEI *sei, uint8_t *dst_data, size_t dst_size)
Copies NAL units and replaces length codes with H.264 Annex B start codes.
static int get_cv_pixel_info(AVCodecContext *avctx, const AVFrame *frame, int *color, int *plane_count, size_t *widths, size_t *heights, size_t *strides, size_t *contiguous_buf_size)
static int get_cv_pixel_format(AVCodecContext *avctx, enum AVPixelFormat fmt, enum AVColorRange range, int *av_pixel_format, int *range_guessed)
CFStringRef kVTProfileLevel_H264_High_5_2
CFStringRef kVTProfileLevel_H264_High_5_1
static int create_cv_pixel_buffer(AVCodecContext *avctx, const AVFrame *frame, CVPixelBufferRef *cv_img)
static int get_sei_msg_bytes(const ExtraSEI *sei, int type)
Returns a sufficient number of bytes to contain the sei data.
#define GET_SYM(symbol, defaultVal)
static int get_params_size(AVCodecContext *avctx, CMVideoFormatDescriptionRef vid_fmt, size_t *size)
Get the parameter sets from a CMSampleBufferRef.
CFStringRef kVTProfileLevel_HEVC_Main10_AutoLevel
static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix)
CFStringRef kVTProfileLevel_H264_Baseline_5_0
CFStringRef kVTProfileLevel_H264_Baseline_AutoLevel
static int vtenc_configure_encoder(AVCodecContext *avctx)
CFStringRef kVTProfileLevel_H264_Main_4_2
@ kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange
CFStringRef kVTProfileLevel_H264_Extended_5_0
int len