summaryrefslogtreecommitdiff
path: root/libavdevice/avfoundation.m (plain)
blob: e2ddf47dbe8dc7dad799d0dd77b274efcf8f487f
1/*
2 * AVFoundation input device
3 * Copyright (c) 2014 Thilo Borgmann <thilo.borgmann@mail.de>
4 *
5 * This file is part of FFmpeg.
6 *
7 * FFmpeg is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * FFmpeg is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with FFmpeg; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22/**
23 * @file
24 * AVFoundation input device
25 * @author Thilo Borgmann <thilo.borgmann@mail.de>
26 */
27
28#import <AVFoundation/AVFoundation.h>
29#include <pthread.h>
30
31#include "libavutil/pixdesc.h"
32#include "libavutil/opt.h"
33#include "libavutil/avstring.h"
34#include "libavformat/internal.h"
35#include "libavutil/internal.h"
36#include "libavutil/parseutils.h"
37#include "libavutil/time.h"
38#include "avdevice.h"
39
40static const int avf_time_base = 1000000;
41
42static const AVRational avf_time_base_q = {
43 .num = 1,
44 .den = avf_time_base
45};
46
47struct AVFPixelFormatSpec {
48 enum AVPixelFormat ff_id;
49 OSType avf_id;
50};
51
52static const struct AVFPixelFormatSpec avf_pixel_formats[] = {
53 { AV_PIX_FMT_MONOBLACK, kCVPixelFormatType_1Monochrome },
54 { AV_PIX_FMT_RGB555BE, kCVPixelFormatType_16BE555 },
55 { AV_PIX_FMT_RGB555LE, kCVPixelFormatType_16LE555 },
56 { AV_PIX_FMT_RGB565BE, kCVPixelFormatType_16BE565 },
57 { AV_PIX_FMT_RGB565LE, kCVPixelFormatType_16LE565 },
58 { AV_PIX_FMT_RGB24, kCVPixelFormatType_24RGB },
59 { AV_PIX_FMT_BGR24, kCVPixelFormatType_24BGR },
60 { AV_PIX_FMT_0RGB, kCVPixelFormatType_32ARGB },
61 { AV_PIX_FMT_BGR0, kCVPixelFormatType_32BGRA },
62 { AV_PIX_FMT_0BGR, kCVPixelFormatType_32ABGR },
63 { AV_PIX_FMT_RGB0, kCVPixelFormatType_32RGBA },
64 { AV_PIX_FMT_BGR48BE, kCVPixelFormatType_48RGB },
65 { AV_PIX_FMT_UYVY422, kCVPixelFormatType_422YpCbCr8 },
66 { AV_PIX_FMT_YUVA444P, kCVPixelFormatType_4444YpCbCrA8R },
67 { AV_PIX_FMT_YUVA444P16LE, kCVPixelFormatType_4444AYpCbCr16 },
68 { AV_PIX_FMT_YUV444P, kCVPixelFormatType_444YpCbCr8 },
69 { AV_PIX_FMT_YUV422P16, kCVPixelFormatType_422YpCbCr16 },
70 { AV_PIX_FMT_YUV422P10, kCVPixelFormatType_422YpCbCr10 },
71 { AV_PIX_FMT_YUV444P10, kCVPixelFormatType_444YpCbCr10 },
72 { AV_PIX_FMT_YUV420P, kCVPixelFormatType_420YpCbCr8Planar },
73 { AV_PIX_FMT_NV12, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange },
74 { AV_PIX_FMT_YUYV422, kCVPixelFormatType_422YpCbCr8_yuvs },
75#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
76 { AV_PIX_FMT_GRAY8, kCVPixelFormatType_OneComponent8 },
77#endif
78 { AV_PIX_FMT_NONE, 0 }
79};
80
81typedef struct
82{
83 AVClass* class;
84
85 int frames_captured;
86 int audio_frames_captured;
87 int64_t first_pts;
88 int64_t first_audio_pts;
89 pthread_mutex_t frame_lock;
90 pthread_cond_t frame_wait_cond;
91 id avf_delegate;
92 id avf_audio_delegate;
93
94 AVRational framerate;
95 int width, height;
96
97 int capture_cursor;
98 int capture_mouse_clicks;
99
100 int list_devices;
101 int video_device_index;
102 int video_stream_index;
103 int audio_device_index;
104 int audio_stream_index;
105
106 char *video_filename;
107 char *audio_filename;
108
109 int num_video_devices;
110
111 int audio_channels;
112 int audio_bits_per_sample;
113 int audio_float;
114 int audio_be;
115 int audio_signed_integer;
116 int audio_packed;
117 int audio_non_interleaved;
118
119 int32_t *audio_buffer;
120 int audio_buffer_size;
121
122 enum AVPixelFormat pixel_format;
123
124 AVCaptureSession *capture_session;
125 AVCaptureVideoDataOutput *video_output;
126 AVCaptureAudioDataOutput *audio_output;
127 CMSampleBufferRef current_frame;
128 CMSampleBufferRef current_audio_frame;
129} AVFContext;
130
131static void lock_frames(AVFContext* ctx)
132{
133 pthread_mutex_lock(&ctx->frame_lock);
134}
135
136static void unlock_frames(AVFContext* ctx)
137{
138 pthread_mutex_unlock(&ctx->frame_lock);
139}
140
141/** FrameReciever class - delegate for AVCaptureSession
142 */
143@interface AVFFrameReceiver : NSObject
144{
145 AVFContext* _context;
146}
147
148- (id)initWithContext:(AVFContext*)context;
149
150- (void) captureOutput:(AVCaptureOutput *)captureOutput
151 didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
152 fromConnection:(AVCaptureConnection *)connection;
153
154@end
155
156@implementation AVFFrameReceiver
157
158- (id)initWithContext:(AVFContext*)context
159{
160 if (self = [super init]) {
161 _context = context;
162 }
163 return self;
164}
165
166- (void) captureOutput:(AVCaptureOutput *)captureOutput
167 didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
168 fromConnection:(AVCaptureConnection *)connection
169{
170 lock_frames(_context);
171
172 if (_context->current_frame != nil) {
173 CFRelease(_context->current_frame);
174 }
175
176 _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
177
178 pthread_cond_signal(&_context->frame_wait_cond);
179
180 unlock_frames(_context);
181
182 ++_context->frames_captured;
183}
184
185@end
186
187/** AudioReciever class - delegate for AVCaptureSession
188 */
189@interface AVFAudioReceiver : NSObject
190{
191 AVFContext* _context;
192}
193
194- (id)initWithContext:(AVFContext*)context;
195
196- (void) captureOutput:(AVCaptureOutput *)captureOutput
197 didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
198 fromConnection:(AVCaptureConnection *)connection;
199
200@end
201
202@implementation AVFAudioReceiver
203
204- (id)initWithContext:(AVFContext*)context
205{
206 if (self = [super init]) {
207 _context = context;
208 }
209 return self;
210}
211
212- (void) captureOutput:(AVCaptureOutput *)captureOutput
213 didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
214 fromConnection:(AVCaptureConnection *)connection
215{
216 lock_frames(_context);
217
218 if (_context->current_audio_frame != nil) {
219 CFRelease(_context->current_audio_frame);
220 }
221
222 _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
223
224 pthread_cond_signal(&_context->frame_wait_cond);
225
226 unlock_frames(_context);
227
228 ++_context->audio_frames_captured;
229}
230
231@end
232
233static void destroy_context(AVFContext* ctx)
234{
235 [ctx->capture_session stopRunning];
236
237 [ctx->capture_session release];
238 [ctx->video_output release];
239 [ctx->audio_output release];
240 [ctx->avf_delegate release];
241 [ctx->avf_audio_delegate release];
242
243 ctx->capture_session = NULL;
244 ctx->video_output = NULL;
245 ctx->audio_output = NULL;
246 ctx->avf_delegate = NULL;
247 ctx->avf_audio_delegate = NULL;
248
249 av_freep(&ctx->audio_buffer);
250
251 pthread_mutex_destroy(&ctx->frame_lock);
252 pthread_cond_destroy(&ctx->frame_wait_cond);
253
254 if (ctx->current_frame) {
255 CFRelease(ctx->current_frame);
256 }
257}
258
259static void parse_device_name(AVFormatContext *s)
260{
261 AVFContext *ctx = (AVFContext*)s->priv_data;
262 char *tmp = av_strdup(s->filename);
263 char *save;
264
265 if (tmp[0] != ':') {
266 ctx->video_filename = av_strtok(tmp, ":", &save);
267 ctx->audio_filename = av_strtok(NULL, ":", &save);
268 } else {
269 ctx->audio_filename = av_strtok(tmp, ":", &save);
270 }
271}
272
273/**
274 * Configure the video device.
275 *
276 * Configure the video device using a run-time approach to access properties
277 * since formats, activeFormat are available since iOS >= 7.0 or OSX >= 10.7
278 * and activeVideoMaxFrameDuration is available since i0S >= 7.0 and OSX >= 10.9.
279 *
280 * The NSUndefinedKeyException must be handled by the caller of this function.
281 *
282 */
283static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
284{
285 AVFContext *ctx = (AVFContext*)s->priv_data;
286
287 double framerate = av_q2d(ctx->framerate);
288 NSObject *range = nil;
289 NSObject *format = nil;
290 NSObject *selected_range = nil;
291 NSObject *selected_format = nil;
292
293 for (format in [video_device valueForKey:@"formats"]) {
294 CMFormatDescriptionRef formatDescription;
295 CMVideoDimensions dimensions;
296
297 formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
298 dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
299
300 if ((ctx->width == 0 && ctx->height == 0) ||
301 (dimensions.width == ctx->width && dimensions.height == ctx->height)) {
302
303 selected_format = format;
304
305 for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
306 double max_framerate;
307
308 [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
309 if (fabs (framerate - max_framerate) < 0.01) {
310 selected_range = range;
311 break;
312 }
313 }
314 }
315 }
316
317 if (!selected_format) {
318 av_log(s, AV_LOG_ERROR, "Selected video size (%dx%d) is not supported by the device\n",
319 ctx->width, ctx->height);
320 goto unsupported_format;
321 }
322
323 if (!selected_range) {
324 av_log(s, AV_LOG_ERROR, "Selected framerate (%f) is not supported by the device\n",
325 framerate);
326 goto unsupported_format;
327 }
328
329 if ([video_device lockForConfiguration:NULL] == YES) {
330 NSValue *min_frame_duration = [selected_range valueForKey:@"minFrameDuration"];
331
332 [video_device setValue:selected_format forKey:@"activeFormat"];
333 [video_device setValue:min_frame_duration forKey:@"activeVideoMinFrameDuration"];
334 [video_device setValue:min_frame_duration forKey:@"activeVideoMaxFrameDuration"];
335 } else {
336 av_log(s, AV_LOG_ERROR, "Could not lock device for configuration");
337 return AVERROR(EINVAL);
338 }
339
340 return 0;
341
342unsupported_format:
343
344 av_log(s, AV_LOG_ERROR, "Supported modes:\n");
345 for (format in [video_device valueForKey:@"formats"]) {
346 CMFormatDescriptionRef formatDescription;
347 CMVideoDimensions dimensions;
348
349 formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
350 dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
351
352 for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
353 double min_framerate;
354 double max_framerate;
355
356 [[range valueForKey:@"minFrameRate"] getValue:&min_framerate];
357 [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
358 av_log(s, AV_LOG_ERROR, " %dx%d@[%f %f]fps\n",
359 dimensions.width, dimensions.height,
360 min_framerate, max_framerate);
361 }
362 }
363 return AVERROR(EINVAL);
364}
365
366static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
367{
368 AVFContext *ctx = (AVFContext*)s->priv_data;
369 int ret;
370 NSError *error = nil;
371 AVCaptureInput* capture_input = nil;
372 struct AVFPixelFormatSpec pxl_fmt_spec;
373 NSNumber *pixel_format;
374 NSDictionary *capture_dict;
375 dispatch_queue_t queue;
376
377 if (ctx->video_device_index < ctx->num_video_devices) {
378 capture_input = (AVCaptureInput*) [[[AVCaptureDeviceInput alloc] initWithDevice:video_device error:&error] autorelease];
379 } else {
380 capture_input = (AVCaptureInput*) video_device;
381 }
382
383 if (!capture_input) {
384 av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
385 [[error localizedDescription] UTF8String]);
386 return 1;
387 }
388
389 if ([ctx->capture_session canAddInput:capture_input]) {
390 [ctx->capture_session addInput:capture_input];
391 } else {
392 av_log(s, AV_LOG_ERROR, "can't add video input to capture session\n");
393 return 1;
394 }
395
396 // Attaching output
397 ctx->video_output = [[AVCaptureVideoDataOutput alloc] init];
398
399 if (!ctx->video_output) {
400 av_log(s, AV_LOG_ERROR, "Failed to init AV video output\n");
401 return 1;
402 }
403
404 // Configure device framerate and video size
405 @try {
406 if ((ret = configure_video_device(s, video_device)) < 0) {
407 return ret;
408 }
409 } @catch (NSException *exception) {
410 if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
411 av_log (s, AV_LOG_ERROR, "An error occurred: %s", [exception.reason UTF8String]);
412 return AVERROR_EXTERNAL;
413 }
414 }
415
416 // select pixel format
417 pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
418
419 for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
420 if (ctx->pixel_format == avf_pixel_formats[i].ff_id) {
421 pxl_fmt_spec = avf_pixel_formats[i];
422 break;
423 }
424 }
425
426 // check if selected pixel format is supported by AVFoundation
427 if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
428 av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by AVFoundation.\n",
429 av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
430 return 1;
431 }
432
433 // check if the pixel format is available for this device
434 if ([[ctx->video_output availableVideoCVPixelFormatTypes] indexOfObject:[NSNumber numberWithInt:pxl_fmt_spec.avf_id]] == NSNotFound) {
435 av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by the input device.\n",
436 av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
437
438 pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;
439
440 av_log(s, AV_LOG_ERROR, "Supported pixel formats:\n");
441 for (NSNumber *pxl_fmt in [ctx->video_output availableVideoCVPixelFormatTypes]) {
442 struct AVFPixelFormatSpec pxl_fmt_dummy;
443 pxl_fmt_dummy.ff_id = AV_PIX_FMT_NONE;
444 for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
445 if ([pxl_fmt intValue] == avf_pixel_formats[i].avf_id) {
446 pxl_fmt_dummy = avf_pixel_formats[i];
447 break;
448 }
449 }
450
451 if (pxl_fmt_dummy.ff_id != AV_PIX_FMT_NONE) {
452 av_log(s, AV_LOG_ERROR, " %s\n", av_get_pix_fmt_name(pxl_fmt_dummy.ff_id));
453
454 // select first supported pixel format instead of user selected (or default) pixel format
455 if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
456 pxl_fmt_spec = pxl_fmt_dummy;
457 }
458 }
459 }
460
461 // fail if there is no appropriate pixel format or print a warning about overriding the pixel format
462 if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
463 return 1;
464 } else {
465 av_log(s, AV_LOG_WARNING, "Overriding selected pixel format to use %s instead.\n",
466 av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
467 }
468 }
469
470 ctx->pixel_format = pxl_fmt_spec.ff_id;
471 pixel_format = [NSNumber numberWithUnsignedInt:pxl_fmt_spec.avf_id];
472 capture_dict = [NSDictionary dictionaryWithObject:pixel_format
473 forKey:(id)kCVPixelBufferPixelFormatTypeKey];
474
475 [ctx->video_output setVideoSettings:capture_dict];
476 [ctx->video_output setAlwaysDiscardsLateVideoFrames:YES];
477
478 ctx->avf_delegate = [[AVFFrameReceiver alloc] initWithContext:ctx];
479
480 queue = dispatch_queue_create("avf_queue", NULL);
481 [ctx->video_output setSampleBufferDelegate:ctx->avf_delegate queue:queue];
482 dispatch_release(queue);
483
484 if ([ctx->capture_session canAddOutput:ctx->video_output]) {
485 [ctx->capture_session addOutput:ctx->video_output];
486 } else {
487 av_log(s, AV_LOG_ERROR, "can't add video output to capture session\n");
488 return 1;
489 }
490
491 return 0;
492}
493
494static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
495{
496 AVFContext *ctx = (AVFContext*)s->priv_data;
497 NSError *error = nil;
498 AVCaptureDeviceInput* audio_dev_input = [[[AVCaptureDeviceInput alloc] initWithDevice:audio_device error:&error] autorelease];
499 dispatch_queue_t queue;
500
501 if (!audio_dev_input) {
502 av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
503 [[error localizedDescription] UTF8String]);
504 return 1;
505 }
506
507 if ([ctx->capture_session canAddInput:audio_dev_input]) {
508 [ctx->capture_session addInput:audio_dev_input];
509 } else {
510 av_log(s, AV_LOG_ERROR, "can't add audio input to capture session\n");
511 return 1;
512 }
513
514 // Attaching output
515 ctx->audio_output = [[AVCaptureAudioDataOutput alloc] init];
516
517 if (!ctx->audio_output) {
518 av_log(s, AV_LOG_ERROR, "Failed to init AV audio output\n");
519 return 1;
520 }
521
522 ctx->avf_audio_delegate = [[AVFAudioReceiver alloc] initWithContext:ctx];
523
524 queue = dispatch_queue_create("avf_audio_queue", NULL);
525 [ctx->audio_output setSampleBufferDelegate:ctx->avf_audio_delegate queue:queue];
526 dispatch_release(queue);
527
528 if ([ctx->capture_session canAddOutput:ctx->audio_output]) {
529 [ctx->capture_session addOutput:ctx->audio_output];
530 } else {
531 av_log(s, AV_LOG_ERROR, "adding audio output to capture session failed\n");
532 return 1;
533 }
534
535 return 0;
536}
537
538static int get_video_config(AVFormatContext *s)
539{
540 AVFContext *ctx = (AVFContext*)s->priv_data;
541 CVImageBufferRef image_buffer;
542 CGSize image_buffer_size;
543 AVStream* stream = avformat_new_stream(s, NULL);
544
545 if (!stream) {
546 return 1;
547 }
548
549 // Take stream info from the first frame.
550 while (ctx->frames_captured < 1) {
551 CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
552 }
553
554 lock_frames(ctx);
555
556 ctx->video_stream_index = stream->index;
557
558 avpriv_set_pts_info(stream, 64, 1, avf_time_base);
559
560 image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
561 image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);
562
563 stream->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
564 stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
565 stream->codecpar->width = (int)image_buffer_size.width;
566 stream->codecpar->height = (int)image_buffer_size.height;
567 stream->codecpar->format = ctx->pixel_format;
568
569 CFRelease(ctx->current_frame);
570 ctx->current_frame = nil;
571
572 unlock_frames(ctx);
573
574 return 0;
575}
576
577static int get_audio_config(AVFormatContext *s)
578{
579 AVFContext *ctx = (AVFContext*)s->priv_data;
580 CMFormatDescriptionRef format_desc;
581 AVStream* stream = avformat_new_stream(s, NULL);
582
583 if (!stream) {
584 return 1;
585 }
586
587 // Take stream info from the first frame.
588 while (ctx->audio_frames_captured < 1) {
589 CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
590 }
591
592 lock_frames(ctx);
593
594 ctx->audio_stream_index = stream->index;
595
596 avpriv_set_pts_info(stream, 64, 1, avf_time_base);
597
598 format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
599 const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);
600
601 if (!basic_desc) {
602 av_log(s, AV_LOG_ERROR, "audio format not available\n");
603 return 1;
604 }
605
606 stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
607 stream->codecpar->sample_rate = basic_desc->mSampleRate;
608 stream->codecpar->channels = basic_desc->mChannelsPerFrame;
609 stream->codecpar->channel_layout = av_get_default_channel_layout(stream->codecpar->channels);
610
611 ctx->audio_channels = basic_desc->mChannelsPerFrame;
612 ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
613 ctx->audio_float = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat;
614 ctx->audio_be = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian;
615 ctx->audio_signed_integer = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger;
616 ctx->audio_packed = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked;
617 ctx->audio_non_interleaved = basic_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved;
618
619 if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
620 ctx->audio_float &&
621 ctx->audio_bits_per_sample == 32 &&
622 ctx->audio_packed) {
623 stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE;
624 } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
625 ctx->audio_signed_integer &&
626 ctx->audio_bits_per_sample == 16 &&
627 ctx->audio_packed) {
628 stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE;
629 } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
630 ctx->audio_signed_integer &&
631 ctx->audio_bits_per_sample == 24 &&
632 ctx->audio_packed) {
633 stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE;
634 } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
635 ctx->audio_signed_integer &&
636 ctx->audio_bits_per_sample == 32 &&
637 ctx->audio_packed) {
638 stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE;
639 } else {
640 av_log(s, AV_LOG_ERROR, "audio format is not supported\n");
641 return 1;
642 }
643
644 if (ctx->audio_non_interleaved) {
645 CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
646 ctx->audio_buffer_size = CMBlockBufferGetDataLength(block_buffer);
647 ctx->audio_buffer = av_malloc(ctx->audio_buffer_size);
648 if (!ctx->audio_buffer) {
649 av_log(s, AV_LOG_ERROR, "error allocating audio buffer\n");
650 return 1;
651 }
652 }
653
654 CFRelease(ctx->current_audio_frame);
655 ctx->current_audio_frame = nil;
656
657 unlock_frames(ctx);
658
659 return 0;
660}
661
662static int avf_read_header(AVFormatContext *s)
663{
664 NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
665 int capture_screen = 0;
666 uint32_t num_screens = 0;
667 AVFContext *ctx = (AVFContext*)s->priv_data;
668 AVCaptureDevice *video_device = nil;
669 AVCaptureDevice *audio_device = nil;
670 // Find capture device
671 NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
672 ctx->num_video_devices = [devices count];
673
674 ctx->first_pts = av_gettime();
675 ctx->first_audio_pts = av_gettime();
676
677 pthread_mutex_init(&ctx->frame_lock, NULL);
678 pthread_cond_init(&ctx->frame_wait_cond, NULL);
679
680#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
681 CGGetActiveDisplayList(0, NULL, &num_screens);
682#endif
683
684 // List devices if requested
685 if (ctx->list_devices) {
686 int index = 0;
687 av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
688 for (AVCaptureDevice *device in devices) {
689 const char *name = [[device localizedName] UTF8String];
690 index = [devices indexOfObject:device];
691 av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
692 index++;
693 }
694#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
695 if (num_screens > 0) {
696 CGDirectDisplayID screens[num_screens];
697 CGGetActiveDisplayList(num_screens, screens, &num_screens);
698 for (int i = 0; i < num_screens; i++) {
699 av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", index + i, i);
700 }
701 }
702#endif
703
704 av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
705 devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
706 for (AVCaptureDevice *device in devices) {
707 const char *name = [[device localizedName] UTF8String];
708 int index = [devices indexOfObject:device];
709 av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
710 }
711 goto fail;
712 }
713
714 // parse input filename for video and audio device
715 parse_device_name(s);
716
717 // check for device index given in filename
718 if (ctx->video_device_index == -1 && ctx->video_filename) {
719 sscanf(ctx->video_filename, "%d", &ctx->video_device_index);
720 }
721 if (ctx->audio_device_index == -1 && ctx->audio_filename) {
722 sscanf(ctx->audio_filename, "%d", &ctx->audio_device_index);
723 }
724
725 if (ctx->video_device_index >= 0) {
726 if (ctx->video_device_index < ctx->num_video_devices) {
727 video_device = [devices objectAtIndex:ctx->video_device_index];
728 } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) {
729#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
730 CGDirectDisplayID screens[num_screens];
731 CGGetActiveDisplayList(num_screens, screens, &num_screens);
732 AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[ctx->video_device_index - ctx->num_video_devices]] autorelease];
733
734 if (ctx->framerate.num > 0) {
735 capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
736 }
737
738#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
739 if (ctx->capture_cursor) {
740 capture_screen_input.capturesCursor = YES;
741 } else {
742 capture_screen_input.capturesCursor = NO;
743 }
744#endif
745
746 if (ctx->capture_mouse_clicks) {
747 capture_screen_input.capturesMouseClicks = YES;
748 } else {
749 capture_screen_input.capturesMouseClicks = NO;
750 }
751
752 video_device = (AVCaptureDevice*) capture_screen_input;
753 capture_screen = 1;
754#endif
755 } else {
756 av_log(ctx, AV_LOG_ERROR, "Invalid device index\n");
757 goto fail;
758 }
759 } else if (ctx->video_filename &&
760 strncmp(ctx->video_filename, "none", 4)) {
761 if (!strncmp(ctx->video_filename, "default", 7)) {
762 video_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
763 } else {
764 // looking for video inputs
765 for (AVCaptureDevice *device in devices) {
766 if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
767 video_device = device;
768 break;
769 }
770 }
771
772#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
773 // looking for screen inputs
774 if (!video_device) {
775 int idx;
776 if(sscanf(ctx->video_filename, "Capture screen %d", &idx) && idx < num_screens) {
777 CGDirectDisplayID screens[num_screens];
778 CGGetActiveDisplayList(num_screens, screens, &num_screens);
779 AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease];
780 video_device = (AVCaptureDevice*) capture_screen_input;
781 ctx->video_device_index = ctx->num_video_devices + idx;
782 capture_screen = 1;
783
784 if (ctx->framerate.num > 0) {
785 capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
786 }
787
788#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
789 if (ctx->capture_cursor) {
790 capture_screen_input.capturesCursor = YES;
791 } else {
792 capture_screen_input.capturesCursor = NO;
793 }
794#endif
795
796 if (ctx->capture_mouse_clicks) {
797 capture_screen_input.capturesMouseClicks = YES;
798 } else {
799 capture_screen_input.capturesMouseClicks = NO;
800 }
801 }
802 }
803#endif
804 }
805
806 if (!video_device) {
807 av_log(ctx, AV_LOG_ERROR, "Video device not found\n");
808 goto fail;
809 }
810 }
811
812 // get audio device
813 if (ctx->audio_device_index >= 0) {
814 NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
815
816 if (ctx->audio_device_index >= [devices count]) {
817 av_log(ctx, AV_LOG_ERROR, "Invalid audio device index\n");
818 goto fail;
819 }
820
821 audio_device = [devices objectAtIndex:ctx->audio_device_index];
822 } else if (ctx->audio_filename &&
823 strncmp(ctx->audio_filename, "none", 4)) {
824 if (!strncmp(ctx->audio_filename, "default", 7)) {
825 audio_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
826 } else {
827 NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
828
829 for (AVCaptureDevice *device in devices) {
830 if (!strncmp(ctx->audio_filename, [[device localizedName] UTF8String], strlen(ctx->audio_filename))) {
831 audio_device = device;
832 break;
833 }
834 }
835 }
836
837 if (!audio_device) {
838 av_log(ctx, AV_LOG_ERROR, "Audio device not found\n");
839 goto fail;
840 }
841 }
842
843 // Video nor Audio capture device not found, looking for AVMediaTypeVideo/Audio
844 if (!video_device && !audio_device) {
845 av_log(s, AV_LOG_ERROR, "No AV capture device found\n");
846 goto fail;
847 }
848
849 if (video_device) {
850 if (ctx->video_device_index < ctx->num_video_devices) {
851 av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device localizedName] UTF8String]);
852 } else {
853 av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device description] UTF8String]);
854 }
855 }
856 if (audio_device) {
857 av_log(s, AV_LOG_DEBUG, "audio device '%s' opened\n", [[audio_device localizedName] UTF8String]);
858 }
859
860 // Initialize capture session
861 ctx->capture_session = [[AVCaptureSession alloc] init];
862
863 if (video_device && add_video_device(s, video_device)) {
864 goto fail;
865 }
866 if (audio_device && add_audio_device(s, audio_device)) {
867 }
868
869 [ctx->capture_session startRunning];
870
871 /* Unlock device configuration only after the session is started so it
872 * does not reset the capture formats */
873 if (!capture_screen) {
874 [video_device unlockForConfiguration];
875 }
876
877 if (video_device && get_video_config(s)) {
878 goto fail;
879 }
880
881 // set audio stream
882 if (audio_device && get_audio_config(s)) {
883 goto fail;
884 }
885
886 [pool release];
887 return 0;
888
889fail:
890 [pool release];
891 destroy_context(ctx);
892 return AVERROR(EIO);
893}
894
895static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
896{
897 AVFContext* ctx = (AVFContext*)s->priv_data;
898
899 do {
900 CVImageBufferRef image_buffer;
901 lock_frames(ctx);
902
903 image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
904
905 if (ctx->current_frame != nil) {
906 void *data;
907 if (av_new_packet(pkt, (int)CVPixelBufferGetDataSize(image_buffer)) < 0) {
908 return AVERROR(EIO);
909 }
910
911 CMItemCount count;
912 CMSampleTimingInfo timing_info;
913
914 if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {
915 AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
916 pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
917 }
918
919 pkt->stream_index = ctx->video_stream_index;
920 pkt->flags |= AV_PKT_FLAG_KEY;
921
922 CVPixelBufferLockBaseAddress(image_buffer, 0);
923
924 data = CVPixelBufferGetBaseAddress(image_buffer);
925 memcpy(pkt->data, data, pkt->size);
926
927 CVPixelBufferUnlockBaseAddress(image_buffer, 0);
928 CFRelease(ctx->current_frame);
929 ctx->current_frame = nil;
930 } else if (ctx->current_audio_frame != nil) {
931 CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
932 int block_buffer_size = CMBlockBufferGetDataLength(block_buffer);
933
934 if (!block_buffer || !block_buffer_size) {
935 return AVERROR(EIO);
936 }
937
938 if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) {
939 return AVERROR_BUFFER_TOO_SMALL;
940 }
941
942 if (av_new_packet(pkt, block_buffer_size) < 0) {
943 return AVERROR(EIO);
944 }
945
946 CMItemCount count;
947 CMSampleTimingInfo timing_info;
948
949 if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {
950 AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
951 pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
952 }
953
954 pkt->stream_index = ctx->audio_stream_index;
955 pkt->flags |= AV_PKT_FLAG_KEY;
956
957 if (ctx->audio_non_interleaved) {
958 int sample, c, shift, num_samples;
959
960 OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer);
961 if (ret != kCMBlockBufferNoErr) {
962 return AVERROR(EIO);
963 }
964
965 num_samples = pkt->size / (ctx->audio_channels * (ctx->audio_bits_per_sample >> 3));
966
967 // transform decoded frame into output format
968 #define INTERLEAVE_OUTPUT(bps) \
969 { \
970 int##bps##_t **src; \
971 int##bps##_t *dest; \
972 src = av_malloc(ctx->audio_channels * sizeof(int##bps##_t*)); \
973 if (!src) return AVERROR(EIO); \
974 for (c = 0; c < ctx->audio_channels; c++) { \
975 src[c] = ((int##bps##_t*)ctx->audio_buffer) + c * num_samples; \
976 } \
977 dest = (int##bps##_t*)pkt->data; \
978 shift = bps - ctx->audio_bits_per_sample; \
979 for (sample = 0; sample < num_samples; sample++) \
980 for (c = 0; c < ctx->audio_channels; c++) \
981 *dest++ = src[c][sample] << shift; \
982 av_freep(&src); \
983 }
984
985 if (ctx->audio_bits_per_sample <= 16) {
986 INTERLEAVE_OUTPUT(16)
987 } else {
988 INTERLEAVE_OUTPUT(32)
989 }
990 } else {
991 OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
992 if (ret != kCMBlockBufferNoErr) {
993 return AVERROR(EIO);
994 }
995 }
996
997 CFRelease(ctx->current_audio_frame);
998 ctx->current_audio_frame = nil;
999 } else {
1000 pkt->data = NULL;
1001 pthread_cond_wait(&ctx->frame_wait_cond, &ctx->frame_lock);
1002 }
1003
1004 unlock_frames(ctx);
1005 } while (!pkt->data);
1006
1007 return 0;
1008}
1009
1010static int avf_close(AVFormatContext *s)
1011{
1012 AVFContext* ctx = (AVFContext*)s->priv_data;
1013 destroy_context(ctx);
1014 return 0;
1015}
1016
1017static const AVOption options[] = {
1018 { "list_devices", "list available devices", offsetof(AVFContext, list_devices), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM, "list_devices" },
1019 { "true", "", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 0, AV_OPT_FLAG_DECODING_PARAM, "list_devices" },
1020 { "false", "", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 0, AV_OPT_FLAG_DECODING_PARAM, "list_devices" },
1021 { "video_device_index", "select video device by index for devices with same name (starts at 0)", offsetof(AVFContext, video_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1022 { "audio_device_index", "select audio device by index for devices with same name (starts at 0)", offsetof(AVFContext, audio_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1023 { "pixel_format", "set pixel format", offsetof(AVFContext, pixel_format), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_YUV420P}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM},
1024 { "framerate", "set frame rate", offsetof(AVFContext, framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "ntsc"}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1025 { "video_size", "set video size", offsetof(AVFContext, width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM },
1026 { "capture_cursor", "capture the screen cursor", offsetof(AVFContext, capture_cursor), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1027 { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1028
1029 { NULL },
1030};
1031
1032static const AVClass avf_class = {
1033 .class_name = "AVFoundation input device",
1034 .item_name = av_default_item_name,
1035 .option = options,
1036 .version = LIBAVUTIL_VERSION_INT,
1037 .category = AV_CLASS_CATEGORY_DEVICE_VIDEO_INPUT,
1038};
1039
1040AVInputFormat ff_avfoundation_demuxer = {
1041 .name = "avfoundation",
1042 .long_name = NULL_IF_CONFIG_SMALL("AVFoundation input device"),
1043 .priv_data_size = sizeof(AVFContext),
1044 .read_header = avf_read_header,
1045 .read_packet = avf_read_packet,
1046 .read_close = avf_close,
1047 .flags = AVFMT_NOFILE,
1048 .priv_class = &avf_class,
1049};
1050
generated by cgit at 2024-05-17 03:25:47 (GMT)