blob: 81fcebce232349cde0aa6516a2bc56fdb200beb2
1 | /* |
2 | * MMAL Video Decoder |
3 | * Copyright (c) 2015 Rodger Combs |
4 | * |
5 | * This file is part of FFmpeg. |
6 | * |
7 | * FFmpeg is free software; you can redistribute it and/or |
8 | * modify it under the terms of the GNU Lesser General Public |
9 | * License as published by the Free Software Foundation; either |
10 | * version 2.1 of the License, or (at your option) any later version. |
11 | * |
12 | * FFmpeg is distributed in the hope that it will be useful, |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
15 | * Lesser General Public License for more details. |
16 | * |
17 | * You should have received a copy of the GNU Lesser General Public |
18 | * License along with FFmpeg; if not, write to the Free Software |
19 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
20 | */ |
21 | |
22 | /** |
23 | * @file |
24 | * MMAL Video Decoder |
25 | */ |
26 | |
27 | #include <bcm_host.h> |
28 | #include <interface/mmal/mmal.h> |
29 | #include <interface/mmal/mmal_parameters_video.h> |
30 | #include <interface/mmal/util/mmal_util.h> |
31 | #include <interface/mmal/util/mmal_util_params.h> |
32 | #include <interface/mmal/util/mmal_default_components.h> |
33 | #include <interface/mmal/vc/mmal_vc_api.h> |
34 | #include <stdatomic.h> |
35 | |
36 | #include "avcodec.h" |
37 | #include "internal.h" |
38 | #include "libavutil/avassert.h" |
39 | #include "libavutil/buffer.h" |
40 | #include "libavutil/common.h" |
41 | #include "libavutil/imgutils.h" |
42 | #include "libavutil/opt.h" |
43 | #include "libavutil/log.h" |
44 | |
45 | typedef struct FFBufferEntry { |
46 | AVBufferRef *ref; |
47 | void *data; |
48 | size_t length; |
49 | int64_t pts, dts; |
50 | int flags; |
51 | struct FFBufferEntry *next; |
52 | } FFBufferEntry; |
53 | |
54 | // MMAL_POOL_T destroys all of its MMAL_BUFFER_HEADER_Ts. If we want correct |
55 | // refcounting for AVFrames, we can free the MMAL_POOL_T only after all AVFrames |
56 | // have been unreferenced. |
57 | typedef struct FFPoolRef { |
58 | atomic_int refcount; |
59 | MMAL_POOL_T *pool; |
60 | } FFPoolRef; |
61 | |
62 | typedef struct FFBufferRef { |
63 | MMAL_BUFFER_HEADER_T *buffer; |
64 | FFPoolRef *pool; |
65 | } FFBufferRef; |
66 | |
67 | typedef struct MMALDecodeContext { |
68 | AVClass *av_class; |
69 | int extra_buffers; |
70 | int extra_decoder_buffers; |
71 | |
72 | MMAL_COMPONENT_T *decoder; |
73 | MMAL_QUEUE_T *queue_decoded_frames; |
74 | MMAL_POOL_T *pool_in; |
75 | FFPoolRef *pool_out; |
76 | |
77 | // Waiting input packets. Because the libavcodec API requires decoding and |
78 | // returning packets in lockstep, it can happen that queue_decoded_frames |
79 | // contains almost all surfaces - then the decoder input queue can quickly |
80 | // fill up and won't accept new input either. Without consuming input, the |
81 | // libavcodec API can't return new frames, and we have a logical deadlock. |
82 | // This is avoided by queuing such buffers here. |
83 | FFBufferEntry *waiting_buffers, *waiting_buffers_tail; |
84 | |
85 | int64_t packets_sent; |
86 | atomic_int packets_buffered; |
87 | int64_t frames_output; |
88 | int eos_received; |
89 | int eos_sent; |
90 | int extradata_sent; |
91 | int interlaced_frame; |
92 | int top_field_first; |
93 | } MMALDecodeContext; |
94 | |
95 | // Assume decoder is guaranteed to produce output after at least this many |
96 | // packets (where each packet contains 1 frame). |
97 | #define MAX_DELAYED_FRAMES 16 |
98 | |
99 | static void ffmmal_poolref_unref(FFPoolRef *ref) |
100 | { |
101 | if (ref && |
102 | atomic_fetch_add_explicit(&ref->refcount, -1, memory_order_acq_rel) == 1) { |
103 | mmal_pool_destroy(ref->pool); |
104 | av_free(ref); |
105 | } |
106 | } |
107 | |
108 | static void ffmmal_release_frame(void *opaque, uint8_t *data) |
109 | { |
110 | FFBufferRef *ref = (void *)data; |
111 | |
112 | mmal_buffer_header_release(ref->buffer); |
113 | ffmmal_poolref_unref(ref->pool); |
114 | |
115 | av_free(ref); |
116 | } |
117 | |
118 | // Setup frame with a new reference to buffer. The buffer must have been |
119 | // allocated from the given pool. |
120 | static int ffmmal_set_ref(AVFrame *frame, FFPoolRef *pool, |
121 | MMAL_BUFFER_HEADER_T *buffer) |
122 | { |
123 | FFBufferRef *ref = av_mallocz(sizeof(*ref)); |
124 | if (!ref) |
125 | return AVERROR(ENOMEM); |
126 | |
127 | ref->pool = pool; |
128 | ref->buffer = buffer; |
129 | |
130 | frame->buf[0] = av_buffer_create((void *)ref, sizeof(*ref), |
131 | ffmmal_release_frame, NULL, |
132 | AV_BUFFER_FLAG_READONLY); |
133 | if (!frame->buf[0]) { |
134 | av_free(ref); |
135 | return AVERROR(ENOMEM); |
136 | } |
137 | |
138 | atomic_fetch_add_explicit(&ref->pool->refcount, 1, memory_order_relaxed); |
139 | mmal_buffer_header_acquire(buffer); |
140 | |
141 | frame->format = AV_PIX_FMT_MMAL; |
142 | frame->data[3] = (uint8_t *)ref->buffer; |
143 | return 0; |
144 | } |
145 | |
146 | static void ffmmal_stop_decoder(AVCodecContext *avctx) |
147 | { |
148 | MMALDecodeContext *ctx = avctx->priv_data; |
149 | MMAL_COMPONENT_T *decoder = ctx->decoder; |
150 | MMAL_BUFFER_HEADER_T *buffer; |
151 | |
152 | mmal_port_disable(decoder->input[0]); |
153 | mmal_port_disable(decoder->output[0]); |
154 | mmal_port_disable(decoder->control); |
155 | |
156 | mmal_port_flush(decoder->input[0]); |
157 | mmal_port_flush(decoder->output[0]); |
158 | mmal_port_flush(decoder->control); |
159 | |
160 | while ((buffer = mmal_queue_get(ctx->queue_decoded_frames))) |
161 | mmal_buffer_header_release(buffer); |
162 | |
163 | while (ctx->waiting_buffers) { |
164 | FFBufferEntry *buffer = ctx->waiting_buffers; |
165 | |
166 | ctx->waiting_buffers = buffer->next; |
167 | |
168 | if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END) |
169 | atomic_fetch_add(&ctx->packets_buffered, -1); |
170 | |
171 | av_buffer_unref(&buffer->ref); |
172 | av_free(buffer); |
173 | } |
174 | ctx->waiting_buffers_tail = NULL; |
175 | |
176 | av_assert0(atomic_load(&ctx->packets_buffered) == 0); |
177 | |
178 | ctx->frames_output = ctx->eos_received = ctx->eos_sent = ctx->packets_sent = ctx->extradata_sent = 0; |
179 | } |
180 | |
181 | static av_cold int ffmmal_close_decoder(AVCodecContext *avctx) |
182 | { |
183 | MMALDecodeContext *ctx = avctx->priv_data; |
184 | |
185 | if (ctx->decoder) |
186 | ffmmal_stop_decoder(avctx); |
187 | |
188 | mmal_component_destroy(ctx->decoder); |
189 | ctx->decoder = NULL; |
190 | mmal_queue_destroy(ctx->queue_decoded_frames); |
191 | mmal_pool_destroy(ctx->pool_in); |
192 | ffmmal_poolref_unref(ctx->pool_out); |
193 | |
194 | mmal_vc_deinit(); |
195 | |
196 | return 0; |
197 | } |
198 | |
199 | static void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) |
200 | { |
201 | AVCodecContext *avctx = (AVCodecContext*)port->userdata; |
202 | MMALDecodeContext *ctx = avctx->priv_data; |
203 | |
204 | if (!buffer->cmd) { |
205 | FFBufferEntry *entry = buffer->user_data; |
206 | av_buffer_unref(&entry->ref); |
207 | if (entry->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END) |
208 | atomic_fetch_add(&ctx->packets_buffered, -1); |
209 | av_free(entry); |
210 | } |
211 | mmal_buffer_header_release(buffer); |
212 | } |
213 | |
214 | static void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) |
215 | { |
216 | AVCodecContext *avctx = (AVCodecContext*)port->userdata; |
217 | MMALDecodeContext *ctx = avctx->priv_data; |
218 | |
219 | mmal_queue_put(ctx->queue_decoded_frames, buffer); |
220 | } |
221 | |
222 | static void control_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer) |
223 | { |
224 | AVCodecContext *avctx = (AVCodecContext*)port->userdata; |
225 | MMAL_STATUS_T status; |
226 | |
227 | if (buffer->cmd == MMAL_EVENT_ERROR) { |
228 | status = *(uint32_t *)buffer->data; |
229 | av_log(avctx, AV_LOG_ERROR, "MMAL error %d on control port\n", (int)status); |
230 | } else { |
231 | av_log(avctx, AV_LOG_WARNING, "Unknown MMAL event %s on control port\n", |
232 | av_fourcc2str(buffer->cmd)); |
233 | } |
234 | |
235 | mmal_buffer_header_release(buffer); |
236 | } |
237 | |
238 | // Feed free output buffers to the decoder. |
239 | static int ffmmal_fill_output_port(AVCodecContext *avctx) |
240 | { |
241 | MMALDecodeContext *ctx = avctx->priv_data; |
242 | MMAL_BUFFER_HEADER_T *buffer; |
243 | MMAL_STATUS_T status; |
244 | |
245 | if (!ctx->pool_out) |
246 | return AVERROR_UNKNOWN; // format change code failed with OOM previously |
247 | |
248 | while ((buffer = mmal_queue_get(ctx->pool_out->pool->queue))) { |
249 | if ((status = mmal_port_send_buffer(ctx->decoder->output[0], buffer))) { |
250 | mmal_buffer_header_release(buffer); |
251 | av_log(avctx, AV_LOG_ERROR, "MMAL error %d when sending output buffer.\n", (int)status); |
252 | return AVERROR_UNKNOWN; |
253 | } |
254 | } |
255 | |
256 | return 0; |
257 | } |
258 | |
259 | static enum AVColorSpace ffmmal_csp_to_av_csp(MMAL_FOURCC_T fourcc) |
260 | { |
261 | switch (fourcc) { |
262 | case MMAL_COLOR_SPACE_BT470_2_BG: |
263 | case MMAL_COLOR_SPACE_BT470_2_M: |
264 | case MMAL_COLOR_SPACE_ITUR_BT601: return AVCOL_SPC_BT470BG; |
265 | case MMAL_COLOR_SPACE_ITUR_BT709: return AVCOL_SPC_BT709; |
266 | case MMAL_COLOR_SPACE_FCC: return AVCOL_SPC_FCC; |
267 | case MMAL_COLOR_SPACE_SMPTE240M: return AVCOL_SPC_SMPTE240M; |
268 | default: return AVCOL_SPC_UNSPECIFIED; |
269 | } |
270 | } |
271 | |
272 | static int ffmal_update_format(AVCodecContext *avctx) |
273 | { |
274 | MMALDecodeContext *ctx = avctx->priv_data; |
275 | MMAL_STATUS_T status; |
276 | int ret = 0; |
277 | MMAL_COMPONENT_T *decoder = ctx->decoder; |
278 | MMAL_ES_FORMAT_T *format_out = decoder->output[0]->format; |
279 | MMAL_PARAMETER_VIDEO_INTERLACE_TYPE_T interlace_type; |
280 | |
281 | ffmmal_poolref_unref(ctx->pool_out); |
282 | if (!(ctx->pool_out = av_mallocz(sizeof(*ctx->pool_out)))) { |
283 | ret = AVERROR(ENOMEM); |
284 | goto fail; |
285 | } |
286 | atomic_store(&ctx->pool_out->refcount, 1); |
287 | |
288 | if (!format_out) |
289 | goto fail; |
290 | |
291 | if ((status = mmal_port_parameter_set_uint32(decoder->output[0], MMAL_PARAMETER_EXTRA_BUFFERS, ctx->extra_buffers))) |
292 | goto fail; |
293 | |
294 | if ((status = mmal_port_parameter_set_boolean(decoder->output[0], MMAL_PARAMETER_VIDEO_INTERPOLATE_TIMESTAMPS, 0))) |
295 | goto fail; |
296 | |
297 | if (avctx->pix_fmt == AV_PIX_FMT_MMAL) { |
298 | format_out->encoding = MMAL_ENCODING_OPAQUE; |
299 | } else { |
300 | format_out->encoding_variant = format_out->encoding = MMAL_ENCODING_I420; |
301 | } |
302 | |
303 | if ((status = mmal_port_format_commit(decoder->output[0]))) |
304 | goto fail; |
305 | |
306 | interlace_type.hdr.id = MMAL_PARAMETER_VIDEO_INTERLACE_TYPE; |
307 | interlace_type.hdr.size = sizeof(MMAL_PARAMETER_VIDEO_INTERLACE_TYPE_T); |
308 | status = mmal_port_parameter_get(decoder->output[0], &interlace_type.hdr); |
309 | if (status != MMAL_SUCCESS) { |
310 | av_log(avctx, AV_LOG_ERROR, "Cannot read MMAL interlace information!\n"); |
311 | } else { |
312 | ctx->interlaced_frame = (interlace_type.eMode != MMAL_InterlaceProgressive); |
313 | ctx->top_field_first = (interlace_type.eMode == MMAL_InterlaceFieldsInterleavedUpperFirst); |
314 | } |
315 | |
316 | if ((ret = ff_set_dimensions(avctx, format_out->es->video.crop.x + format_out->es->video.crop.width, |
317 | format_out->es->video.crop.y + format_out->es->video.crop.height)) < 0) |
318 | goto fail; |
319 | |
320 | if (format_out->es->video.par.num && format_out->es->video.par.den) { |
321 | avctx->sample_aspect_ratio.num = format_out->es->video.par.num; |
322 | avctx->sample_aspect_ratio.den = format_out->es->video.par.den; |
323 | } |
324 | if (format_out->es->video.frame_rate.num && format_out->es->video.frame_rate.den) { |
325 | avctx->framerate.num = format_out->es->video.frame_rate.num; |
326 | avctx->framerate.den = format_out->es->video.frame_rate.den; |
327 | } |
328 | |
329 | avctx->colorspace = ffmmal_csp_to_av_csp(format_out->es->video.color_space); |
330 | |
331 | decoder->output[0]->buffer_size = |
332 | FFMAX(decoder->output[0]->buffer_size_min, decoder->output[0]->buffer_size_recommended); |
333 | decoder->output[0]->buffer_num = |
334 | FFMAX(decoder->output[0]->buffer_num_min, decoder->output[0]->buffer_num_recommended) + ctx->extra_buffers; |
335 | ctx->pool_out->pool = mmal_pool_create(decoder->output[0]->buffer_num, |
336 | decoder->output[0]->buffer_size); |
337 | if (!ctx->pool_out->pool) { |
338 | ret = AVERROR(ENOMEM); |
339 | goto fail; |
340 | } |
341 | |
342 | return 0; |
343 | |
344 | fail: |
345 | return ret < 0 ? ret : AVERROR_UNKNOWN; |
346 | } |
347 | |
348 | static av_cold int ffmmal_init_decoder(AVCodecContext *avctx) |
349 | { |
350 | MMALDecodeContext *ctx = avctx->priv_data; |
351 | MMAL_STATUS_T status; |
352 | MMAL_ES_FORMAT_T *format_in; |
353 | MMAL_COMPONENT_T *decoder; |
354 | int ret = 0; |
355 | |
356 | bcm_host_init(); |
357 | |
358 | if (mmal_vc_init()) { |
359 | av_log(avctx, AV_LOG_ERROR, "Cannot initialize MMAL VC driver!\n"); |
360 | return AVERROR(ENOSYS); |
361 | } |
362 | |
363 | if ((ret = ff_get_format(avctx, avctx->codec->pix_fmts)) < 0) |
364 | return ret; |
365 | |
366 | avctx->pix_fmt = ret; |
367 | |
368 | if ((status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_DECODER, &ctx->decoder))) |
369 | goto fail; |
370 | |
371 | decoder = ctx->decoder; |
372 | |
373 | format_in = decoder->input[0]->format; |
374 | format_in->type = MMAL_ES_TYPE_VIDEO; |
375 | switch (avctx->codec_id) { |
376 | case AV_CODEC_ID_MPEG2VIDEO: |
377 | format_in->encoding = MMAL_ENCODING_MP2V; |
378 | break; |
379 | case AV_CODEC_ID_MPEG4: |
380 | format_in->encoding = MMAL_ENCODING_MP4V; |
381 | break; |
382 | case AV_CODEC_ID_VC1: |
383 | format_in->encoding = MMAL_ENCODING_WVC1; |
384 | break; |
385 | case AV_CODEC_ID_H264: |
386 | default: |
387 | format_in->encoding = MMAL_ENCODING_H264; |
388 | break; |
389 | } |
390 | format_in->es->video.width = FFALIGN(avctx->width, 32); |
391 | format_in->es->video.height = FFALIGN(avctx->height, 16); |
392 | format_in->es->video.crop.width = avctx->width; |
393 | format_in->es->video.crop.height = avctx->height; |
394 | format_in->es->video.frame_rate.num = 24000; |
395 | format_in->es->video.frame_rate.den = 1001; |
396 | format_in->es->video.par.num = avctx->sample_aspect_ratio.num; |
397 | format_in->es->video.par.den = avctx->sample_aspect_ratio.den; |
398 | format_in->flags = MMAL_ES_FORMAT_FLAG_FRAMED; |
399 | |
400 | av_log(avctx, AV_LOG_DEBUG, "Using MMAL %s encoding.\n", |
401 | av_fourcc2str(format_in->encoding)); |
402 | |
403 | #if HAVE_MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS |
404 | if (mmal_port_parameter_set_uint32(decoder->input[0], MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS, |
405 | -1 - ctx->extra_decoder_buffers)) { |
406 | av_log(avctx, AV_LOG_WARNING, "Could not set input buffering limit.\n"); |
407 | } |
408 | #endif |
409 | |
410 | if ((status = mmal_port_format_commit(decoder->input[0]))) |
411 | goto fail; |
412 | |
413 | decoder->input[0]->buffer_num = |
414 | FFMAX(decoder->input[0]->buffer_num_min, 20); |
415 | decoder->input[0]->buffer_size = |
416 | FFMAX(decoder->input[0]->buffer_size_min, 512 * 1024); |
417 | ctx->pool_in = mmal_pool_create(decoder->input[0]->buffer_num, 0); |
418 | if (!ctx->pool_in) { |
419 | ret = AVERROR(ENOMEM); |
420 | goto fail; |
421 | } |
422 | |
423 | if ((ret = ffmal_update_format(avctx)) < 0) |
424 | goto fail; |
425 | |
426 | ctx->queue_decoded_frames = mmal_queue_create(); |
427 | if (!ctx->queue_decoded_frames) |
428 | goto fail; |
429 | |
430 | decoder->input[0]->userdata = (void*)avctx; |
431 | decoder->output[0]->userdata = (void*)avctx; |
432 | decoder->control->userdata = (void*)avctx; |
433 | |
434 | if ((status = mmal_port_enable(decoder->control, control_port_cb))) |
435 | goto fail; |
436 | if ((status = mmal_port_enable(decoder->input[0], input_callback))) |
437 | goto fail; |
438 | if ((status = mmal_port_enable(decoder->output[0], output_callback))) |
439 | goto fail; |
440 | |
441 | if ((status = mmal_component_enable(decoder))) |
442 | goto fail; |
443 | |
444 | return 0; |
445 | |
446 | fail: |
447 | ffmmal_close_decoder(avctx); |
448 | return ret < 0 ? ret : AVERROR_UNKNOWN; |
449 | } |
450 | |
451 | static void ffmmal_flush(AVCodecContext *avctx) |
452 | { |
453 | MMALDecodeContext *ctx = avctx->priv_data; |
454 | MMAL_COMPONENT_T *decoder = ctx->decoder; |
455 | MMAL_STATUS_T status; |
456 | |
457 | ffmmal_stop_decoder(avctx); |
458 | |
459 | if ((status = mmal_port_enable(decoder->control, control_port_cb))) |
460 | goto fail; |
461 | if ((status = mmal_port_enable(decoder->input[0], input_callback))) |
462 | goto fail; |
463 | if ((status = mmal_port_enable(decoder->output[0], output_callback))) |
464 | goto fail; |
465 | |
466 | return; |
467 | |
468 | fail: |
469 | av_log(avctx, AV_LOG_ERROR, "MMAL flush error: %i\n", (int)status); |
470 | } |
471 | |
472 | // Split packets and add them to the waiting_buffers list. We don't queue them |
473 | // immediately, because it can happen that the decoder is temporarily blocked |
474 | // (due to us not reading/returning enough output buffers) and won't accept |
475 | // new input. (This wouldn't be an issue if MMAL input buffers always were |
476 | // complete frames - then the input buffer just would have to be big enough.) |
477 | // If is_extradata is set, send it as MMAL_BUFFER_HEADER_FLAG_CONFIG. |
478 | static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt, |
479 | int is_extradata) |
480 | { |
481 | MMALDecodeContext *ctx = avctx->priv_data; |
482 | AVBufferRef *buf = NULL; |
483 | int size = 0; |
484 | uint8_t *data = (uint8_t *)""; |
485 | uint8_t *start; |
486 | int ret = 0; |
487 | |
488 | if (avpkt->size) { |
489 | if (avpkt->buf) { |
490 | buf = av_buffer_ref(avpkt->buf); |
491 | size = avpkt->size; |
492 | data = avpkt->data; |
493 | } else { |
494 | buf = av_buffer_alloc(avpkt->size); |
495 | if (buf) { |
496 | memcpy(buf->data, avpkt->data, avpkt->size); |
497 | size = buf->size; |
498 | data = buf->data; |
499 | } |
500 | } |
501 | if (!buf) { |
502 | ret = AVERROR(ENOMEM); |
503 | goto done; |
504 | } |
505 | if (!is_extradata) |
506 | ctx->packets_sent++; |
507 | } else { |
508 | if (ctx->eos_sent) |
509 | goto done; |
510 | if (!ctx->packets_sent) { |
511 | // Short-cut the flush logic to avoid upsetting MMAL. |
512 | ctx->eos_sent = 1; |
513 | ctx->eos_received = 1; |
514 | goto done; |
515 | } |
516 | } |
517 | |
518 | start = data; |
519 | |
520 | do { |
521 | FFBufferEntry *buffer = av_mallocz(sizeof(*buffer)); |
522 | if (!buffer) { |
523 | ret = AVERROR(ENOMEM); |
524 | goto done; |
525 | } |
526 | |
527 | buffer->data = data; |
528 | buffer->length = FFMIN(size, ctx->decoder->input[0]->buffer_size); |
529 | |
530 | if (is_extradata) |
531 | buffer->flags |= MMAL_BUFFER_HEADER_FLAG_CONFIG; |
532 | |
533 | if (data == start) |
534 | buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_START; |
535 | |
536 | data += buffer->length; |
537 | size -= buffer->length; |
538 | |
539 | buffer->pts = avpkt->pts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->pts; |
540 | buffer->dts = avpkt->dts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->dts; |
541 | |
542 | if (!size) { |
543 | buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_END; |
544 | atomic_fetch_add(&ctx->packets_buffered, 1); |
545 | } |
546 | |
547 | if (!buffer->length) { |
548 | buffer->flags |= MMAL_BUFFER_HEADER_FLAG_EOS; |
549 | ctx->eos_sent = 1; |
550 | } |
551 | |
552 | if (buf) { |
553 | buffer->ref = av_buffer_ref(buf); |
554 | if (!buffer->ref) { |
555 | av_free(buffer); |
556 | ret = AVERROR(ENOMEM); |
557 | goto done; |
558 | } |
559 | } |
560 | |
561 | // Insert at end of the list |
562 | if (!ctx->waiting_buffers) |
563 | ctx->waiting_buffers = buffer; |
564 | if (ctx->waiting_buffers_tail) |
565 | ctx->waiting_buffers_tail->next = buffer; |
566 | ctx->waiting_buffers_tail = buffer; |
567 | } while (size); |
568 | |
569 | done: |
570 | av_buffer_unref(&buf); |
571 | return ret; |
572 | } |
573 | |
574 | // Move prepared/split packets from waiting_buffers to the MMAL decoder. |
575 | static int ffmmal_fill_input_port(AVCodecContext *avctx) |
576 | { |
577 | MMALDecodeContext *ctx = avctx->priv_data; |
578 | |
579 | while (ctx->waiting_buffers) { |
580 | MMAL_BUFFER_HEADER_T *mbuffer; |
581 | FFBufferEntry *buffer; |
582 | MMAL_STATUS_T status; |
583 | |
584 | mbuffer = mmal_queue_get(ctx->pool_in->queue); |
585 | if (!mbuffer) |
586 | return 0; |
587 | |
588 | buffer = ctx->waiting_buffers; |
589 | |
590 | mmal_buffer_header_reset(mbuffer); |
591 | mbuffer->cmd = 0; |
592 | mbuffer->pts = buffer->pts; |
593 | mbuffer->dts = buffer->dts; |
594 | mbuffer->flags = buffer->flags; |
595 | mbuffer->data = buffer->data; |
596 | mbuffer->length = buffer->length; |
597 | mbuffer->user_data = buffer; |
598 | mbuffer->alloc_size = ctx->decoder->input[0]->buffer_size; |
599 | |
600 | // Remove from start of the list |
601 | ctx->waiting_buffers = buffer->next; |
602 | if (ctx->waiting_buffers_tail == buffer) |
603 | ctx->waiting_buffers_tail = NULL; |
604 | |
605 | if ((status = mmal_port_send_buffer(ctx->decoder->input[0], mbuffer))) { |
606 | mmal_buffer_header_release(mbuffer); |
607 | av_buffer_unref(&buffer->ref); |
608 | if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END) |
609 | atomic_fetch_add(&ctx->packets_buffered, -1); |
610 | av_free(buffer); |
611 | } |
612 | |
613 | if (status) { |
614 | av_log(avctx, AV_LOG_ERROR, "MMAL error %d when sending input\n", (int)status); |
615 | return AVERROR_UNKNOWN; |
616 | } |
617 | } |
618 | |
619 | return 0; |
620 | } |
621 | |
622 | static int ffmal_copy_frame(AVCodecContext *avctx, AVFrame *frame, |
623 | MMAL_BUFFER_HEADER_T *buffer) |
624 | { |
625 | MMALDecodeContext *ctx = avctx->priv_data; |
626 | int ret = 0; |
627 | |
628 | frame->interlaced_frame = ctx->interlaced_frame; |
629 | frame->top_field_first = ctx->top_field_first; |
630 | |
631 | if (avctx->pix_fmt == AV_PIX_FMT_MMAL) { |
632 | if (!ctx->pool_out) |
633 | return AVERROR_UNKNOWN; // format change code failed with OOM previously |
634 | |
635 | if ((ret = ff_decode_frame_props(avctx, frame)) < 0) |
636 | goto done; |
637 | |
638 | if ((ret = ffmmal_set_ref(frame, ctx->pool_out, buffer)) < 0) |
639 | goto done; |
640 | } else { |
641 | int w = FFALIGN(avctx->width, 32); |
642 | int h = FFALIGN(avctx->height, 16); |
643 | uint8_t *src[4]; |
644 | int linesize[4]; |
645 | |
646 | if ((ret = ff_get_buffer(avctx, frame, 0)) < 0) |
647 | goto done; |
648 | |
649 | av_image_fill_arrays(src, linesize, |
650 | buffer->data + buffer->type->video.offset[0], |
651 | avctx->pix_fmt, w, h, 1); |
652 | av_image_copy(frame->data, frame->linesize, src, linesize, |
653 | avctx->pix_fmt, avctx->width, avctx->height); |
654 | } |
655 | |
656 | frame->pts = buffer->pts == MMAL_TIME_UNKNOWN ? AV_NOPTS_VALUE : buffer->pts; |
657 | #if FF_API_PKT_PTS |
658 | FF_DISABLE_DEPRECATION_WARNINGS |
659 | frame->pkt_pts = frame->pts; |
660 | FF_ENABLE_DEPRECATION_WARNINGS |
661 | #endif |
662 | frame->pkt_dts = AV_NOPTS_VALUE; |
663 | |
664 | done: |
665 | return ret; |
666 | } |
667 | |
668 | // Fetch a decoded buffer and place it into the frame parameter. |
669 | static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_frame) |
670 | { |
671 | MMALDecodeContext *ctx = avctx->priv_data; |
672 | MMAL_BUFFER_HEADER_T *buffer = NULL; |
673 | MMAL_STATUS_T status = 0; |
674 | int ret = 0; |
675 | |
676 | if (ctx->eos_received) |
677 | goto done; |
678 | |
679 | while (1) { |
680 | // To ensure decoding in lockstep with a constant delay between fed packets |
681 | // and output frames, we always wait until an output buffer is available. |
682 | // Except during start we don't know after how many input packets the decoder |
683 | // is going to return the first buffer, and we can't distinguish decoder |
684 | // being busy from decoder waiting for input. So just poll at the start and |
685 | // keep feeding new data to the buffer. |
686 | // We are pretty sure the decoder will produce output if we sent more input |
687 | // frames than what a H.264 decoder could logically delay. This avoids too |
688 | // excessive buffering. |
689 | // We also wait if we sent eos, but didn't receive it yet (think of decoding |
690 | // stream with a very low number of frames). |
691 | if (atomic_load(&ctx->packets_buffered) > MAX_DELAYED_FRAMES || |
692 | (ctx->packets_sent && ctx->eos_sent)) { |
693 | // MMAL will ignore broken input packets, which means the frame we |
694 | // expect here may never arrive. Dealing with this correctly is |
695 | // complicated, so here's a hack to avoid that it freezes forever |
696 | // in this unlikely situation. |
697 | buffer = mmal_queue_timedwait(ctx->queue_decoded_frames, 100); |
698 | if (!buffer) { |
699 | av_log(avctx, AV_LOG_ERROR, "Did not get output frame from MMAL.\n"); |
700 | ret = AVERROR_UNKNOWN; |
701 | goto done; |
702 | } |
703 | } else { |
704 | buffer = mmal_queue_get(ctx->queue_decoded_frames); |
705 | if (!buffer) |
706 | goto done; |
707 | } |
708 | |
709 | ctx->eos_received |= !!(buffer->flags & MMAL_BUFFER_HEADER_FLAG_EOS); |
710 | if (ctx->eos_received) |
711 | goto done; |
712 | |
713 | if (buffer->cmd == MMAL_EVENT_FORMAT_CHANGED) { |
714 | MMAL_COMPONENT_T *decoder = ctx->decoder; |
715 | MMAL_EVENT_FORMAT_CHANGED_T *ev = mmal_event_format_changed_get(buffer); |
716 | MMAL_BUFFER_HEADER_T *stale_buffer; |
717 | |
718 | av_log(avctx, AV_LOG_INFO, "Changing output format.\n"); |
719 | |
720 | if ((status = mmal_port_disable(decoder->output[0]))) |
721 | goto done; |
722 | |
723 | while ((stale_buffer = mmal_queue_get(ctx->queue_decoded_frames))) |
724 | mmal_buffer_header_release(stale_buffer); |
725 | |
726 | mmal_format_copy(decoder->output[0]->format, ev->format); |
727 | |
728 | if ((ret = ffmal_update_format(avctx)) < 0) |
729 | goto done; |
730 | |
731 | if ((status = mmal_port_enable(decoder->output[0], output_callback))) |
732 | goto done; |
733 | |
734 | if ((ret = ffmmal_fill_output_port(avctx)) < 0) |
735 | goto done; |
736 | |
737 | if ((ret = ffmmal_fill_input_port(avctx)) < 0) |
738 | goto done; |
739 | |
740 | mmal_buffer_header_release(buffer); |
741 | continue; |
742 | } else if (buffer->cmd) { |
743 | av_log(avctx, AV_LOG_WARNING, "Unknown MMAL event %s on output port\n", |
744 | av_fourcc2str(buffer->cmd)); |
745 | goto done; |
746 | } else if (buffer->length == 0) { |
747 | // Unused output buffer that got drained after format change. |
748 | mmal_buffer_header_release(buffer); |
749 | continue; |
750 | } |
751 | |
752 | ctx->frames_output++; |
753 | |
754 | if ((ret = ffmal_copy_frame(avctx, frame, buffer)) < 0) |
755 | goto done; |
756 | |
757 | *got_frame = 1; |
758 | break; |
759 | } |
760 | |
761 | done: |
762 | if (buffer) |
763 | mmal_buffer_header_release(buffer); |
764 | if (status && ret >= 0) |
765 | ret = AVERROR_UNKNOWN; |
766 | return ret; |
767 | } |
768 | |
769 | static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame, |
770 | AVPacket *avpkt) |
771 | { |
772 | MMALDecodeContext *ctx = avctx->priv_data; |
773 | AVFrame *frame = data; |
774 | int ret = 0; |
775 | |
776 | if (avctx->extradata_size && !ctx->extradata_sent) { |
777 | AVPacket pkt = {0}; |
778 | av_init_packet(&pkt); |
779 | pkt.data = avctx->extradata; |
780 | pkt.size = avctx->extradata_size; |
781 | ctx->extradata_sent = 1; |
782 | if ((ret = ffmmal_add_packet(avctx, &pkt, 1)) < 0) |
783 | return ret; |
784 | } |
785 | |
786 | if ((ret = ffmmal_add_packet(avctx, avpkt, 0)) < 0) |
787 | return ret; |
788 | |
789 | if ((ret = ffmmal_fill_input_port(avctx)) < 0) |
790 | return ret; |
791 | |
792 | if ((ret = ffmmal_fill_output_port(avctx)) < 0) |
793 | return ret; |
794 | |
795 | if ((ret = ffmmal_read_frame(avctx, frame, got_frame)) < 0) |
796 | return ret; |
797 | |
798 | // ffmmal_read_frame() can block for a while. Since the decoder is |
799 | // asynchronous, it's a good idea to fill the ports again. |
800 | |
801 | if ((ret = ffmmal_fill_output_port(avctx)) < 0) |
802 | return ret; |
803 | |
804 | if ((ret = ffmmal_fill_input_port(avctx)) < 0) |
805 | return ret; |
806 | |
807 | return ret; |
808 | } |
809 | |
810 | AVHWAccel ff_h264_mmal_hwaccel = { |
811 | .name = "h264_mmal", |
812 | .type = AVMEDIA_TYPE_VIDEO, |
813 | .id = AV_CODEC_ID_H264, |
814 | .pix_fmt = AV_PIX_FMT_MMAL, |
815 | }; |
816 | |
817 | AVHWAccel ff_mpeg2_mmal_hwaccel = { |
818 | .name = "mpeg2_mmal", |
819 | .type = AVMEDIA_TYPE_VIDEO, |
820 | .id = AV_CODEC_ID_MPEG2VIDEO, |
821 | .pix_fmt = AV_PIX_FMT_MMAL, |
822 | }; |
823 | |
824 | AVHWAccel ff_mpeg4_mmal_hwaccel = { |
825 | .name = "mpeg4_mmal", |
826 | .type = AVMEDIA_TYPE_VIDEO, |
827 | .id = AV_CODEC_ID_MPEG4, |
828 | .pix_fmt = AV_PIX_FMT_MMAL, |
829 | }; |
830 | |
831 | AVHWAccel ff_vc1_mmal_hwaccel = { |
832 | .name = "vc1_mmal", |
833 | .type = AVMEDIA_TYPE_VIDEO, |
834 | .id = AV_CODEC_ID_VC1, |
835 | .pix_fmt = AV_PIX_FMT_MMAL, |
836 | }; |
837 | |
838 | static const AVOption options[]={ |
839 | {"extra_buffers", "extra buffers", offsetof(MMALDecodeContext, extra_buffers), AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0}, |
840 | {"extra_decoder_buffers", "extra MMAL internal buffered frames", offsetof(MMALDecodeContext, extra_decoder_buffers), AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0}, |
841 | {NULL} |
842 | }; |
843 | |
844 | #define FFMMAL_DEC_CLASS(NAME) \ |
845 | static const AVClass ffmmal_##NAME##_dec_class = { \ |
846 | .class_name = "mmal_" #NAME "_dec", \ |
847 | .option = options, \ |
848 | .version = LIBAVUTIL_VERSION_INT, \ |
849 | }; |
850 | |
851 | #define FFMMAL_DEC(NAME, ID) \ |
852 | FFMMAL_DEC_CLASS(NAME) \ |
853 | AVCodec ff_##NAME##_mmal_decoder = { \ |
854 | .name = #NAME "_mmal", \ |
855 | .long_name = NULL_IF_CONFIG_SMALL(#NAME " (mmal)"), \ |
856 | .type = AVMEDIA_TYPE_VIDEO, \ |
857 | .id = ID, \ |
858 | .priv_data_size = sizeof(MMALDecodeContext), \ |
859 | .init = ffmmal_init_decoder, \ |
860 | .close = ffmmal_close_decoder, \ |
861 | .decode = ffmmal_decode, \ |
862 | .flush = ffmmal_flush, \ |
863 | .priv_class = &ffmmal_##NAME##_dec_class, \ |
864 | .capabilities = AV_CODEC_CAP_DELAY, \ |
865 | .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS, \ |
866 | .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_MMAL, \ |
867 | AV_PIX_FMT_YUV420P, \ |
868 | AV_PIX_FMT_NONE}, \ |
869 | }; |
870 | |
871 | FFMMAL_DEC(h264, AV_CODEC_ID_H264) |
872 | FFMMAL_DEC(mpeg2, AV_CODEC_ID_MPEG2VIDEO) |
873 | FFMMAL_DEC(mpeg4, AV_CODEC_ID_MPEG4) |
874 | FFMMAL_DEC(vc1, AV_CODEC_ID_VC1) |
875 |