25 #include <stdatomic.h>
29 #include <camera/NdkCameraDevice.h>
30 #include <camera/NdkCameraManager.h>
31 #include <media/NdkImage.h>
32 #include <media/NdkImageReader.h>
50 #define IMAGE_FORMAT_ANDROID AIMAGE_FORMAT_YUV_420_888
52 #define MAX_BUF_COUNT 2
53 #define VIDEO_STREAM_INDEX 0
54 #define VIDEO_TIMEBASE_ANDROID 1000000000
56 #define RETURN_CASE(x) case x: return AV_STRINGIFY(x);
57 #define RETURN_DEFAULT(x) default: return AV_STRINGIFY(x);
154 return "ERROR_CAMERA_UNKNOWN";
164 ACameraDevice_getId(device));
180 ACameraIdList *camera_ids;
182 ret = ACameraManager_getCameraIdList(
ctx->camera_mgr, &camera_ids);
183 if (ret != ACAMERA_OK) {
189 if (
ctx->camera_index < camera_ids->numCameras) {
191 if (!
ctx->camera_id) {
201 ACameraManager_deleteCameraIdList(camera_ids);
203 ret = ACameraManager_getCameraCharacteristics(
ctx->camera_mgr,
204 ctx->camera_id, &
ctx->camera_metadata);
205 if (ret != ACAMERA_OK) {
206 av_log(avctx,
AV_LOG_ERROR,
"Failed to get metadata for camera with id %s, error: %s.\n",
211 ctx->camera_state_callbacks.context = avctx;
215 ret = ACameraManager_openCamera(
ctx->camera_mgr,
ctx->camera_id,
216 &
ctx->camera_state_callbacks, &
ctx->camera_dev);
217 if (ret != ACAMERA_OK) {
229 ACameraMetadata_const_entry lens_facing;
230 ACameraMetadata_const_entry sensor_orientation;
232 ACameraMetadata_getConstEntry(
ctx->camera_metadata,
233 ACAMERA_LENS_FACING, &lens_facing);
234 ACameraMetadata_getConstEntry(
ctx->camera_metadata,
235 ACAMERA_SENSOR_ORIENTATION, &sensor_orientation);
237 ctx->lens_facing = lens_facing.data.u8[0];
238 ctx->sensor_orientation = sensor_orientation.data.i32[0];
244 ACameraMetadata_const_entry available_configs;
247 ACameraMetadata_getConstEntry(
ctx->camera_metadata,
248 ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS,
251 for (
int i = 0;
i < available_configs.count;
i++) {
252 int32_t input = available_configs.data.i32[
i * 4 + 3];
274 if (!found ||
ctx->width == 0 ||
ctx->height == 0) {
275 ctx->width = available_configs.data.i32[1];
276 ctx->height = available_configs.data.i32[2];
279 "Requested video_size %dx%d not available, falling back to %dx%d\n",
280 ctx->requested_width,
ctx->requested_height,
ctx->width,
ctx->height);
289 ACameraMetadata_const_entry available_framerates;
291 int current_best_match = -1;
292 int requested_framerate =
av_q2d(
ctx->framerate);
294 ACameraMetadata_getConstEntry(
ctx->camera_metadata,
295 ACAMERA_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES,
296 &available_framerates);
298 for (
int i = 0;
i < available_framerates.count;
i++) {
299 int32_t min = available_framerates.data.i32[
i * 2 + 0];
300 int32_t max = available_framerates.data.i32[
i * 2 + 1];
302 if (requested_framerate ==
max) {
304 ctx->framerate_range[0] =
min;
305 ctx->framerate_range[1] =
max;
308 }
else if (current_best_match >= 0) {
309 int32_t current_best_match_min = available_framerates.data.i32[current_best_match * 2 + 0];
310 if (
min > current_best_match_min) {
311 current_best_match =
i;
314 current_best_match =
i;
320 if (current_best_match >= 0) {
321 ctx->framerate_range[0] = available_framerates.data.i32[current_best_match * 2 + 0];
322 ctx->framerate_range[1] = available_framerates.data.i32[current_best_match * 2 + 1];
325 ctx->framerate_range[0] = available_framerates.data.i32[0];
326 ctx->framerate_range[1] = available_framerates.data.i32[1];
330 "Requested framerate %d not available, falling back to min: %d and max: %d fps\n",
331 requested_framerate,
ctx->framerate_range[0],
ctx->framerate_range[1]);
342 int plane_data_length[2];
344 for (
int i = 0;
i < 2;
i++) {
345 AImage_getPlanePixelStride(image,
i + 1, &image_pixelstrides[
i]);
346 AImage_getPlaneData(image,
i + 1, &image_plane_data[
i], &plane_data_length[
i]);
349 if (image_pixelstrides[0] != image_pixelstrides[1]) {
351 "Pixel strides of U and V plane should have been the same.\n");
355 switch (image_pixelstrides[0]) {
360 if (image_plane_data[0] < image_plane_data[1]) {
368 "Unknown pixel stride %d of U and V plane, cannot determine camera image format.\n",
369 image_pixelstrides[0]);
380 media_status_t media_status;
384 int64_t image_timestamp;
387 int plane_data_length[4];
390 int pkt_buffer_size = 0;
392 media_status = AImageReader_acquireLatestImage(reader, &image);
393 if (media_status != AMEDIA_OK) {
394 if (media_status == AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE) {
396 "An image reader frame was discarded");
399 "Failed to acquire latest image from image reader, error: %s.\n",
416 "Could not get image format of camera.\n");
424 AImage_getTimestamp(image, &image_timestamp);
426 AImage_getPlaneRowStride(image, 0, &image_linestrides[0]);
427 AImage_getPlaneData(image, 0, &image_plane_data[0], &plane_data_length[0]);
429 switch (
ctx->image_format) {
431 AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);
432 AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);
433 AImage_getPlaneRowStride(image, 2, &image_linestrides[2]);
434 AImage_getPlaneData(image, 2, &image_plane_data[2], &plane_data_length[2]);
437 AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);
438 AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);
441 AImage_getPlaneRowStride(image, 2, &image_linestrides[1]);
442 AImage_getPlaneData(image, 2, &image_plane_data[1], &plane_data_length[1]);
453 "Failed to create new av packet, error: %s.\n",
av_err2str(ret));
458 pkt.
pts = image_timestamp;
460 (
const uint8_t *
const *) image_plane_data,
461 image_linestrides,
ctx->image_format,
462 ctx->width,
ctx->height, 32);
470 "Error while processing new image, error: %s.\n",
av_err2str(ret));
475 "Input queue was full, dropping frame, consider raising the input_queue_size option (current value: %d)\n",
476 ctx->input_queue_size);
478 if (pkt_buffer_size) {
483 AImage_delete(image);
495 if (ret != AMEDIA_OK) {
501 ctx->image_listener.context = avctx;
504 ret = AImageReader_setImageListener(
ctx->image_reader, &
ctx->image_listener);
505 if (ret != AMEDIA_OK) {
507 "Failed to set image listener on image reader, error: %s.\n",
512 ret = AImageReader_getWindow(
ctx->image_reader, &
ctx->image_reader_window);
513 if (ret != AMEDIA_OK) {
515 "Could not get image reader window, error: %s.\n",
543 ret = ACaptureSessionOutputContainer_create(&
ctx->capture_session_output_container);
544 if (ret != ACAMERA_OK) {
546 "Failed to create capture session output container, error: %s.\n",
551 ANativeWindow_acquire(
ctx->image_reader_window);
553 ret = ACaptureSessionOutput_create(
ctx->image_reader_window, &
ctx->capture_session_output);
554 if (ret != ACAMERA_OK) {
556 "Failed to create capture session container, error: %s.\n",
561 ret = ACaptureSessionOutputContainer_add(
ctx->capture_session_output_container,
562 ctx->capture_session_output);
563 if (ret != ACAMERA_OK) {
565 "Failed to add output to output container, error: %s.\n",
570 ret = ACameraOutputTarget_create(
ctx->image_reader_window, &
ctx->camera_output_target);
571 if (ret != ACAMERA_OK) {
573 "Failed to create camera output target, error: %s.\n",
578 ret = ACameraDevice_createCaptureRequest(
ctx->camera_dev, TEMPLATE_RECORD, &
ctx->capture_request);
579 if (ret != ACAMERA_OK) {
581 "Failed to create capture request, error: %s.\n",
586 ret = ACaptureRequest_setEntry_i32(
ctx->capture_request, ACAMERA_CONTROL_AE_TARGET_FPS_RANGE,
587 2,
ctx->framerate_range);
588 if (ret != ACAMERA_OK) {
590 "Failed to set target fps range in capture request, error: %s.\n",
595 ret = ACaptureRequest_addTarget(
ctx->capture_request,
ctx->camera_output_target);
596 if (ret != ACAMERA_OK) {
598 "Failed to add capture request capture request, error: %s.\n",
603 ctx->capture_session_state_callbacks.context = avctx;
608 ret = ACameraDevice_createCaptureSession(
ctx->camera_dev,
ctx->capture_session_output_container,
609 &
ctx->capture_session_state_callbacks, &
ctx->capture_session);
610 if (ret != ACAMERA_OK) {
612 "Failed to create capture session, error: %s.\n",
617 ret = ACameraCaptureSession_setRepeatingRequest(
ctx->capture_session,
NULL, 1, &
ctx->capture_request,
NULL);
618 if (ret != ACAMERA_OK) {
620 "Failed to set repeating request on capture session, error: %s.\n",
648 if (
ctx->lens_facing == ACAMERA_LENS_FACING_FRONT) {
659 memcpy(side_data, display_matrix,
sizeof(display_matrix));
701 if (
ctx->capture_session) {
702 ACameraCaptureSession_stopRepeating(
ctx->capture_session);
706 ACameraCaptureSession_close(
ctx->capture_session);
710 if (
ctx->capture_request) {
711 ACaptureRequest_removeTarget(
ctx->capture_request,
ctx->camera_output_target);
712 ACaptureRequest_free(
ctx->capture_request);
716 if (
ctx->camera_output_target) {
717 ACameraOutputTarget_free(
ctx->camera_output_target);
718 ctx->camera_output_target =
NULL;
721 if (
ctx->capture_session_output) {
722 ACaptureSessionOutputContainer_remove(
ctx->capture_session_output_container,
723 ctx->capture_session_output);
724 ACaptureSessionOutput_free(
ctx->capture_session_output);
725 ctx->capture_session_output =
NULL;
728 if (
ctx->image_reader_window) {
729 ANativeWindow_release(
ctx->image_reader_window);
730 ctx->image_reader_window =
NULL;
733 if (
ctx->capture_session_output_container) {
734 ACaptureSessionOutputContainer_free(
ctx->capture_session_output_container);
735 ctx->capture_session_output_container =
NULL;
738 if (
ctx->camera_dev) {
739 ACameraDevice_close(
ctx->camera_dev);
743 if (
ctx->image_reader) {
744 AImageReader_delete(
ctx->image_reader);
748 if (
ctx->camera_metadata) {
749 ACameraMetadata_free(
ctx->camera_metadata);
755 if (
ctx->camera_mgr) {
756 ACameraManager_delete(
ctx->camera_mgr);
760 if (
ctx->input_queue) {
783 "Failed to allocate input queue, error: %s.\n",
av_err2str(ret));
787 ctx->camera_mgr = ACameraManager_create();
788 if (!
ctx->camera_mgr) {
844 #define OFFSET(x) offsetof(AndroidCameraCtx, x)
845 #define DEC AV_OPT_FLAG_DECODING_PARAM
849 {
"camera_index",
"set index of camera to use",
OFFSET(camera_index),
AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX,
DEC },
850 {
"input_queue_size",
"set maximum number of frames to buffer",
OFFSET(input_queue_size),
AV_OPT_TYPE_INT, {.i64 = 5}, 0, INT_MAX,
DEC },
863 .
name =
"android_camera",
static double val(void *priv, double ch)
static const char *const format[]
static void image_available(void *context, AImageReader *reader)
static int android_camera_read_header(AVFormatContext *avctx)
AVInputFormat ff_android_camera_demuxer
static void capture_session_active(void *context, ACameraCaptureSession *session)
#define RETURN_DEFAULT(x)
static const AVOption options[]
static int add_video_stream(AVFormatContext *avctx)
static int get_image_format(AVFormatContext *avctx, AImage *image)
static void camera_dev_error(void *context, ACameraDevice *device, int error)
#define IMAGE_FORMAT_ANDROID
#define VIDEO_STREAM_INDEX
static void camera_dev_disconnected(void *context, ACameraDevice *device)
static const char * media_status_string(media_status_t val)
#define VIDEO_TIMEBASE_ANDROID
static int android_camera_read_packet(AVFormatContext *avctx, AVPacket *pkt)
static int create_image_reader(AVFormatContext *avctx)
static void get_sensor_orientation(AVFormatContext *avctx)
static int open_camera(AVFormatContext *avctx)
static const char * error_state_callback_string(int val)
static void capture_session_ready(void *context, ACameraCaptureSession *session)
static int wait_for_image_format(AVFormatContext *avctx)
static void capture_session_closed(void *context, ACameraCaptureSession *session)
static int android_camera_read_close(AVFormatContext *avctx)
static int create_capture_session(AVFormatContext *avctx)
static const AVClass android_camera_class
static void match_framerate(AVFormatContext *avctx)
static void match_video_size(AVFormatContext *avctx)
static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
static const char * camera_status_string(camera_status_t val)
static int read_packet(void *opaque, uint8_t *buf, int buf_size)
#define flags(name, subs,...)
#define atomic_store(object, desired)
#define atomic_load(object)
#define atomic_init(obj, value)
static int read_header(FFV1Context *f)
@ AV_OPT_TYPE_IMAGE_SIZE
offset must point to two consecutive integers
@ AV_OPT_TYPE_VIDEO_RATE
offset must point to AVRational
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
int av_new_packet(AVPacket *pkt, int size)
Allocate the payload of a packet and initialize its fields with default values.
@ AV_PKT_DATA_DISPLAYMATRIX
This side data contains a 3x3 transformation matrix describing an affine transformation that needs to...
uint8_t * av_stream_new_side_data(AVStream *stream, enum AVPacketSideDataType type, size_t size)
Allocate new information from stream.
AVStream * avformat_new_stream(AVFormatContext *s, const AVCodec *c)
Add a new stream to a media file.
#define AVERROR_EXTERNAL
Generic error in an external library.
#define AVERROR_EOF
End of file.
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
#define AV_LOG_WARNING
Something somehow does not look correct.
#define AV_LOG_INFO
Standard information.
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
const char * av_default_item_name(void *ptr)
Return the context name.
static double av_q2d(AVRational a)
Convert an AVRational to a double.
char * av_strdup(const char *s)
Duplicate a string.
int av_image_get_buffer_size(enum AVPixelFormat pix_fmt, int width, int height, int align)
Return the size in bytes of the amount of data required to store an image with the given parameters.
int av_image_copy_to_buffer(uint8_t *dst, int dst_size, const uint8_t *const src_data[4], const int src_linesize[4], enum AVPixelFormat pix_fmt, int width, int height, int align)
Copy image data from an image into a buffer.
#define LIBAVUTIL_VERSION_INT
void av_display_rotation_set(int32_t matrix[9], double angle)
Initialize a transformation matrix describing a pure counterclockwise rotation by the specified angle...
void av_display_matrix_flip(int32_t matrix[9], int hflip, int vflip)
Flip the input matrix horizontally and/or vertically.
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
static av_cold int read_close(AVFormatContext *ctx)
@ AV_CLASS_CATEGORY_DEVICE_VIDEO_INPUT
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
@ AV_PIX_FMT_NV21
as above, but U and V bytes are swapped
Describe the class of an AVClass context structure.
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
This struct describes the properties of an encoded stream.
enum AVMediaType codec_type
General type of the encoded data.
enum AVCodecID codec_id
Specific type of the encoded data (the codec used).
int flags
Flags modifying the (de)muxer behaviour.
void * priv_data
Format private data.
This structure stores compressed data.
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Rational number (pair of numerator and denominator).
AVCodecParameters * codecpar
Codec parameters associated with this stream.
int id
Format-specific stream ID.
AVRational avg_frame_rate
Average framerate.
AVRational r_frame_rate
Real base framerate of the stream.
ACameraDevice_StateCallbacks camera_state_callbacks
atomic_int got_image_format
ANativeWindow * image_reader_window
ACameraCaptureSession_stateCallbacks capture_session_state_callbacks
ACameraCaptureSession * capture_session
AImageReader_ImageListener image_listener
ACameraManager * camera_mgr
int32_t framerate_range[2]
ACameraDevice * camera_dev
ACameraOutputTarget * camera_output_target
AImageReader * image_reader
ACameraMetadata * camera_metadata
AVThreadMessageQueue * input_queue
int32_t sensor_orientation
ACaptureRequest * capture_request
ACaptureSessionOutputContainer * capture_session_output_container
ACaptureSessionOutput * capture_session_output
static void error(const char *err)
void av_thread_message_queue_set_err_send(AVThreadMessageQueue *mq, int err)
Set the sending error code.
int av_thread_message_queue_recv(AVThreadMessageQueue *mq, void *msg, unsigned flags)
Receive a message from the queue.
int av_thread_message_queue_alloc(AVThreadMessageQueue **mq, unsigned nelem, unsigned elsize)
Allocate a new message queue.
void av_thread_message_queue_set_err_recv(AVThreadMessageQueue *mq, int err)
Set the receiving error code.
int av_thread_message_queue_send(AVThreadMessageQueue *mq, void *msg, unsigned flags)
Send a message on the queue.
void av_thread_message_queue_free(AVThreadMessageQueue **mq)
Free a message queue.
@ AV_THREAD_MESSAGE_NONBLOCK
Perform non-blocking operation.