/* * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application * Copyright (C) 2005-2015, Anthony Minessale II * * Version: MPL 1.1 * * The contents of this file are subject to the Mozilla Public License Version * 1.1 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * http://www.mozilla.org/MPL/ * * Software distributed under the License is distributed on an "AS IS" basis, * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License * for the specific language governing rights and limitations under the * License. * * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application * * The Initial Developer of the Original Code is * Seven Du * Portions created by the Initial Developer are Copyright (C) * the Initial Developer. All Rights Reserved. * * Contributor(s): * * Seven Du * Anthony Minessale * Jakub Karolczyk * * mod_avformat -- File Formats with libav.org * */ #include #include "mod_av.h" GCC_DIAG_OFF(deprecated-declarations) #include #ifdef _MSC_VER #include /* LIBAVCODEC_VERSION_INT */ #endif #include #ifdef _MSC_VER #include /* LIBAVFORMAT_VERSION_INT */ #endif #include #include #include #include #include #ifdef USE_AVRESAMPLE #include #define SwrContext AVAudioResampleContext #define swr_alloc avresample_alloc_context #define swr_init avresample_open #define swr_free avresample_free #define swr_get_out_samples avresample_get_out_samples #define swr_get_out_samples avresample_get_out_samples #define swr_convert(ctx, odata, osamples, idata, isamples) \ avresample_convert(ctx, odata, 0, osamples, (uint8_t **)idata, 0, isamples) #else #include #endif #define SLICE_SIZE (SWITCH_DEFAULT_VIDEO_SIZE + 100) GCC_DIAG_ON(deprecated-declarations) #define SCALE_FLAGS SWS_BICUBIC #define DFT_RECORD_OFFSET 0 #ifndef AVUTIL_TIMESTAMP_H #define AVUTIL_TIMESTAMP_H #define AV_TS_MAX_STRING_SIZE 32 #define UINTVAL(v) (v > 0 ? v : 0); // Compatibility with old libav on Debian Jessie // Not required if libavcodec version > 56.34.1 #ifndef AV_CODEC_FLAG_LOOP_FILTER #define AV_CODEC_FLAG_LOOP_FILTER CODEC_FLAG_LOOP_FILTER #define AV_CODEC_FLAG_GLOBAL_HEADER CODEC_FLAG_GLOBAL_HEADER #define AV_CODEC_CAP_VARIABLE_FRAME_SIZE CODEC_CAP_VARIABLE_FRAME_SIZE #endif struct avformat_globals { enum AVColorSpace colorspace; }; struct avformat_globals avformat_globals = { 0 }; /* App interface */ // a wrapper around a single output AVStream typedef struct MediaStream { AVStream *st; AVFrame *frame; AVFrame *tmp_frame; #if (LIBAVFORMAT_VERSION_MAJOR >= LIBAVFORMAT_V) AVCodecContext *codec; #endif // audio int channels; int sample_rate; struct SwrContext *resample_ctx; //video int width; int height; struct SwsContext *sws_ctx; int64_t next_pts; int active; int r; } MediaStream; typedef struct record_helper_s { switch_mutex_t *mutex; AVFormatContext *fc; MediaStream *video_st; switch_timer_t *video_timer; int in_callback; switch_queue_t *video_queue; switch_thread_t *video_thread; switch_mm_t *mm; int finalize; switch_file_handle_t *fh; switch_time_t record_timer_paused; uint64_t last_ts; } record_helper_t; /* file interface */ struct av_file_context { switch_memory_pool_t *pool; switch_mutex_t *mutex; switch_thread_cond_t *cond; switch_buffer_t *buf; switch_buffer_t *audio_buffer; switch_timer_t video_timer; int offset; int audio_start; int aud_ready; int vid_ready; int audio_ready; int closed; MediaStream video_st; MediaStream audio_st[2]; AVFormatContext *fc; #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) AVCodec *audio_codec; AVCodec *video_codec; #else const AVCodec *audio_codec; const AVCodec *video_codec; #endif enum AVColorSpace colorspace; int has_audio; int has_video; record_helper_t eh; switch_thread_t *file_read_thread; int file_read_thread_running; int file_read_thread_started; switch_time_t video_start_time; switch_image_t *last_img; int read_fps; switch_time_t last_vid_push; int64_t seek_ts; switch_bool_t read_paused; int errs; switch_file_handle_t *handle; int16_t *mux_buf; switch_size_t mux_buf_len; switch_time_t last_vid_write; int audio_timer; switch_bool_t no_video_decode; switch_queue_t *video_pkt_queue; switch_packetizer_t *packetizer; AVPacket *last_read_pkt; }; typedef struct av_file_context av_file_context_t; /** * Fill the provided buffer with a string containing a timestamp * representation. * * @param buf a buffer with size in bytes of at least AV_TS_MAX_STRING_SIZE * @param ts the timestamp to represent * @return the buffer in input */ static inline char *av_ts_make_string(char *buf, int64_t ts) { if (ts == AV_NOPTS_VALUE) snprintf(buf, AV_TS_MAX_STRING_SIZE, "NOPTS"); else snprintf(buf, AV_TS_MAX_STRING_SIZE, "%"PRId64"", ts); return buf; } /** * Convenience macro, the return value should be used only directly in * function arguments but never stand-alone. */ #define av_ts2str(ts) av_ts_make_string((char[AV_TS_MAX_STRING_SIZE]){0}, ts) /** * Fill the provided buffer with a string containing a timestamp time * representation. * * @param buf a buffer with size in bytes of at least AV_TS_MAX_STRING_SIZE * @param ts the timestamp to represent * @param tb the timebase of the timestamp * @return the buffer in input */ static inline char *av_ts_make_time_string(char *buf, int64_t ts, AVRational *tb) { if (ts == AV_NOPTS_VALUE) snprintf(buf, AV_TS_MAX_STRING_SIZE, "NOPTS"); else snprintf(buf, AV_TS_MAX_STRING_SIZE, "%.6g", av_q2d(*tb) * ts); return buf; } /** * Convenience macro, the return value should be used only directly in * function arguments but never stand-alone. */ #define av_ts2timestr(ts, tb) av_ts_make_time_string((char[AV_TS_MAX_STRING_SIZE]){0}, ts, tb) #endif /* AVUTIL_TIMESTAMP_H */ static switch_status_t av_file_close(switch_file_handle_t *handle); SWITCH_MODULE_LOAD_FUNCTION(mod_avformat_load); static inline AVCodecContext *av_get_codec_context(MediaStream *stream) { AVCodecContext *c = NULL; #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) GCC_DIAG_OFF(deprecated-declarations) if (stream->st) { c = stream->st->codec; } GCC_DIAG_ON(deprecated-declarations) #else c = stream->codec; #endif return c; } static inline enum AVCodecID av_get_codec_id(AVStream *av_stream) { if (!av_stream) { return AV_CODEC_ID_NONE; } #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) GCC_DIAG_OFF(deprecated-declarations) return av_stream->codec->codec_id; GCC_DIAG_ON(deprecated-declarations) #else return av_stream->codecpar->codec_id; #endif } static inline enum AVMediaType av_get_codec_type(AVStream *av_stream) { if (!av_stream) { return AVMEDIA_TYPE_UNKNOWN; } #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) GCC_DIAG_OFF(deprecated-declarations) return av_stream->codec->codec_type; GCC_DIAG_ON(deprecated-declarations) #else return av_stream->codecpar->codec_type; #endif } static char *const get_error_text(const int error, char *error_buffer, switch_size_t error_buflen) { av_strerror(error, error_buffer, error_buflen); return error_buffer; } static void av_unused fill_avframe(AVFrame *pict, switch_image_t *img) { int i; uint8_t *y = img->planes[0]; uint8_t *u = img->planes[1]; uint8_t *v = img->planes[2]; /* Y */ for (i = 0; i < pict->height; i++) { memcpy(&pict->data[0][i * pict->linesize[0]], y + i * img->stride[0], pict->width); } /* U/V */ for(i = 0; i < pict->height / 2; i++) { memcpy(&pict->data[1][i * pict->linesize[1]], u + i * img->stride[1], pict->width / 2); memcpy(&pict->data[2][i * pict->linesize[2]], v + i * img->stride[2], pict->width / 2); } } static void av_unused avframe2img(AVFrame *pict, switch_image_t *img) { int i, j; if (img->fmt == SWITCH_IMG_FMT_I420) { if (pict->format == AV_PIX_FMT_YUV420P) { switch_I420_copy2(pict->data, pict->linesize, img->planes, img->stride, img->d_w, img->d_h); } else if (pict->format == AV_PIX_FMT_YUVA420P) { int linesize[3]; linesize[0] = pict->linesize[0]; linesize[1] = pict->linesize[1]; linesize[2] = pict->linesize[2] + pict->linesize[0]; switch_I420_copy2(pict->data, linesize, img->planes, img->stride, img->d_w, img->d_h); } } else if (img->fmt == SWITCH_IMG_FMT_ARGB) { if (pict->format == AV_PIX_FMT_YUV420P) { switch_rgb_color_t *color = (switch_rgb_color_t *)img->planes[SWITCH_PLANE_PACKED]; uint8_t *alpha = pict->data[3]; /*!\brief I420 to ARGB Convertion*/ switch_I420ToARGB(pict->data[0], pict->linesize[0], pict->data[1], pict->linesize[1], pict->data[2], pict->linesize[2], img->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED], img->d_w, img->d_h); for (j = 0; j < img->d_h; j++) { for (i = 0; i < img->d_w; i++) { color->a = *alpha++; color++; } color = (switch_rgb_color_t *)(img->planes[SWITCH_PLANE_PACKED] + img->stride[SWITCH_PLANE_PACKED] * j); } } else if (pict->format == AV_PIX_FMT_RGBA) { #if SWITCH_BYTE_ORDER == __BIG_ENDIAN switch_RGBAToARGB(pict->data[0], pict->linesize[0], mg->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED], img->d_w, img->d_h); #else switch_ABGRToARGB(pict->data[0], pict->linesize[0], img->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED], img->d_w, img->d_h); #endif } else if (pict->format == AV_PIX_FMT_BGRA) { #if SWITCH_BYTE_ORDER == __BIG_ENDIAN switch_BGRAToARGB(pict->data[0], pict->linesize[0], , img->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED], , img->d_w, img->d_h); #else switch_ARGBToARGB(pict->data[0], pict->linesize[0], img->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED], img->d_w, img->d_h); #endif } } } static void av_unused avframe2fd(AVFrame *pict, int fd) { int i; uint8_t *y = pict->data[0]; uint8_t *u = pict->data[1]; uint8_t *v = pict->data[2]; /* Y */ for (i = 0; i < pict->height; i++) { write(fd, y + i * pict->linesize[0], pict->width); } /* U/V */ for(i = 0; i < pict->height / 2; i++) { write(fd, u + i * pict->linesize[1], pict->width / 2); } for(i = 0; i < pict->height / 2; i++) { write(fd, v + i * pict->linesize[2], pict->width / 2); } } static void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt) { if (mod_av_globals.debug < 2) return; { AVRational *time_base = &fmt_ctx->streams[pkt->stream_index]->time_base; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pts:%s pts_time:%s dts:%s dts_time:%s duration:%s duration_time:%s stream_index:%d\n", av_ts2str(pkt->pts), av_ts2timestr(pkt->pts, time_base), av_ts2str(pkt->dts), av_ts2timestr(pkt->dts, time_base), av_ts2str(pkt->duration), av_ts2timestr(pkt->duration, time_base), pkt->stream_index); } } static int interrupt_cb(void *cp) { av_file_context_t *context = (av_file_context_t *) cp; if (context->closed) { return 1; } return 0; } #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_6_V) static int mod_avformat_alloc_output_context2(AVFormatContext **avctx, const char *format, const char *filename, av_file_context_t *context) { AVFormatContext *s = avformat_alloc_context(); int ret = 0; #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) AVOutputFormat *oformat = NULL; #else const AVOutputFormat *oformat = NULL; #endif s->interrupt_callback.callback = interrupt_cb; s->interrupt_callback.opaque = context; *avctx = NULL; if (!s) { goto nomem; } if (!oformat) { if (format) { oformat = av_guess_format(format, NULL, NULL); if (!oformat) { av_log(s, AV_LOG_ERROR, "Requested output format '%s' is not a suitable output format\n", format); ret = AVERROR(EINVAL); goto error; } } else { oformat = av_guess_format(NULL, filename, NULL); if (!oformat) { ret = AVERROR(EINVAL); av_log(s, AV_LOG_ERROR, "Unable to find a suitable output format for '%s'\n", filename); goto error; } } } s->oformat = oformat; if (s->oformat->priv_data_size > 0) { s->priv_data = av_mallocz(s->oformat->priv_data_size); if (!s->priv_data) { goto nomem; } if (s->oformat->priv_class) { *(const AVClass**)s->priv_data= s->oformat->priv_class; av_opt_set_defaults(s->priv_data); } } else { s->priv_data = NULL; } if (filename) { #if (LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58,7,100)) av_strlcpy(s->filename, filename, sizeof(s->filename)); #else s->url = av_strdup(filename); switch_assert(s->url); #endif } *avctx = s; return 0; nomem: av_log(s, AV_LOG_ERROR, "Out of memory\n"); ret = AVERROR(ENOMEM); error: avformat_free_context(s); return ret; } #endif static int write_frame(AVFormatContext *fmt_ctx, const AVRational *time_base, AVStream *st, AVPacket *pkt) { /* rescale output packet timestamp values from codec to stream timebase */ av_packet_rescale_ts(pkt, *time_base, st->time_base); pkt->stream_index = st->index; /* Write the compressed frame to the media file. */ log_packet(fmt_ctx, pkt); return av_interleaved_write_frame(fmt_ctx, pkt); } /* Add an output stream. */ #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) static switch_status_t add_stream(av_file_context_t *context, MediaStream *mst, AVFormatContext *fc, AVCodec **codec, enum AVCodecID codec_id, switch_mm_t *mm) #else static switch_status_t add_stream(av_file_context_t *context, MediaStream *mst, AVFormatContext *fc, const AVCodec **codec, enum AVCodecID codec_id, switch_mm_t *mm) #endif { AVCodecContext *c = NULL; switch_status_t status = SWITCH_STATUS_FALSE; //int threads = switch_core_cpu_count(); int buffer_bytes = 2097152; /* 2 mb */ int fps = 15; //if (mm->try_hardware_encoder && codec_id == AV_CODEC_ID_H264) { // *codec = avcodec_find_encoder_by_name("nvenc_h264"); //} if (!*codec) { /* find the encoder */ *codec = avcodec_find_encoder(codec_id); } if (!(*codec)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not find encoder\n"); return status; } mst->st = avformat_new_stream(fc, *codec); if (!mst->st) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate stream\n"); return status; } mst->st->id = fc->nb_streams - 1; #if (LIBAVCODEC_VERSION_MAJOR >= LIBAVCODEC_V) mst->codec = avcodec_alloc_context3(*codec); if (!mst->codec) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate codec context\n"); return status; } #endif c = av_get_codec_context(mst); //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "id:%d den:%d num:%d\n", mst->st->id, mst->st->time_base.den, mst->st->time_base.num); //if (threads > 4) { // threads = 4; // } switch ((*codec)->type) { case AVMEDIA_TYPE_AUDIO: c->sample_fmt = (*codec)->sample_fmts ? (*codec)->sample_fmts[0] : AV_SAMPLE_FMT_FLTP; c->bit_rate = 128000; c->sample_rate = mst->sample_rate = context->handle->samplerate; #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) c->channels = mst->channels; c->channel_layout = av_get_default_channel_layout(c->channels); #else av_channel_layout_default(&c->ch_layout, mst->channels); #endif if (mm) { if (mm->ab) { c->bit_rate = mm->ab * 1024; } if (mm->samplerate) { c->sample_rate = mst->sample_rate = mm->samplerate; } } if (context && context->has_video && !context->handle->stream_name) { mst->st->time_base.den = c->sample_rate; mst->st->time_base.num = 1; c->time_base.den = c->sample_rate; c->time_base.num = 1; } else { // nothing to do for audio only recording, just leave the time base as is // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "stream_timebase: %d/%d codec_timebase: %d/%d nb_samples: %d\n", // mst->st->time_base.num, mst->st->time_base.den, c->time_base.num, c->time_base.den, c->frame_size); } break; case AVMEDIA_TYPE_VIDEO: switch_assert(mm); if (mm->vbuf) { buffer_bytes = mm->vbuf; } if (mm->fps) { fps = mm->fps; } else { mm->fps = fps; } if (mm->vw && mm->vh) { mst->width = mm->vw; mst->height = mm->vh; } c->codec_id = codec_id; /* Resolution must be a multiple of two. */ c->width = mst->width; c->height = mst->height; c->bit_rate = mm->vb * 1024; mst->st->time_base.den = 90000; mst->st->time_base.num = 1; c->time_base.den = 90000; c->time_base.num = 1; c->gop_size = fps * 10; /* emit one intra frame every 10 frames at most */ c->pix_fmt = AV_PIX_FMT_YUV420P; //c->thread_count = threads; c->rc_initial_buffer_occupancy = buffer_bytes * 8; if (codec_id == AV_CODEC_ID_H264) { c->ticks_per_frame = 2; c->flags|=AV_CODEC_FLAG_LOOP_FILTER; // flags=+loop c->me_cmp|= 1; // cmp=+chroma, where CHROMA = 1 c->me_range = 16; // me_range=16 c->max_b_frames = 3; // bf=3 av_opt_set_int(c->priv_data, "b_strategy", 1, 0); //av_opt_set_int(c->priv_data, "motion_est", ME_HEX, 0); av_opt_set(c->priv_data, "motion_est", "hex", 0); av_opt_set_int(c->priv_data, "coder", 1, 0); switch (mm->vprofile) { case SWITCH_VIDEO_PROFILE_BASELINE: av_opt_set(c->priv_data, "profile", "baseline", 0); c->level = 41; break; case SWITCH_VIDEO_PROFILE_MAIN: av_opt_set(c->priv_data, "profile", "main", 0); av_opt_set(c->priv_data, "level", "5", 0); c->level = 5; break; case SWITCH_VIDEO_PROFILE_HIGH: av_opt_set(c->priv_data, "profile", "high", 0); av_opt_set(c->priv_data, "level", "52", 0); c->level = 52; break; } switch (mm->vencspd) { case SWITCH_VIDEO_ENCODE_SPEED_SLOW: av_opt_set(c->priv_data, "preset", "veryslow", 0); break; case SWITCH_VIDEO_ENCODE_SPEED_MEDIUM: av_opt_set(c->priv_data, "preset", "medium", 0); break; case SWITCH_VIDEO_ENCODE_SPEED_FAST: //av_opt_set_int(c->priv_data, "intra-refresh", 1, 0); av_opt_set(c->priv_data, "preset", "veryfast", 0); //av_opt_set(c->priv_data, "tune", "animation+zerolatency", 0); av_opt_set(c->priv_data, "tune", "fastdecode", 0); break; default: break; } } if (mm->cbr) { c->rc_min_rate = c->bit_rate; c->rc_max_rate = c->bit_rate; c->rc_buffer_size = c->bit_rate; c->qcompress = 0; c->gop_size = fps * 2; c->keyint_min = fps * 2; } else { c->gop_size = fps * 10; c->keyint_min = fps; c->i_quant_factor = 0.71; // i_qfactor=0.71 c->qcompress = 0.6; // qcomp=0.6 c->qmin = 10; // qmin=10 c->qmax = 31; // qmax=31 c->max_qdiff = 4; // qdiff=4 av_opt_set_int(c->priv_data, "crf", 18, 0); } if (mm->vb) { c->bit_rate = mm->vb * 1024; } if (mm->keyint) { c->gop_size = mm->keyint; } if (codec_id == AV_CODEC_ID_VP8) { av_set_options_string(c, "quality=realtime", "=", ":"); } // av_opt_set_int(c->priv_data, "slice-max-size", SWITCH_DEFAULT_VIDEO_SIZE, 0); c->colorspace = context->colorspace; c->color_range = AVCOL_RANGE_JPEG; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "colorspace = %d\n", c->colorspace); break; default: break; } /* Some formats want stream headers to be separate. */ if (fc->oformat->flags & AVFMT_GLOBALHEADER) { c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; } mst->active = 1; return SWITCH_STATUS_SUCCESS; } static AVFrame *alloc_picture(enum AVPixelFormat pix_fmt, int width, int height) { AVFrame *picture; int ret; picture = av_frame_alloc(); if (!picture) return NULL; picture->format = pix_fmt; picture->width = width; picture->height = height; /* allocate the buffers for the frame data */ ret = av_frame_get_buffer(picture, 32); if (ret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate frame data.\n"); return NULL; } return picture; } #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) static switch_status_t open_video(AVFormatContext *fc, AVCodec *codec, MediaStream *mst) #else static switch_status_t open_video(AVFormatContext *fc, const AVCodec *codec, MediaStream *mst) #endif { int ret; AVCodecContext *c = NULL; switch_status_t status = SWITCH_STATUS_FALSE; //int threads = switch_core_cpu_count(); // if (threads > 4) threads = 4; // c->thread_count = threads; c = av_get_codec_context(mst); /* open the codec */ ret = avcodec_open2(c, codec, NULL); if (ret < 0) { char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open video codec: %s\n", get_error_text(ret, ebuf, sizeof(ebuf))); return status; } /* allocate and init a re-usable frame */ mst->frame = alloc_picture(c->pix_fmt, c->width, c->height); switch_assert(mst->frame); mst->frame->pts = 0; // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pix_fmt: %d\n", c->pix_fmt); switch_assert(c->pix_fmt == AV_PIX_FMT_YUV420P); // always I420 for NOW #if (LIBAVCODEC_VERSION_MAJOR >= LIBAVCODEC_V) if (((ret = avcodec_parameters_from_context(mst->st->codecpar, mst->codec)) < 0)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not copy to codec params ret=%d\n", ret); return SWITCH_STATUS_FALSE; } #endif return SWITCH_STATUS_SUCCESS; } #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) static switch_status_t open_audio(AVFormatContext *fc, AVCodec *codec, MediaStream *mst) #else static switch_status_t open_audio(AVFormatContext *fc, const AVCodec *codec, MediaStream *mst) #endif { AVCodecContext *c = NULL; int ret; switch_status_t status = SWITCH_STATUS_FALSE; c = av_get_codec_context(mst); ret = avcodec_open2(c, codec, NULL); if (ret == AVERROR_EXPERIMENTAL) { const AVCodecDescriptor *desc = avcodec_descriptor_get(c->codec_id); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Codec [%s] is experimental feature in libavcodec, never mind\n", desc->name); c->strict_std_compliance = FF_COMPLIANCE_EXPERIMENTAL; ret = avcodec_open2(c, codec, NULL); } if (ret < 0) { const AVCodecDescriptor *desc = avcodec_descriptor_get(c->codec_id); char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open audio codec [%s], error: %s\n", desc->name, get_error_text(ret, ebuf, sizeof(ebuf))); return status; } mst->frame = av_frame_alloc(); switch_assert(mst->frame); mst->frame->sample_rate = c->sample_rate; mst->frame->format = AV_SAMPLE_FMT_S16; #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) mst->frame->channel_layout = c->channel_layout; #else mst->frame->ch_layout = c->ch_layout; #endif if (c->codec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE) { //mst->frame->nb_samples = 10000; #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) mst->frame->nb_samples = (mst->frame->sample_rate / 50) * c->channels; #else mst->frame->nb_samples = (mst->frame->sample_rate / 50) * c->ch_layout.nb_channels; #endif } else { mst->frame->nb_samples = c->frame_size; } switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "sample_rate: %d nb_samples: %d\n", mst->frame->sample_rate, mst->frame->nb_samples); if (c->sample_fmt != AV_SAMPLE_FMT_S16 || c->sample_rate != mst->sample_rate) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "sample_fmt %d != AV_SAMPLE_FMT_S16, start resampler\n", c->sample_fmt); mst->resample_ctx = swr_alloc(); if (!mst->resample_ctx) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate resampler context\n"); return status; } /* set options */ #if (LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(59,27,100)) /* FFmpeg 5.0 */ av_opt_set_int(mst->resample_ctx, "in_channel_count", c->channels, 0); #else /* FFmpeg 5.1 */ av_opt_set_chlayout(mst->resample_ctx, "in_chlayout", &c->ch_layout, 0); #endif av_opt_set_int(mst->resample_ctx, "in_sample_rate", c->sample_rate, 0); #if (LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(59,27,100)) av_opt_set_int(mst->resample_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0); av_opt_set_int(mst->resample_ctx, "in_channel_layout", c->channel_layout, 0); av_opt_set_int(mst->resample_ctx, "out_channel_count", c->channels, 0); #else av_opt_set_sample_fmt(mst->resample_ctx, "in_sample_fmt", AV_SAMPLE_FMT_S16, 0); av_opt_set_chlayout(mst->resample_ctx, "out_chlayout", &c->ch_layout, 0); #endif av_opt_set_int(mst->resample_ctx, "out_sample_rate", c->sample_rate, 0); #if (LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(59,27,100)) av_opt_set_int(mst->resample_ctx, "out_sample_fmt", c->sample_fmt, 0); av_opt_set_int(mst->resample_ctx, "out_channel_layout", c->channel_layout, 0); #else av_opt_set_sample_fmt(mst->resample_ctx, "out_sample_fmt", c->sample_fmt, 0); #endif if (swr_init(mst->resample_ctx) < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Failed to initialize the resampling context\n"); av_free(mst->resample_ctx); mst->resample_ctx = NULL; return status; } } ret = av_frame_get_buffer(mst->frame, 0); if (ret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate audio frame.\n"); return status; } if (mst->resample_ctx) { mst->tmp_frame = av_frame_alloc(); switch_assert(mst->tmp_frame); mst->tmp_frame->sample_rate = c->sample_rate; mst->tmp_frame->format = c->sample_fmt; #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) mst->tmp_frame->channel_layout = c->channel_layout; #else mst->tmp_frame->ch_layout = c->ch_layout; #endif mst->tmp_frame->nb_samples = mst->frame->nb_samples; ret = av_frame_get_buffer(mst->tmp_frame, 0); if (ret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate audio frame.\n"); return status; } } #if (LIBAVCODEC_VERSION_MAJOR >= LIBAVCODEC_V) if (((ret = avcodec_parameters_from_context(mst->st->codecpar, mst->codec)) < 0)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not copy to codec params ret=%d\n", ret); return SWITCH_STATUS_FALSE; } #endif return SWITCH_STATUS_SUCCESS; } static int flush_video_queue(switch_queue_t *q, int min) { void *pop; if (switch_queue_size(q) > min) { while (switch_queue_trypop(q, &pop) == SWITCH_STATUS_SUCCESS) { switch_image_t *img = (switch_image_t *) pop; switch_img_free(&img); if (min && switch_queue_size(q) <= min) { break; } } } return switch_queue_size(q); } static void flush_video_pkt_queue(switch_queue_t *q) { AVPacket *pkt; while (switch_queue_trypop(q, (void **)&pkt) == SWITCH_STATUS_SUCCESS) { av_packet_free(&pkt); } } static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *obj) { av_file_context_t *context = (av_file_context_t *) obj; void *pop = NULL; switch_image_t *img = NULL; int d_w = context->eh.video_st->width, d_h = context->eh.video_st->height; int size = 0, skip = 0, skip_freq = 0, skip_count = 0, skip_total = 0, skip_total_count = 0; uint64_t delta_avg = 0, delta_sum = 0, delta_i = 0, delta = 0; int first = 1; AVCodecContext *c = NULL; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "video thread start\n"); switch_assert(context->eh.video_queue); for(;;) { AVPacket *pkt = NULL; #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) int got_packet; #endif int ret = -1; top: switch_assert(context->eh.video_queue); while(switch_queue_size(context->eh.video_queue) > 1) { switch_image_t *tmp_img; switch_queue_pop(context->eh.video_queue, &pop); tmp_img = (switch_image_t *) pop; switch_img_free(&tmp_img); } if (switch_queue_pop(context->eh.video_queue, &pop) == SWITCH_STATUS_SUCCESS) { switch_img_free(&img); if (!pop) { goto endfor; } img = (switch_image_t *) pop; if (!d_w) d_w = img->d_w; if (!d_h) d_h = img->d_h; if (d_w && d_h && (d_w != img->d_w || d_h != img->d_h)) { /* scale to match established stream */ switch_img_fit(&img, d_w, d_h, SWITCH_FIT_SIZE); } } else { continue; } if (skip) { if ((skip_total_count > 0 && !--skip_total_count) || ++skip_count >= skip_freq) { skip_total_count = skip_total; skip_count = 0; skip--; goto top; } } else { size = switch_queue_size(context->eh.video_queue); if (size > 5 && !context->eh.finalize) { skip = size; if (size > 10) { skip_freq = 3; skip_total = 1; } else { skip_freq = 2; skip_total = 1; } } } //switch_mutex_lock(context->eh.mutex); context->eh.in_callback = 1; if (context->eh.video_st->frame) { ret = av_frame_make_writable(context->eh.video_st->frame); } if (ret < 0) { continue; } if (context->eh.record_timer_paused) { context->eh.last_ts = 0; continue; } fill_avframe(context->eh.video_st->frame, img); if (first) { first = 0; // pts = 0; } else if (context->eh.finalize) { if (delta_i && !delta_avg) { delta_avg = (int)(double)(delta_sum / delta_i); delta_i = 1; delta_sum = delta_avg; } if (delta_avg) { delta = delta_avg; } else if (context->eh.mm->fps) { delta = 1000 / context->eh.mm->fps; } else { delta = 33; } context->eh.video_st->frame->pts += delta * 90; } else { uint64_t delta_tmp; switch_core_timer_next(context->eh.video_timer); delta_tmp = (context->eh.video_timer->samplecount * 90) - context->eh.last_ts; if (delta_tmp != 0) { delta_sum += delta_tmp; delta_i++; if (delta_i == UINT64_MAX) { delta_i = 1; delta_sum = delta_avg; } if ((delta_i % 10) == 0) { delta_avg = (int)(double)(delta_sum / delta_i); } context->eh.video_st->frame->pts = context->eh.video_timer->samplecount * 90; } else { context->eh.video_st->frame->pts = ((context->eh.video_timer->samplecount) * 90) + 1; } } pkt = av_packet_alloc(); context->eh.last_ts = context->eh.video_st->frame->pts; // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pts: %" SWITCH_INT64_T_FMT "\n", context->eh.video_st->frame->pts); /* encode the image */ c = av_get_codec_context(context->eh.video_st); #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) GCC_DIAG_OFF(deprecated-declarations) ret = avcodec_encode_video2(c, pkt, context->eh.video_st->frame, &got_packet); GCC_DIAG_ON(deprecated-declarations) if (ret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoding Error %d\n", ret); c = NULL; av_packet_free(&pkt); continue; } if (got_packet) { #else ret = avcodec_send_frame(c, context->eh.video_st->frame); if (ret == AVERROR_EOF) { ret = 0; } else if (ret == AVERROR(EAGAIN)) { /* we fully drain all the output in each encode call, so this should not ever happen */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "Error sending frame to encoder AVERROR_BUG - should never happen\n"); ret = AVERROR_BUG; } else if (ret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "Error sending frame to encoder\n"); } while (ret >= 0) { ret = avcodec_receive_packet(c, pkt); if (ret == AVERROR(EAGAIN)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "No more video packets at the moment\n"); break; } else if (ret == AVERROR_EOF) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "No more video packets at all\n"); break; } else if (ret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoding error\n"); break; } #endif switch_mutex_lock(context->eh.mutex); write_frame(context->eh.fc, &c->time_base, context->eh.video_st->st, pkt); switch_mutex_unlock(context->eh.mutex); } context->eh.in_callback = 0; av_packet_free(&pkt); c = NULL; //switch_mutex_unlock(context->eh.mutex); } endfor: for(;;) { AVPacket *pkt = av_packet_alloc(); int ret = 0; #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) int got_packet = 0; #else int wret = 0; #endif c = av_get_codec_context(context->eh.video_st); #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) GCC_DIAG_OFF(deprecated-declarations) ret = avcodec_encode_video2(c, pkt, NULL, &got_packet); GCC_DIAG_ON(deprecated-declarations) if (ret < 0) { goto do_break; } else if (got_packet) { #else ret = avcodec_send_frame(c, NULL); if (ret == AVERROR_EOF) { ret = 0; } else if (ret == AVERROR(EAGAIN)) { /* we fully drain all the output in each encode call, so this should not ever happen */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "Error sending frame to encoder on draining AVERROR_BUG - should never happen\n"); goto do_break; } else if (ret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "Error sending frame to encoder on draining\n"); c = NULL; goto do_break; } while (ret >= 0) { ret = avcodec_receive_packet(c, pkt); if (ret == AVERROR(EAGAIN)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "No more video packets at the moment on draining\n"); break; } else if (ret == AVERROR_EOF) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "No more video packets at all on draining \n"); break; } else if (ret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoding error on draining\n"); break; } #endif switch_mutex_lock(context->eh.mutex); #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) ret = write_frame(context->eh.fc, &c->time_base, context->eh.video_st->st, pkt); #else wret = write_frame(context->eh.fc, &c->time_base, context->eh.video_st->st, pkt); #endif switch_mutex_unlock(context->eh.mutex); #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) if (ret < 0) { goto do_break; } } else { goto do_break; #else if (wret < 0) { goto do_break; } #endif } av_packet_free(&pkt); c = NULL; #if (LIBAVCODEC_VERSION_MAJOR >= LIBAVCODEC_V) if (ret < 0 && ret != AVERROR(EAGAIN)) { break; } #endif continue; do_break: av_packet_free(&pkt); break; } while(switch_queue_trypop(context->eh.video_queue, &pop) == SWITCH_STATUS_SUCCESS) { if (!pop) break; img = (switch_image_t *) pop; switch_img_free(&img); } switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "video thread done\n"); return NULL; } static void close_stream(AVFormatContext *fc, MediaStream *mst) { AVCodecContext *c = NULL; if (!mst->active) return; if (mst->resample_ctx) swr_free(&mst->resample_ctx); if (mst->sws_ctx) sws_freeContext(mst->sws_ctx); if (mst->frame) av_frame_free(&mst->frame); if (mst->tmp_frame) av_frame_free(&mst->tmp_frame); c = av_get_codec_context(mst); #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) avcodec_close(c); #else /* avcodec_close() will be called in avcodec_free_context() */ avcodec_free_context(&c); #endif mst->active = 0; } static int is_device(const AVClass *avclass) { #if defined (AV_CLASS_CATEGORY_DEVICE_VIDEO_OUTPUT) if (!avclass) return 0; return avclass->category == AV_CLASS_CATEGORY_DEVICE_VIDEO_OUTPUT || avclass->category == AV_CLASS_CATEGORY_DEVICE_VIDEO_INPUT || avclass->category == AV_CLASS_CATEGORY_DEVICE_AUDIO_OUTPUT || avclass->category == AV_CLASS_CATEGORY_DEVICE_AUDIO_INPUT || avclass->category == AV_CLASS_CATEGORY_DEVICE_OUTPUT || avclass->category == AV_CLASS_CATEGORY_DEVICE_INPUT; #endif return 0; } void show_formats(switch_stream_handle_t *stream) { const AVInputFormat *ifmt = NULL; const AVOutputFormat *ofmt = NULL; const char *last_name; // int is_dev; stream->write_function(stream, "============= File Formats ==============================:\n" " D. = Demuxing supported\n" " .M = Muxing supported\n" "----------------------\n"); last_name = "000"; for (;;) { int decode = 0; int encode = 0; int is_dev = 0; const char *name = NULL; const char *long_name = NULL; #if (LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(58,9,100)) while ((ofmt = av_oformat_next(ofmt))) { #else void *i = 0; while ((ofmt = av_muxer_iterate(&i))) { #endif is_dev = is_device(ofmt->priv_class); if ((name == NULL || strcmp(ofmt->name, name) < 0) && strcmp(ofmt->name, last_name) > 0) { name = ofmt->name; long_name = ofmt->long_name; encode = 1; } } #if (LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(58,9,100)) while ((ifmt = av_iformat_next(ifmt))) { #else i = 0; while ((ifmt = av_demuxer_iterate(&i))) { #endif is_dev = is_device(ifmt->priv_class); if ((name == NULL || strcmp(ifmt->name, name) < 0) && strcmp(ifmt->name, last_name) > 0) { name = ifmt->name; long_name = ifmt->long_name; encode = 0; } if (name && strcmp(ifmt->name, name) == 0) decode = 1; } if (name == NULL) break; last_name = name; stream->write_function(stream, "%s%s%s %-15s %s\n", is_dev ? "*" : " ", decode ? "D" : " ", encode ? "M" : " ", name, long_name ? long_name:" "); } } static void mod_avformat_destroy_output_context(av_file_context_t *context) { close_stream(context->fc, &context->video_st); close_stream(context->fc, &context->audio_st[0]); close_stream(context->fc, &context->audio_st[1]); if (context->audio_st[0].resample_ctx) { swr_free(&context->audio_st[0].resample_ctx); } if (context->audio_st[1].resample_ctx) { swr_free(&context->audio_st[1].resample_ctx); } avformat_close_input(&context->fc); context->fc = NULL; context->audio_st[0].st = NULL; context->audio_st[1].st = NULL; context->video_st.st = NULL; } static switch_status_t open_input_file(av_file_context_t *context, switch_file_handle_t *handle, const char *filename) { #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) AVCodec *audio_codec = NULL; AVCodec *video_codec = NULL; #else const AVCodec *audio_codec = NULL; const AVCodec *video_codec = NULL; #endif enum AVCodecID codec_id; AVCodecContext *cc = NULL; AVDictionary *opts = NULL; int error; int i, idx = 0; switch_status_t status = SWITCH_STATUS_SUCCESS; // av_dict_set(&opts, "c:v", "libvpx", 0); /** Open the input file to read from it. */ if (!context->fc) { context->fc = avformat_alloc_context(); } if (!context->fc) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open input file '%s' (error '%s')\n", filename, "NO MEM"); switch_goto_status(SWITCH_STATUS_FALSE, err); } context->fc->interrupt_callback.callback = interrupt_cb; context->fc->interrupt_callback.opaque = context; if ((error = avformat_open_input(&context->fc, filename, NULL, NULL)) < 0) { char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not open input file '%s' (error '%s')\n", filename, get_error_text(error, ebuf, sizeof(ebuf))); avformat_free_context(context->fc); context->fc = NULL; switch_goto_status(SWITCH_STATUS_FALSE, err); } handle->seekable = context->fc->iformat->read_seek2 ? 1 : (context->fc->iformat->read_seek ? 1 : 0); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "file %s is %sseekable\n", filename, handle->seekable ? "" : "not "); /** Get information on the input file (number of streams etc.). */ if ((error = avformat_find_stream_info(context->fc, opts ? &opts : NULL)) < 0) { char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not find stream info in file: %s, error = %s)\n", filename, get_error_text(error, ebuf, sizeof(ebuf))); if (opts) av_dict_free(&opts); switch_goto_status(SWITCH_STATUS_FALSE, err); } if (opts) av_dict_free(&opts); av_dump_format(context->fc, 0, filename, 0); for (i = 0; i < context->fc->nb_streams; i++) { enum AVMediaType codec_type = av_get_codec_type(context->fc->streams[i]); if (codec_type == AVMEDIA_TYPE_AUDIO && context->has_audio < 2 && idx < 2) { context->audio_st[idx++].st = context->fc->streams[i]; context->has_audio++; } else if (codec_type == AVMEDIA_TYPE_VIDEO && !context->has_video) { context->video_st.st = context->fc->streams[i]; if (switch_test_flag(handle, SWITCH_FILE_FLAG_VIDEO)) { context->has_video = 1; #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) handle->duration = av_rescale_q(context->video_st.st->duration != AV_NOPTS_VALUE ? context->video_st.st->duration : context->fc->duration / AV_TIME_BASE * 1000, context->video_st.st->time_base, AV_TIME_BASE_Q); } if (context->fc->bit_rate) { handle->mm.source_kps = context->fc->bit_rate / 1024; } if (context->video_st.st->avg_frame_rate.num) { handle->mm.source_fps = ceil(av_q2d(context->video_st.st->avg_frame_rate)); } else { handle->mm.source_fps = 25; } context->read_fps = (int)handle->mm.source_fps; #else } #endif } } /** Find a decoder for the audio stream. */ if (context->has_audio && !(audio_codec = avcodec_find_decoder((codec_id = av_get_codec_id(context->audio_st[0].st))))) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not find input codec %d\n", codec_id); context->has_audio = 0; } if (context->has_video && !(video_codec = avcodec_find_decoder((codec_id = av_get_codec_id(context->video_st.st))))) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not find input codec %d\n", codec_id); context->has_video = 0; } #if (LIBAVFORMAT_VERSION_MAJOR >= LIBAVFORMAT_V) if (context->has_audio == 2) { context->audio_st[1].codec = avcodec_alloc_context3(audio_codec); context->audio_st[0].codec = avcodec_alloc_context3(audio_codec); } else if (context->has_audio) { context->audio_st[0].codec = avcodec_alloc_context3(audio_codec); } if (context->has_video) { context->video_st.codec = avcodec_alloc_context3(video_codec); } #endif cc = av_get_codec_context(&context->audio_st[0]); #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) if (context->has_audio && ((error = avcodec_open2(cc, audio_codec, NULL)) < 0)) { #else if (context->has_audio && (((error = avcodec_parameters_to_context(cc, context->audio_st[0].st->codecpar)) < 0) || ((error = avcodec_open2(cc, audio_codec, NULL)) < 0))) { #endif char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open input audio codec (error '%s')\n", get_error_text(error, ebuf, sizeof(ebuf))); context->has_audio = 0; } cc = av_get_codec_context(&context->audio_st[1]); #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) if (context->has_audio == 2 && ((error = avcodec_open2(cc, audio_codec, NULL)) < 0)) { #else if (context->has_audio == 2 && (((error = avcodec_parameters_to_context(cc, context->audio_st[1].st->codecpar)) < 0) || ((error = avcodec_open2(cc, audio_codec, NULL)) < 0))) { #endif char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open input audio codec channel 2 (error '%s')\n", get_error_text(error, ebuf, sizeof(ebuf))); if ((cc = av_get_codec_context(&context->audio_st[0]))) { avcodec_close(cc); } context->has_audio = 0; } cc = av_get_codec_context(&context->video_st); #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) if (context->has_video && (error = avcodec_open2(cc, video_codec, NULL)) < 0) { #else if (context->has_video) { if (((error = avcodec_parameters_to_context(cc, context->video_st.st->codecpar)) < 0) || ((error = avcodec_open2(cc, video_codec, NULL)) < 0)) { #endif char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open input codec (error '%s')\n", get_error_text(error, ebuf, sizeof(ebuf))); context->has_video = 0; } #if (LIBAVFORMAT_VERSION_MAJOR >= LIBAVFORMAT_V) handle->duration = av_rescale_q(context->video_st.st->duration != AV_NOPTS_VALUE ? context->video_st.st->duration : context->fc->duration / AV_TIME_BASE * 1000, context->video_st.st->time_base, AV_TIME_BASE_Q); if (context->fc->bit_rate) { handle->mm.source_kps = context->fc->bit_rate / 1024; } if (context->video_st.st->avg_frame_rate.num) { handle->mm.source_fps = ceil(av_q2d(context->video_st.st->avg_frame_rate)); } else { handle->mm.source_fps = 25; } context->read_fps = (int)handle->mm.source_fps; } #endif context->video_st.active = 1; // printf("has audio:%d has_video:%d\n", context->has_audio, context->has_video); if ((!context->has_audio) && (!context->has_video)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Neither audio nor video stream found in file %s\n", filename); switch_goto_status(SWITCH_STATUS_FALSE, err); } if (context->has_audio) { AVCodecContext *c[2] = { NULL }; if (!(c[0] = av_get_codec_context(&context->audio_st[0]))) { switch_goto_status(SWITCH_STATUS_FALSE, err); } if ((cc = av_get_codec_context(&context->audio_st[1]))) { c[1] = cc; } context->audio_st[0].frame = av_frame_alloc(); switch_assert(context->audio_st[0].frame); context->audio_st[0].active = 1; if (c[1]) { context->audio_st[1].frame = av_frame_alloc(); switch_assert(context->audio_st[1].frame); context->audio_st[0].channels = 1; context->audio_st[1].channels = 1; } else { #if (LIBAVCODEC_VERSION_MAJOR < LIBAVFORMAT_V) handle->channels = c[0]->channels > 2 ? 2 : c[0]->channels; #else handle->channels = c[0]->ch_layout.nb_channels > 2 ? 2 : c[0]->ch_layout.nb_channels; #endif context->audio_st[0].channels = handle->channels; } context->audio_st[0].sample_rate = handle->samplerate; context->audio_st[1].sample_rate = handle->samplerate; if (c[0]->sample_fmt != AV_SAMPLE_FMT_S16 || c[0]->sample_rate != handle->samplerate) { int x; for (x = 0; x < context->has_audio && x < 2 && c[x]; x++) { struct SwrContext *resample_ctx = swr_alloc(); if (resample_ctx) { int ret; #if (LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(59,27,100)) /* FFmpeg 5.0 */ av_opt_set_int(resample_ctx, "in_channel_count", c[x]->channels, 0); #else /* FFmpeg 5.1 */ av_opt_set_chlayout(resample_ctx, "in_chlayout", &c[x]->ch_layout, 0); #endif av_opt_set_int(resample_ctx, "in_sample_rate", c[x]->sample_rate, 0); #if (LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(59,27,100)) av_opt_set_int(resample_ctx, "in_sample_fmt", c[x]->sample_fmt, 0); av_opt_set_int(resample_ctx, "in_channel_layout", (c[x]->channel_layout == 0 && c[x]->channels == 2) ? AV_CH_LAYOUT_STEREO : c[x]->channel_layout, 0); av_opt_set_int(resample_ctx, "out_channel_count", handle->channels, 0); #else av_opt_set_sample_fmt(resample_ctx, "in_sample_fmt", c[x]->sample_fmt, 0); av_opt_set_chlayout(resample_ctx, "out_chlayout", &c[x]->ch_layout, 0); #endif av_opt_set_int(resample_ctx, "out_sample_rate", handle->samplerate,0); #if (LIBAVFORMAT_VERSION_INT < AV_VERSION_INT(59,27,100)) av_opt_set_int(resample_ctx, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0); av_opt_set_int(resample_ctx, "out_channel_layout", handle->channels == 2 ? AV_CH_LAYOUT_STEREO : AV_CH_LAYOUT_MONO, 0); #else av_opt_set_sample_fmt(resample_ctx, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0); #endif if ((ret = swr_init(resample_ctx)) < 0) { char errbuf[1024]; av_strerror(ret, errbuf, 1024); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Failed to initialize the resampling context, ret=%d: %s\n", ret, errbuf); av_free(resample_ctx); switch_goto_status(SWITCH_STATUS_FALSE, err); } context->audio_st[x].resample_ctx = resample_ctx; } } } } if (!context->has_video) { switch_clear_flag(handle, SWITCH_FILE_FLAG_VIDEO); } else { if (!(cc = av_get_codec_context(&context->video_st))) { goto err; } switch (cc->pix_fmt) { case AV_PIX_FMT_YUVA420P: case AV_PIX_FMT_RGBA: case AV_PIX_FMT_ARGB: case AV_PIX_FMT_BGRA: context->handle->mm.fmt = SWITCH_IMG_FMT_ARGB; break; default: context->handle->mm.fmt = SWITCH_IMG_FMT_I420; break; } switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opening file in mode: %s\n", context->handle->mm.fmt == SWITCH_IMG_FMT_ARGB ? "ARGB" : "I420"); } return status; err: if (context->fc) { int nb_streams = context->fc->nb_streams; if (nb_streams > 2) nb_streams = 2; if (context->has_video) close_stream(context->fc, &context->video_st); for (i = 0; i < nb_streams; i++) { close_stream(context->fc, &context->audio_st[i]); } avformat_free_context(context->fc); context->fc = NULL; } return status; } //#define ALT_WAY #define AUDIO_BUF_SEC 5 static void *SWITCH_THREAD_FUNC file_read_thread_run(switch_thread_t *thread, void *obj) { av_file_context_t *context = (av_file_context_t *) obj; AVPacket *pkt = NULL; #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) int got_data = 0; #else int dret = -1; #endif int error; int sync = 0; int eof = 0; AVCodecContext *c = NULL; #if (LIBAVCODEC_VERSION_MAJOR >= LIBAVCODEC_V) AVFrame *vframe = NULL; #endif switch_mutex_lock(context->mutex); context->file_read_thread_started = 1; context->file_read_thread_running = 1; switch_thread_cond_signal(context->cond); switch_mutex_unlock(context->mutex); while (context->file_read_thread_running && !context->closed) { int vid_frames = 0; if (context->seek_ts >= 0) { int stream_id = -1; switch_mutex_lock(context->mutex); switch_buffer_zero(context->audio_buffer); switch_mutex_unlock(context->mutex); // if (context->has_audio) stream_id = context->audio_st.st->index; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "seeking to %" SWITCH_INT64_T_FMT "\n", context->seek_ts); avformat_seek_file(context->fc, stream_id, 0, context->seek_ts, INT64_MAX, 0); context->seek_ts = -2; if (context->has_video) { void *pop; context->video_st.next_pts = 0; context->video_start_time = 0; if (!(c = av_get_codec_context(&context->video_st))) { break; } avcodec_flush_buffers(c); while(switch_queue_trypop(context->eh.video_queue, &pop) == SWITCH_STATUS_SUCCESS) { switch_image_t *img; if (!pop) break; img = (switch_image_t *) pop; switch_img_free(&img); } } } if (context->has_video) { vid_frames = switch_queue_size(context->eh.video_queue); } if (switch_buffer_inuse(context->audio_buffer) > AUDIO_BUF_SEC * context->handle->samplerate * context->handle->channels * 2 && (!context->has_video || vid_frames > 5)) { switch_yield(context->has_video ? 1000 : 10000); continue; } if (pkt) av_packet_free(&pkt); pkt = av_packet_alloc(); if ((error = av_read_frame(context->fc, pkt)) < 0) { if (error == AVERROR_EOF) { if (!context->has_video) break; eof = 1; /* just make sure*/ pkt->data = NULL; pkt->size = 0; pkt->stream_index = context->video_st.st->index; } else { char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not read frame (error '%s')\n", get_error_text(error, ebuf, sizeof(ebuf))); continue; } } // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "stream: %d, pkt size %d\n", pkt.stream_index, pkt.size); if (context->has_video && pkt->stream_index == context->video_st.st->index) { #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) AVFrame *vframe = NULL; #endif switch_image_t *img; if (context->no_video_decode) { if (eof) { break; } else { switch_status_t status; AVPacket *new_pkt = av_packet_alloc(); if (0) { // debug uint8_t *p = pkt->data; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "size = %u %x %x %x %x %x %x\n", pkt->size, *p, *(p+1), *(p+2), *(p+3), *(p+4), *(p+5)); } av_packet_ref(new_pkt, pkt); status = switch_queue_push(context->video_pkt_queue, new_pkt); // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "size = %4u flag=%x pts=%" SWITCH_INT64_T_FMT " dts=%" SWITCH_INT64_T_FMT "\n", pkt.size, pkt.flags, pkt.pts, pkt.dts); context->vid_ready = 1; if (status != SWITCH_STATUS_SUCCESS) { av_packet_free(&new_pkt); } continue; } } if (!sync) { switch_buffer_zero(context->audio_buffer); sync = 1; } again: vframe = av_frame_alloc(); switch_assert(vframe); if (!(c = av_get_codec_context(&context->video_st))) { break; } #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) GCC_DIAG_OFF(deprecated-declarations) error = avcodec_decode_video2(c, vframe, &got_data, pkt); GCC_DIAG_ON(deprecated-declarations) #else if (eof) { dret = avcodec_send_packet(c, NULL); } else { dret = avcodec_send_packet(c, pkt); } if (dret == AVERROR_EOF) { dret = 0; } else if (dret == AVERROR(EAGAIN)) { /* we fully drain all the output in each decode call, so this should not ever happen */ dret = AVERROR_BUG; goto check_errors; } else if (dret < 0) { goto check_errors; } while (dret >= 0) { dret = avcodec_receive_frame(c, vframe); if (dret == AVERROR(EAGAIN)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "No more video frames at the moment\n"); } else if (dret == AVERROR_EOF) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "No more video frames at all\n"); } else if (dret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "Video decoding error\n"); } #endif #if (LIBAVCODEC_VERSION_MAJOR >= LIBAVCODEC_V) check_errors: if (dret < 0 && dret != AVERROR(EAGAIN) && dret != AVERROR_EOF) { #else if (error < 0) { #endif char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not decode frame (error '%s')\n", get_error_text(error, ebuf, sizeof(ebuf))); av_frame_free(&vframe); continue; } // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pkt: %d, pts: %lld dts: %lld\n", pkt.size, pkt.pts, pkt.dts); av_packet_unref(pkt); //if (switch_queue_size(context->eh.video_queue) > 300) { // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Dropping frames\n"); // av_frame_free(&vframe); // continue; //} // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got_data=%d, error=%d\n", got_data, error); #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) if (got_data && error >= 0) { #else if (dret >= 0) { #endif switch_img_fmt_t fmt = SWITCH_IMG_FMT_I420; if (( vframe->format == AV_PIX_FMT_YUVA420P || vframe->format == AV_PIX_FMT_RGBA || vframe->format == AV_PIX_FMT_ARGB || vframe->format == AV_PIX_FMT_BGRA )) { fmt = SWITCH_IMG_FMT_ARGB; } else if (vframe->format != AV_PIX_FMT_YUV420P) { AVFrame *frm = vframe; int ret; if (!context->video_st.sws_ctx) { context->video_st.sws_ctx = sws_getContext(frm->width, frm->height, frm->format, frm->width, frm->height, AV_PIX_FMT_YUV420P, SCALE_FLAGS, NULL, NULL, NULL); if (!context->video_st.sws_ctx) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Cannot init sws context\n"); av_frame_free(&frm); goto do_continue; } } vframe = av_frame_alloc(); switch_assert(vframe); vframe->format = AV_PIX_FMT_YUV420P; vframe->width = frm->width; vframe->height = frm->height; vframe->pts = frm->pts; #if (LIBAVUTIL_VERSION_MAJOR < LIBAVUTIL_V) GCC_DIAG_OFF(deprecated-declarations) vframe->pkt_pts = frm->pkt_pts; GCC_DIAG_ON(deprecated-declarations) #endif vframe->pkt_dts = frm->pkt_dts; ret = av_frame_get_buffer(vframe, 32); switch_assert(ret >= 0); ret = sws_scale(context->video_st.sws_ctx, (const uint8_t *const *)frm->data, frm->linesize, 0, frm->height, vframe->data, vframe->linesize); av_frame_free(&frm); if (ret <= 0 ) { av_frame_free(&vframe); goto do_continue; } } context->handle->mm.fmt = fmt; img = switch_img_alloc(NULL, fmt, vframe->width, vframe->height, 1); if (img) { int64_t *pts = malloc(sizeof(int64_t)); if (pts) { #ifdef ALT_WAY int diff; int sleep = 66000; #endif #if (LIBAVUTIL_VERSION_MAJOR < LIBAVUTIL_V) GCC_DIAG_OFF(deprecated-declarations) *pts = vframe->pkt_pts; GCC_DIAG_ON(deprecated-declarations) #else *pts = vframe->pts; #endif avframe2img(vframe, img); img->user_priv = pts; #ifdef ALT_WAY diff = sleep - (switch_time_now() - context->last_vid_push); if (diff > 0 && diff <= sleep) { switch_core_timer_next(&context->video_timer); } else { switch_core_timer_sync(&context->video_timer); } #endif context->vid_ready = 1; switch_queue_push(context->eh.video_queue, img); context->last_vid_push = switch_time_now(); } else { switch_img_free(&img); } } } #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) av_frame_free(&vframe); #endif if (eof) { #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) if (got_data) { #else if (dret != AVERROR_EOF) { #endif #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) goto again; // to get all delayed video frames in decoder #else av_frame_free(&vframe); goto again; // to get all delayed video frames in decoder #endif } else { goto do_break; } } #if (LIBAVCODEC_VERSION_MAJOR >= LIBAVCODEC_V) } av_frame_free(&vframe); #endif goto do_continue; } else if (context->has_audio && pkt->stream_index == context->audio_st[0].st->index) { AVFrame in_frame = { { 0 } }; if (!(c = av_get_codec_context(&context->audio_st[0]))) { continue; } #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) GCC_DIAG_OFF(deprecated-declarations) if ((error = avcodec_decode_audio4(c, &in_frame, &got_data, pkt)) < 0) { GCC_DIAG_ON(deprecated-declarations) char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not decode frame (error '%s')\n", get_error_text(error, ebuf, sizeof(ebuf))); continue; } #else dret = avcodec_send_packet(c, pkt); if (dret == AVERROR_EOF) { dret = 0; } else if (dret == AVERROR(EAGAIN)) { /* we fully drain all the output in each decode call, so this should not ever happen */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "Error sending audio packet to decoder - BUG, should never happen\n"); goto do_continue; } else if (dret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "Error sending audio packet to decoder\n"); goto do_continue; } while (dret >= 0) { dret = avcodec_receive_frame(c, &in_frame); if (dret == AVERROR(EAGAIN)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "No more audio frames at the moment\n"); } else if (dret == AVERROR_EOF) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "No more audio frames at all\n"); } else if (dret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "Video decoding error\n"); goto do_continue; } #endif // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pkt: %d, decodedddd: %d pts: %lld dts: %lld\n", pkt.size, error, pkt.pts, pkt.dts); av_packet_unref(pkt); #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) if (got_data) { #else if (dret >= 0) { #endif // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got data frm->format: %d samples: %d\n", in_frame.format, in_frame.nb_samples); if (context->audio_st[0].resample_ctx) { int out_samples = swr_get_out_samples(context->audio_st[0].resample_ctx, in_frame.nb_samples); int ret; uint8_t *data[2] = { 0 }; data[0] = malloc(out_samples * context->audio_st[0].channels * 2); switch_assert(data[0]); ret = swr_convert(context->audio_st[0].resample_ctx, data, out_samples, (const uint8_t **)in_frame.data, in_frame.nb_samples); // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "out_samples: %d ret: %d delay: %ld buffer: %zu\n", out_samples, ret, swr_get_delay(context->audio_st[0].resample_ctx, 8000), switch_buffer_inuse(context->audio_buffer)); if (ret) { switch_mutex_lock(context->mutex); switch_buffer_write(context->audio_buffer, data[0], ret * 2 * context->audio_st[0].channels); switch_mutex_unlock(context->mutex); } // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "decoded samples: %d\n", ret); free(data[0]); // if (ret == 0 && avresample_get_delay(context->audio_st[0].resample_ctx)) { // frameP = NULL; // goto again; // } } else { //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "this block is not tested samples: %d\n", in_frame.nb_samples); switch_mutex_lock(context->mutex); switch_buffer_write(context->audio_buffer, in_frame.data[0], in_frame.nb_samples * 2 * context->audio_st[0].channels); switch_mutex_unlock(context->mutex); } } #if (LIBAVCODEC_VERSION_MAJOR >= LIBAVCODEC_V) } } #else } else { av_packet_unref(pkt); } #endif do_continue: continue; do_break: break; } av_packet_free(&pkt); #if (LIBAVCODEC_VERSION_MAJOR >= LIBAVCODEC_V) av_frame_free(&vframe); #endif if (context->has_video) switch_queue_push(context->eh.video_queue, NULL); context->file_read_thread_running = 0; return NULL; } static switch_status_t av_file_open(switch_file_handle_t *handle, const char *path) { av_file_context_t *context = NULL; char *ext; const char *tmp = NULL; #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) AVOutputFormat *fmt; #else const AVOutputFormat *fmt; enum AVCodecID video_codec = AV_CODEC_ID_NONE; enum AVCodecID audio_codec = AV_CODEC_ID_NONE; #endif const char *format = NULL; int ret; char file[1024]; int disable_write_buffer = 0; switch_status_t status = SWITCH_STATUS_SUCCESS; switch_set_string(file, path); if (handle->stream_name) { disable_write_buffer = 1; } if (handle->stream_name && (!strcasecmp(handle->stream_name, "rtmp") || !strcasecmp(handle->stream_name, "rtmps") || !strcasecmp(handle->stream_name, "youtube"))) { char *secure = ""; format = "flv"; if ((ext = strrchr((char *)path, '.')) == 0) { ext = ".flv"; } if (!strcasecmp(handle->stream_name, "rtmps")) secure = "s"; // meh really silly format for the user / pass libav..... if (handle->mm.auth_username && handle->mm.auth_password) { switch_snprintf(file, sizeof(file), "rtmp%s://%s pubUser=%s pubPasswd=%s flashver=FMLE/3.0", secure, path, handle->mm.auth_username, handle->mm.auth_password); } else { switch_snprintf(file, sizeof(file), "rtmp%s://%s", secure, path); } } else if (handle->stream_name && !strcasecmp(handle->stream_name, "rtsp")) { format = "rtsp"; if ((ext = strrchr((char *)path, '.')) == 0) { ext = ".rtsp"; } switch_snprintf(file, sizeof(file), "rtsp://%s", path); disable_write_buffer = 1; } else if ((ext = strrchr((char *)path, '.')) == 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid Format\n"); return SWITCH_STATUS_GENERR; } ext++; if ((context = (av_file_context_t *)switch_core_alloc(handle->memory_pool, sizeof(av_file_context_t))) == 0) { switch_goto_status(SWITCH_STATUS_MEMERR, end); } memset(context, 0, sizeof(av_file_context_t)); handle->private_info = context; context->pool = handle->memory_pool; context->seek_ts = -1; context->offset = DFT_RECORD_OFFSET; context->handle = handle; context->audio_timer = 1; context->colorspace = avformat_globals.colorspace; if (handle->params) { if ((tmp = switch_event_get_header(handle->params, "av_video_offset"))) { context->offset = atoi(tmp); } if ((tmp = switch_event_get_header(handle->params, "video_time_audio"))) { if (switch_false(tmp)) { context->audio_timer = 0; } } if ((tmp = switch_event_get_header(handle->params, "colorspace"))) { int value = atoi(tmp); enum AVColorSpace colorspace = UINTVAL(value); if (colorspace <= AVCOL_SPC_NB) { context->colorspace = colorspace; } } } switch_mutex_init(&context->mutex, SWITCH_MUTEX_NESTED, handle->memory_pool); switch_thread_cond_create(&context->cond, handle->memory_pool); switch_buffer_create_dynamic(&context->audio_buffer, 512, 512, 0); if (!context->audio_buffer) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not allocate buffer for %s\n", path); switch_goto_status(SWITCH_STATUS_MEMERR, end); } switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "sample rate: %d, channels: %d\n", handle->samplerate, handle->channels); if (switch_test_flag(handle, SWITCH_FILE_FLAG_READ)) { if (open_input_file(context, handle, path) != SWITCH_STATUS_SUCCESS) { //clean up; switch_goto_status(SWITCH_STATUS_GENERR, end); } if (context->has_video) { switch_fps_t fps_data = { 0 }; switch_queue_create(&context->eh.video_queue, context->read_fps, handle->memory_pool); context->no_video_decode = handle->params && switch_true(switch_event_get_header(handle->params, "no_video_decode")); if (context->no_video_decode) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Opening video in no decode mode\n"); switch_queue_create(&context->video_pkt_queue, 120 * 5, handle->memory_pool); } switch_mutex_init(&context->eh.mutex, SWITCH_MUTEX_NESTED, handle->memory_pool); switch_calc_video_fps(&fps_data, context->read_fps); switch_core_timer_init(&context->video_timer, "soft", fps_data.ms, fps_data.samples, context->pool); } { switch_threadattr_t *thd_attr = NULL; switch_threadattr_create(&thd_attr, handle->memory_pool); switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); switch_thread_create(&context->file_read_thread, thd_attr, file_read_thread_run, context, handle->memory_pool); } return SWITCH_STATUS_SUCCESS; } #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_6_V) mod_avformat_alloc_output_context2(&context->fc, format, (char *)file, context); #else avformat_alloc_output_context2(&context->fc, NULL, format, (char *)file); if (context->fc) { context->fc->interrupt_callback.callback = interrupt_cb; context->fc->interrupt_callback.opaque = context; } #endif if (!context->fc) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Could not deduce output format from file extension\n"); switch_goto_status(SWITCH_STATUS_GENERR, end); } #if (LIBAVFORMAT_VERSION_MAJOR >= LIBAVFORMAT_V) fmt = context->fc->oformat; video_codec = fmt->video_codec; audio_codec = fmt->audio_codec; #endif fmt = context->fc->oformat; if (handle->params && (tmp = switch_event_get_header(handle->params, "av_audio_codec"))) { if ((context->audio_codec = avcodec_find_encoder_by_name(tmp))) { #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) fmt->audio_codec = context->audio_codec->id; #else audio_codec = context->audio_codec->id; #endif switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "specified audio codec %s %s [%s]\n", tmp, context->audio_codec->name, context->audio_codec->long_name); if (!strcasecmp(tmp, "pcm_mulaw")) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "specified audio codec requires 8000hz\n"); handle->mm.samplerate = 8000; handle->mm.ab = 64; if (!switch_event_get_header(handle->params, "channelsplit")) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "specified audio codec requires mono\n"); handle->real_channels = handle->channels; handle->channels = 1; handle->mm.channels = 1; } } } } if (handle->params && (tmp = switch_event_get_header(handle->params, "av_video_codec"))) { if ((context->video_codec = avcodec_find_encoder_by_name(tmp))) { #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) fmt->video_codec = context->video_codec->id; #else video_codec = context->video_codec->id; #endif switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "specified video codec %s %s [%s]\n", tmp, context->video_codec->name, context->video_codec->long_name); } } if (!strcasecmp(ext, "wav") || (handle->params && switch_true(switch_event_get_header(handle->params, "av_record_audio_only")))) { context->has_video = 0; switch_clear_flag(handle, SWITCH_FILE_FLAG_VIDEO); } /* open the output file, if needed */ if (!(fmt->flags & AVFMT_NOFILE)) { ret = avio_open(&context->fc->pb, file, AVIO_FLAG_WRITE); if (ret < 0) { char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open '%s': %s\n", file, get_error_text(ret, ebuf, sizeof(ebuf))); switch_goto_status(SWITCH_STATUS_GENERR, end); } } else { avformat_network_init(); } if (handle->mm.samplerate) { handle->samplerate = handle->mm.samplerate; } else { handle->mm.samplerate = handle->samplerate; } if (!handle->mm.ab) { handle->mm.ab = 128; } if (!handle->mm.vb) { handle->mm.vb = switch_calc_bitrate(handle->mm.vw, handle->mm.vh, 1, handle->mm.fps); } #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) if (switch_test_flag(handle, SWITCH_FILE_FLAG_VIDEO) && fmt->video_codec != AV_CODEC_ID_NONE) { #else if (switch_test_flag(handle, SWITCH_FILE_FLAG_VIDEO) && video_codec != AV_CODEC_ID_NONE) { #endif const AVCodecDescriptor *desc; if ((handle->stream_name && (!strcasecmp(handle->stream_name, "rtmp") || !strcasecmp(handle->stream_name, "rtmps") || !strcasecmp(handle->stream_name, "youtube")))) { #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) if (fmt->video_codec != AV_CODEC_ID_H264 ) { fmt->video_codec = AV_CODEC_ID_H264; // force H264 #else if (video_codec != AV_CODEC_ID_H264 ) { video_codec = AV_CODEC_ID_H264; // force H264 #endif } #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) fmt->audio_codec = AV_CODEC_ID_AAC; #else audio_codec = AV_CODEC_ID_AAC; #endif handle->samplerate = 44100; handle->mm.samplerate = 44100; handle->mm.ab = 128; handle->mm.cbr = 1; handle->mm.vencspd = SWITCH_VIDEO_ENCODE_SPEED_FAST; handle->mm.vprofile = SWITCH_VIDEO_PROFILE_BASELINE; if (!handle->mm.vb && handle->mm.vw && handle->mm.vh) { switch(handle->mm.vh) { case 240: handle->mm.vb = 400; break; case 360: handle->mm.vb = 750; break; case 480: handle->mm.vb = 1000; break; case 720: handle->mm.vb = 2500; break; case 1080: handle->mm.vb = 4500; break; default: handle->mm.vb = switch_calc_bitrate(handle->mm.vw, handle->mm.vh, 1, handle->mm.fps); break; } } if (handle->mm.fps > 0.0f) { handle->mm.keyint = (int) 2.0f * handle->mm.fps; } } #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) desc = avcodec_descriptor_get(fmt->video_codec); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "use video codec: [%d] %s (%s)\n", fmt->video_codec, desc->name, desc->long_name); #else desc = avcodec_descriptor_get(video_codec); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "use video codec: [%d] %s (%s)\n", video_codec, desc->name, desc->long_name); #endif } #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) if (fmt->audio_codec != AV_CODEC_ID_NONE) { #else if (audio_codec != AV_CODEC_ID_NONE) { #endif const char *issplit = 0; context->audio_st[0].channels = handle->channels; context->audio_st[1].sample_rate = handle->samplerate; if (handle->channels > 1 && handle->params && (issplit = switch_event_get_header(handle->params, "channelsplit"))) { int lr = (!strcasecmp(issplit, "lr") || switch_true(issplit)); int rl = !strcasecmp(issplit, "rl"); if (lr || rl) { context->audio_st[0].channels = 1; context->audio_st[1].channels = 1; #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) add_stream(context, &context->audio_st[0], context->fc, &context->audio_codec, fmt->audio_codec, &handle->mm); add_stream(context, &context->audio_st[1], context->fc, &context->audio_codec, fmt->audio_codec, &handle->mm); #else add_stream(context, &context->audio_st[0], context->fc, &context->audio_codec, audio_codec, &handle->mm); add_stream(context, &context->audio_st[1], context->fc, &context->audio_codec, audio_codec, &handle->mm); #endif } if (lr) { context->audio_st[0].r = 1; } else if (rl) { context->audio_st[1].r = 1; } } if (!context->audio_st[0].active) { #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V) add_stream(context, &context->audio_st[0], context->fc, &context->audio_codec, fmt->audio_codec, &handle->mm); #else add_stream(context, &context->audio_st[0], context->fc, &context->audio_codec, audio_codec, &handle->mm); #endif } if (open_audio(context->fc, context->audio_codec, &context->audio_st[0]) != SWITCH_STATUS_SUCCESS) { switch_goto_status(SWITCH_STATUS_GENERR, end); } context->has_audio = 1; if (context->audio_st[1].active) { if (open_audio(context->fc, context->audio_codec, &context->audio_st[1]) != SWITCH_STATUS_SUCCESS) { switch_goto_status(SWITCH_STATUS_GENERR, end); } context->has_audio++; } } av_dump_format(context->fc, 0, file, 1); handle->format = 0; handle->sections = 0; handle->seekable = 0; handle->speed = 0; handle->pos = 0; if (disable_write_buffer) { handle->pre_buffer_datalen = 0; } switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "Opening File [%s] %dhz %s\n", file, handle->samplerate, context->has_video ? " with VIDEO" : ""); return SWITCH_STATUS_SUCCESS; end: if (!context) { return status; } if (context->fc) { mod_avformat_destroy_output_context(context); } if (context->video_timer.interval) { switch_core_timer_destroy(&context->video_timer); } if (context->audio_buffer) { switch_buffer_destroy(&context->audio_buffer); } return status; } static switch_status_t av_file_truncate(switch_file_handle_t *handle, int64_t offset) { return SWITCH_STATUS_FALSE; } static switch_status_t av_file_write(switch_file_handle_t *handle, void *data, size_t *len) { uint32_t datalen = 0; switch_status_t status = SWITCH_STATUS_SUCCESS; // uint8_t buf[SWITCH_RECOMMENDED_BUFFER_SIZE] = { 0 }, *bp = buf; // uint32_t encoded_rate; av_file_context_t *context = (av_file_context_t *)handle->private_info; // uint32_t size = 0; uint32_t bytes; int inuse; int sample_start = 0; AVCodecContext *c = NULL; if (!switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE)) { return SWITCH_STATUS_FALSE; } if (!context->has_audio) { return SWITCH_STATUS_SUCCESS; } if (!context->vid_ready) { if (switch_test_flag(handle, SWITCH_FILE_FLAG_VIDEO)) { switch_buffer_zero(context->audio_buffer); return status; } else if (!context->aud_ready) { // audio only recording int ret = avformat_write_header(context->fc, NULL); if (ret < 0) { char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error occurred when opening output file: %s\n", get_error_text(ret, ebuf, sizeof(ebuf))); return SWITCH_STATUS_FALSE; } context->aud_ready = 1; } } if (data && len) { datalen = *len * 2 * handle->channels; if (context->offset) { switch_size_t samples = *len; int fps = handle->samplerate / samples; int lead_frames = (context->offset * fps) / 1000; for (int x = 0; x < lead_frames; x++) { switch_buffer_zero_fill(context->audio_buffer, datalen); } context->offset = 0; } switch_buffer_write(context->audio_buffer, data, datalen); } bytes = context->audio_st[0].frame->nb_samples * 2 * context->handle->channels; //context->audio_st[0].st->codec->channels; //{ // int inuse = switch_buffer_inuse(context->audio_buffer); // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "inuse: %d samples: %d bytes: %d\n", inuse, context->audio_st[0].frame->nb_samples, bytes); //} if (context->closed) { inuse = switch_buffer_inuse(context->audio_buffer); if (inuse < bytes) { switch_buffer_zero_fill(context->audio_buffer, bytes - inuse); } } if (context->video_timer.interval) { int delta; switch_core_timer_sync(&context->video_timer); delta = context->video_timer.samplecount - context->last_vid_write; if (context->audio_timer || delta >= 200) { uint32_t new_pts = context->video_timer.samplecount * (handle->samplerate / 1000); if (!context->audio_timer) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Delta of %d detected. Video timer sync: %" SWITCH_UINT64_T_FMT "/%d %" SWITCH_UINT64_T_FMT "\n", delta, context->audio_st[0].next_pts, context->video_timer.samplecount, new_pts - context->audio_st[0].next_pts); } sample_start = new_pts; } context->last_vid_write = context->video_timer.samplecount; } if (sample_start) { int j = 0; for (j = 0; j < 2; j++) { if (context->audio_st[j].active) { context->audio_st[j].next_pts = sample_start; } } } while (switch_buffer_inuse(context->audio_buffer) >= bytes) { AVPacket *pkt[2]; #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) int got_packet[2] = {0}; int result[2] = {0}; #else int dret = -1; #endif int j = 0, ret = -1, audio_stream_count = 1; AVFrame *use_frame = NULL; pkt[0] = av_packet_alloc(); pkt[1] = av_packet_alloc(); if (context->audio_st[1].active) { switch_size_t len = 0; int i = 0, j = 0; int l = 0, r = 1; if (!context->mux_buf || context->mux_buf_len < bytes / 2) { context->mux_buf_len = bytes / 2; context->mux_buf = (int16_t *)realloc((void *)context->mux_buf, context->mux_buf_len * 2); } audio_stream_count = 2; len = switch_buffer_read(context->audio_buffer, (uint8_t *)context->mux_buf, bytes); if (context->audio_st[0].r) { l = 1; r = 0; } for (i = 0; i < len / 4; i++) { *((int16_t *)context->audio_st[l].frame->data[0] + i) = context->mux_buf[j++]; *((int16_t *)context->audio_st[r].frame->data[0] + i) = context->mux_buf[j++]; } } else { switch_buffer_read(context->audio_buffer, context->audio_st[0].frame->data[0], bytes); } for (j = 0; j < audio_stream_count; j++) { av_frame_make_writable(context->audio_st[j].frame); use_frame = context->audio_st[j].frame; if (context->audio_st[j].resample_ctx) { int out_samples = swr_get_out_samples(context->audio_st[j].resample_ctx, context->audio_st[j].frame->nb_samples); av_frame_make_writable(context->audio_st[j].tmp_frame); /* convert to destination format */ ret = swr_convert(context->audio_st[j].resample_ctx, context->audio_st[j].tmp_frame->data, out_samples, (const uint8_t **)context->audio_st[j].frame->data, context->audio_st[j].frame->nb_samples); if (ret < 0) { char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error while converting %d samples, error text: %s\n", context->audio_st[j].frame->nb_samples, get_error_text(ret, ebuf, sizeof(ebuf))); continue; } use_frame = context->audio_st[j].tmp_frame; } use_frame->pts = context->audio_st[j].next_pts; // context->audio_st[j].next_pts = use_frame->pts + use_frame->nb_samples; if (!(c = av_get_codec_context(&context->audio_st[j]))) { continue; } #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) GCC_DIAG_OFF(deprecated-declarations) result[j] = avcodec_encode_audio2(c, pkt[j], use_frame, &got_packet[j]); GCC_DIAG_ON(deprecated-declarations) #else dret = avcodec_send_frame(c, use_frame); if (dret == AVERROR_EOF) { dret = 0; } else if (dret == AVERROR(EAGAIN)) { /* we fully drain all the output in each decode call, so this should not ever happen */ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "Encoding error for channel %d on sending frame to encode - BUG, should never happen\n", j); dret = AVERROR_BUG; } else if (dret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoding error for channel %d on sending frame to encode\n", j); } while (dret >= 0) { dret = avcodec_receive_packet(c, pkt[j]); if (dret == AVERROR(EAGAIN)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "No more audio packets at the moment for channel %d\n", j); break; } else if (dret == AVERROR_EOF) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG9, "No more audio packets at all for channel %d\n", j); break; } else if (dret < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoding error for channel %d\n", j); break; } if (context->mutex) switch_mutex_lock(context->mutex); ret = write_frame(context->fc, &c->time_base, context->audio_st[j].st, pkt[j]); if (context->mutex) switch_mutex_unlock(context->mutex); if (ret < 0) { context->errs++; if ((context->errs % 10) == 0) { char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error while writing audio frame: %d %s\n", ret, get_error_text(ret, ebuf, sizeof(ebuf))); if ((ret == -5 || ret == -104) && handle->stream_name) { context->errs = 1001; } } //switch_goto_status(SWITCH_STATUS_FALSE, end); } else { context->errs = 0; } if (context->errs > 1000) { av_packet_free(&pkt[0]); av_packet_free(&pkt[1]); switch_goto_status(SWITCH_STATUS_FALSE, end); } } #endif context->audio_st[j].next_pts += use_frame->nb_samples; } #if (LIBAVCODEC_VERSION_MAJOR < LIBAVCODEC_V) if (result[0] < 0 || result[1] < 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Error encoding audio frame: %d %d \n", result[0], result[1]); av_packet_free(&pkt[0]); av_packet_free(&pkt[1]); continue; } for (j = 0; j < audio_stream_count; j++) { if (got_packet[j]) { if (context->mutex) switch_mutex_lock(context->mutex); if (!(c = av_get_codec_context(&context->audio_st[j]))) { if (context->mutex) switch_mutex_unlock(context->mutex); continue; } ret = write_frame(context->fc, &c->time_base, context->audio_st[j].st, pkt[j]); if (context->mutex) switch_mutex_unlock(context->mutex); if (ret < 0) { context->errs++; if ((context->errs % 10) == 0) { char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error while writing audio frame: %s\n", get_error_text(ret, ebuf, sizeof(ebuf))); } //switch_goto_status(SWITCH_STATUS_FALSE, end); } else { context->errs = 0; } if (context->errs > 1000) { status = SWITCH_STATUS_FALSE; goto end; } } } #endif av_packet_free(&pkt[0]); av_packet_free(&pkt[1]); } end: return status; } static switch_status_t av_file_command(switch_file_handle_t *handle, switch_file_command_t command) { av_file_context_t *context = (av_file_context_t *)handle->private_info; uint32_t offset = 0; switch(command) { case SCFC_FLUSH_AUDIO: switch_mutex_lock(context->mutex); switch_buffer_zero(context->audio_buffer); switch_mutex_unlock(context->mutex); break; case SCFC_PAUSE_READ: if (context->read_paused) { context->read_paused = SWITCH_FALSE; context->video_st.next_pts = 0; context->video_start_time = 0; } else { context->read_paused = SWITCH_TRUE; } break; case SCFC_PAUSE_WRITE: context->vid_ready = 0; context->eh.record_timer_paused = switch_micro_time_now(); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s pause write\n", handle->file_path); break; case SCFC_RESUME_WRITE: if (context->eh.record_timer_paused) { context->eh.last_ts = 0; offset = (uint32_t)(switch_micro_time_now() - context->eh.record_timer_paused); context->video_timer.start += offset; switch_core_timer_sync(&context->video_timer); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s resume write\n", handle->file_path); context->eh.record_timer_paused = 0; } break; default: break; } return SWITCH_STATUS_SUCCESS; } static switch_status_t av_file_close(switch_file_handle_t *handle) { av_file_context_t *context = (av_file_context_t *)handle->private_info; switch_status_t status; context->closed = 1; context->eh.finalize = 1; if (context->eh.video_queue) { if (!switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE)) { flush_video_queue(context->eh.video_queue, 0); switch_queue_push(context->eh.video_queue, NULL); switch_queue_term(context->eh.video_queue); } else { switch_queue_push(context->eh.video_queue, NULL); } } if (context->eh.video_thread) { switch_thread_join(&status, context->eh.video_thread); } if (switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE)) { av_file_write(handle, NULL, NULL); } if (context->file_read_thread_running) { context->file_read_thread_running = 0; } if (context->video_pkt_queue) { flush_video_pkt_queue(context->video_pkt_queue); switch_queue_term(context->video_pkt_queue); } if (context->packetizer) { switch_packetizer_close(&context->packetizer); } if (context->file_read_thread) { switch_thread_join(&status, context->file_read_thread); context->file_read_thread = NULL; } if (context->eh.video_queue) { flush_video_queue(context->eh.video_queue, 0); } if (context->fc) { if ((context->aud_ready || context->has_video) && switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE)) { av_write_trailer(context->fc); } mod_avformat_destroy_output_context(context); } if (context->video_timer.interval) { switch_core_timer_destroy(&context->video_timer); } switch_img_free(&context->last_img); switch_buffer_destroy(&context->audio_buffer); switch_safe_free(context->mux_buf); return SWITCH_STATUS_SUCCESS; } static switch_status_t av_file_seek(switch_file_handle_t *handle, unsigned int *cur_sample, int64_t samples, int whence) { av_file_context_t *context = (av_file_context_t *)handle->private_info; if (whence == SEEK_SET) { handle->pos = handle->offset_pos = samples; } context->seek_ts = samples / handle->native_rate * AV_TIME_BASE; *cur_sample = context->seek_ts; context->closed = 0; if (!context->file_read_thread_running) { switch_threadattr_t *thd_attr = NULL; if (context->file_read_thread) { switch_status_t status; switch_thread_join(&status, context->file_read_thread); context->file_read_thread = NULL; } switch_threadattr_create(&thd_attr, handle->memory_pool); switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); switch_thread_create(&context->file_read_thread, thd_attr, file_read_thread_run, context, handle->memory_pool); } return SWITCH_STATUS_FALSE; } static switch_status_t av_file_read(switch_file_handle_t *handle, void *data, size_t *len) { av_file_context_t *context = (av_file_context_t *)handle->private_info; int size; size_t need = *len * 2 * context->audio_st[0].channels; if (!context->has_audio && context->has_video && context->file_read_thread_running) { memset(data, 0, *len * handle->channels * 2); return SWITCH_STATUS_SUCCESS; } switch_mutex_lock(context->mutex); while (!context->file_read_thread_started) { switch_thread_cond_wait(context->cond, context->mutex); } switch_mutex_unlock(context->mutex); if (context->closed || (!context->file_read_thread_running && switch_buffer_inuse(context->audio_buffer) == 0)) { *len = 0; return SWITCH_STATUS_FALSE; } while (context->has_video && !context->vid_ready && !context->closed) { switch_yield(1000); } switch_mutex_lock(context->mutex); size = switch_buffer_read(context->audio_buffer, data, need); switch_mutex_unlock(context->mutex); if (size == 0) { size_t blank = (handle->samplerate / 20) * 2 * handle->real_channels; if (need > blank) { need = blank; } memset(data, 0, need); *len = need / 2 / handle->real_channels; } else { *len = size / context->audio_st[0].channels / 2; } handle->pos += *len; handle->sample_count += *len; return *len == 0 ? SWITCH_STATUS_FALSE : SWITCH_STATUS_SUCCESS; } #ifdef ALT_WAY static switch_status_t av_file_read_video(switch_file_handle_t *handle, switch_frame_t *frame, switch_video_read_flag_t flags) { void *pop; av_file_context_t *context = (av_file_context_t *)handle->private_info; switch_status_t status; if (!context->has_video || context->closed) return SWITCH_STATUS_FALSE; if ((flags & SVR_CHECK)) { return SWITCH_STATUS_BREAK; } if ((flags & SVR_FLUSH)) { flush_video_queue(context->eh.video_queue, 1); } if ((flags & SVR_BLOCK)) { status = switch_queue_pop(context->eh.video_queue, &pop); } else { status = switch_queue_trypop(context->eh.video_queue, &pop); } if (status == SWITCH_STATUS_SUCCESS) { if (!pop) { return SWITCH_STATUS_FALSE; } context->vid_ready = 1; frame->img = (switch_image_t *) pop; if (frame->img) { if (frame->img && context->handle->mm.scale_w && context->handle->mm.scale_h) { if (frame->img->d_w != context->handle->mm.scale_w || frame->img->d_h != context->handle->mm.scale_h) { switch_img_fit(&frame->img, context->handle->mm.scale_w, context->handle->mm.scale_h, SWITCH_FIT_SIZE); } } context->vid_ready = 1; } return SWITCH_STATUS_SUCCESS; } return (flags & SVR_FLUSH) ? SWITCH_STATUS_BREAK : status; } #else static switch_status_t no_video_decode_packets(switch_file_handle_t *handle, switch_frame_t *frame, switch_video_read_flag_t flags) { av_file_context_t *context = (av_file_context_t *)handle->private_info; MediaStream *mst = &context->video_st; AVStream *st = mst->st; // AVCodecContext *ctx = st->codec; // int ticks = 0; // int64_t max_delta = 1 * AV_TIME_BASE; // 1 second switch_status_t status = SWITCH_STATUS_SUCCESS; AVPacket *pkt; int64_t pts; if (!context->packetizer) { // uint8_t *p = st->codecpar->extradata; // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "size = %u %x %x %x %x %x %x\n", st->codecpar->extradata_size, *p, *(p+1), *(p+2), *(p+3), *(p+4), *(p+5)); context->packetizer = switch_packetizer_create(SPT_H264_SIZED_BITSTREAM, SLICE_SIZE); if (!context->packetizer) return SWITCH_STATUS_FALSE; switch_packetizer_feed_extradata(context->packetizer, st->codecpar->extradata, st->codecpar->extradata_size); } if (context->last_read_pkt) { status = switch_packetizer_read(context->packetizer, frame); if (status == SWITCH_STATUS_SUCCESS) { av_packet_free(&context->last_read_pkt); context->last_read_pkt = NULL; } return status; } status = switch_queue_trypop(context->video_pkt_queue, (void **)&pkt); if (status != SWITCH_STATUS_SUCCESS || !pkt) { switch_cond_next(); return SWITCH_STATUS_BREAK; } context->last_read_pkt = pkt; switch_packetizer_feed(context->packetizer, pkt->data, pkt->size); status = switch_packetizer_read(context->packetizer, frame); pts = av_rescale_q(pkt->pts, st->time_base, AV_TIME_BASE_Q); frame->timestamp = pts * 9 / 100; // scale to sample 900000 // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "pts=%" SWITCH_INT64_T_FMT " status = %d\n", pts, status); if (status == SWITCH_STATUS_SUCCESS) { av_packet_free(&context->last_read_pkt); context->last_read_pkt = NULL; } if (status == SWITCH_STATUS_SUCCESS || status == SWITCH_STATUS_MORE_DATA) { if (!context->video_start_time) { context->video_start_time = switch_time_now() - pts; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "set start time: %" SWITCH_INT64_T_FMT " now: %" SWITCH_INT64_T_FMT " pts: %" SWITCH_INT64_T_FMT "\n", context->video_start_time, switch_time_now(), pts); } else if (flags & SVR_BLOCK) { int64_t sleep = pts - (switch_time_now() - context->video_start_time); if (sleep > 0) { // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "zzZ... %" SWITCH_INT64_T_FMT "\n", sleep); if (sleep > 1000000) { sleep = 1000000; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "but zzZ... %" SWITCH_INT64_T_FMT " at most\n", sleep); } switch_yield(sleep); } else { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "video is late %" SWITCH_INT64_T_FMT "\n", sleep); } } } return status; } static switch_status_t av_file_read_video(switch_file_handle_t *handle, switch_frame_t *frame, switch_video_read_flag_t flags) { av_file_context_t *context = (av_file_context_t *)handle->private_info; void *pop; MediaStream *mst = &context->video_st; AVStream *st = mst->st; int ticks = 0; int64_t max_delta = 1 * AV_TIME_BASE; // 1 second switch_status_t status = SWITCH_STATUS_SUCCESS; double fl_to = 0.02; int do_fl = 0; int smaller_ts = context->read_fps; AVCodecContext *c = NULL; AVCodecParserContext *cp = NULL; if (!context->has_video) return SWITCH_STATUS_FALSE; if ((flags & SVR_CHECK)) { return SWITCH_STATUS_BREAK; } if (context->no_video_decode) { switch_set_flag(frame, SFF_ENCODED); status = no_video_decode_packets(handle, frame, flags); // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "return len=%4u nalu=%02x m=%d ts=%u\n", frame->datalen, *(uint8_t *)frame->data, frame->m, frame->timestamp); return status; } if (handle->mm.fps > 0 && handle->mm.fps < smaller_ts) { smaller_ts = handle->mm.fps; } fl_to = (1000 / smaller_ts) * 1000; //printf("WTF %d (%f)\n",switch_queue_size(context->eh.video_queue), fl_to); if (flags & SVR_FLUSH) { max_delta = fl_to; do_fl = 1; } if (!context->file_read_thread_running && switch_queue_size(context->eh.video_queue) == 0) { return SWITCH_STATUS_FALSE; } if (context->read_paused || context->seek_ts == -2) { int sanity = 10; if (context->seek_ts == -2) { // just seeked, try read a new img again1: status = switch_queue_trypop(context->eh.video_queue, &pop); if (pop && status == SWITCH_STATUS_SUCCESS) { context->seek_ts = -1; switch_img_free(&context->last_img); context->last_img = (switch_image_t *)pop; switch_img_copy(context->last_img, &frame->img); context->vid_ready = 1; goto resize_check; } if (context->last_img) { // repeat the last img switch_img_copy(context->last_img, &frame->img); context->vid_ready = 1; context->seek_ts = -1; goto resize_check; } if ((flags & SVR_BLOCK) && sanity-- > 0) { switch_yield(10000); goto again1; } return SWITCH_STATUS_BREAK; } if (context->last_img) { // repeat the last img if ((flags & SVR_BLOCK)) switch_yield(100000); switch_img_copy(context->last_img, &frame->img); context->vid_ready = 1; goto resize_check; } if ((flags & SVR_BLOCK)) { status = switch_queue_pop(context->eh.video_queue, &pop); } else { status = switch_queue_trypop(context->eh.video_queue, &pop); } if (pop && status == SWITCH_STATUS_SUCCESS) { switch_img_free(&context->last_img); context->last_img = (switch_image_t *)pop; switch_img_copy(context->last_img, &frame->img); context->vid_ready = 1; goto resize_check; } return SWITCH_STATUS_BREAK; } #if 0 if (context->last_img) { if (mst->next_pts && (switch_time_now() - mst->next_pts > max_delta)) { switch_img_free(&context->last_img); // too late } else if (mst->next_pts && (switch_time_now() - mst->next_pts > -10000)) { frame->img = context->last_img; context->last_img = NULL; context->vid_ready = 1; return SWITCH_STATUS_SUCCESS; } if (!(flags & SVR_BLOCK) && !do_fl) { if (!mst->next_pts) { frame->img = context->last_img; context->last_img = NULL; context->vid_ready = 1; return SWITCH_STATUS_SUCCESS; } return SWITCH_STATUS_BREAK; } } #endif if ((c = av_get_codec_context(mst)) && c->time_base.num) { cp = av_stream_get_parser(st); ticks = cp ? cp->repeat_pict + 1 : c->ticks_per_frame; // mst->next_pts += ((int64_t)AV_TIME_BASE * st->codec->time_base.num * ticks) / st->codec->time_base.den; } if (!context->video_start_time) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "start: %" SWITCH_INT64_T_FMT " ticks: %d ticks_per_frame: %d st num:%d st den:%d codec num:%d codec den:%d start: %" SWITCH_TIME_T_FMT ", duration:%" SWITCH_INT64_T_FMT " nb_frames:%" SWITCH_INT64_T_FMT " q2d:%f\n", context->video_start_time, ticks, c ? c->ticks_per_frame : -1, st->time_base.num, st->time_base.den, c ? c->time_base.num : -1, c ? c->time_base.den : -1, st->start_time, st->duration == AV_NOPTS_VALUE ? context->fc->duration / AV_TIME_BASE * 1000 : st->duration, st->nb_frames, av_q2d(st->time_base)); } again: if (context->last_img) { pop = (void *) context->last_img; context->last_img = NULL; status = SWITCH_STATUS_SUCCESS; } else { if ((flags & SVR_BLOCK)) { status = switch_queue_pop(context->eh.video_queue, &pop); } else { status = switch_queue_trypop(context->eh.video_queue, &pop); } } if (pop && status == SWITCH_STATUS_SUCCESS) { switch_image_t *img = (switch_image_t *)pop; int64_t pts; int64_t now = switch_time_now(); pts = av_rescale_q(*((uint64_t *)img->user_priv), st->time_base, AV_TIME_BASE_Q); // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "pkt_pts: %lld pts: %lld queue size: %u\n", *((uint64_t *)img->user_priv), pts, switch_queue_size(context->eh.video_queue)); handle->vpos = pts; if (!context->video_start_time) { context->video_start_time = now - pts; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "set start time: %" SWITCH_INT64_T_FMT " now: %" SWITCH_INT64_T_FMT " pts: %" SWITCH_INT64_T_FMT "\n", context->video_start_time, now, pts); } if (st->time_base.num == 0) { mst->next_pts = 0; } else { // int64_t last_pts = mst->next_pts; mst->next_pts = context->video_start_time + pts; // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "pts: %" SWITCH_INT64_T_FMT " last_pts: %" SWITCH_INT64_T_FMT " delta: %" SWITCH_INT64_T_FMT " frame_pts: %" SWITCH_INT64_T_FMT " nextpts: %" SWITCH_INT64_T_FMT ", num: %d, den:%d num:%d den:%d sleep: %" SWITCH_INT64_T_FMT "\n", // pts, last_pts, mst->next_pts - last_pts, *((uint64_t *)img->user_priv), mst->next_pts, st->time_base.num, st->time_base.den, st->codec->time_base.num, st->codec->time_base.den, mst->next_pts - now); } if (pts == 0 || context->video_start_time == 0) mst->next_pts = 0; if ((mst->next_pts && (now - mst->next_pts) > max_delta)) { //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "picture is too late, off: %" SWITCH_INT64_T_FMT " max delta: %" SWITCH_INT64_T_FMT " queue size:%u fps:%u/%0.2f\n", (int64_t)(now - mst->next_pts), max_delta, switch_queue_size(context->eh.video_queue), context->read_fps, handle->mm.fps); switch_img_free(&img); //max_delta = AV_TIME_BASE; if (switch_queue_size(context->eh.video_queue) > 0) { // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "WTF again\n"); goto again; } else if (!(flags & SVR_BLOCK) && !do_fl) { mst->next_pts = 0; context->video_start_time = 0; return SWITCH_STATUS_BREAK; } } if ((flags & SVR_BLOCK)) { while (switch_micro_time_now() - mst->next_pts < -10000) { // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "yield, delta=%" SWITCH_INT64_T_FMT "\n", switch_micro_time_now() - mst->next_pts); switch_yield(1000); } frame->img = img; } else { if (switch_micro_time_now() - mst->next_pts > -10000) { frame->img = img; } else { switch_img_free(&context->last_img); context->last_img = img; return SWITCH_STATUS_BREAK; } } } else { return SWITCH_STATUS_BREAK; } resize_check: if (frame->img) { if (context->handle->mm.scale_w && context->handle->mm.scale_h) { if (frame->img->d_w != context->handle->mm.scale_w || frame->img->d_h != context->handle->mm.scale_h) { switch_img_fit(&frame->img, context->handle->mm.scale_w, context->handle->mm.scale_h, SWITCH_FIT_SCALE); } } context->vid_ready = 1; } if ((flags & SVR_BLOCK)) { if (!frame->img) context->closed = 1; return frame->img ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_FALSE; } else { return frame->img ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_BREAK; } } #endif static switch_status_t av_file_write_video(switch_file_handle_t *handle, switch_frame_t *frame) { switch_status_t status = SWITCH_STATUS_SUCCESS; av_file_context_t *context = (av_file_context_t *)handle->private_info; switch_image_t *img = NULL; AVCodecContext *c = NULL; if (!switch_test_flag(handle, SWITCH_FILE_FLAG_VIDEO)) { return SWITCH_STATUS_FALSE; } if (!frame->img) { switch_goto_status(SWITCH_STATUS_FALSE, end); } if (!context->has_video) { context->video_st.width = frame->img->d_w; context->video_st.height = frame->img->d_h; context->video_st.next_pts = switch_time_now() / 1000; if (add_stream(context, &context->video_st, context->fc, &context->video_codec, context->fc->oformat->video_codec, &handle->mm) == SWITCH_STATUS_SUCCESS && open_video(context->fc, context->video_codec, &context->video_st) == SWITCH_STATUS_SUCCESS) { char codec_str[256] = ""; int ret; if ((c = av_get_codec_context(&context->video_st))) { avcodec_string(codec_str, sizeof(codec_str), c, 1); } switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "use video codec implementation %s\n", codec_str); context->has_video = 1; // av_dump_format(context->fc, 0, "/tmp/av.mp4", 1); ret = avformat_write_header(context->fc, NULL); if (ret < 0) { char ebuf[255] = ""; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error occurred when opening output file: %s\n", get_error_text(ret, ebuf, sizeof(ebuf))); switch_goto_status(SWITCH_STATUS_FALSE, end); } } else { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error adding video stream\n"); switch_goto_status(SWITCH_STATUS_FALSE, end); } } if (!context->eh.video_thread) { switch_threadattr_t *thd_attr = NULL; switch_mutex_init(&context->mutex, SWITCH_MUTEX_NESTED, handle->memory_pool); context->eh.mutex = context->mutex; context->eh.video_st = &context->video_st; context->eh.fc = context->fc; context->eh.mm = &handle->mm; switch_queue_create(&context->eh.video_queue, SWITCH_CORE_QUEUE_LEN, handle->memory_pool); switch_threadattr_create(&thd_attr, handle->memory_pool); //switch_threadattr_priority_set(thd_attr, SWITCH_PRI_REALTIME); switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); switch_core_timer_init(&context->video_timer, "soft", 1, 1, context->pool); context->eh.video_timer = &context->video_timer; if (context->has_audio) { context->audio_st[0].frame->pts = 0; context->audio_st[0].next_pts = 0; } switch_thread_create(&context->eh.video_thread, thd_attr, video_thread_run, context, handle->memory_pool); } switch_img_copy(frame->img, &img); switch_queue_push(context->eh.video_queue, img); if (!context->vid_ready) { switch_mutex_lock(context->mutex); switch_buffer_zero(context->audio_buffer); switch_mutex_unlock(context->mutex); context->vid_ready = 1; } end: return status; } static switch_status_t av_file_set_string(switch_file_handle_t *handle, switch_audio_col_t col, const char *string) { av_file_context_t *context = (av_file_context_t *)handle->private_info; if (context->fc) { const char *field = switch_parse_audio_col(col); if (field) { av_dict_set(&context->fc->metadata, field, string, 0); return SWITCH_STATUS_SUCCESS; } } return SWITCH_STATUS_FALSE; } static switch_status_t av_file_get_string(switch_file_handle_t *handle, switch_audio_col_t col, const char **string) { av_file_context_t *context = (av_file_context_t *)handle->private_info; if (context->fc) { AVDictionaryEntry *tag = NULL; const char *field = switch_parse_audio_col(col); if (field && (tag = av_dict_get(context->fc->metadata, field, tag, 0))) { *string = tag->value; return SWITCH_STATUS_SUCCESS; } } return SWITCH_STATUS_FALSE; } static char *supported_formats[SWITCH_MAX_CODECS] = { 0 }; static const char modname[] = "mod_av"; static switch_status_t load_config(void) { char *cf = "avformat.conf"; switch_xml_t cfg, xml, param, settings; if (!(xml = switch_xml_open_cfg(cf, &cfg, NULL))) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "open of %s failed\n", cf); return SWITCH_STATUS_TERM; } if ((settings = switch_xml_child(cfg, "settings"))) { for (param = switch_xml_child(settings, "param"); param; param = param->next) { char *var = (char *) switch_xml_attr_soft(param, "name"); char *val = (char *) switch_xml_attr_soft(param, "value"); if (!strcasecmp(var, "colorspace")) { int value = atoi(val); avformat_globals.colorspace = UINTVAL(value); if (avformat_globals.colorspace > AVCOL_SPC_NB) { avformat_globals.colorspace = AVCOL_SPC_RGB; } } } } switch_xml_free(xml); return SWITCH_STATUS_SUCCESS; } SWITCH_MODULE_LOAD_FUNCTION(mod_avformat_load) { switch_file_interface_t *file_interface; int i = 0; memset(&avformat_globals, 0, sizeof(struct avformat_globals)); load_config(); supported_formats[i++] = "av"; supported_formats[i++] = "rtmp"; supported_formats[i++] = "rtmps"; supported_formats[i++] = "rtsp"; supported_formats[i++] = "mp4"; supported_formats[i++] = "m4a"; supported_formats[i++] = "mov"; supported_formats[i++] = "mkv"; supported_formats[i++] = "webm"; //supported_formats[i++] = "wav"; file_interface = (switch_file_interface_t *)switch_loadable_module_create_interface(*module_interface, SWITCH_FILE_INTERFACE); file_interface->interface_name = modname; file_interface->extens = supported_formats; file_interface->file_open = av_file_open; file_interface->file_close = av_file_close; file_interface->file_truncate = av_file_truncate; file_interface->file_read = av_file_read; file_interface->file_write = av_file_write; file_interface->file_read_video = av_file_read_video; file_interface->file_write_video = av_file_write_video; file_interface->file_seek = av_file_seek; file_interface->file_set_string = av_file_set_string; file_interface->file_get_string = av_file_get_string; file_interface->file_command = av_file_command; /* indicate that the module should continue to be loaded */ return SWITCH_STATUS_SUCCESS; } /* For Emacs: * Local Variables: * mode:c * indent-tabs-mode:t * tab-width:4 * c-basic-offset:4 * End: * For VIM: * vim:set softtabstop=4 shiftwidth=4 tabstop=4: */