kenmirrors
/
freeswitch
mirror of https://github.com/signalwire/freeswitch.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

FS-10249: [mod_av] Audio gradually falls behind video in recordings

This commit is contained in:
Anthony Minessale 2017-05-19 17:38:08 -05:00
parent a1fc18aee5
commit e76ccc3955
7 changed files with 330 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/include/private/switch_core_pvt.h
View File

@ -239,7 +239,7 @@ struct switch_media_bug {
switch_buffer_t *text_buffer; switch_buffer_t *text_buffer;
char *text_framedata; char *text_framedata;
uint32_t text_framesize; uint32_t text_framesize;
switch_mm_t mm;
struct switch_media_bug *next; struct switch_media_bug *next;
}; };

3
src/include/switch_core.h
View File

@ -351,6 +351,9 @@ SWITCH_DECLARE(void) switch_core_media_bug_set_read_demux_frame(_In_ switch_medi
*/ */
SWITCH_DECLARE(switch_core_session_t *) switch_core_media_bug_get_session(_In_ switch_media_bug_t *bug); SWITCH_DECLARE(switch_core_session_t *) switch_core_media_bug_get_session(_In_ switch_media_bug_t *bug);
SWITCH_DECLARE(void) switch_core_media_bug_set_media_params(switch_media_bug_t *bug, switch_mm_t *mm);
SWITCH_DECLARE(void) switch_core_media_bug_get_media_params(switch_media_bug_t *bug, switch_mm_t *mm);
SWITCH_DECLARE(const char *) switch_core_media_bug_get_text(switch_media_bug_t *bug); SWITCH_DECLARE(const char *) switch_core_media_bug_get_text(switch_media_bug_t *bug);
/*! /*!

35
src/include/switch_module_interfaces.h
View File

@ -309,41 +309,6 @@ struct switch_file_interface {
struct switch_file_interface *next; struct switch_file_interface *next;
}; };
typedef enum {
SWITCH_VIDEO_ENCODE_SPEED_DEFAULT = 0,
SWITCH_VIDEO_ENCODE_SPEED_FAST = 0,
SWITCH_VIDEO_ENCODE_SPEED_MEDIUM,
SWITCH_VIDEO_ENCODE_SPEED_SLOW
} switch_video_encode_speed_t;
typedef enum {
SWITCH_VIDEO_PROFILE_BASELINE,
SWITCH_VIDEO_PROFILE_MAIN,
SWITCH_VIDEO_PROFILE_HIGH
} switch_video_profile_t;
typedef struct switch_mm_s {
int samplerate;
int channels;
int keyint;
int ab;
int vb;
int vw;
int vh;
int cbr;
float fps;
float source_fps;
int vbuf;
switch_video_profile_t vprofile;
switch_video_encode_speed_t vencspd;
uint8_t try_hardware_encoder;
int scale_w;
int scale_h;
switch_img_fmt_t fmt;
char *auth_username;
char *auth_password;
} switch_mm_t;
/*! an abstract representation of a file handle (some parameters based on compat with libsndfile) */ /*! an abstract representation of a file handle (some parameters based on compat with libsndfile) */
struct switch_file_handle { struct switch_file_handle {
/*! the interface of the module that implemented the current file type */ /*! the interface of the module that implemented the current file type */

36
src/include/switch_types.h
View File

@ -2670,6 +2670,42 @@ typedef struct switch_agc_s switch_agc_t;
struct switch_chromakey_s; struct switch_chromakey_s;
typedef struct switch_chromakey_s switch_chromakey_t; typedef struct switch_chromakey_s switch_chromakey_t;
typedef enum {
SWITCH_VIDEO_ENCODE_SPEED_DEFAULT = 0,
SWITCH_VIDEO_ENCODE_SPEED_FAST = 0,
SWITCH_VIDEO_ENCODE_SPEED_MEDIUM,
SWITCH_VIDEO_ENCODE_SPEED_SLOW
} switch_video_encode_speed_t;
typedef enum {
SWITCH_VIDEO_PROFILE_BASELINE,
SWITCH_VIDEO_PROFILE_MAIN,
SWITCH_VIDEO_PROFILE_HIGH
} switch_video_profile_t;
typedef struct switch_mm_s {
int samplerate;
int channels;
int keyint;
int ab;
int vb;
int vw;
int vh;
int cbr;
float fps;
float source_fps;
int vbuf;
switch_video_profile_t vprofile;
switch_video_encode_speed_t vencspd;
uint8_t try_hardware_encoder;
int scale_w;
int scale_h;
switch_img_fmt_t fmt;
char *auth_username;
char *auth_password;
} switch_mm_t;
SWITCH_END_EXTERN_C SWITCH_END_EXTERN_C
#endif #endif
/* For Emacs: /* For Emacs:

336
src/mod/applications/mod_av/avformat.c
View File

@ -49,6 +49,88 @@
#define AV_TS_MAX_STRING_SIZE 32 #define AV_TS_MAX_STRING_SIZE 32
/* App interface */
// a wrapper around a single output AVStream
typedef struct MediaStream {
AVStream *st;
AVFrame *frame;
AVFrame *tmp_frame;
// audio
int channels;
int sample_rate;
struct AVAudioResampleContext *resample_ctx;
//video
int width;
int height;
struct SwsContext *sws_ctx;
int64_t next_pts;
} MediaStream;
typedef struct record_helper_s {
switch_mutex_t *mutex;
AVFormatContext *fc;
MediaStream *video_st;
switch_timer_t *timer;
int in_callback;
switch_queue_t *video_queue;
switch_thread_t *video_thread;
switch_mm_t *mm;
int finalize;
} record_helper_t;
/* file interface */
struct av_file_context {
switch_memory_pool_t *pool;
switch_mutex_t *mutex;
switch_thread_cond_t *cond;
switch_buffer_t *buf;
switch_buffer_t *audio_buffer;
switch_timer_t video_timer;
int offset;
int audio_start;
int aud_ready;
int vid_ready;
int audio_ready;
int closed;
MediaStream video_st;
MediaStream audio_st;
AVFormatContext *fc;
AVCodec *audio_codec;
AVCodec *video_codec;
int has_audio;
int has_video;
record_helper_t eh;
switch_thread_t *file_read_thread;
int file_read_thread_running;
int file_read_thread_started;
switch_time_t video_start_time;
switch_image_t *last_img;
int read_fps;
switch_time_t last_vid_push;
int64_t seek_ts;
switch_bool_t read_paused;
int errs;
switch_file_handle_t *handle;
};
typedef struct av_file_context av_file_context_t;
//FUCK
/** /**
* Fill the provided buffer with a string containing a timestamp * Fill the provided buffer with a string containing a timestamp
* representation. * representation.
@ -206,39 +288,6 @@ static void av_unused avframe2fd(AVFrame *pict, int fd)
} }
} }
/* App interface */
// a wrapper around a single output AVStream
typedef struct MediaStream {
AVStream *st;
AVFrame *frame;
AVFrame *tmp_frame;
// audio
int channels;
int sample_rate;
struct AVAudioResampleContext *resample_ctx;
//video
int width;
int height;
struct SwsContext *sws_ctx;
int64_t next_pts;
} MediaStream;
typedef struct record_helper_s {
switch_mutex_t *mutex;
AVFormatContext *fc;
MediaStream *video_st;
switch_timer_t *timer;
int in_callback;
switch_queue_t *video_queue;
switch_thread_t *video_thread;
switch_mm_t *mm;
int finalize;
} record_helper_t;
static void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt) static void log_packet(const AVFormatContext *fmt_ctx, const AVPacket *pkt)
{ {
AVRational *time_base = &fmt_ctx->streams[pkt->stream_index]->time_base; AVRational *time_base = &fmt_ctx->streams[pkt->stream_index]->time_base;
@ -651,15 +700,15 @@ static int flush_video_queue(switch_queue_t *q, int min)
static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *obj) static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *obj)
{ {
record_helper_t *eh = (record_helper_t *) obj; av_file_context_t *context = (av_file_context_t *) obj;
void *pop = NULL; void *pop = NULL;
switch_image_t *img = NULL, *tmp_img = NULL; switch_image_t *img = NULL, *tmp_img = NULL;
int d_w = eh->video_st->width, d_h = eh->video_st->height; int d_w = context->eh.video_st->width, d_h = context->eh.video_st->height;
int size = 0, skip = 0, skip_freq = 0, skip_count = 0, skip_total = 0, skip_total_count = 0; int size = 0, skip = 0, skip_freq = 0, skip_count = 0, skip_total = 0, skip_total_count = 0;
uint64_t delta_avg = 0, delta_sum = 0, delta_i = 0, delta = 0, last_ts = 0; uint64_t delta_avg = 0, delta_sum = 0, delta_i = 0, delta = 0, last_ts = 0;
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "video thread start\n"); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "video thread start\n");
switch_assert(context->eh.video_queue);
for(;;) { for(;;) {
AVPacket pkt = { 0 }; AVPacket pkt = { 0 };
int got_packet; int got_packet;
@ -667,23 +716,29 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
top: top:
if (switch_queue_pop(eh->video_queue, &pop) == SWITCH_STATUS_SUCCESS) { switch_assert(context->eh.video_queue);
if (switch_queue_pop(context->eh.video_queue, &pop) == SWITCH_STATUS_SUCCESS) {
switch_img_free(&img); switch_img_free(&img);
if (!pop) { if (!pop) {
goto endfor; goto endfor;
} }
img = (switch_image_t *) pop;
img = (switch_image_t *) pop;
if (!d_w) d_w = img->d_w; if (!d_w) d_w = img->d_w;
if (!d_h) d_h = img->d_h; if (!d_h) d_h = img->d_h;
if (d_w && d_h && (d_w != img->d_w || d_h != img->d_h)) { if (d_w && d_h && (d_w != img->d_w || d_h != img->d_h)) {
/* scale to match established stream */ /* scale to match established stream */
switch_img_scale(img, &tmp_img, d_w, d_h); switch_img_scale(img, &tmp_img, d_w, d_h);
switch_img_free(&img); if (tmp_img) {
img = tmp_img; switch_img_free(&img);
tmp_img = NULL; img = tmp_img;
tmp_img = NULL;
}
} }
} else { } else {
continue; continue;
@ -699,9 +754,9 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
} }
} else { } else {
size = switch_queue_size(eh->video_queue); size = switch_queue_size(context->eh.video_queue);
if (size > 5 && !eh->finalize) { if (size > 5 && !context->eh.finalize) {
skip = size; skip = size;
if (size > 10) { if (size > 10) {
@ -714,65 +769,71 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
} }
} }
//switch_mutex_lock(eh->mutex); //switch_mutex_lock(context->eh.mutex);
eh->in_callback = 1; context->eh.in_callback = 1;
av_init_packet(&pkt); av_init_packet(&pkt);
if (eh->video_st->frame) { if (context->eh.video_st->frame) {
ret = av_frame_make_writable(eh->video_st->frame); ret = av_frame_make_writable(context->eh.video_st->frame);
} }
if (ret < 0) { if (ret < 0) {
continue; continue;
} }
fill_avframe(eh->video_st->frame, img); fill_avframe(context->eh.video_st->frame, img);
if (eh->finalize) { if (context->eh.finalize) {
if (delta_i && !delta_avg) { if (delta_i && !delta_avg) {
delta_avg = (int)(double)(delta_sum / delta_i); delta_avg = (int)(double)(delta_sum / delta_i);
delta_i = 0; delta_i = 1;
delta_sum = delta_avg; delta_sum = delta_avg;
} }
if (delta_avg) { if (delta_avg) {
delta = delta_avg; delta = delta_avg;
} else if (eh->mm->fps) { } else if (context->eh.mm->fps) {
delta = 1000 / eh->mm->fps; delta = 1000 / context->eh.mm->fps;
} else { } else {
delta = 33; delta = 33;
} }
eh->video_st->frame->pts += delta; context->eh.video_st->frame->pts += delta;
} else { } else {
uint64_t delta_tmp; uint64_t delta_tmp;
switch_core_timer_sync(eh->timer); switch_core_timer_sync(context->eh.timer);
delta_tmp = eh->timer->samplecount - last_ts; delta_tmp = context->eh.timer->samplecount - last_ts;
if (delta_tmp != last_ts) { if (delta_tmp != 0) {
delta_sum += delta_tmp; delta_sum += delta_tmp;
delta_i++; delta_i++;
if (delta_i >= 60) { if (delta_i == UINT64_MAX) {
delta_avg = (int)(double)(delta_sum / delta_i); delta_i = 1;
delta_i = 0;
delta_sum = delta_avg; delta_sum = delta_avg;
} }
if ((delta_i % 10) == 0) {
delta_avg = (int)(double)(delta_sum / delta_i);
}
eh->video_st->frame->pts = eh->timer->samplecount; context->eh.video_st->frame->pts = context->eh.timer->samplecount;
} else {
context->eh.video_st->frame->pts = context->eh.timer->samplecount + 1;
} }
} }
last_ts = eh->video_st->frame->pts; last_ts = context->eh.video_st->frame->pts;
//eh->video_st->frame->pts = switch_time_now() / 1000 - eh->video_st->next_pts; //context->eh.video_st->frame->pts = switch_time_now() / 1000 - context->eh.video_st->next_pts;
//switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pts: %ld\n", eh->video_st->frame->pts); //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pts: %ld\n", context->eh.video_st->frame->pts);
/* encode the image */ /* encode the image */
ret = avcodec_encode_video2(eh->video_st->st->codec, &pkt, eh->video_st->frame, &got_packet); ret = avcodec_encode_video2(context->eh.video_st->st->codec, &pkt, context->eh.video_st->frame, &got_packet);
if (ret < 0) { if (ret < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoding Error %d\n", ret); switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoding Error %d\n", ret);
@ -780,14 +841,14 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
} }
if (got_packet) { if (got_packet) {
switch_mutex_lock(eh->mutex); switch_mutex_lock(context->eh.mutex);
ret = write_frame(eh->fc, &eh->video_st->st->codec->time_base, eh->video_st->st, &pkt); ret = write_frame(context->eh.fc, &context->eh.video_st->st->codec->time_base, context->eh.video_st->st, &pkt);
switch_mutex_unlock(eh->mutex); switch_mutex_unlock(context->eh.mutex);
av_packet_unref(&pkt); av_packet_unref(&pkt);
} }
eh->in_callback = 0; context->eh.in_callback = 0;
//switch_mutex_unlock(eh->mutex); //switch_mutex_unlock(context->eh.mutex);
} }
endfor: endfor:
@ -799,14 +860,14 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
av_init_packet(&pkt); av_init_packet(&pkt);
ret = avcodec_encode_video2(eh->video_st->st->codec, &pkt, NULL, &got_packet); ret = avcodec_encode_video2(context->eh.video_st->st->codec, &pkt, NULL, &got_packet);
if (ret < 0) { if (ret < 0) {
break; break;
} else if (got_packet) { } else if (got_packet) {
switch_mutex_lock(eh->mutex); switch_mutex_lock(context->eh.mutex);
ret = write_frame(eh->fc, &eh->video_st->st->codec->time_base, eh->video_st->st, &pkt); ret = write_frame(context->eh.fc, &context->eh.video_st->st->codec->time_base, context->eh.video_st->st, &pkt);
switch_mutex_unlock(eh->mutex); switch_mutex_unlock(context->eh.mutex);
av_packet_unref(&pkt); av_packet_unref(&pkt);
if (ret < 0) break; if (ret < 0) break;
} else { } else {
@ -814,7 +875,7 @@ static void *SWITCH_THREAD_FUNC video_thread_run(switch_thread_t *thread, void *
} }
} }
while(switch_queue_trypop(eh->video_queue, &pop) == SWITCH_STATUS_SUCCESS) { while(switch_queue_trypop(context->eh.video_queue, &pop) == SWITCH_STATUS_SUCCESS) {
if (!pop) break; if (!pop) break;
img = (switch_image_t *) pop; img = (switch_image_t *) pop;
switch_img_free(&img); switch_img_free(&img);
@ -854,7 +915,6 @@ SWITCH_STANDARD_APP(record_av_function)
switch_status_t status; switch_status_t status;
switch_frame_t *read_frame; switch_frame_t *read_frame;
switch_channel_t *channel = switch_core_session_get_channel(session); switch_channel_t *channel = switch_core_session_get_channel(session);
record_helper_t eh = { 0 };
switch_timer_t timer = { 0 }; switch_timer_t timer = { 0 };
switch_mutex_t *mutex = NULL; switch_mutex_t *mutex = NULL;
switch_codec_t codec;//, *vid_codec; switch_codec_t codec;//, *vid_codec;
@ -872,6 +932,7 @@ SWITCH_STANDARD_APP(record_av_function)
AVCodec *audio_codec, *video_codec; AVCodec *audio_codec, *video_codec;
int has_audio = 0, has_video = 0; int has_audio = 0, has_video = 0;
int ret; int ret;
av_file_context_t context = { 0 };
switch_channel_answer(channel); switch_channel_answer(channel);
switch_core_session_get_read_impl(session, &read_impl); switch_core_session_get_read_impl(session, &read_impl);
@ -990,23 +1051,22 @@ SWITCH_STANDARD_APP(record_av_function)
if (has_video) { if (has_video) {
switch_threadattr_t *thd_attr = NULL; switch_threadattr_t *thd_attr = NULL;
switch_mutex_init(&mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session)); switch_mutex_init(&mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(session));
eh.mutex = mutex; context.eh.mutex = mutex;
eh.video_st = &video_st; context.eh.video_st = &video_st;
eh.fc = fc; context.eh.fc = fc;
if (switch_core_timer_init(&timer, "soft", 1, 1, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) { if (switch_core_timer_init(&timer, "soft", 1, 1, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Timer Activation Fail\n"); switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Timer Activation Fail\n");
goto end; goto end;
} }
eh.timer = &timer; context.eh.timer = &timer;
switch_queue_create(&eh.video_queue, SWITCH_CORE_QUEUE_LEN, switch_core_session_get_pool(session)); switch_queue_create(&context.eh.video_queue, SWITCH_CORE_QUEUE_LEN, switch_core_session_get_pool(session));
switch_core_session_set_video_read_callback(session, video_read_callback, (void *)&eh); switch_core_session_set_video_read_callback(session, video_read_callback, (void *)&context.eh);
switch_threadattr_create(&thd_attr, switch_core_session_get_pool(session)); switch_threadattr_create(&thd_attr, switch_core_session_get_pool(session));
//switch_threadattr_priority_set(thd_attr, SWITCH_PRI_REALTIME); //switch_threadattr_priority_set(thd_attr, SWITCH_PRI_REALTIME);
switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
switch_thread_create(&eh.video_thread, thd_attr, video_thread_run, &eh, switch_core_session_get_pool(session)); switch_thread_create(&context.eh.video_thread, thd_attr, video_thread_run, &context, switch_core_session_get_pool(session));
} }
switch_core_session_set_read_codec(session, &codec); switch_core_session_set_read_codec(session, &codec);
@ -1137,8 +1197,8 @@ SWITCH_STANDARD_APP(record_av_function)
int ret = 0; int ret = 0;
switch_status_t st; switch_status_t st;
switch_queue_push(eh.video_queue, NULL); switch_queue_push(context.eh.video_queue, NULL);
switch_thread_join(&st, eh.video_thread); switch_thread_join(&st, context.eh.video_thread);
again: again:
av_init_packet(&pkt); av_init_packet(&pkt);
@ -1293,46 +1353,6 @@ SWITCH_STANDARD_API(av_format_api_function)
return SWITCH_STATUS_SUCCESS; return SWITCH_STATUS_SUCCESS;
} }
/* file interface */
struct av_file_context {
switch_memory_pool_t *pool;
switch_mutex_t *mutex;
switch_thread_cond_t *cond;
switch_buffer_t *buf;
switch_buffer_t *audio_buffer;
switch_timer_t video_timer;
int offset;
int audio_start;
int aud_ready;
int vid_ready;
int audio_ready;
int closed;
MediaStream video_st;
MediaStream audio_st;
AVFormatContext *fc;
AVCodec *audio_codec;
AVCodec *video_codec;
int has_audio;
int has_video;
record_helper_t eh;
switch_thread_t *file_read_thread;
int file_read_thread_running;
int file_read_thread_started;
switch_time_t video_start_time;
switch_image_t *last_img;
int read_fps;
switch_time_t last_vid_push;
int64_t seek_ts;
switch_bool_t read_paused;
int errs;
switch_file_handle_t *handle;
};
typedef struct av_file_context av_file_context_t;
static void mod_avformat_destroy_output_context(av_file_context_t *context) static void mod_avformat_destroy_output_context(av_file_context_t *context)
{ {
@ -2142,9 +2162,13 @@ static switch_status_t av_file_close(switch_file_handle_t *handle)
context->eh.finalize = 1; context->eh.finalize = 1;
if (context->eh.video_queue) { if (context->eh.video_queue) {
flush_video_queue(context->eh.video_queue, 0); if (!switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE)) {
switch_queue_push(context->eh.video_queue, NULL); flush_video_queue(context->eh.video_queue, 0);
switch_queue_term(context->eh.video_queue); switch_queue_push(context->eh.video_queue, NULL);
switch_queue_term(context->eh.video_queue);
} else {
switch_queue_push(context->eh.video_queue, NULL);
}
} }
if (context->eh.video_thread) { if (context->eh.video_thread) {
@ -2164,6 +2188,10 @@ static switch_status_t av_file_close(switch_file_handle_t *handle)
context->file_read_thread = NULL; context->file_read_thread = NULL;
} }
if (context->eh.video_queue) {
flush_video_queue(context->eh.video_queue, 0);
}
if (context->fc) { if (context->fc) {
if (switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE)) av_write_trailer(context->fc); if (switch_test_flag(handle, SWITCH_FILE_FLAG_WRITE)) av_write_trailer(context->fc);
@ -2553,35 +2581,39 @@ static switch_status_t av_file_write_video(switch_file_handle_t *handle, switch_
} }
} }
if (!context->vid_ready) {
switch_threadattr_t *thd_attr = NULL;
switch_mutex_init(&context->mutex, SWITCH_MUTEX_NESTED, handle->memory_pool);
context->eh.mutex = context->mutex;
context->eh.video_st = &context->video_st;
context->eh.fc = context->fc;
context->eh.mm = &handle->mm;
context->eh.timer = &context->video_timer;
switch_queue_create(&context->eh.video_queue, SWITCH_CORE_QUEUE_LEN, handle->memory_pool);
switch_threadattr_create(&thd_attr, handle->memory_pool);
//switch_threadattr_priority_set(thd_attr, SWITCH_PRI_REALTIME);
switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
switch_thread_create(&context->eh.video_thread, thd_attr, video_thread_run, &context->eh, handle->memory_pool);
switch_core_timer_init(&context->video_timer, "soft", 1, 1, context->pool);
switch_buffer_zero(context->audio_buffer);
context->audio_st.frame->pts = 0;
context->audio_st.next_pts = 0;
}
if (context->has_video) { if (context->has_video) {
switch_image_t *img = NULL; switch_image_t *img = NULL;
if (!context->eh.video_thread) {
switch_threadattr_t *thd_attr = NULL;
switch_mutex_init(&context->mutex, SWITCH_MUTEX_NESTED, handle->memory_pool);
context->eh.mutex = context->mutex;
context->eh.video_st = &context->video_st;
context->eh.fc = context->fc;
context->eh.mm = &handle->mm;
switch_queue_create(&context->eh.video_queue, SWITCH_CORE_QUEUE_LEN, handle->memory_pool);
switch_threadattr_create(&thd_attr, handle->memory_pool);
//switch_threadattr_priority_set(thd_attr, SWITCH_PRI_REALTIME);
switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE);
switch_core_timer_init(&context->video_timer, "soft", 1, 1, context->pool);
context->eh.timer = &context->video_timer;
context->audio_st.frame->pts = 0;
context->audio_st.next_pts = 0;
switch_thread_create(&context->eh.video_thread, thd_attr, video_thread_run, context, handle->memory_pool);
}
switch_img_copy(frame->img, &img); switch_img_copy(frame->img, &img);
switch_queue_push(context->eh.video_queue, img); switch_queue_push(context->eh.video_queue, img);
}
if (!context->vid_ready) {
switch_mutex_lock(context->mutex);
switch_buffer_zero(context->audio_buffer);
switch_mutex_unlock(context->mutex);
context->vid_ready = 1;
}
context->vid_ready = 1; }
end: end:
return status; return status;

126
src/switch_core_media_bug.c
View File

@ -83,6 +83,16 @@ SWITCH_DECLARE(uint32_t) switch_core_media_bug_clear_flag(switch_media_bug_t *bu
return switch_clear_flag(bug, flag); return switch_clear_flag(bug, flag);
} }
SWITCH_DECLARE(void) switch_core_media_bug_set_media_params(switch_media_bug_t *bug, switch_mm_t *mm)
{
bug->mm = *mm;
}
SWITCH_DECLARE(void) switch_core_media_bug_get_media_params(switch_media_bug_t *bug, switch_mm_t *mm)
{
*mm = bug->mm;
}
SWITCH_DECLARE(switch_core_session_t *) switch_core_media_bug_get_session(switch_media_bug_t *bug) SWITCH_DECLARE(switch_core_session_t *) switch_core_media_bug_get_session(switch_media_bug_t *bug)
{ {
return bug->session; return bug->session;
@ -553,6 +563,9 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *
uint8_t *buf; uint8_t *buf;
switch_size_t buflen = SWITCH_RTP_MAX_BUF_LEN; switch_size_t buflen = SWITCH_RTP_MAX_BUF_LEN;
switch_frame_t frame = { 0 }; switch_frame_t frame = { 0 };
switch_timer_t timer = { 0 };
switch_mm_t mm = { 0 };
int fps = 15;
buf = switch_core_session_alloc(bug->session, buflen); buf = switch_core_session_alloc(bug->session, buflen);
frame.packet = buf; frame.packet = buf;
@ -573,65 +586,114 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *
return NULL; return NULL;
} }
switch_core_media_bug_get_media_params(bug, &mm);
if (mm.fps) {
fps = (int) mm.fps;
}
switch_core_timer_init(&timer, "soft", 1000 / fps, (90000 / (1000 / fps)), NULL);
while (bug->ready) { while (bug->ready) {
switch_status_t status; switch_status_t status;
int w = 0, h = 0, ok = 1; int w = 0, h = 0, ok = 1, new_main = 0, new_other = 0, new_canvas = 0;
switch_core_timer_next(&timer);
if (!switch_channel_test_flag(bug->session->channel, CF_ANSWERED) && switch_core_media_bug_test_flag(bug, SMBF_ANSWER_REQ)) {
flush_video_queue(main_q, 0);
if (other_q) flush_video_queue(other_q, 0);
continue;
}
flush_video_queue(main_q, 1);
if ((status = switch_queue_trypop(main_q, &pop)) == SWITCH_STATUS_SUCCESS) {
switch_img_free(&img);
if ((status = switch_queue_pop(main_q, &pop)) == SWITCH_STATUS_SUCCESS) {
if (!pop) { if (!pop) {
goto end; goto end;
} }
img = (switch_image_t *) pop; img = (switch_image_t *) pop;
new_main = 1;
w = img->d_w; w = img->d_w;
h = img->d_h; h = img->d_h;
}
if (other_q) {
flush_video_queue(other_q, 1);
if (other_q) { if ((status = switch_queue_trypop(other_q, &other_pop)) == SWITCH_STATUS_SUCCESS) {
flush_video_queue(other_q, 1); switch_img_free(&other_img);
other_img = (switch_image_t *) other_pop;
new_other = 1;
}
if ((status = switch_queue_trypop(other_q, &other_pop)) == SWITCH_STATUS_SUCCESS) { if (other_img) {
if (!(other_img = (switch_image_t *) other_pop)) { if (!w) w = other_img->d_w;
goto end; if (!h) h = other_img->d_h;
}
} if (other_img->d_w != w || other_img->d_h != h) {
switch_image_t *tmp_img = NULL;
if (other_img) { switch_img_scale(other_img, &tmp_img, w, h);
if (other_img->d_w != w || other_img->d_h != h) {
switch_image_t *tmp_img = NULL;
switch_img_scale(other_img, &tmp_img, w, h);
switch_img_free(&other_img);
other_img = tmp_img;
}
}
w *= 2;
if (!IMG || IMG->d_h != h || IMG->d_w != w) {
switch_img_free(&IMG);
IMG = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, w, h, 1);
}
switch_img_patch(IMG, img, 0, 0);
if (other_img) {
switch_img_patch(IMG, other_img, w / 2, 0);
switch_img_free(&other_img); switch_img_free(&other_img);
other_img = tmp_img;
} }
} }
if (!(w&&h)) continue;
if (img) {
if (img->d_w != w || img->d_h != h) {
switch_image_t *tmp_img = NULL;
switch_img_scale(img, &tmp_img, w, h);
switch_img_free(&img);
img = tmp_img;
}
}
w *= 2;
if (!IMG || IMG->d_h != h || IMG->d_w != w) {
switch_rgb_color_t color = { 0 };
switch_img_free(&IMG);
IMG = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, w, h, 1);
new_canvas = 1;
switch_color_set_rgb(&color, "#000000");
switch_img_fill(IMG, 0, 0, IMG->d_w, IMG->d_h, &color);
}
}
if (IMG) {
if (img && (new_canvas || new_main)) {
switch_img_patch(IMG, img, 0, 0);
}
if (other_img && (new_canvas || new_other)) {
switch_img_patch(IMG, other_img, w / 2, 0);
}
}
if (IMG || img) {
switch_thread_rwlock_rdlock(bug->session->bug_rwlock); switch_thread_rwlock_rdlock(bug->session->bug_rwlock);
frame.img = other_q ? IMG : img; frame.img = other_q ? IMG : img;
bug->video_ping_frame = &frame; bug->video_ping_frame = &frame;
if (bug->callback) { if (bug->callback) {
if (bug->callback(bug, bug->user_data, SWITCH_ABC_TYPE_STREAM_VIDEO_PING) == SWITCH_FALSE if (bug->callback(bug, bug->user_data, SWITCH_ABC_TYPE_STREAM_VIDEO_PING) == SWITCH_FALSE
|| (bug->stop_time && bug->stop_time <= switch_epoch_time_now(NULL))) { || (bug->stop_time && bug->stop_time <= switch_epoch_time_now(NULL))) {
ok = SWITCH_FALSE; ok = SWITCH_FALSE;
} }
} }
bug->video_ping_frame = NULL; bug->video_ping_frame = NULL;
switch_img_free(&img);
switch_thread_rwlock_unlock(bug->session->bug_rwlock); switch_thread_rwlock_unlock(bug->session->bug_rwlock);
if (!ok) { if (!ok) {
@ -643,6 +705,8 @@ static void *SWITCH_THREAD_FUNC video_bug_thread(switch_thread_t *thread, void *
end: end:
switch_core_timer_destroy(&timer);
switch_img_free(&IMG); switch_img_free(&IMG);
switch_img_free(&img); switch_img_free(&img);
switch_img_free(&other_img); switch_img_free(&other_img);

14
src/switch_ivr_async.c
View File

@ -1229,6 +1229,11 @@ static switch_bool_t record_callback(switch_media_bug_t *bug, void *user_data, s
rh->completion_cause = NULL; rh->completion_cause = NULL;
switch_core_session_get_read_impl(session, &rh->read_impl); switch_core_session_get_read_impl(session, &rh->read_impl);
if (switch_core_file_has_video(rh->fh, SWITCH_TRUE)) {
switch_core_media_bug_set_media_params(bug, &rh->fh->mm);
}
} }
break; break;
case SWITCH_ABC_TYPE_TAP_NATIVE_READ: case SWITCH_ABC_TYPE_TAP_NATIVE_READ:
@ -2536,7 +2541,6 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_record_session(switch_core_session_t
file_flags |= SWITCH_FILE_WRITE_APPEND; file_flags |= SWITCH_FILE_WRITE_APPEND;
} }
fh->samplerate = 0; fh->samplerate = 0;
if ((vval = switch_channel_get_variable(channel, "record_sample_rate"))) { if ((vval = switch_channel_get_variable(channel, "record_sample_rate"))) {
int tmp = 0; int tmp = 0;
@ -2554,6 +2558,8 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_record_session(switch_core_session_t
fh->channels = channels; fh->channels = channels;
if ((vval = switch_channel_get_variable(channel, "enable_file_write_buffering"))) { if ((vval = switch_channel_get_variable(channel, "enable_file_write_buffering"))) {
int tmp = atoi(vval); int tmp = atoi(vval);
@ -2640,11 +2646,13 @@ SWITCH_DECLARE(switch_status_t) switch_ivr_record_session(switch_core_session_t
//switch_core_media_set_video_file(session, fh, SWITCH_RW_READ); //switch_core_media_set_video_file(session, fh, SWITCH_RW_READ);
//switch_channel_set_flag_recursive(session->channel, CF_VIDEO_DECODED_READ); //switch_channel_set_flag_recursive(session->channel, CF_VIDEO_DECODED_READ);
if ((vval = switch_channel_get_variable(channel, "record_concat_video")) && switch_true(vval)) { if (switch_channel_var_true(channel, "record_concat_video")) {
flags |= SMBF_READ_VIDEO_STREAM; flags |= SMBF_READ_VIDEO_STREAM;
flags |= SMBF_WRITE_VIDEO_STREAM; flags |= SMBF_WRITE_VIDEO_STREAM;
} else if (switch_channel_var_true(channel, "record_bleg_video")) {
flags |= SMBF_WRITE_VIDEO_STREAM;
} else { } else {
flags |= SMBF_READ_VIDEO_PING; flags |= SMBF_READ_VIDEO_STREAM;
} }
} else { } else {
flags &= ~SMBF_READ_VIDEO_PING; flags &= ~SMBF_READ_VIDEO_PING;