diff --git a/src/include/switch_core_media.h b/src/include/switch_core_media.h index 2238fc14b1..37631fce70 100644 --- a/src/include/switch_core_media.h +++ b/src/include/switch_core_media.h @@ -192,6 +192,7 @@ SWITCH_DECLARE(int) switch_core_session_check_incoming_crypto(switch_core_sessio switch_media_type_t type, const char *crypto, int crypto_tag, switch_sdp_type_t sdp_type); +SWITCH_DECLARE(uint32_t) switch_core_media_get_video_fps(switch_core_session_t *session); SWITCH_DECLARE(void) switch_core_media_set_rtp_session(switch_core_session_t *session, switch_media_type_t type, switch_rtp_t *rtp_session); SWITCH_DECLARE(const char *)switch_core_media_get_codec_string(switch_core_session_t *session); diff --git a/src/include/switch_rtp.h b/src/include/switch_rtp.h index 9d6d4c6942..53e26c5a87 100644 --- a/src/include/switch_rtp.h +++ b/src/include/switch_rtp.h @@ -327,6 +327,8 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_activate_rtcp(switch_rtp_t *rtp_sessi SWITCH_DECLARE(switch_timer_t *) switch_rtp_get_media_timer(switch_rtp_t *rtp_session); +SWITCH_DECLARE(switch_status_t) switch_rtp_set_video_buffer_size(switch_rtp_t *rtp_session, uint32_t frames); + /*! \brief Acvite a jitter buffer on an RTP session \param rtp_session the rtp session diff --git a/src/include/switch_vidderbuffer.h b/src/include/switch_vidderbuffer.h index d59ad3c46b..132a7bf177 100644 --- a/src/include/switch_vidderbuffer.h +++ b/src/include/switch_vidderbuffer.h @@ -40,6 +40,7 @@ typedef enum { SWITCH_BEGIN_EXTERN_C SWITCH_DECLARE(switch_status_t) switch_vb_create(switch_vb_t **vbp, uint32_t min_frame_len, uint32_t max_frame_len, switch_memory_pool_t *pool); +SWITCH_DECLARE(switch_status_t) switch_vb_set_frames(switch_vb_t *vb, uint32_t min_frame_len, uint32_t max_frame_len); SWITCH_DECLARE(switch_status_t) switch_vb_destroy(switch_vb_t **vbp); SWITCH_DECLARE(void) switch_vb_reset(switch_vb_t *vb); SWITCH_DECLARE(void) switch_vb_debug_level(switch_vb_t *vb, uint8_t level); diff --git a/src/mod/endpoints/mod_verto/mod_verto.c b/src/mod/endpoints/mod_verto/mod_verto.c index 928201931e..89c9a77769 100644 --- a/src/mod/endpoints/mod_verto/mod_verto.c +++ b/src/mod/endpoints/mod_verto/mod_verto.c @@ -2263,7 +2263,7 @@ static void verto_set_media_options(verto_pvt_t *tech_pvt, verto_profile_t *prof tech_pvt->mparams->inbound_codec_string = switch_core_session_strdup(tech_pvt->session, profile->inbound_codec_string); tech_pvt->mparams->outbound_codec_string = switch_core_session_strdup(tech_pvt->session, profile->outbound_codec_string); - tech_pvt->mparams->jb_msec = "-1"; + //tech_pvt->mparams->jb_msec = "-1"; switch_media_handle_set_media_flag(tech_pvt->smh, SCMF_SUPPRESS_CNG); switch_media_handle_set_media_flag(tech_pvt->smh, SCMF_RENEG_ON_REINVITE); diff --git a/src/mod/formats/mod_avformat/mod_avformat.c b/src/mod/formats/mod_avformat/mod_avformat.c index 06d779aa92..46a59f90a9 100644 --- a/src/mod/formats/mod_avformat/mod_avformat.c +++ b/src/mod/formats/mod_avformat/mod_avformat.c @@ -38,6 +38,7 @@ #include // #include #include +#define DFT_RECORD_OFFSET 350 SWITCH_MODULE_LOAD_FUNCTION(mod_avformat_load); SWITCH_MODULE_DEFINITION(mod_avformat, mod_avformat_load, NULL, NULL); @@ -665,7 +666,7 @@ SWITCH_STANDARD_APP(record_av_function) char buf[SWITCH_RECOMMENDED_BUFFER_SIZE] = {0}; switch_size_t datalen = codec.implementation->decoded_bytes_per_packet; switch_size_t samples = datalen / 2 / codec.implementation->number_of_channels; - int offset = 1200; + int offset = DFT_RECORD_OFFSET; int fps = codec.implementation->actual_samples_per_second / samples; int lead_frames = (offset * fps) / 1000; @@ -1139,7 +1140,7 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa memset(context, 0, sizeof(av_file_context_t)); - context->offset = 1200; + context->offset = DFT_RECORD_OFFSET; if (handle->params && (tmp = switch_event_get_header(handle->params, "av_video_offset"))) { context->offset = atoi(tmp); } diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 0efb43b04c..79132a8c99 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -208,6 +208,9 @@ struct switch_media_handle_s { switch_vid_params_t vid_params; switch_file_handle_t *video_read_fh; switch_file_handle_t *video_write_fh; + + uint64_t vid_frames; + time_t vid_started; }; @@ -309,6 +312,46 @@ static void _switch_core_media_pass_zrtp_hash2(switch_core_session_t *aleg_sessi } } +static uint32_t round_to_step(uint32_t num, uint32_t step) +{ + uint32_t r; + uint32_t x; + + if (!num) return 0; + + r = (num % step); + x = num - r; + + if (r > step / 2) { + x += step; + } + + return x; +} +SWITCH_DECLARE(uint32_t) switch_core_media_get_video_fps(switch_core_session_t *session) +{ + switch_media_handle_t *smh; + time_t now; + + switch_assert(session); + + if (!(smh = session->media_handle)) { + return SWITCH_STATUS_FALSE; + } + + if (!switch_channel_test_flag(session->channel, CF_VIDEO)) { + return 0; + } + + now = switch_epoch_time_now(NULL); + + if (!(smh->vid_started && smh->vid_frames && smh->vid_started < now)) { + return 0; + } + + return round_to_step(smh->vid_frames / (now - smh->vid_started), 5); +} + SWITCH_DECLARE(void) switch_core_media_pass_zrtp_hash2(switch_core_session_t *aleg_session, switch_core_session_t *bleg_session) { _switch_core_media_pass_zrtp_hash2(aleg_session, bleg_session, SWITCH_MEDIA_TYPE_AUDIO); @@ -1766,7 +1809,8 @@ SWITCH_DECLARE(void) switch_core_media_prepare_codecs(switch_core_session_t *ses } -static void check_jb(switch_core_session_t *session, const char *input) + +static void check_jb(switch_core_session_t *session, const char *input, int32_t jb_msec, int32_t maxlen) { const char *val; switch_media_handle_t *smh; @@ -1820,18 +1864,20 @@ static void check_jb(switch_core_session_t *session, const char *input) } - if ((val = switch_channel_get_variable(session->channel, "jitterbuffer_msec")) || (val = smh->mparams->jb_msec)) { - int jb_msec = atoi(val); - int maxlen = 0, max_drift = 0; + if (jb_msec || (val = switch_channel_get_variable(session->channel, "jitterbuffer_msec")) || (val = smh->mparams->jb_msec)) { + int max_drift = 0; char *p, *q; + if (!jb_msec) { + jb_msec = atoi(val); - if ((p = strchr(val, ':'))) { - p++; - maxlen = atoi(p); - if ((q = strchr(p, ':'))) { - q++; - max_drift = abs(atoi(q)); + if ((p = strchr(val, ':'))) { + p++; + maxlen = atoi(p); + if ((q = strchr(p, ':'))) { + q++; + max_drift = abs(atoi(q)); + } } } @@ -1878,6 +1924,61 @@ static void check_jb(switch_core_session_t *session, const char *input) } +static void check_jb_sync(switch_core_session_t *session) +{ + int32_t jb_sync_msec = 200; + uint32_t fps, frames = 0; + switch_media_handle_t *smh; + switch_rtp_engine_t *v_engine = NULL; + const char *var; + + switch_assert(session); + + if (!switch_channel_test_flag(session->channel, CF_VIDEO)) { + return; + } + + if (!(smh = session->media_handle)) { + return; + } + + v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO]; + + if ((var = switch_channel_get_variable(session->channel, "jb_sync_msec"))) { + int tmp; + + if (!strcasecmp(var, "disabled")) { + return; + } + + tmp = atol(var); + + if (tmp > -50 && tmp < 10000) { + jb_sync_msec = tmp; + } + } + + fps = switch_core_media_get_video_fps(session); + + if (!fps) return; + + if (jb_sync_msec < 0) { + frames = abs(jb_sync_msec); + jb_sync_msec = 1000 / (fps / frames); + } else { + frames = fps / (1000 / jb_sync_msec); + } + + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), + SWITCH_LOG_DEBUG, "%s Sync Audio and Video Jitterbuffer to %dms %u Video Frames FPS %u\n", + switch_channel_get_name(session->channel), + jb_sync_msec, frames, fps); + + switch_rtp_set_video_buffer_size(v_engine->rtp_session, frames); + check_jb(session, NULL, jb_sync_msec, jb_sync_msec); +} + + //? SWITCH_DECLARE(switch_status_t) switch_core_media_read_lock_unlock(switch_core_session_t *session, switch_media_type_t type, switch_bool_t lock) { @@ -1991,6 +2092,18 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_read_frame(switch_core_session goto end; } + if (type == SWITCH_MEDIA_TYPE_VIDEO && engine->read_frame.m) { + if (!smh->vid_started) { + smh->vid_started = switch_epoch_time_now(NULL); + } + smh->vid_frames++; + + if (smh->vid_frames == 45) { + check_jb_sync(session); + } + } + + /* re-set codec if necessary */ if (engine->reset_codec > 0) { const char *val; @@ -2039,7 +2152,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_read_frame(switch_core_session } } - check_jb(session, NULL); + check_jb(session, NULL, 0, 0); engine->check_frames = 0; engine->last_ts = 0; @@ -5919,7 +6032,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_activate_rtp(switch_core_sessi } - check_jb(session, NULL); + check_jb(session, NULL, 0, 0); if ((val = switch_channel_get_variable(session->channel, "rtp_timeout_sec"))) { int v = atoi(val); @@ -8467,7 +8580,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_receive_message(switch_core_se case SWITCH_MESSAGE_INDICATE_JITTER_BUFFER: { if (switch_rtp_ready(a_engine->rtp_session)) { - check_jb(session, msg->string_arg); + check_jb(session, msg->string_arg, 0, 0); } } break; diff --git a/src/switch_rtp.c b/src/switch_rtp.c index 40141098a5..28d6f14558 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -3549,9 +3549,12 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_create(switch_rtp_t **new_rtp_session switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Starting video timer.\n"); } - switch_vb_create(&rtp_session->vb, 5, 30, rtp_session->pool); + //switch_vb_create(&rtp_session->vb, 5, 30, rtp_session->pool); //switch_vb_debug_level(rtp_session->vb, 10); - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Starting video buffer.\n"); + + + + } else { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Not using a timer\n"); @@ -3834,6 +3837,23 @@ static void jb_logger(const char *file, const char *func, int line, int level, c va_end(ap); } +SWITCH_DECLARE(switch_status_t) switch_rtp_set_video_buffer_size(switch_rtp_t *rtp_session, uint32_t frames) +{ + if (!switch_rtp_ready(rtp_session)) { + return SWITCH_STATUS_FALSE; + } + + if (!rtp_session->vb) { + switch_vb_create(&rtp_session->vb, frames, frames * 3, rtp_session->pool); + } else { + switch_vb_set_frames(rtp_session->vb, frames, frames * 3); + } + + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "Setting video buffer %u Frames.\n", frames); + + return SWITCH_STATUS_SUCCESS; +} + SWITCH_DECLARE(switch_status_t) switch_rtp_debug_jitter_buffer(switch_rtp_t *rtp_session, const char *name) { diff --git a/src/switch_vidderbuffer.c b/src/switch_vidderbuffer.c index 1b0c1b7e4d..415f4f7cfc 100644 --- a/src/switch_vidderbuffer.c +++ b/src/switch_vidderbuffer.c @@ -415,6 +415,16 @@ SWITCH_DECLARE(void) switch_vb_reset(switch_vb_t *vb) switch_mutex_unlock(vb->mutex); } +SWITCH_DECLARE(switch_status_t) switch_vb_set_frames(switch_vb_t *vb, uint32_t min_frame_len, uint32_t max_frame_len) +{ + switch_mutex_lock(vb->mutex); + vb->min_frame_len = vb->frame_len = min_frame_len; + vb->max_frame_len = max_frame_len; + switch_mutex_unlock(vb->mutex); + + return SWITCH_STATUS_SUCCESS; +} + SWITCH_DECLARE(switch_status_t) switch_vb_create(switch_vb_t **vbp, uint32_t min_frame_len, uint32_t max_frame_len, switch_memory_pool_t *pool) { switch_vb_t *vb;