From 0e6e53f15ce4e86785687fb313085cf9e6933ce6 Mon Sep 17 00:00:00 2001
From: Dragos Oancea <droancea@yahoo.com>
Date: Fri, 23 Oct 2015 20:27:25 -0400
Subject: [PATCH] FS-8644: OPUS_SET_BITRATE(), codec control and estimators for
packet loss and RTT (with Kalman filters) to detect a slow or congested link.
Feature enabled with "adjust-bitrate" in opus.conf.xml - it's a feedback loop
with incoming RTCP.
---
Makefile.am | 2 +
src/include/switch.h | 1 +
src/include/switch_core_media.h | 4 +
src/include/switch_estimators.h | 100 +++++++++++
src/include/switch_types.h | 10 ++
src/mod/codecs/mod_opus/mod_opus.c | 84 ++++++++-
src/switch_core_media.c | 29 ++++
src/switch_estimators.c | 263 +++++++++++++++++++++++++++++
src/switch_rtp.c | 151 ++++++++++++++++-
9 files changed, 636 insertions(+), 8 deletions(-)
create mode 100644 src/include/switch_estimators.h
create mode 100644 src/switch_estimators.c
diff --git a/Makefile.am b/Makefile.am
index ef5747e124..7a6f3b7bd6 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -286,6 +286,7 @@ library_include_HEADERS = \
src/include/switch_utils.h \
src/include/switch_rtp.h \
src/include/switch_jitterbuffer.h \
+ src/include/switch_estimators.h \
src/include/switch_rtcp_frame.h \
src/include/switch_stun.h \
src/include/switch_nat.h \
@@ -351,6 +352,7 @@ libfreeswitch_la_SOURCES = \
src/switch_regex.c \
src/switch_rtp.c \
src/switch_jitterbuffer.c \
+ src/switch_estimators.c \
src/switch_ivr_bridge.c \
src/switch_ivr_originate.c \
src/switch_ivr_async.c \
diff --git a/src/include/switch.h b/src/include/switch.h
index d612532612..e38846d6eb 100644
--- a/src/include/switch.h
+++ b/src/include/switch.h
@@ -144,6 +144,7 @@
#include "switch_core_media.h"
#include "switch_core_video.h"
#include "switch_jitterbuffer.h"
+#include "switch_estimators.h"
#include <libteletone.h>
diff --git a/src/include/switch_core_media.h b/src/include/switch_core_media.h
index 096bc815fc..6e4c44af2f 100644
--- a/src/include/switch_core_media.h
+++ b/src/include/switch_core_media.h
@@ -322,6 +322,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
switch_codec_control_type_t *rtype,
void **ret_data);
+SWITCH_DECLARE(switch_bool_t) switch_core_media_codec_get_cap(switch_core_session_t *session,
+ switch_media_type_t mtype,
+ switch_codec_flag_t flag);
+
#define switch_core_media_gen_key_frame(_session) switch_core_media_codec_control(_session, SWITCH_MEDIA_TYPE_VIDEO, SWITCH_IO_WRITE, \
SCC_VIDEO_GEN_KEYFRAME, SCCT_NONE, NULL, SCCT_NONE, NULL, NULL, NULL) \
diff --git a/src/include/switch_estimators.h b/src/include/switch_estimators.h
new file mode 100644
index 0000000000..a8c92c3f02
--- /dev/null
+++ b/src/include/switch_estimators.h
@@ -0,0 +1,100 @@
+/*
+ * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ * Copyright (C) 2005-2015, Anthony Minessale II <anthm@freeswitch.org>
+ *
+ * Version: MPL 1.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ *
+ * The Initial Developer of the Original Code is
+ * Anthony Minessale II <anthm@freeswitch.org>
+ * Portions created by the Initial Developer are Copyright (C)
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Dragos Oancea <droancea@yahoo.com>
+ *
+ * switch_estimators.h -- Estimators for Packet Loss, Jitter, RTT , etc
+ *
+ */
+
+
+#ifndef SWITCH_ESTIMATORS_H
+#define SWITCH_ESTIMATORS_H
+
+
+#include <switch.h>
+
+
+SWITCH_BEGIN_EXTERN_C
+
+struct kalman_estimator_s {
+ /* initial values for the Kalman filter */
+ float val_estimate_last ;
+ float P_last ;
+ /* the noise in the system:
+ The amount of noise in your measurements and the state-transitions
+ (e.g. the standard deviation of the signal noise, and how 'wrong' your simplified model
+ of the state-transitions are) => These are Q and R matrices */
+ float Q ; /* the process noise covariance matrix */
+ float R ; /* the measurement noise covariance matrix */
+ float K; /* P_temp * H^T * (H* P_temp * H^T + R)^-1 */
+ float P; /* the Kalman gain (calculated) */
+ float val_estimate; /* x_temp_est + K * (z_measured - H * x_temp_est) */
+ float val_measured; /* the 'noisy' value we measured */
+};
+
+struct cusum_kalman_detector_s {
+ /* initial values for the CUSUM Kalman filter */
+ float val_estimate_last;
+ float val_desired_last;
+ float P_last;
+ float K_last;
+ float delta;
+ float measurement_noise_e;
+ float variance_Re;
+ float measurement_noise_v;
+ float variance_Rv;
+ float g_last;
+ /*constants per model*/
+ float epsilon;
+ float h;
+ /* for calculating variance */
+ float last_average;
+ float last_q;
+ float N; /*how many samples we have so far (eg: how many RTCP we received, granted that we can calculate RTT for each one of them)*/
+};
+
+typedef struct kalman_estimator_s kalman_estimator_t;
+typedef struct cusum_kalman_detector_s cusum_kalman_detector_t;
+
+SWITCH_DECLARE(void) switch_kalman_init(kalman_estimator_t *est, float Q, float R);
+SWITCH_DECLARE(switch_bool_t) switch_kalman_cusum_init(cusum_kalman_detector_t *detect_change, float epsilon,float h);
+SWITCH_DECLARE(switch_bool_t) switch_kalman_estimate(kalman_estimator_t * est, float measurement, int system_model);
+SWITCH_DECLARE (switch_bool_t) switch_kalman_cusum_detect_change(cusum_kalman_detector_t * detector, float measurement, float rtt_avg);
+SWITCH_DECLARE(switch_bool_t) switch_kalman_is_slow_link(kalman_estimator_t * est_loss, kalman_estimator_t * est_rtt);
+
+SWITCH_END_EXTERN_C
+#endif
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */
diff --git a/src/include/switch_types.h b/src/include/switch_types.h
index cc8539085e..b31366f9b5 100644
--- a/src/include/switch_types.h
+++ b/src/include/switch_types.h
@@ -755,6 +755,8 @@ typedef enum {
SWITCH_ZRTP_FLAG_SECURE_MITM_RECV,
SWITCH_RTP_FLAG_DEBUG_RTP_READ,
SWITCH_RTP_FLAG_DEBUG_RTP_WRITE,
+ SWITCH_RTP_FLAG_ESTIMATORS,
+ SWITCH_RTP_FLAG_ADJ_BITRATE_CAP,
SWITCH_RTP_FLAG_VIDEO,
SWITCH_RTP_FLAG_ENABLE_RTCP,
SWITCH_RTP_FLAG_RTCP_MUX,
@@ -1631,6 +1633,7 @@ typedef enum {
SWITCH_CODEC_FLAG_AAL2 = (1 << 6),
SWITCH_CODEC_FLAG_PASSTHROUGH = (1 << 7),
SWITCH_CODEC_FLAG_READY = (1 << 8),
+ SWITCH_CODEC_FLAG_HAS_ADJ_BITRATE = (1 << 14),
SWITCH_CODEC_FLAG_HAS_PLC = (1 << 15),
SWITCH_CODEC_FLAG_VIDEO_PATCHING = (1 << 16)
} switch_codec_flag_enum_t;
@@ -2268,6 +2271,7 @@ typedef enum {
SCC_VIDEO_BANDWIDTH,
SCC_VIDEO_RESET,
SCC_AUDIO_PACKET_LOSS,
+ SCC_AUDIO_ADJUST_BITRATE,
SCC_DEBUG,
SCC_CODEC_SPECIFIC
} switch_codec_control_command_t;
@@ -2586,6 +2590,12 @@ typedef struct switch_waitlist_s {
struct switch_jb_s;
typedef struct switch_jb_s switch_jb_t;
+//struct kalman_estimator_s;
+//typedef struct kalman_estimator_s kalman_estimator_t;
+
+//struct cusum_kalman_detector_s;
+//typedef struct cusum_kalman_detector_s cusum_kalman_detector_t;
+
struct switch_img_txt_handle_s;
typedef struct switch_img_txt_handle_s switch_img_txt_handle_t;
diff --git a/src/mod/codecs/mod_opus/mod_opus.c b/src/mod/codecs/mod_opus/mod_opus.c
index 959311b58b..4a46c3f1e3 100644
--- a/src/mod/codecs/mod_opus/mod_opus.c
+++ b/src/mod/codecs/mod_opus/mod_opus.c
@@ -34,6 +34,11 @@
#include "switch.h"
#include "opus.h"
+#define SWITCH_OPUS_MIN_BITRATE 6000
+#define SWITCH_OPUS_MAX_BITRATE 510000
+
+#define SWITCH_OPUS_MIN_FEC_BITRATE 12400
+
SWITCH_MODULE_LOAD_FUNCTION(mod_opus_load);
SWITCH_MODULE_DEFINITION(mod_opus, mod_opus_load, NULL, NULL);
@@ -91,6 +96,15 @@ struct dec_stats {
};
typedef struct dec_stats dec_stats_t;
+struct codec_control_state {
+ int keep_fec;
+ opus_int32 current_bitrate;
+ opus_int32 wanted_bitrate;
+ uint32_t increase_step;
+ uint32_t decrease_step;
+};
+typedef struct codec_control_state codec_control_state_t;
+
struct opus_context {
OpusEncoder *encoder_object;
OpusDecoder *decoder_object;
@@ -103,6 +117,7 @@ struct opus_context {
int look_check;
int look_ts;
dec_stats_t decoder_stats;
+ codec_control_state_t control_state;
};
struct {
@@ -116,6 +131,7 @@ struct {
int asymmetric_samplerates;
int keep_fec;
int fec_decode;
+ int adjust_bitrate;
int debuginfo;
uint32_t use_jb_lookahead;
switch_mutex_t *mutex;
@@ -253,7 +269,7 @@ static switch_status_t switch_opus_fmtp_parse(const char *fmtp, switch_codec_fmt
if (!strcasecmp(data, "maxaveragebitrate")) {
codec_settings->maxaveragebitrate = atoi(arg);
- if (codec_settings->maxaveragebitrate < 6000 || codec_settings->maxaveragebitrate > 510000) {
+ if (codec_settings->maxaveragebitrate < SWITCH_OPUS_MIN_BITRATE || codec_settings->maxaveragebitrate > SWITCH_OPUS_MAX_BITRATE) {
codec_settings->maxaveragebitrate = 0; /* values outside the range between 6000 and 510000 SHOULD be ignored */
}
}
@@ -600,6 +616,7 @@ static switch_status_t switch_opus_init(switch_codec_t *codec, switch_codec_flag
opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(fec_bitrate));
/* will override the maxaveragebitrate set in opus.conf.xml */
opus_codec_settings.maxaveragebitrate = fec_bitrate;
+ context->control_state.keep_fec = opus_prefs.keep_fec ;
}
}
}
@@ -607,6 +624,10 @@ static switch_status_t switch_opus_init(switch_codec_t *codec, switch_codec_flag
if (opus_codec_settings.usedtx) {
opus_encoder_ctl(context->encoder_object, OPUS_SET_DTX(opus_codec_settings.usedtx));
}
+
+ if (opus_prefs.adjust_bitrate) {
+ switch_set_flag(codec, SWITCH_CODEC_FLAG_HAS_ADJ_BITRATE);
+ }
}
if (decoding) {
@@ -972,9 +993,11 @@ static switch_status_t opus_load_config(switch_bool_t reload)
opus_prefs.keep_fec = atoi(val);
} else if (!strcasecmp(key, "advertise-useinbandfec")) { /*decoder, has meaning only for FMTP: useinbandfec=1 by default */
opus_prefs.fec_decode = atoi(val);
+ } else if (!strcasecmp(key, "adjust-bitrate")) { /* encoder, this setting will make the encoder adjust its bitrate based on a feedback loop (RTCP). This is not "VBR".*/
+ opus_prefs.adjust_bitrate = atoi(val);
} else if (!strcasecmp(key, "maxaveragebitrate")) {
opus_prefs.maxaveragebitrate = atoi(val);
- if (opus_prefs.maxaveragebitrate < 6000 || opus_prefs.maxaveragebitrate > 510000) {
+ if (opus_prefs.maxaveragebitrate < SWITCH_OPUS_MIN_BITRATE || opus_prefs.maxaveragebitrate > SWITCH_OPUS_MAX_BITRATE) {
opus_prefs.maxaveragebitrate = 0; /* values outside the range between 6000 and 510000 SHOULD be ignored */
}
} else if (!strcasecmp(key, "maxplaybackrate")) {
@@ -1132,13 +1155,68 @@ static switch_status_t switch_opus_control(switch_codec_t *codec,
}
if (globals.debug || context->debug) {
- switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus Adjusting packet loss percent from %d%% to %d%%!\n",
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting packet loss percent from %d%% to %d%%!\n",
context->old_plpct, plpct);
}
}
context->old_plpct = plpct;
}
break;
+ case SCC_AUDIO_ADJUST_BITRATE:
+ {
+ const char *cmd = (const char *)cmd_data;
+
+ if (!zstr(cmd)) {
+ opus_int32 current_bitrate=context->control_state.current_bitrate;
+ if (!strcasecmp(cmd, "increase")) {
+ /* https://wiki.xiph.org/OpusFAQ
+ "[...]Opus scales from about 6 to 512 kb/s, in increments of 0.4 kb/s (one byte with 20 ms frames).
+ Opus can have more than 1200 possible bitrates[...]" */
+ int br_step = context->control_state.increase_step?context->control_state.increase_step:400;
+ opus_encoder_ctl(context->encoder_object, OPUS_GET_BITRATE(¤t_bitrate));
+ if (opus_prefs.maxaveragebitrate > current_bitrate) {
+ opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(current_bitrate+br_step));
+ if ((context->control_state.keep_fec) && (current_bitrate > SWITCH_OPUS_MIN_FEC_BITRATE)) {
+ opus_prefs.keep_fec = 1; /* enable back FEC if it was disabled by SCC_AUDIO_ADJUST_BITRATE, we have enough network bandwidth now */
+ }
+ if (globals.debug || context->debug) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (increase)\n", current_bitrate+br_step);
+ }
+ }
+ } else if (!strcasecmp(cmd, "decrease")) {
+ int br_step = context->control_state.decrease_step?context->control_state.decrease_step:400;
+ opus_encoder_ctl(context->encoder_object, OPUS_GET_BITRATE(¤t_bitrate));
+ if (current_bitrate > SWITCH_OPUS_MIN_BITRATE) {
+ if ((context->control_state.keep_fec) && (current_bitrate < SWITCH_OPUS_MIN_FEC_BITRATE)) {
+ opus_prefs.keep_fec = 0; /* no point to try to keep FEC enabled anymore, we're low on network bandwidth (that's why we ended up here) */
+ }
+ opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(current_bitrate-br_step));
+ if (globals.debug || context->debug) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (decrease)\n", current_bitrate-br_step);
+ }
+ }
+ } else if (!strcasecmp(cmd, "default")) {
+ /*restore default bitrate */
+ opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(opus_prefs.maxaveragebitrate));
+ if (context->control_state.keep_fec) {
+ opus_prefs.keep_fec = 1; /* enable back FEC, we have enough network bandwidth now */
+ }
+ if (globals.debug || context->debug) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (configured maxaveragebitrate)\n", opus_prefs.maxaveragebitrate);
+ }
+ } else {
+ /* set Opus minimum bitrate */
+ opus_encoder_ctl(context->encoder_object, OPUS_SET_BITRATE(SWITCH_OPUS_MIN_BITRATE));
+ if (context->control_state.keep_fec) {
+ opus_prefs.keep_fec = 0; /* do not enforce FEC anymore, we're low on network bandwidth */
+ }
+ if (globals.debug || context->debug) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Opus encoder: Adjusting bitrate to %d (minimum)\n", SWITCH_OPUS_MIN_BITRATE);
+ }
+ }
+ }
+ }
+ break;
default:
break;
}
diff --git a/src/switch_core_media.c b/src/switch_core_media.c
index 82be0945bd..5fb25f5772 100644
--- a/src/switch_core_media.c
+++ b/src/switch_core_media.c
@@ -11322,6 +11322,35 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess
return SWITCH_STATUS_FALSE;
}
+SWITCH_DECLARE(switch_bool_t) switch_core_media_codec_get_cap(switch_core_session_t *session,
+ switch_media_type_t mtype,
+ switch_codec_flag_t flag) {
+ switch_rtp_engine_t *engine = NULL;
+ switch_media_handle_t *smh = NULL;
+ switch_codec_t *codec = NULL;
+
+ switch_assert(session);
+
+ if (!(smh = session->media_handle)) {
+ return SWITCH_FALSE;
+ }
+
+ if (!(engine = &smh->engines[mtype])) {
+ return SWITCH_FALSE;
+ }
+
+ codec = &engine->write_codec;
+
+ if (!switch_core_codec_ready(codec)) {
+ return SWITCH_FALSE;
+ }
+
+ if (switch_test_flag(codec, flag)){
+ return SWITCH_TRUE;
+ }
+
+ return SWITCH_FALSE;
+}
SWITCH_DECLARE(switch_status_t) switch_core_session_write_encoded_video_frame(switch_core_session_t *session,
switch_frame_t *frame, switch_io_flag_t flags, int stream_id)
diff --git a/src/switch_estimators.c b/src/switch_estimators.c
new file mode 100644
index 0000000000..3effb6cfa7
--- /dev/null
+++ b/src/switch_estimators.c
@@ -0,0 +1,263 @@
+/*
+ * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ * Copyright (C) 2005-2015, Anthony Minessale II <anthm@freeswitch.org>
+ *
+ * Version: MPL 1.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application
+ *
+ * The Initial Developer of the Original Code is
+ * Anthony Minessale II <anthm@freeswitch.org>
+ * Portions created by the Initial Developer are Copyright (C)
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *
+ * Dragos Oancea <droancea@yahoo.com>
+ *
+ * switch_estimators.c -- Estimators and Detectors (try to read into the future: packet loss, jitter, RTT, etc)
+ *
+ */
+
+#include <switch_estimators.h>
+
+#include <switch.h>
+#ifndef _MSC_VER
+#include <switch_private.h>
+#endif
+#undef PACKAGE_NAME
+#undef PACKAGE_STRING
+#undef PACKAGE_TARNAME
+#undef PACKAGE_VERSION
+#undef PACKAGE_BUGREPORT
+#undef VERSION
+#undef PACKAGE
+#undef inline
+#include <datatypes.h>
+#include <switch_types.h>
+
+#define KALMAN_SYSTEM_MODELS 3 /*loss, jitter, rtt*/
+#define EST_LOSS 0
+#define EST_JITTER 1
+#define EST_RTT 2
+
+/* This function initializes the Kalman System Model
+ *
+ * xk+1 = A*xk + wk
+ * zk = H*xk + vk
+ * xk = state variable (must exist in physical world - measurable )
+ * zk = measurment
+ * wk,vk - white noise
+ * A = state trasition matrix , (n x n ) matrix
+ * H = state-to-measurment matrix , ( n x n ) matrix
+ * Noise covariance:
+ * Q: Covariance matrix of wk, ( n x n ) diagonal matrix
+ * R: Covariance matrix of vk , ( m x m ) diagonal matrix
+ * R: if you want to be affected less by the measurement and get the estimate with less variation, increase R
+ * Q: if you want to be affected more by the measurement and get the estimate with more variation, decrease Q
+ *
+ * (Phil Kim book)
+ *
+ */
+void switch_kalman_init(kalman_estimator_t *est, float Q, float R)
+{
+ est -> val_estimate_last = 0 ;
+ est -> P_last = 0;
+ est -> Q = Q; /*accuracy of system model */ /* SYSTEM MODEL: TO BE DEDUCTED */
+ est -> R = R; /*accuracy of measurement*/ /* SYSTEM MODEL: TO BE DEDUCTED */
+ est -> K = 0;
+ est -> val_estimate = 0 ;
+ est -> val_measured = 0 ; // [0-100 %] or [0-5000] or [0-2sec]
+}
+
+/*
+CUSUM Kalman functions to detect sudden change over a predefined thereshold.
+
+y(t) = sampled RTT
+x(t)= desired RTT
+
+Model:
+x(t+1) = x(t) + delta(t)*v(t)
+y(t) = x(t) + e(t)
+
+Noisy characteristic of RTT captured by measurment noise e(t) with variance Re.
+The step changes in the desired RTT x(t) is modeled as the process noise v(t)
+with variance Rv and the discrete variable delta(t) .
+If a change occurs at time t, then delta(t) = 1 otherwise delta(t) = 0.
+
+avg(x(t)) = avg(x(t-1)) + K(t)(y(t) - avg(x(t-1)))
+K(t) = P(t-1)/(P(t-1) + Re))
+P(t) = (1-K(t))P(t-1) + delta(t-1)* Rv
+e(t) = y(t) - avg(x(t))
+g(t) = max(g(t-1) + e(t) - epsilon,0)
+if g(t) > 0 then
+ delta(t) = 1 // alarm
+ g(t) = 0
+else
+ delta(t) = 0
+endif
+
+constants:
+
+epsilon = 0.005
+h = 0.05
+*/
+switch_bool_t switch_kalman_cusum_init (cusum_kalman_detector_t *detect_change, float epsilon, float h)
+{
+ cusum_kalman_detector_t *detector_change = detect_change;
+
+
+ if (epsilon < 0 || h < 0) {
+ return FALSE;
+ }
+
+ detector_change -> val_estimate_last = 0;
+ detector_change -> val_desired_last = 0;
+ detector_change -> P_last = 0;
+ detector_change -> K_last = 0;
+ detector_change -> delta = 0;
+ detector_change -> measurement_noise_e = 0;
+ detector_change -> variance_Re = 0;
+ detector_change -> measurement_noise_v = 0;
+ detector_change -> variance_Rv = 0;
+ detector_change -> g_last = 0;
+ /*per system model*/
+ detector_change -> epsilon = epsilon;
+ detector_change -> h = h;
+ /*variance*/
+ detector_change -> last_average = 0;
+ detector_change -> last_q = 0;
+ detector_change -> N = 0;
+ return TRUE;
+}
+
+switch_bool_t switch_kalman_cusum_detect_change(cusum_kalman_detector_t * detector, float measurement, float avg)
+{
+ float K=0;
+ float P=0;
+ float g=0;
+ float desired_val;
+ float current_average;
+ float current_q;
+ float sample_variance_Re = 0;
+
+ /*variance*/
+
+ detector->N++;
+ current_average = detector->last_average + (measurement - detector->last_average)/detector->N ;
+ if (avg > current_average) {
+ current_average = avg;
+ }
+ current_q = detector-> last_q + (measurement - detector->last_average) * (measurement - current_average);
+ if (detector->N != 0)
+ sample_variance_Re = sqrt(current_q/detector->N);
+
+ detector->variance_Re = sample_variance_Re;
+ detector->variance_Rv = sample_variance_Re;
+
+ if (sample_variance_Re != 0) {
+ K = detector->P_last / (detector->P_last + detector->variance_Re);
+ desired_val = detector->val_desired_last + K * (measurement - detector->variance_Re);
+ P = (1 - K) * detector->P_last + detector->delta * detector->variance_Rv;
+ detector->measurement_noise_e = measurement - desired_val;
+ g = detector->g_last + detector->measurement_noise_e - detector->epsilon;
+ if (g > detector->h) {
+ detector->delta = 1;
+ g = 0;
+ } else {
+ detector->delta = 0;
+ }
+
+ /* update last vals for calculating variance */
+ detector->last_average = current_average;
+ /* update lasts (cusum)*/
+ detector -> g_last = g;
+ detector -> P_last = P;
+ detector -> val_desired_last = desired_val;
+ }
+ if (detector->delta == 1) {
+ return TRUE;
+ }
+ return FALSE;
+}
+
+/* Kalman filter abstract ( measure and estimate 1 single value per system model )
+ * Given the measurment and the system model together with the current state ,
+ * the function puts an estimate in the estimator struct */
+switch_bool_t switch_kalman_estimate (kalman_estimator_t * est, float measurement, int system_model)
+{
+ /*system model can be about: loss, jitter, rtt*/
+ float val_estimate;
+ float val_temp_est = est->val_estimate_last;
+ float P_temp = est->P_last + est->Q;
+
+ if (system_model >= KALMAN_SYSTEM_MODELS) {
+ return SWITCH_FALSE ;
+ }
+
+ /*sanitize input a little bit, just in case */
+ if (system_model == EST_LOSS ) {
+ if ((measurement > 100) && (measurement < 0)) {
+ return SWITCH_FALSE ;
+ }
+ }
+
+ if (system_model == EST_JITTER) {
+ if ((measurement > 10000) && (measurement < 0)) {
+ return SWITCH_FALSE;
+ }
+ }
+
+ if (system_model == EST_RTT) {
+ if ((measurement > 2 ) && (measurement < 0)) {
+ return SWITCH_FALSE;
+ }
+ }
+
+ /* calculate the Kalman gain */
+ est->K = P_temp * (1.0/(P_temp + est->R));
+ /* real life measurement */
+ est->val_measured = measurement ;
+ val_estimate = val_temp_est + est->K * (est->val_measured - val_temp_est);
+ est->P = (1 - est->K) * P_temp;
+ /*update lasts*/
+ est->P_last = est->P;
+ /* save the estimated value (future) */
+ est->val_estimate_last = val_estimate;
+ return SWITCH_TRUE;
+}
+
+switch_bool_t switch_kalman_is_slow_link(kalman_estimator_t * est_loss, kalman_estimator_t * est_rtt)
+{
+ float thresh_packet_loss = 5; /* % */
+ float thresh_rtt = 0.8 ; /*seconds*/
+
+ if ((est_loss->val_estimate_last > thresh_packet_loss) &&
+ (est_rtt->val_estimate_last > thresh_rtt )) {
+ return SWITCH_TRUE;
+ }
+
+ return SWITCH_FALSE;
+}
+
+/* For Emacs:
+ * Local Variables:
+ * mode:c
+ * indent-tabs-mode:t
+ * tab-width:4
+ * c-basic-offset:4
+ * End:
+ * For VIM:
+ * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet:
+ */
+
diff --git a/src/switch_rtp.c b/src/switch_rtp.c
index 398d3233c9..697c302d4a 100644
--- a/src/switch_rtp.c
+++ b/src/switch_rtp.c
@@ -35,6 +35,7 @@
//#define DEBUG_MISSED_SEQ
//#define DEBUG_EXTRA
//#define DEBUG_RTCP
+#define DEBUG_ESTIMATORS
#include <switch.h>
#ifndef _MSC_VER
@@ -55,6 +56,8 @@
#include <srtp_priv.h>
#include <switch_ssl.h>
#include <switch_jitterbuffer.h>
+#include <switch_estimators.h>
+
#define JITTER_LEAD_FRAMES 10
#define READ_INC(rtp_session) switch_mutex_lock(rtp_session->read_mutex); rtp_session->reading++
@@ -170,6 +173,10 @@ typedef struct {
#pragma pack(pop, r1)
#endif
+#define KALMAN_SYSTEM_MODELS 3 /*loss, jitter, rtt*/
+#define EST_LOSS 0
+#define EST_JITTER 1
+#define EST_RTT 2
typedef struct {
switch_rtcp_ext_hdr_t header;
@@ -446,6 +453,8 @@ struct switch_rtp {
switch_core_session_t *session;
payload_map_t **pmaps;
payload_map_t *pmap_tail;
+ kalman_estimator_t *estimators[KALMAN_SYSTEM_MODELS];
+ cusum_kalman_detector_t *detectors[KALMAN_SYSTEM_MODELS];
int ice_adj;
uint8_t has_rtp;
uint8_t has_rtcp;
@@ -1860,6 +1869,24 @@ static void rtcp_stats_init(switch_rtp_t *rtp_session)
} else {
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG, "rtcp_stats_init: %s ssrc[%u] base_seq[%u]\n", rtp_type(rtp_session), stats->ssrc, stats->base_seq);
}
+
+ if (rtp_session->flags[SWITCH_RTP_FLAG_ENABLE_RTCP] && (switch_core_media_codec_get_cap(rtp_session->session,
+ SWITCH_MEDIA_TYPE_AUDIO, SWITCH_CODEC_FLAG_HAS_ADJ_BITRATE))) {
+ kalman_estimator_t *estimators[KALMAN_SYSTEM_MODELS];
+ cusum_kalman_detector_t *detectors[KALMAN_SYSTEM_MODELS];
+
+ rtp_session->flags[SWITCH_RTP_FLAG_ADJ_BITRATE_CAP] = 1;
+ rtp_session->flags[SWITCH_RTP_FLAG_ESTIMATORS] = 1;
+
+ rtp_session->estimators[EST_LOSS] = switch_core_alloc(rtp_session->pool, sizeof(*estimators[0]));
+ switch_kalman_init(rtp_session->estimators[EST_LOSS],0.1,0.1);
+ rtp_session->estimators[EST_RTT] = switch_core_alloc(rtp_session->pool, sizeof(*estimators[0]));
+ switch_kalman_init(rtp_session->estimators[EST_RTT],0.03,1);
+ rtp_session->detectors[EST_RTT] = switch_core_alloc(rtp_session->pool, sizeof(*detectors[0]));
+ switch_kalman_cusum_init(rtp_session->detectors[EST_RTT],0.005,0.5);
+ rtp_session->detectors[EST_LOSS] = switch_core_alloc(rtp_session->pool, sizeof(*detectors[0]));
+ switch_kalman_cusum_init(rtp_session->detectors[EST_LOSS],0.005,0.5);
+ }
}
static int rtcp_stats(switch_rtp_t *rtp_session)
@@ -5908,6 +5935,8 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t
uint32_t sec, ntp_sec, ntp_usec, lsr_now;
uint32_t lsr;
uint32_t packet_ssrc;
+ double rtt_now = 0;
+ int rtt_increase = 0, packet_loss_increase=0;
now = switch_time_now(); /* number of microseconds since 00:00:00 january 1, 1970 UTC */
sec = (uint32_t)(now/1000000); /* converted to second (NTP most significant bits) */
@@ -5962,10 +5991,6 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t
((float)(uint8_t)percent_fraction * .3));
}
- if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->rtcp_frame.reports[i].loss_avg != old_avg) {
- switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO, SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT, (void *)&rtp_session->rtcp_frame.reports[i].loss_avg, SCCT_NONE, NULL, NULL, NULL);
- }
-
rtp_session->rtcp_frame.reports[i].ssrc = ntohl(report->ssrc);
rtp_session->rtcp_frame.reports[i].fraction = (uint8_t)report->fraction;
rtp_session->rtcp_frame.reports[i].lost = ntohl(report->lost);
@@ -5974,7 +5999,6 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t
rtp_session->rtcp_frame.reports[i].lsr = ntohl(report->lsr);
rtp_session->rtcp_frame.reports[i].dlsr = ntohl(report->dlsr);
if (rtp_session->rtcp_frame.reports[i].lsr && !rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU]) {
- double rtt_now;
switch_time_exp_gmt(&now_hr,now);
/* Calculating RTT = A - DLSR - LSR */
rtt_now = (double)(lsr_now - rtp_session->rtcp_frame.reports[i].dlsr - rtp_session->rtcp_frame.reports[i].lsr)/65536;
@@ -5992,6 +6016,123 @@ static switch_status_t process_rtcp_report(switch_rtp_t *rtp_session, rtcp_msg_t
switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "RTT average %f\n",
rtp_session->rtcp_frame.reports[i].rtt_avg);
}
+
+ if (rtp_session->flags[SWITCH_RTP_FLAG_ADJ_BITRATE_CAP] && rtp_session->flags[SWITCH_RTP_FLAG_ESTIMATORS] && !rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) {
+
+ /* SWITCH_RTP_FLAG_ADJ_BITRATE_CAP : Can the codec change its bitrate on the fly per API command ? */
+#ifdef DEBUG_ESTIMATORS
+ switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "Current packet loss: [%d %%] Current RTT: [%f ms]\n", percent_fraction, rtt_now);
+#endif
+
+ switch_kalman_estimate(rtp_session->estimators[EST_RTT], rtt_now, EST_RTT);
+
+ if (switch_kalman_cusum_detect_change(rtp_session->detectors[EST_RTT], rtt_now, rtp_session->estimators[EST_RTT]->val_estimate_last)) {
+ /* sudden change in the mean value of RTT */
+#ifdef DEBUG_ESTIMATORS
+ switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"Sudden change in the mean value of RTT !\n");
+#endif
+ rtt_increase = 1;
+ }
+
+ switch_kalman_estimate(rtp_session->estimators[EST_LOSS], percent_fraction, EST_LOSS);
+
+ if (switch_kalman_cusum_detect_change(rtp_session->detectors[EST_LOSS], percent_fraction, rtp_session->estimators[EST_LOSS]->val_estimate_last)){
+ /* sudden change in the mean value of packet loss */
+#ifdef DEBUG_ESTIMATORS
+ switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"Sudden change in the mean value of packet loss!\n");
+#endif
+ packet_loss_increase = 1;
+ }
+#ifdef DEBUG_ESTIMATORS
+ switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "ESTIMATORS: Packet loss will be: [%f] RTT will be: [%f ms]\n",
+ rtp_session->estimators[EST_LOSS]->val_estimate_last, rtp_session->estimators[EST_RTT]->val_estimate_last);
+#endif
+
+ if (rtp_session->rtcp_frame.reports[i].loss_avg != old_avg) {
+ /*getting bad*/
+ if (switch_kalman_is_slow_link(rtp_session->estimators[EST_LOSS],
+ rtp_session->estimators[EST_RTT])) {
+ /* going to minimum bitrate */
+#ifdef DEBUG_ESTIMATORS
+ switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "Slow link conditions: Loss average: [%d %%], Previous loss: [%d %%]. \
+ Going to minimum bitrate!",rtp_session->rtcp_frame.reports[i].loss_avg, old_avg);
+#endif
+ switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
+ SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE, SCCT_STRING, "minimum", SCCT_NONE, NULL, NULL, NULL);
+ /* if after going to minimum bitrate we still have packet loss then we increase ptime. TODO */
+
+ } else if (packet_loss_increase && (rtp_session->estimators[EST_LOSS]->val_estimate_last >= 5)) {
+ /* sudden change in the mean value of packet loss percentage */
+ switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
+ SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE,
+ SCCT_STRING, "decrease",
+ SCCT_NONE, NULL, NULL, NULL);
+#ifdef DEBUG_ESTIMATORS
+ switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"Sudden change in the mean value of packet loss percentage !\n");
+#endif
+ switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
+ SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
+ (void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
+ SCCT_NONE, NULL, NULL, NULL);
+
+ } else if (!rtt_increase && rtp_session->estimators[EST_LOSS]->val_estimate_last >= rtp_session->rtcp_frame.reports[i].loss_avg ) {
+ /* lossy because of congestion (queues full somewhere -> some packets are dropped , but RTT is good ), packet loss with many small gaps */
+#ifdef DEBUG_ESTIMATORS
+ switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "packet loss, but RTT is not bad\n");
+#endif
+ switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
+ SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
+ (void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
+ SCCT_NONE, NULL, NULL, NULL);
+
+ } else if ((rtp_session->estimators[EST_LOSS]->val_estimate_last < 1) && packet_loss_increase) {
+#ifdef DEBUG_ESTIMATORS
+ switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3, "small packet loss average\n");
+#endif
+ /*small loss_avg*/
+ switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
+ SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE,
+ SCCT_STRING, "default",
+ SCCT_NONE, NULL, NULL, NULL);
+
+ switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
+ SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
+ (void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
+ SCCT_NONE, NULL, NULL, NULL);
+
+ } else if ((rtp_session->estimators[EST_LOSS]->val_estimate_last < 5) &&
+ (rtp_session->rtcp_frame.reports[i].rtt_avg < rtp_session->estimators[EST_RTT]->val_estimate_last)) {
+
+ /* estimate that packet loss will decrease, we can increase the bitrate */
+ switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
+ SWITCH_IO_WRITE, SCC_AUDIO_ADJUST_BITRATE,
+ SCCT_STRING, "increase",
+ SCCT_NONE, NULL, NULL, NULL);
+
+ switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
+ SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
+ (void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
+ SCCT_NONE, NULL, NULL, NULL);
+
+ } else {
+ /* *do nothing about bitrate, just pass the packet loss to the codec */
+#ifdef DEBUG_ESTIMATORS
+ switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG3,"do nothing about bitrate, just pass the packet loss to the codec\n");
+#endif
+ switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
+ SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
+ (void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
+ SCCT_NONE, NULL, NULL, NULL);
+ }
+ }
+ } else {
+ if (!rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && rtp_session->rtcp_frame.reports[i].loss_avg != old_avg) {
+ switch_core_media_codec_control(rtp_session->session, SWITCH_MEDIA_TYPE_AUDIO,
+ SWITCH_IO_WRITE, SCC_AUDIO_PACKET_LOSS, SCCT_INT,
+ (void *)&rtp_session->rtcp_frame.reports[i].loss_avg,
+ SCCT_NONE, NULL, NULL, NULL);
+ }
+ }
}
rtp_session->rtcp_frame.report_count = (uint16_t)i;