diff --git a/src/include/switch_core_media.h b/src/include/switch_core_media.h index b0f88acf49..8e4f0b74a1 100644 --- a/src/include/switch_core_media.h +++ b/src/include/switch_core_media.h @@ -157,6 +157,9 @@ typedef struct switch_core_media_params_s { switch_bool_t external_video_source; + uint32_t video_key_freq; + uint32_t video_key_first; + } switch_core_media_params_t; static inline const char *switch_media_type2str(switch_media_type_t type) @@ -305,6 +308,11 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess switch_codec_control_type_t *rtype, void **ret_data); + +#define switch_core_media_gen_key_frame(_session) switch_core_media_codec_control(_session, SWITCH_MEDIA_TYPE_VIDEO, SWITCH_IO_WRITE, \ + SCC_VIDEO_REFRESH, SCCT_NONE, NULL, NULL, NULL) \ + + SWITCH_DECLARE(switch_timer_t *) switch_core_media_get_timer(switch_core_session_t *session, switch_media_type_t mtype); SWITCH_END_EXTERN_C diff --git a/src/include/switch_module_interfaces.h b/src/include/switch_module_interfaces.h index f989b2808c..0a2dc0946a 100644 --- a/src/include/switch_module_interfaces.h +++ b/src/include/switch_module_interfaces.h @@ -591,12 +591,21 @@ struct switch_directory_handle { void *private_info; }; -/* nobody has more setting than speex so we will let them set the standard */ -/*! \brief Various codec settings (currently only relevant to speex) */ -struct switch_codec_settings { +struct switch_audio_codec_settings { int unused; }; +struct switch_video_codec_settings { + uint32_t bandwidth; + int32_t width; + int32_t height; +}; + +union switch_codec_settings { + struct switch_audio_codec_settings audio; + struct switch_video_codec_settings video; +}; + /*! an abstract handle of a fmtp parsed by codec */ struct switch_codec_fmtp { /*! actual samples transferred per second for those who are not moron g722 RFC writers */ diff --git a/src/include/switch_rtp.h b/src/include/switch_rtp.h index 3046946657..9e7f9198d8 100644 --- a/src/include/switch_rtp.h +++ b/src/include/switch_rtp.h @@ -517,6 +517,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_del_dtls(switch_rtp_t *rtp_session, d SWITCH_DECLARE(int) switch_rtp_has_dtls(void); SWITCH_DECLARE(void) switch_rtp_video_refresh(switch_rtp_t *rtp_session); +SWITCH_DECLARE(void) switch_rtp_video_loss(switch_rtp_t *rtp_session); /*! \} diff --git a/src/include/switch_types.h b/src/include/switch_types.h index 2dbc4a20a6..783ed02220 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -1496,7 +1496,8 @@ typedef enum { SFF_RTCP = (1 << 10), SFF_MARKER = (1 << 11), SFF_WAIT_KEY_FRAME = (1 << 12), - SFF_RAW_RTP_PARSE_FRAME = (1 << 13) + SFF_RAW_RTP_PARSE_FRAME = (1 << 13), + SFF_PICTURE_RESET = (1 << 14) } switch_frame_flag_enum_t; typedef uint32_t switch_frame_flag_t; @@ -2101,7 +2102,7 @@ typedef struct switch_codec switch_codec_t; typedef struct switch_core_thread_session switch_core_thread_session_t; typedef struct switch_codec_implementation switch_codec_implementation_t; typedef struct switch_buffer switch_buffer_t; -typedef struct switch_codec_settings switch_codec_settings_t; +typedef union switch_codec_settings switch_codec_settings_t; typedef struct switch_codec_fmtp switch_codec_fmtp_t; typedef struct switch_odbc_handle switch_odbc_handle_t; typedef struct switch_pgsql_handle switch_pgsql_handle_t; diff --git a/src/mod/codecs/mod_vpx/mod_vpx.c b/src/mod/codecs/mod_vpx/mod_vpx.c index 0fe2b0a139..2c0028cde5 100644 --- a/src/mod/codecs/mod_vpx/mod_vpx.c +++ b/src/mod/codecs/mod_vpx/mod_vpx.c @@ -44,18 +44,21 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_vpx_load); SWITCH_MODULE_DEFINITION(mod_vpx, mod_vpx_load, NULL, NULL); + +#define encoder_interface (vpx_codec_vp8_cx()) +#define decoder_interface (vpx_codec_vp8_dx()) + struct vpx_context { switch_codec_t *codec; unsigned int flags; - + switch_codec_settings_t codec_settings; + unsigned int bandwidth; vpx_codec_enc_cfg_t config; vpx_codec_ctx_t encoder; + uint8_t encoder_init; vpx_image_t *pic; switch_bool_t force_key_frame; - int width; - int height; - int bitrate; int fps; int format; int intra_period; @@ -65,9 +68,8 @@ struct vpx_context { const vpx_codec_cx_pkt_t *pkt; int pkt_pos; vpx_codec_iter_t iter; - switch_time_t last_ts; - vpx_codec_ctx_t decoder; + uint8_t decoder_init; switch_buffer_t *vpx_packet_buffer; int got_key_frame; switch_size_t last_received_timestamp; @@ -76,48 +78,17 @@ struct vpx_context { }; typedef struct vpx_context vpx_context_t; -static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_t flags, const switch_codec_settings_t *codec_settings) + +static switch_status_t init_codec(switch_codec_t *codec) { - vpx_context_t *context = NULL; - int encoding, decoding; - vpx_codec_ctx_t *encoder = NULL; - vpx_codec_ctx_t *decoder = NULL; - vpx_codec_enc_cfg_t *config; - const vpx_codec_iface_t* encoder_interface = vpx_codec_vp8_cx(); - const vpx_codec_iface_t* decoder_interface = vpx_codec_vp8_dx(); - - - encoding = (flags & SWITCH_CODEC_FLAG_ENCODE); - decoding = (flags & SWITCH_CODEC_FLAG_DECODE); - - if (!(encoding || decoding) || ((context = switch_core_alloc(codec->memory_pool, sizeof(*context))) == 0)) { - return SWITCH_STATUS_FALSE; - } - memset(context, 0, sizeof(*context)); - context->flags = flags; - codec->private_info = context; - - if (codec->fmtp_in) { - codec->fmtp_out = switch_core_strdup(codec->memory_pool, codec->fmtp_in); - } - - config = &context->config; - - if (vpx_codec_enc_config_default(encoder_interface, config, 0) != VPX_CODEC_OK) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoder config Error\n"); - return SWITCH_STATUS_FALSE; - } - - // very big defaults till we know why scaling segs it - context->width = 3840; - context->height = 2160; - context->bitrate = 3840000; + vpx_context_t *context = (vpx_context_t *)codec->private_info; + vpx_codec_enc_cfg_t *config = &context->config; // settings config->g_profile = 1; - config->g_w = context->width; - config->g_h = context->height; - config->rc_target_bitrate = context->bitrate; + config->g_w = context->codec_settings.video.width; + config->g_h = context->codec_settings.video.height; + config->rc_target_bitrate = context->bandwidth; config->g_timebase.num = 1; config->g_timebase.den = 1000; config->g_error_resilient = VPX_ERROR_RESILIENT_PARTITIONS; @@ -127,11 +98,11 @@ static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_ config->rc_dropframe_thresh = 0; config->rc_end_usage = VPX_CBR; config->g_pass = VPX_RC_ONE_PASS; - // config->kf_mode = VPX_KF_DISABLED; - config->kf_mode = VPX_KF_AUTO; - // config->kf_min_dist = FPS;// Intra Period 3 seconds; - // config->kf_max_dist = FPS * 3; - config->rc_resize_allowed = 0; + config->kf_mode = VPX_KF_DISABLED; + //config->kf_mode = VPX_KF_AUTO; + //config->kf_min_dist = FPS;// Intra Period 3 seconds; + //config->kf_max_dist = FPS; + config->rc_resize_allowed = 1; config->rc_min_quantizer = 2; config->rc_max_quantizer = 56; //Rate control adaptation undershoot control. @@ -157,59 +128,74 @@ static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_ // indicates that the client will buffer (at least) 5000ms worth // of encoded data. Use the target bitrate (rc_target_bitrate) to // convert to bits/bytes, if necessary. - config->rc_buf_sz = 1000; + config->rc_buf_sz = 5000; //Decoder Buffer Initial Size. // This value indicates the amount of data that will be buffered // by the decoding application prior to beginning playback. // This value is expressed in units of time (milliseconds). // Use the target bitrate (rc_target_bitrate) to convert to // bits/bytes, if necessary. - config->rc_buf_initial_sz = 500; + config->rc_buf_initial_sz = 1000; //Decoder Buffer Optimal Size. // This value indicates the amount of data that the encoder should // try to maintain in the decoder's buffer. This value is expressed // in units of time (milliseconds). // Use the target bitrate (rc_target_bitrate) to convert to // bits/bytes, if necessary. - config->rc_buf_optimal_sz = 600; + config->rc_buf_optimal_sz = 1000; + + if (context->flags & SWITCH_CODEC_FLAG_ENCODE) { + + if (context->encoder_init) { + vpx_codec_destroy(&context->encoder); + context->encoder_init = 0; + } - if (encoding) { if (vpx_codec_enc_init(&context->encoder, encoder_interface, config, 0 & VPX_CODEC_USE_OUTPUT_PARTITION) != VPX_CODEC_OK) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", encoder->err, encoder->err_detail); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", context->encoder.err, context->encoder.err_detail); return SWITCH_STATUS_FALSE; } + context->encoder_init = 1; + // The static threshold imposes a change threshold on blocks below which they will be skipped by the encoder. - vpx_codec_control(encoder, VP8E_SET_STATIC_THRESHOLD, 100); + vpx_codec_control(&context->encoder, VP8E_SET_STATIC_THRESHOLD, 100); //Set cpu usage, a bit lower than normal (-6) but higher than android (-12) - vpx_codec_control(encoder, VP8E_SET_CPUUSED, -8); + vpx_codec_control(&context->encoder, VP8E_SET_CPUUSED, -6); // Only one partition - // vpx_codec_control(encoder, VP8E_SET_TOKEN_PARTITIONS, VP8_ONE_TOKENPARTITION); + // vpx_codec_control(&context->encoder, VP8E_SET_TOKEN_PARTITIONS, VP8_ONE_TOKENPARTITION); // Enable noise reduction - vpx_codec_control(encoder, VP8E_SET_NOISE_SENSITIVITY, 0); + vpx_codec_control(&context->encoder, VP8E_SET_NOISE_SENSITIVITY, 1); //Set max data rate for Intra frames. // This value controls additional clamping on the maximum size of a keyframe. // It is expressed as a percentage of the average per-frame bitrate, with the // special (and default) value 0 meaning unlimited, or no additional clamping // beyond the codec's built-in algorithm. // For example, to allocate no more than 4.5 frames worth of bitrate to a keyframe, set this to 450. - vpx_codec_control(encoder, VP8E_SET_MAX_INTRA_BITRATE_PCT, 0); + vpx_codec_control(&context->encoder, VP8E_SET_MAX_INTRA_BITRATE_PCT, 0); } - if (decoding) { + if (context->flags & SWITCH_CODEC_FLAG_DECODE) { vp8_postproc_cfg_t ppcfg; + if (context->decoder_init) { + vpx_codec_destroy(&context->decoder); + context->decoder_init = 0; + } + if (vpx_codec_dec_init(&context->decoder, decoder_interface, NULL, VPX_CODEC_USE_POSTPROC) != VPX_CODEC_OK) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", encoder->err, encoder->err_detail); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", context->encoder.err, context->encoder.err_detail); return SWITCH_STATUS_FALSE; } + context->decoder_init = 1; + // the types of post processing to be done, should be combination of "vp8_postproc_level" ppcfg.post_proc_flag = VP8_DEMACROBLOCK | VP8_DEBLOCK; // the strength of deblocking, valid range [0, 16] ppcfg.deblocking_level = 3; // Set deblocking settings - vpx_codec_control(decoder, VP8_SET_POSTPROC, &ppcfg); + vpx_codec_control(&context->decoder, VP8_SET_POSTPROC, &ppcfg); switch_buffer_create_dynamic(&context->vpx_packet_buffer, 512, 512, 1024000); } @@ -217,6 +203,39 @@ static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_ return SWITCH_STATUS_SUCCESS; } +static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_t flags, const switch_codec_settings_t *codec_settings) +{ + vpx_context_t *context = NULL; + int encoding, decoding; + + + encoding = (flags & SWITCH_CODEC_FLAG_ENCODE); + decoding = (flags & SWITCH_CODEC_FLAG_DECODE); + + if (!(encoding || decoding) || ((context = switch_core_alloc(codec->memory_pool, sizeof(*context))) == 0)) { + return SWITCH_STATUS_FALSE; + } + + memset(context, 0, sizeof(*context)); + context->flags = flags; + codec->private_info = context; + + if (codec_settings) { + context->codec_settings = *codec_settings; + } + + if (codec->fmtp_in) { + codec->fmtp_out = switch_core_strdup(codec->memory_pool, codec->fmtp_in); + } + + if (vpx_codec_enc_config_default(encoder_interface, &context->config, 0) != VPX_CODEC_OK) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoder config Error\n"); + return SWITCH_STATUS_FALSE; + } + + return SWITCH_STATUS_SUCCESS; +} + /* http://tools.ietf.org/html/draft-ietf-payload-vp8-10 The first octets after the RTP header are the VP8 payload descriptor, with the following structure. @@ -313,8 +332,8 @@ static switch_status_t consume_partition(vpx_context_t *context, void *data, uin } static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_image_t *img, - void *encoded_data, uint32_t *encoded_data_len, - unsigned int *flag) + void *encoded_data, uint32_t *encoded_data_len, + unsigned int *flag) { vpx_context_t *context = (vpx_context_t *)codec->private_info; uint32_t duration = 90000 / FPS; @@ -332,6 +351,8 @@ static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_image_t * //d_w and d_h are messed up + //printf("WTF %d %d\n", img->d_w, img->d_h); + width = img->w; height = img->h; @@ -339,28 +360,31 @@ static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_image_t * //switch_assert(height > 0 && (height % 4 == 0)); if (context->config.g_w != width || context->config.g_h != height) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "VPX reset encoder picture from %dx%d to %dx%d\n", context->config.g_w, context->config.g_h, width, height); - context->config.g_w = width; - context->config.g_h = height; - if (vpx_codec_enc_config_set(&context->encoder, &context->config) != VPX_CODEC_OK) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "VPX reset config error!"); + context->codec_settings.video.width = width; + context->codec_settings.video.height = height; + if (context->codec_settings.video.bandwidth) { + context->bandwidth = context->codec_settings.video.bandwidth; + } else { + context->bandwidth = width * height * 8; } - } - if (context->last_ts == 0) context->last_ts = switch_micro_time_now(); + if (context->bandwidth > 1250000) { + context->bandwidth = 1250000; + } - if ((switch_micro_time_now() - context->last_ts) > 2 * 1000000) { - // the config params doesn't seems work for generate regular key frames, - // so we do some trick here to force a key frame every 2 sec - // vpx_flags = VPX_EFLAG_FORCE_KF; - context->last_ts = switch_micro_time_now(); + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(codec->session), SWITCH_LOG_NOTICE, + "VPX reset encoder picture from %dx%d to %dx%d %u BW\n", + context->config.g_w, context->config.g_h, width, height, context->bandwidth); + + init_codec(codec); + *flag |= SFF_PICTURE_RESET; + context->need_key_frame = 1; } if (context->need_key_frame > 0) { // force generate a key frame switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "VPX KEYFRAME REQ\n"); vpx_flags |= VPX_EFLAG_FORCE_KF; - context->last_ts = switch_micro_time_now(); context->need_key_frame--; } @@ -539,7 +563,7 @@ static switch_status_t switch_vpx_destroy(switch_codec_t *codec) if (context) { if ((codec->flags & SWITCH_CODEC_FLAG_ENCODE)) { - vpx_codec_destroy(&context->encoder); // TODO fix crash + vpx_codec_destroy(&context->encoder); } if ((codec->flags & SWITCH_CODEC_FLAG_DECODE)) { diff --git a/src/mod/formats/mod_vlc/mod_vlc.c b/src/mod/formats/mod_vlc/mod_vlc.c index 9c50f6de92..28c5e576b8 100644 --- a/src/mod/formats/mod_vlc/mod_vlc.c +++ b/src/mod/formats/mod_vlc/mod_vlc.c @@ -102,6 +102,7 @@ struct vlc_video_context { switch_mutex_t *video_mutex; switch_core_session_t *session; + switch_channel_t *channel; switch_frame_t *aud_frame; switch_frame_t *vid_frame; uint8_t video_packet[1500 + 12]; @@ -230,20 +231,11 @@ static void *vlc_video_lock_callback(void *data, void **p_pixels) return NULL; /* picture identifier, not needed here */ } -/* dummy callback so it should be good when no video on channel */ -static void vlc_video_unlock_dummy_callback(void *data, void *id, void *const *p_pixels) -{ - vlc_video_context_t *context = (vlc_video_context_t *)data; - assert(id == NULL); /* picture identifier, not needed here */ - switch_mutex_unlock(context->video_mutex); -} - static void vlc_video_unlock_callback(void *data, void *id, void *const *p_pixels) { vlc_video_context_t *context = (vlc_video_context_t *) data; - switch_frame_t *frame = context->vid_frame; - switch_assert(id == NULL); /* picture identifier, not needed here */ + if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return; if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0); @@ -251,14 +243,11 @@ static void vlc_video_unlock_callback(void *data, void *id, void *const *p_pixel yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height); - switch_core_session_write_video_image(context->session, frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, NULL); - switch_mutex_unlock(context->video_mutex); } -static void do_buffer_frame(vlc_video_context_t *context) +static void do_buffer_frame(vlc_video_context_t *context, switch_frame_t *frame) { - switch_frame_t *frame = context->vid_frame; uint32_t size = sizeof(*frame) + frame->packetlen; switch_mutex_lock(context->video_mutex); @@ -277,32 +266,35 @@ static void do_buffer_frame(vlc_video_context_t *context) static void vlc_video_channel_unlock_callback(void *data, void *id, void *const *p_pixels) { vlc_video_context_t *context = (vlc_video_context_t *)data; - uint32_t flag = 0; - switch_frame_t *frame = context->vid_frame; + switch_assert(id == NULL); /* picture identifier, not needed here */ + if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return; + if (!context->img) context->img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, context->width, context->height, 0); switch_assert(context->img); yuyv_to_i420(*p_pixels, context->img->img_data, context->width, context->height); + switch_mutex_unlock(context->video_mutex); +} + +static void vlc_video_display_callback(void *data, void *id) +{ + vlc_video_context_t *context = (vlc_video_context_t *) data; + int32_t flag = 0; + + /* VLC wants to display the video */ + + if (context->channel && !switch_channel_test_flag(context->channel, CF_VIDEO)) return; if (context->video_refresh_req > 0) { flag |= SFF_WAIT_KEY_FRAME; context->video_refresh_req--; } - switch_core_session_write_video_image(context->session, frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, &flag); - - switch_mutex_unlock(context->video_mutex); -} - -static void vlc_video_display_callback(void *data, void *id) -{ - /* VLC wants to display the video */ - (void) data; - assert(id == NULL); + switch_core_session_write_video_image(context->session, context->vid_frame, context->img, SWITCH_DEFAULT_VIDEO_SIZE, NULL); } unsigned video_format_setup_callback(void **opaque, char *chroma, unsigned *width, unsigned *height, unsigned *pitches, unsigned *lines) @@ -723,7 +715,9 @@ SWITCH_STANDARD_APP(play_video_function) audio_datalen = read_impl.decoded_bytes_per_packet; //codec.implementation->actual_samples_per_second / 1000 * (read_impl.microseconds_per_packet / 1000); + context->session = session; + context->channel = channel; context->pool = pool; context->aud_frame = &audio_frame; context->vid_frame = &video_frame; @@ -770,14 +764,8 @@ SWITCH_STANDARD_APP(play_video_function) libvlc_audio_set_format(context->mp, "S16N", read_impl.actual_samples_per_second, read_impl.number_of_channels); libvlc_audio_set_callbacks(context->mp, vlc_play_audio_callback, NULL,NULL,NULL,NULL, (void *) context); - if (switch_channel_test_flag(channel, CF_VIDEO)) { - // libvlc_video_set_format(context->mp, "YUYV", VIDEOWIDTH, VIDEOHEIGHT, VIDEOWIDTH * 2); - libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback); - libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_callback, vlc_video_display_callback, context); - } else { - libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback); - libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_dummy_callback, vlc_video_display_callback, context); - } + libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback); + libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_callback, vlc_video_display_callback, context); // start play if (-1 == libvlc_media_player_play(context->mp)) { @@ -928,6 +916,15 @@ switch_io_routines_t vlc_io_routines = { /*state_run*/ NULL }; +static switch_status_t vlc_channel_img_callback(switch_core_session_t *session, switch_frame_t *frame, switch_image_t *img, void *user_data) +{ + vlc_video_context_t *context = (vlc_video_context_t *) user_data; + + do_buffer_frame(context, frame); + + return SWITCH_STATUS_SUCCESS; +} + static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char *path) { switch_channel_t *channel = switch_core_session_get_channel(session); @@ -952,11 +949,12 @@ static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char memset(context, 0, sizeof(vlc_file_context_t)); tech_pvt->context = context; + switch_buffer_create_dynamic(&(context->audio_buffer), VLC_BUFFER_SIZE, VLC_BUFFER_SIZE * 8, 0); - if (switch_channel_test_flag(channel, CF_VIDEO)) { - switch_buffer_create_dynamic(&(context->video_buffer), VLC_BUFFER_SIZE * 2, VLC_BUFFER_SIZE * 16, 0); - } + switch_buffer_create_dynamic(&(context->video_buffer), VLC_BUFFER_SIZE * 2, VLC_BUFFER_SIZE * 16, 0); + + switch_core_session_set_image_write_callback(session, vlc_channel_img_callback, tech_pvt->context); if (switch_core_timer_init(&tech_pvt->timer, "soft", 20, 8000 / (1000 / 20), pool) != SWITCH_STATUS_SUCCESS) { @@ -969,6 +967,9 @@ static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char context->pool = pool; context->aud_frame = &tech_pvt->read_frame; context->vid_frame = &tech_pvt->read_video_frame; + context->vid_frame->packet = context->video_packet; + context->vid_frame->data = context->video_packet + 12; + context->playing = 0; // context->err = 0; @@ -1010,14 +1011,10 @@ static switch_status_t setup_tech_pvt(switch_core_session_t *session, const char libvlc_audio_set_format(context->mp, "S16N", 8000, 1); libvlc_audio_set_callbacks(context->mp, vlc_play_audio_callback, NULL,NULL,NULL,NULL, (void *) context); - if (switch_channel_test_flag(channel, CF_VIDEO)) { - // libvlc_video_set_format(context->mp, "YUYV", VIDEOWIDTH, VIDEOHEIGHT, VIDEOWIDTH * 2); - libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback); - libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_channel_unlock_callback, vlc_video_display_callback, context); - } else { - libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback); - libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_unlock_dummy_callback, vlc_video_display_callback, context); - } + + libvlc_video_set_format_callbacks(context->mp, video_format_setup_callback, video_format_clean_callback); + libvlc_video_set_callbacks(context->mp, vlc_video_lock_callback, vlc_video_channel_unlock_callback, vlc_video_display_callback, context); + return SWITCH_STATUS_SUCCESS; @@ -1026,25 +1023,13 @@ fail: return status; } -static switch_status_t vlc_channel_img_callback(switch_core_session_t *session, switch_frame_t *frame, switch_image_t *img, void *user_data) -{ - vlc_video_context_t *context = (vlc_video_context_t *) user_data; - - do_buffer_frame(context); - - return SWITCH_STATUS_SUCCESS; -} - - static switch_status_t channel_on_init(switch_core_session_t *session) { switch_channel_t *channel = switch_core_session_get_channel(session); - vlc_private_t *tech_pvt = switch_core_session_get_private(session); + //vlc_private_t *tech_pvt = switch_core_session_get_private(session); switch_channel_set_state(channel, CS_CONSUME_MEDIA); - switch_core_session_set_image_write_callback(session, vlc_channel_img_callback, tech_pvt->context); - return SWITCH_STATUS_SUCCESS; } @@ -1096,6 +1081,10 @@ static switch_status_t channel_on_destroy(switch_core_session_t *session) switch_buffer_destroy(&tech_pvt->context->audio_buffer); } + if (tech_pvt->context->video_buffer) { + switch_buffer_destroy(&tech_pvt->context->video_buffer); + } + if (tech_pvt->timer.interval) { switch_core_timer_destroy(&tech_pvt->timer); } diff --git a/src/switch_core_codec.c b/src/switch_core_codec.c index ef776c790f..0e392edcbb 100644 --- a/src/switch_core_codec.c +++ b/src/switch_core_codec.c @@ -646,6 +646,10 @@ SWITCH_DECLARE(switch_status_t) switch_core_codec_init_with_bitrate(switch_codec memset(codec, 0, sizeof(*codec)); + if (pool) { + codec->session = switch_core_memory_pool_get_data(pool, "__session"); + } + if ((codec_interface = switch_loadable_module_get_codec_interface(codec_name)) == 0) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid codec %s!\n", codec_name); return SWITCH_STATUS_GENERR; diff --git a/src/switch_core_io.c b/src/switch_core_io.c index f7515b9264..85e6b8ce0e 100644 --- a/src/switch_core_io.c +++ b/src/switch_core_io.c @@ -36,174 +36,6 @@ #include #include "private/switch_core_pvt.h" -SWITCH_DECLARE(void) switch_core_session_set_image_write_callback(switch_core_session_t *session, switch_image_write_callback_t callback, void *user_data) -{ - session->image_write_callback = callback; - session->image_write_callback_user_data = user_data; -} - -SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame, - switch_image_t *img, switch_size_t size, uint32_t *flag) -{ - uint32_t encoded_data_len = size, lflag = 0, *flagp = flag; - switch_codec_t *codec = switch_core_session_get_video_write_codec(session); - switch_timer_t *timer; - - switch_assert(session); - - if (!flag) { - flagp = &lflag; - } - - timer = switch_core_media_get_timer(session, SWITCH_MEDIA_TYPE_VIDEO); - switch_assert(timer); - - switch_core_codec_encode_video(codec, img, frame->data, &encoded_data_len, flagp); - - while(encoded_data_len) { - - frame->datalen = encoded_data_len; - frame->packetlen = frame->datalen + 12; - frame->m = (*flagp & SFF_MARKER) ? 1 : 0; - frame->timestamp = timer->samplecount; - - switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME); - - switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0); - - if (session->image_write_callback) { - session->image_write_callback(session, frame, img, session->image_write_callback_user_data); - } - - encoded_data_len = size; - switch_core_codec_encode_video(codec, NULL, frame->data, &encoded_data_len, flagp); - } - - return SWITCH_STATUS_SUCCESS; -} - - -SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags, - int stream_id) -{ - switch_io_event_hook_video_write_frame_t *ptr; - switch_status_t status = SWITCH_STATUS_FALSE; - - if (switch_channel_down(session->channel)) { - return SWITCH_STATUS_FALSE; - } - - if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE)) { - return SWITCH_STATUS_SUCCESS; - } - - if (session->endpoint_interface->io_routines->write_video_frame) { - if ((status = session->endpoint_interface->io_routines->write_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) { - for (ptr = session->event_hooks.video_write_frame; ptr; ptr = ptr->next) { - if ((status = ptr->video_write_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) { - break; - } - } - } - } - return status; -} - -SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags, - int stream_id) -{ - switch_status_t status = SWITCH_STATUS_FALSE; - switch_io_event_hook_video_read_frame_t *ptr; - - switch_assert(session != NULL); - - if (switch_channel_down(session->channel)) { - return SWITCH_STATUS_FALSE; - } - - if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE)) { - *frame = &runtime.dummy_cng_frame; - switch_yield(20000); - return SWITCH_STATUS_SUCCESS; - } - - if (session->endpoint_interface->io_routines->read_video_frame) { - if ((status = session->endpoint_interface->io_routines->read_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) { - for (ptr = session->event_hooks.video_read_frame; ptr; ptr = ptr->next) { - if ((status = ptr->video_read_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) { - break; - } - } - } - } - - if (status == SWITCH_STATUS_INUSE) { - *frame = &runtime.dummy_cng_frame; - switch_yield(20000); - return SWITCH_STATUS_SUCCESS; - } - - if (status != SWITCH_STATUS_SUCCESS) { - goto done; - } - - if (!(*frame)) { - goto done; - } - - if (switch_test_flag(*frame, SFF_CNG)) { - status = SWITCH_STATUS_SUCCESS; - goto done; - } - - if (session->bugs) { - switch_media_bug_t *bp; - switch_bool_t ok = SWITCH_TRUE; - int prune = 0; - switch_thread_rwlock_rdlock(session->bug_rwlock); - for (bp = session->bugs; bp; bp = bp->next) { - if (switch_channel_test_flag(session->channel, CF_PAUSE_BUGS) && !switch_core_media_bug_test_flag(bp, SMBF_NO_PAUSE)) { - continue; - } - - if (!switch_channel_test_flag(session->channel, CF_ANSWERED) && switch_core_media_bug_test_flag(bp, SMBF_ANSWER_REQ)) { - continue; - } - - if (switch_test_flag(bp, SMBF_PRUNE)) { - prune++; - continue; - } - - if (bp->ready && switch_test_flag(bp, SMBF_READ_PING)) { - switch_mutex_lock(bp->read_mutex); - bp->ping_frame = *frame; - if (bp->callback) { - if (bp->callback(bp, bp->user_data, SWITCH_ABC_TYPE_READ_VIDEO_PING) == SWITCH_FALSE - || (bp->stop_time && bp->stop_time <= switch_epoch_time_now(NULL))) { - ok = SWITCH_FALSE; - } - } - bp->ping_frame = NULL;; - switch_mutex_unlock(bp->read_mutex); - } - - if (ok == SWITCH_FALSE) { - switch_set_flag(bp, SMBF_PRUNE); - prune++; - } - } - switch_thread_rwlock_unlock(session->bug_rwlock); - if (prune) { - switch_core_media_bug_prune(session); - } - } - - done: - - return status; -} - SWITCH_DECLARE(void) switch_core_gen_encoded_silence(unsigned char *data, const switch_codec_implementation_t *read_impl, switch_size_t len) { unsigned char g729_filler[] = { diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 5b620548c1..99a9620c34 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -154,6 +154,9 @@ typedef struct switch_rtp_engine_s { uint8_t pli; uint8_t nack; uint8_t no_crypto; + + switch_codec_settings_t codec_settings; + } switch_rtp_engine_t; @@ -190,6 +193,9 @@ struct switch_media_handle_s { switch_rtp_crypto_mode_t crypto_mode; switch_rtp_crypto_key_type_t crypto_suite_order[CRYPTO_INVALID+1]; + switch_time_t video_last_key_time; + switch_time_t video_init; + switch_timer_t video_timer; }; @@ -1436,6 +1442,10 @@ SWITCH_DECLARE(void) switch_media_handle_destroy(switch_core_session_t *session) v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO]; + if (smh->video_timer.timer_interface) { + switch_core_timer_destroy(&smh->video_timer); + } + if (switch_core_codec_ready(&a_engine->read_codec)) { switch_core_codec_destroy(&a_engine->read_codec); } @@ -1498,6 +1508,15 @@ SWITCH_DECLARE(switch_status_t) switch_media_handle_create(switch_media_handle_t } session->media_handle->mparams = params; + + if (!session->media_handle->mparams->video_key_freq) { + session->media_handle->mparams->video_key_freq = 15000000; + } + + if (!session->media_handle->mparams->video_key_first) { + session->media_handle->mparams->video_key_first = 1000000; + } + for (i = 0; i <= CRYPTO_INVALID; i++) { session->media_handle->crypto_suite_order[i] = CRYPTO_INVALID; @@ -2262,6 +2281,43 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_get_offered_pt(switch_core_ses return SWITCH_STATUS_FALSE; } +//#define get_int_value(_var, _set) { const char *__v = switch_channel_get_variable(session->channel, _var); if (__v) { _set = atol(__v);} } +//? +static void switch_core_session_parse_codec_settings(switch_core_session_t *session, switch_media_type_t type) +{ + switch_media_handle_t *smh; + switch_rtp_engine_t *engine; + + switch_assert(session); + + if (!(smh = session->media_handle)) { + return; + } + + if ((engine = &smh->engines[type])); + + switch(type) { + case SWITCH_MEDIA_TYPE_AUDIO: + break; + case SWITCH_MEDIA_TYPE_VIDEO: + { + const char *bwv = switch_channel_get_variable(session->channel, "video_codec_bandwidth"); + uint32_t bw = 0; + + if (bwv && (bw = (uint32_t) atol(bwv))) { + if (switch_stristr("kb", bwv)) { + bw *= 125; + } else if (switch_stristr("mb", bwv)) { + bw *= 125000; + } + engine->codec_settings.video.bandwidth = bw; + } + } + break; + default: + break; + } +} //? @@ -2300,7 +2356,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_codec(switch_core_se } } - + switch_core_session_parse_codec_settings(session, SWITCH_MEDIA_TYPE_VIDEO); if (switch_core_codec_init(&v_engine->read_codec, v_engine->cur_payload_map->rm_encoding, @@ -2309,7 +2365,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_codec(switch_core_se 0, 1, SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE, - NULL, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) { + &v_engine->codec_settings, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Can't load codec?\n"); return SWITCH_STATUS_FALSE; } else { @@ -2320,7 +2376,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_codec(switch_core_se 0, 1, SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE, - NULL, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) { + &v_engine->codec_settings, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Can't load codec?\n"); return SWITCH_STATUS_FALSE; } else { @@ -2361,7 +2417,6 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_video_codec(switch_core_se switch_channel_set_variable(session->channel, "rtp_use_video_codec_fmtp", v_engine->cur_payload_map->rm_fmtp); switch_channel_set_variable_printf(session->channel, "rtp_use_video_codec_rate", "%d", v_engine->cur_payload_map->rm_rate); switch_channel_set_variable_printf(session->channel, "rtp_use_video_codec_ptime", "%d", 0); - } } return SWITCH_STATUS_SUCCESS; @@ -2437,6 +2492,9 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_codec(switch_core_session_ } } + + switch_core_session_parse_codec_settings(session, SWITCH_MEDIA_TYPE_AUDIO); + if (switch_core_codec_init_with_bitrate(&a_engine->read_codec, a_engine->cur_payload_map->iananame, a_engine->cur_payload_map->rm_fmtp, @@ -2445,7 +2503,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_codec(switch_core_session_ a_engine->cur_payload_map->channels, a_engine->cur_payload_map->bitrate, SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE | codec_flags, - NULL, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) { + &a_engine->codec_settings, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Can't load codec?\n"); switch_channel_hangup(session->channel, SWITCH_CAUSE_INCOMPATIBLE_DESTINATION); switch_goto_status(SWITCH_STATUS_FALSE, end); @@ -2462,7 +2520,7 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_set_codec(switch_core_session_ a_engine->cur_payload_map->channels, a_engine->cur_payload_map->bitrate, SWITCH_CODEC_FLAG_ENCODE | SWITCH_CODEC_FLAG_DECODE | codec_flags, - NULL, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) { + &a_engine->codec_settings, switch_core_session_get_pool(session)) != SWITCH_STATUS_SUCCESS) { switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_ERROR, "Can't load codec?\n"); switch_channel_hangup(session->channel, SWITCH_CAUSE_INCOMPATIBLE_DESTINATION); switch_goto_status(SWITCH_STATUS_FALSE, end); @@ -7210,8 +7268,8 @@ SWITCH_DECLARE(void) switch_core_media_gen_local_sdp(switch_core_session_t *sess pli++; nack++; } - - nack = v_engine->nack = pli = v_engine->pli = 0; + + nack = v_engine->nack = 0;//pli = v_engine->pli = 0; if (vp8) { @@ -9502,6 +9560,223 @@ SWITCH_DECLARE(switch_status_t) switch_core_media_codec_control(switch_core_sess return SWITCH_STATUS_FALSE; } + +SWITCH_DECLARE(void) switch_core_session_set_image_write_callback(switch_core_session_t *session, switch_image_write_callback_t callback, void *user_data) +{ + session->image_write_callback = callback; + session->image_write_callback_user_data = user_data; +} + +SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_image(switch_core_session_t *session, switch_frame_t *frame, + switch_image_t *img, switch_size_t size, uint32_t *flag) +{ + uint32_t encoded_data_len = size, lflag = 0, *flagp = flag; + switch_codec_t *codec = switch_core_session_get_video_write_codec(session); + switch_timer_t *timer; + switch_media_handle_t *smh; + //switch_rtp_engine_t *v_engine; + + switch_assert(session); + + if (!(smh = session->media_handle)) { + return SWITCH_STATUS_FALSE; + } + + //v_engine = &smh->engines[SWITCH_MEDIA_TYPE_VIDEO]; + + + if (!flag) { + flagp = &lflag; + } + + if (!(timer = switch_core_media_get_timer(session, SWITCH_MEDIA_TYPE_VIDEO))) { + + if (!smh->video_timer.timer_interface) { + switch_core_timer_init(&smh->video_timer, "soft", 1, 90, switch_core_session_get_pool(session)); + } + + timer = &smh->video_timer; + } + + + do { + encoded_data_len = size; + switch_core_codec_encode_video(codec, img, frame->data, &encoded_data_len, flagp); + + if (*flagp & SFF_PICTURE_RESET) { + smh->video_init = 0; + smh->video_last_key_time = 0; + *flagp &= ~SFF_PICTURE_RESET; + } + + frame->datalen = encoded_data_len; + frame->packetlen = frame->datalen + 12; + frame->m = (*flagp & SFF_MARKER) ? 1 : 0; + frame->timestamp = timer->samplecount; + + switch_set_flag(frame, SFF_RAW_RTP_PARSE_FRAME); + + switch_core_session_write_video_frame(session, frame, SWITCH_IO_FLAG_NONE, 0); + + if (session->image_write_callback) { + session->image_write_callback(session, frame, img, session->image_write_callback_user_data); + } + + img = NULL; + + } while(encoded_data_len); + + return SWITCH_STATUS_SUCCESS; +} + + +SWITCH_DECLARE(switch_status_t) switch_core_session_write_video_frame(switch_core_session_t *session, switch_frame_t *frame, switch_io_flag_t flags, + int stream_id) +{ + switch_io_event_hook_video_write_frame_t *ptr; + switch_status_t status = SWITCH_STATUS_FALSE; + switch_media_handle_t *smh; + + switch_time_t now = switch_micro_time_now(); + + switch_assert(session); + + if (!(smh = session->media_handle)) { + return SWITCH_STATUS_FALSE; + } + + if (switch_channel_down(session->channel)) { + return SWITCH_STATUS_FALSE; + } + + if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE)) { + return SWITCH_STATUS_SUCCESS; + } + + if (!smh->video_init && smh->mparams->video_key_first && (now - smh->video_last_key_time) > smh->mparams->video_key_first) { + switch_core_media_gen_key_frame(smh->session); + + if (smh->video_last_key_time) { + smh->video_init = 1; + } + + smh->video_last_key_time = now; + } + + if (smh->mparams->video_key_freq && (now - smh->video_last_key_time) > smh->mparams->video_key_freq) { + switch_core_media_gen_key_frame(smh->session); + smh->video_last_key_time = now; + } + + if (session->endpoint_interface->io_routines->write_video_frame) { + if ((status = session->endpoint_interface->io_routines->write_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) { + for (ptr = session->event_hooks.video_write_frame; ptr; ptr = ptr->next) { + if ((status = ptr->video_write_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) { + break; + } + } + } + } + return status; +} + +SWITCH_DECLARE(switch_status_t) switch_core_session_read_video_frame(switch_core_session_t *session, switch_frame_t **frame, switch_io_flag_t flags, + int stream_id) +{ + switch_status_t status = SWITCH_STATUS_FALSE; + switch_io_event_hook_video_read_frame_t *ptr; + + switch_assert(session != NULL); + + if (switch_channel_down(session->channel)) { + return SWITCH_STATUS_FALSE; + } + + if (switch_channel_test_flag(session->channel, CF_VIDEO_PAUSE)) { + *frame = &runtime.dummy_cng_frame; + switch_yield(20000); + return SWITCH_STATUS_SUCCESS; + } + + if (session->endpoint_interface->io_routines->read_video_frame) { + if ((status = session->endpoint_interface->io_routines->read_video_frame(session, frame, flags, stream_id)) == SWITCH_STATUS_SUCCESS) { + for (ptr = session->event_hooks.video_read_frame; ptr; ptr = ptr->next) { + if ((status = ptr->video_read_frame(session, frame, flags, stream_id)) != SWITCH_STATUS_SUCCESS) { + break; + } + } + } + } + + if (status == SWITCH_STATUS_INUSE) { + *frame = &runtime.dummy_cng_frame; + switch_yield(20000); + return SWITCH_STATUS_SUCCESS; + } + + if (status != SWITCH_STATUS_SUCCESS) { + goto done; + } + + if (!(*frame)) { + goto done; + } + + if (switch_test_flag(*frame, SFF_CNG)) { + status = SWITCH_STATUS_SUCCESS; + goto done; + } + + if (session->bugs) { + switch_media_bug_t *bp; + switch_bool_t ok = SWITCH_TRUE; + int prune = 0; + switch_thread_rwlock_rdlock(session->bug_rwlock); + for (bp = session->bugs; bp; bp = bp->next) { + if (switch_channel_test_flag(session->channel, CF_PAUSE_BUGS) && !switch_core_media_bug_test_flag(bp, SMBF_NO_PAUSE)) { + continue; + } + + if (!switch_channel_test_flag(session->channel, CF_ANSWERED) && switch_core_media_bug_test_flag(bp, SMBF_ANSWER_REQ)) { + continue; + } + + if (switch_test_flag(bp, SMBF_PRUNE)) { + prune++; + continue; + } + + if (bp->ready && switch_test_flag(bp, SMBF_READ_PING)) { + switch_mutex_lock(bp->read_mutex); + bp->ping_frame = *frame; + if (bp->callback) { + if (bp->callback(bp, bp->user_data, SWITCH_ABC_TYPE_READ_VIDEO_PING) == SWITCH_FALSE + || (bp->stop_time && bp->stop_time <= switch_epoch_time_now(NULL))) { + ok = SWITCH_FALSE; + } + } + bp->ping_frame = NULL;; + switch_mutex_unlock(bp->read_mutex); + } + + if (ok == SWITCH_FALSE) { + switch_set_flag(bp, SMBF_PRUNE); + prune++; + } + } + switch_thread_rwlock_unlock(session->bug_rwlock); + if (prune) { + switch_core_media_bug_prune(session); + } + } + + done: + + return status; +} + + + /* For Emacs: * Local Variables: * mode:c diff --git a/src/switch_rtp.c b/src/switch_rtp.c index f272eecd15..a4db12ae59 100644 --- a/src/switch_rtp.c +++ b/src/switch_rtp.c @@ -56,6 +56,7 @@ #include #define FIR_COUNTDOWN 50 +#define PLI_COUNTDOWN 50 #define JITTER_LEAD_FRAMES 10 #define READ_INC(rtp_session) switch_mutex_lock(rtp_session->read_mutex); rtp_session->reading++ #define READ_DEC(rtp_session) switch_mutex_unlock(rtp_session->read_mutex); rtp_session->reading-- @@ -299,6 +300,7 @@ struct switch_rtp { rtcp_ext_msg_t rtcp_ext_send_msg; uint8_t fir_seq; uint16_t fir_countdown; + uint16_t pli_countdown; ts_normalize_t ts_norm; switch_sockaddr_t *remote_addr, *rtcp_remote_addr; rtp_msg_t recv_msg; @@ -360,6 +362,7 @@ struct switch_rtp { uint32_t samples_per_second; uint32_t conf_samples_per_interval; uint16_t rtcp_send_rate; + switch_time_t rtcp_last_sent; uint32_t rsamples_per_interval; uint32_t ms_per_packet; uint32_t one_second; @@ -948,7 +951,10 @@ static void handle_ice(switch_rtp_t *rtp_session, switch_rtp_ice_t *ice, void *d } else { ice->rready = 1; } - + + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { + switch_core_media_gen_key_frame(rtp_session->session); + } switch_rtp_set_flag(rtp_session, SWITCH_RTP_FLAG_FLUSH); } } @@ -1097,6 +1103,9 @@ static void handle_ice(switch_rtp_t *rtp_session, switch_rtp_ice_t *ice, void *d if (!ice->ready) { ice->ready = 1; + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { + switch_core_media_gen_key_frame(rtp_session->session); + } switch_rtp_set_flag(rtp_session, SWITCH_RTP_FLAG_FLUSH); } @@ -2017,21 +2026,23 @@ static int check_rtcp_and_ice(switch_rtp_t *rtp_session) switch_time_t now = switch_micro_time_now(); if (rtp_session->fir_countdown) { - //if (rtp_session->fir_countdown == FIR_COUNTDOWN) { - // do_flush(rtp_session, SWITCH_TRUE); - //} if (rtp_session->fir_countdown == FIR_COUNTDOWN || (rtp_session->fir_countdown == FIR_COUNTDOWN / 2) || rtp_session->fir_countdown == 1) { - if (rtp_session->flags[SWITCH_RTP_FLAG_PLI]) { - send_pli(rtp_session); - } else { - send_fir(rtp_session); - } + send_fir(rtp_session); } rtp_session->fir_countdown--; } + if (rtp_session->pli_countdown) { + + if (rtp_session->pli_countdown == PLI_COUNTDOWN || (rtp_session->pli_countdown == PLI_COUNTDOWN / 2) || rtp_session->pli_countdown == 1) { + send_pli(rtp_session); + } + + rtp_session->pli_countdown--; + } + if (rtp_session->flags[SWITCH_RTP_FLAG_AUTO_CNG] && rtp_session->send_msg.header.ts && rtp_session->cng_pt && rtp_session->timer.samplecount >= (rtp_session->last_write_samplecount + (rtp_session->samples_per_interval * 60))) { uint8_t data[10] = { 0 }; @@ -2059,17 +2070,16 @@ static int check_rtcp_and_ice(switch_rtp_t *rtp_session) rtcp_ok = 0; } + if (rtp_session->rtcp_sock_output && rtp_session->flags[SWITCH_RTP_FLAG_ENABLE_RTCP] && - !rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU] && - (rtp_session->timer.samplecount - rtp_session->stats.rtcp.last_rpt_ts >= rtp_session->samples_per_second * rtp_session->rtcp_send_rate) ) { - + !rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU] && (now - rtp_session->rtcp_last_sent) > rtp_session->rtcp_send_rate * 1000000) { switch_rtcp_numbers_t * stats = &rtp_session->stats.rtcp; struct switch_rtcp_receiver_report *rr; struct switch_rtcp_sender_report *sr; struct switch_rtcp_report_block *rtcp_report_block; - switch_size_t rtcp_bytes = sizeof(struct switch_rtcp_hdr_s)+sizeof(uint32_t); /* add size of the packet header and the ssrc */ + rtp_session->rtcp_last_sent = now; rtp_session->rtcp_send_msg.header.version = 2; rtp_session->rtcp_send_msg.header.p = 0; rtp_session->rtcp_send_msg.header.count = 1; @@ -3548,6 +3558,7 @@ SWITCH_DECLARE(switch_status_t) switch_rtp_create(switch_rtp_t **new_rtp_session rtp_session->recv_msg.header.cc = 0; rtp_session->payload = payload; + rtp_session->rtcp_last_sent = switch_micro_time_now(); switch_rtp_set_interval(rtp_session, ms_per_packet, samples_per_interval); rtp_session->conf_samples_per_interval = samples_per_interval; @@ -3790,7 +3801,8 @@ static void jb_callback(stfu_instance_t *i, void *udata) SWITCH_DECLARE(switch_timer_t *) switch_rtp_get_media_timer(switch_rtp_t *rtp_session) { - if (rtp_session->timer.timer_interface) { + + if (rtp_session && rtp_session->timer.timer_interface) { if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO]) { switch_core_timer_sync(&rtp_session->timer); } @@ -4066,13 +4078,23 @@ SWITCH_DECLARE(void) switch_rtp_flush(switch_rtp_t *rtp_session) SWITCH_DECLARE(void) switch_rtp_video_refresh(switch_rtp_t *rtp_session) { if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && - (rtp_session->ice.ice_user || rtp_session->flags[SWITCH_RTP_FLAG_FIR] || rtp_session->flags[SWITCH_RTP_FLAG_PLI])) { + (rtp_session->ice.ice_user || rtp_session->flags[SWITCH_RTP_FLAG_FIR])) { if (!rtp_session->fir_countdown) { rtp_session->fir_countdown = FIR_COUNTDOWN; } } } +SWITCH_DECLARE(void) switch_rtp_video_loss(switch_rtp_t *rtp_session) +{ + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && + (rtp_session->ice.ice_user || rtp_session->flags[SWITCH_RTP_FLAG_PLI])) { + if (!rtp_session->pli_countdown) { + rtp_session->pli_countdown = PLI_COUNTDOWN; + } + } +} + SWITCH_DECLARE(void) switch_rtp_break(switch_rtp_t *rtp_session) { if (!switch_rtp_ready(rtp_session)) { @@ -5159,12 +5181,29 @@ static switch_status_t read_rtp_packet(switch_rtp_t *rtp_session, switch_size_t static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_size_t *bytes) { switch_status_t status = SWITCH_STATUS_FALSE; - switch_core_session_t *session = switch_core_memory_pool_get_data(rtp_session->pool, "__session"); - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG10,"Received an RTCP packet of length %" SWITCH_SIZE_T_FMT " bytes\n", *bytes); - if (rtp_session->rtcp_recv_msg.header.version == 2) { - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG10,"RTCP packet type is %d\n", rtp_session->rtcp_recv_msg.header.type); - if (rtp_session->rtcp_recv_msg.header.type == 200 || rtp_session->rtcp_recv_msg.header.type == 201) { + if (rtp_session->rtcp_recv_msg_p->header.version == 2) { + + + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, + "RTCP packet bytes %" SWITCH_SIZE_T_FMT " type %d\n", *bytes, rtp_session->rtcp_recv_msg_p->header.type); + + + + if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && *bytes > 94) { + //(rtp_session->rtcp_recv_msg_p->header.type == 205 || //RTPFB + //rtp_session->rtcp_recv_msg_p->header.type == 206)) {//PSFB + + rtcp_ext_msg_t *extp = (rtcp_ext_msg_t *) rtp_session->rtcp_recv_msg_p; + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "PICKED UP XRTCP type: %d fmt: %d\n", + rtp_session->rtcp_recv_msg_p->header.type, extp->header.fmt); + + if ((extp->header.fmt == 4) || (extp->header.fmt == 1)) { /* FIR || PLI */ + switch_core_media_gen_key_frame(rtp_session->session); + } + } else + + if (rtp_session->rtcp_recv_msg_p->header.type == 200 || rtp_session->rtcp_recv_msg_p->header.type == 201) { struct switch_rtcp_report_block *report_block; switch_time_t now; switch_time_exp_t now_hr; @@ -5177,8 +5216,9 @@ static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_siz ntp_usec = (uint32_t)(now - (sec*1000000)); /* micro seconds */ lsr_now = (uint32_t)(ntp_usec*0.065536) | (ntp_sec&0x0000ffff)<<16; // 0.065536 is used for convertion from useconds - if (rtp_session->rtcp_recv_msg.header.type == 200) { /* Sender report */ - struct switch_rtcp_sender_report* sr = (struct switch_rtcp_sender_report*)rtp_session->rtcp_recv_msg.body; + if (rtp_session->rtcp_recv_msg_p->header.type == 200) { /* Sender report */ + struct switch_rtcp_sender_report* sr = (struct switch_rtcp_sender_report*)rtp_session->rtcp_recv_msg_p->body; + report_block = &sr->report_block; rtp_session->stats.rtcp.packet_count += ntohl(sr->sender_info.pc); rtp_session->stats.rtcp.octet_count += ntohl(sr->sender_info.oc); @@ -5187,7 +5227,7 @@ static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_siz lsr = (ntohl(sr->sender_info.ntp_lsw)&0xffff0000)>>16 | (ntohl(sr->sender_info.ntp_msw)&0x0000ffff)<<16; /* The middle 32 bits out of 64 in the NTP timestamp */ rtp_session->stats.rtcp.last_recv_lsr_peer = htonl(lsr); /* Save it include it in the next SR */ rtp_session->stats.rtcp.last_recv_lsr_local = lsr_now; /* Save it to calculate DLSR when generating next SR */ - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_DEBUG10,"Received a SR with %d report blocks, " \ + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG10,"Received a SR with %d report blocks, " \ "length in words = %d, " \ "SSRC = 0x%X, " \ "NTP MSW = %u, " \ @@ -5195,8 +5235,8 @@ static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_siz "RTP timestamp = %u, " \ "Sender Packet Count = %u, " \ "Sender Octet Count = %u\n", - rtp_session->rtcp_recv_msg.header.count, - ntohs((uint16_t)rtp_session->rtcp_recv_msg.header.length), + rtp_session->rtcp_recv_msg_p->header.count, + ntohs((uint16_t)rtp_session->rtcp_recv_msg_p->header.length), ntohl(sr->ssrc), ntohl(sr->sender_info.ntp_msw), ntohl(sr->sender_info.ntp_lsw), @@ -5204,7 +5244,7 @@ static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_siz ntohl(sr->sender_info.pc), ntohl(sr->sender_info.oc)); } else { /* Receiver report */ - struct switch_rtcp_receiver_report* rr = (struct switch_rtcp_receiver_report*)rtp_session->rtcp_recv_msg.body; + struct switch_rtcp_receiver_report* rr = (struct switch_rtcp_receiver_report*)rtp_session->rtcp_recv_msg_p->body; report_block = &rr->report_block; packet_ssrc = rr->ssrc; } @@ -5213,12 +5253,12 @@ static switch_status_t process_rtcp_packet(switch_rtp_t *rtp_session, switch_siz if (report_block->lsr && !rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU]) { switch_time_exp_gmt(&now_hr,now); /* Calculating RTT = A - DLSR - LSR */ - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(session), SWITCH_LOG_NOTICE, - "Receiving an RTCP packet[%04d-%02d-%02d %02d:%02d:%02d.%d] SSRC[%u]" - "RTT[%f] A[%u] - DLSR[%u] - LSR[%u]\n", - 1900 + now_hr.tm_year, now_hr.tm_mday, now_hr.tm_mon, now_hr.tm_hour, now_hr.tm_min, now_hr.tm_sec, now_hr.tm_usec, - ntohl(packet_ssrc), (double)(lsr_now - ntohl(report_block->dlsr) - ntohl(report_block->lsr))/65536, - lsr_now, ntohl(report_block->dlsr), ntohl(report_block->lsr)); + switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, + "Receiving an RTCP packet\n[%04d-%02d-%02d %02d:%02d:%02d.%d] SSRC[%u]\n" + "RTT[%f] A[%u] - DLSR[%u] - LSR[%u]\n", + 1900 + now_hr.tm_year, now_hr.tm_mday, now_hr.tm_mon, now_hr.tm_hour, now_hr.tm_min, now_hr.tm_sec, now_hr.tm_usec, + ntohl(packet_ssrc), (double)(lsr_now - ntohl(report_block->dlsr) - ntohl(report_block->lsr))/65536, + lsr_now, ntohl(report_block->dlsr), ntohl(report_block->lsr)); } rtp_session->rtcp_fresh_frame = 1; @@ -5673,26 +5713,6 @@ static int rtp_common_read(switch_rtp_t *rtp_session, switch_payload_t *payload_ if (rtcp_status == SWITCH_STATUS_SUCCESS) { switch_rtp_reset_media_timer(rtp_session); - if (rtp_session->flags[SWITCH_RTP_FLAG_VIDEO] && (rtp_session->rtcp_recv_msg_p->header.type == 205 || //RTPFB - rtp_session->rtcp_recv_msg_p->header.type == 206)) {//PSFB - rtcp_ext_msg_t *extp = (rtcp_ext_msg_t *) rtp_session->rtcp_recv_msg_p; - - switch_log_printf(SWITCH_CHANNEL_SESSION_LOG(rtp_session->session), SWITCH_LOG_DEBUG, "PICKED UP XRTCP type: %d fmt: %d\n", - rtp_session->rtcp_recv_msg_p->header.type, extp->header.fmt); - - if ((extp->header.fmt == 4) || (extp->header.fmt == 1)) { /* FIR || PLI */ - - switch_core_media_codec_control(rtp_session->session, - SWITCH_MEDIA_TYPE_VIDEO, - SWITCH_IO_WRITE, - SCC_VIDEO_REFRESH, - SCCT_NONE, - NULL, - NULL, - NULL); - } - } - if (rtp_session->flags[SWITCH_RTP_FLAG_RTCP_PASSTHRU]) { switch_channel_t *channel = switch_core_session_get_channel(rtp_session->session); const char *uuid = switch_channel_get_partner_uuid(channel); @@ -6235,14 +6255,14 @@ SWITCH_DECLARE(switch_status_t) switch_rtcp_zerocopy_read_frame(switch_rtp_t *rt /* A fresh frame has been found! */ if (rtp_session->rtcp_fresh_frame) { - struct switch_rtcp_sender_report* sr = (struct switch_rtcp_sender_report*)rtp_session->rtcp_recv_msg.body; + struct switch_rtcp_sender_report* sr = (struct switch_rtcp_sender_report*)rtp_session->rtcp_recv_msg_p->body; int i = 0; /* turn the flag off! */ rtp_session->rtcp_fresh_frame = 0; frame->ssrc = ntohl(sr->ssrc); - frame->packet_type = (uint16_t)rtp_session->rtcp_recv_msg.header.type; + frame->packet_type = (uint16_t)rtp_session->rtcp_recv_msg_p->header.type; frame->ntp_msw = ntohl(sr->sender_info.ntp_msw); frame->ntp_lsw = ntohl(sr->sender_info.ntp_lsw); frame->timestamp = ntohl(sr->sender_info.ts); @@ -6787,10 +6807,9 @@ static int rtp_common_write(switch_rtp_t *rtp_session, if (rtp_session->flags[SWITCH_RTP_FLAG_USE_TIMER]) { rtp_session->last_write_samplecount = rtp_session->timer.samplecount; - } else { - rtp_session->last_write_timestamp = switch_micro_time_now(); } - + + rtp_session->last_write_timestamp = switch_micro_time_now(); } ret = (int) bytes;