From 416745e2cd8b6ab44c08488fb85b1114f66a7f2f Mon Sep 17 00:00:00 2001 From: Seven Du Date: Fri, 10 Jan 2014 12:00:01 +0800 Subject: [PATCH] FS-7500: move video bridge logic to switch_core_media so possible to do transcoding --- src/include/switch_types.h | 1 + src/switch_core_media.c | 125 ++++++++++++++++++++++++++++++++++++- src/switch_ivr_bridge.c | 15 +++++ 3 files changed, 140 insertions(+), 1 deletion(-) diff --git a/src/include/switch_types.h b/src/include/switch_types.h index adedb0b1be..2b19038907 100644 --- a/src/include/switch_types.h +++ b/src/include/switch_types.h @@ -1446,6 +1446,7 @@ typedef enum { CF_VIDEO_BREAK, CF_AUDIO_PAUSE, CF_VIDEO_PAUSE, + CF_VIDEO_BRIDGE, CF_BYPASS_MEDIA_AFTER_HOLD, CF_HANGUP_HELD, CF_CONFERENCE_RESET_MEDIA, diff --git a/src/switch_core_media.c b/src/switch_core_media.c index 4128b50f69..f4f970ed4a 100644 --- a/src/switch_core_media.c +++ b/src/switch_core_media.c @@ -4344,6 +4344,121 @@ SWITCH_DECLARE(int) switch_core_media_toggle_hold(switch_core_session_t *session return changed; } +#define BUF_SIZE (352 * 288 * 3 / 2 * 4) // big enough for 4CIF, looks like C doesn't like huge array +#define FPS 15 +#define WIDTH 352 +#define HEIGHT 288 +#define SIZE WIDTH * HEIGHT + +static switch_status_t video_bridge_callback(switch_core_session_t *session, switch_bool_t video_transcoding, uint32_t *ts) +{ + switch_frame_t *read_frame; + switch_status_t status = SWITCH_STATUS_SUCCESS; + switch_core_session_t *session_b = switch_channel_get_private(session->channel, "_video_bridged_session_"); + switch_codec_t *codec, *other_codec; + + status = switch_core_session_read_video_frame(session, &read_frame, SWITCH_IO_FLAG_NONE, 0); + + if (!SWITCH_READ_ACCEPTABLE(status)) { + switch_cond_next(); + return status; + } + + if (switch_channel_test_flag(session->channel, CF_VIDEO_REFRESH_REQ)) { + switch_core_session_refresh_video(session); + switch_channel_clear_flag(session->channel, CF_VIDEO_REFRESH_REQ); + } + + if (switch_test_flag(read_frame, SFF_CNG)) { + return status; + } + + if (!session_b || !switch_channel_up_nosig(session_b->channel)) { // echo and return + switch_core_session_write_video_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0); + return status; + } + + codec = read_frame->codec; + other_codec = session_b->video_write_codec; + + if (codec->implementation->impl_id == other_codec->implementation->impl_id) { + switch_core_session_write_video_frame(session_b, read_frame, SWITCH_IO_FLAG_NONE, 0); + return status; + } + + if (!video_transcoding) { // echo back + switch_core_session_write_video_frame(session, read_frame, SWITCH_IO_FLAG_NONE, 0); + return status; + } else { + uint8_t raw_buff[BUF_SIZE]; + uint8_t rtp_buff[1500] = { 0 }; + uint32_t decoded_rate = 0; + uint32_t decoded_data_len = BUF_SIZE; + uint32_t flag = 0; + uint32_t encoded_data_len = 1500; + uint32_t encoded_rate = 0; + switch_frame_t write_frame; + +#if 0 + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%d/%s != %d/%s need transcoding!!!\n", + codec->implementation->impl_id, codec->implementation->iananame, + other_codec->implementation->impl_id, other_codec->implementation->iananame); +#endif + // uncomment to test only one leg + // if (!strcmp(codec->implementation->iananame, "VP8")) return status; + // if (!strcmp(codec->implementation->iananame, "H264")) return status; + + codec->cur_frame = read_frame; + switch_core_codec_decode(codec, NULL, read_frame->data, read_frame->datalen, 0, raw_buff, &decoded_data_len, &decoded_rate, &flag); + + if (decoded_data_len < 3) return SWITCH_STATUS_SUCCESS; + + decoded_data_len = 152064; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "decoded_data_len: %d %s\n", decoded_data_len, codec->implementation->iananame); + + write_frame.packet = rtp_buff; + write_frame.data = rtp_buff + 12; + + encoded_data_len = 1500; + switch_core_codec_encode(other_codec, NULL, raw_buff, decoded_data_len, 0, rtp_buff+12, &encoded_data_len, &encoded_rate, &flag); + + while(encoded_data_len) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "encoded: %s [%d] flag=%d ts=%u\n", other_codec->implementation->iananame, encoded_data_len, flag, *ts); + + write_frame.datalen = encoded_data_len; + write_frame.packetlen = write_frame.datalen + 12; + write_frame.m = flag; + write_frame.timestamp = *ts; + if (write_frame.m) *ts += 90000 / FPS; + + if (1) { + /* set correct mark and ts */ + switch_rtp_hdr_t *rtp = (switch_rtp_hdr_t *)write_frame.packet; + + memset(rtp, 0, 12); + rtp->version = 2; + rtp->m = write_frame.m; + rtp->ts = htonl(write_frame.timestamp); + rtp->ssrc = (uint32_t) ((intptr_t) rtp + (uint32_t) switch_epoch_time_now(NULL)); + + switch_set_flag(&write_frame, SFF_RAW_RTP); + } + + // switch_rtp_set_flag(session_b->media_handle->engines[SWITCH_MEDIA_TYPE_VIDEO].rtp_session, SWITCH_RTP_FLAG_DEBUG_RTP_WRITE); + switch_rtp_set_flag(session_b->media_handle->engines[SWITCH_MEDIA_TYPE_VIDEO].rtp_session, SWITCH_RTP_FLAG_RAW_WRITE); + + switch_core_session_write_video_frame(session_b, &write_frame, SWITCH_IO_FLAG_NONE, 0); + + encoded_data_len = 1500; + switch_core_codec_encode(other_codec, NULL, NULL, 0, 0, rtp_buff+12, &encoded_data_len, &encoded_rate, &flag); + } + } + + return status; +} + + static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, void *obj) { struct media_helper *mh = obj; @@ -4352,6 +4467,8 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi switch_status_t status; switch_frame_t *read_frame; switch_media_handle_t *smh; + uint32_t rtp_ts = 0; + switch_bool_t video_transcoding = SWITCH_FALSE; if (!(smh = session->media_handle)) { return NULL; @@ -4365,6 +4482,7 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s Video thread started. Echo is %s\n", switch_channel_get_name(session->channel), switch_channel_test_flag(channel, CF_VIDEO_ECHO) ? "on" : "off"); switch_core_session_refresh_video(session); + video_transcoding = switch_true(switch_channel_get_variable(channel, "video_transcoding")); while (switch_channel_up_nosig(channel)) { @@ -4375,6 +4493,7 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "%s Video thread resumed Echo is %s\n", switch_channel_get_name(session->channel), switch_channel_test_flag(channel, CF_VIDEO_ECHO) ? "on" : "off"); switch_core_session_refresh_video(session); + video_transcoding = switch_true(switch_channel_get_variable(channel, "video_transcoding")); } if (switch_channel_test_flag(channel, CF_VIDEO_PASSIVE)) { @@ -4386,7 +4505,11 @@ static void *SWITCH_THREAD_FUNC video_helper_thread(switch_thread_t *thread, voi continue; } - + if (switch_channel_test_flag(channel, CF_VIDEO_BRIDGE)) { + video_bridge_callback(session, video_transcoding, &rtp_ts); + continue; + } + status = switch_core_session_read_video_frame(session, &read_frame, SWITCH_IO_FLAG_NONE, 0); if (!SWITCH_READ_ACCEPTABLE(status)) { diff --git a/src/switch_ivr_bridge.c b/src/switch_ivr_bridge.c index 360223c30f..51f0d365b2 100644 --- a/src/switch_ivr_bridge.c +++ b/src/switch_ivr_bridge.c @@ -46,6 +46,7 @@ struct vid_helper { int up; }; +#if 0 static void *SWITCH_THREAD_FUNC video_bridge_thread(switch_thread_t *thread, void *obj) { struct vid_helper *vh = obj; @@ -108,9 +109,17 @@ static void *SWITCH_THREAD_FUNC video_bridge_thread(switch_thread_t *thread, voi vh->up = 0; return NULL; } +#endif static switch_thread_t *launch_video(struct vid_helper *vh) { + switch_channel_t *channel = switch_core_session_get_channel(vh->session_a); + + switch_channel_set_private(channel, "_video_bridged_session_", vh->session_b); + switch_channel_set_flag(channel, CF_VIDEO_BRIDGE); + return NULL; + +#if 0 switch_thread_t *thread; switch_threadattr_t *thd_attr = NULL; @@ -118,6 +127,8 @@ static switch_thread_t *launch_video(struct vid_helper *vh) switch_threadattr_stacksize_set(thd_attr, SWITCH_THREAD_STACKSIZE); switch_thread_create(&thread, thd_attr, video_bridge_thread, vh, switch_core_session_get_pool(vh->session_a)); return thread; +#endif + } #endif @@ -590,6 +601,10 @@ static void *audio_bridge_thread(switch_thread_t *thread, void *obj) end_of_bridge_loop: #ifdef SWITCH_VIDEO_IN_THREADS + + switch_channel_clear_flag(chan_a, CF_VIDEO_BRIDGE); + switch_channel_clear_flag(chan_b, CF_VIDEO_BRIDGE); + if (vid_thread) { vh.up = -1; switch_channel_set_flag(chan_a, CF_NOT_READY);