From 13c99e938b1dee5e1b166716cee9d99cc0915708 Mon Sep 17 00:00:00 2001 From: Seven Du Date: Thu, 4 Feb 2016 08:30:00 +0800 Subject: [PATCH 1/2] FS-8406 #comment add options to scale down cavas size, fps and bandwidth --- .../mod_conference/conference_video.c | 34 +++++++++++++++++++ .../mod_conference/mod_conference.c | 34 ++++++++++++++++++- .../mod_conference/mod_conference.h | 9 +++++ 3 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/mod/applications/mod_conference/conference_video.c b/src/mod/applications/mod_conference/conference_video.c index e4b9ff5b01..cb64053aaf 100644 --- a/src/mod/applications/mod_conference/conference_video.c +++ b/src/mod/applications/mod_conference/conference_video.c @@ -1233,6 +1233,7 @@ void conference_video_write_canvas_image_to_codec_group(conference_obj_t *confer conference_member_t *imember; switch_frame_t write_frame = { 0 }, *frame = NULL; switch_status_t encode_status = SWITCH_STATUS_FALSE; + switch_image_t *scaled_img = codec_set->scaled_img; write_frame = codec_set->frame; frame = &write_frame; @@ -1254,6 +1255,16 @@ void conference_video_write_canvas_image_to_codec_group(conference_obj_t *confer switch_core_codec_control(&codec_set->codec, SCC_VIDEO_GEN_KEYFRAME, SCCT_NONE, NULL, SCCT_NONE, NULL, NULL, NULL); } + if (scaled_img) { + if (!send_keyframe && codec_set->fps_divisor > 1 && (codec_set->frame_count++) % codec_set->fps_divisor) { + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Skip one frame, total: %d\n", codec_set->frame_count); + return; + } + + switch_img_scale(frame->img, &scaled_img, scaled_img->d_w, scaled_img->d_h); + frame->img = scaled_img; + } + do { frame->data = ((unsigned char *)frame->packet) + 12; @@ -2320,6 +2331,28 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr write_codecs[i]->frame.data = ((uint8_t *)write_codecs[i]->frame.packet) + 12; write_codecs[i]->frame.packetlen = buflen; write_codecs[i]->frame.buflen = buflen - 12; + if (conference->scale_h264_canvas_width > 0 && conference->scale_h264_canvas_height > 0 && !strcmp(check_codec->implementation->iananame, "H264")) { + int32_t bw = -1; + + write_codecs[i]->fps_divisor = conference->scale_h264_canvas_fps_divisor; + write_codecs[i]->scaled_img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, conference->scale_h264_canvas_width, conference->scale_h264_canvas_height, 16); + + if (conference->scale_h264_canvas_bandwidth) { + if (strcasecmp(conference->scale_h264_canvas_bandwidth, "auto")) { + bw = switch_parse_bandwidth_string(conference->scale_h264_canvas_bandwidth); + } + } + + if (bw == -1) { + float fps = conference->video_fps.fps; + + if (write_codecs[i]->fps_divisor) fps /= write_codecs[i]->fps_divisor; + + bw = switch_calc_bitrate(conference->scale_h264_canvas_width, conference->scale_h264_canvas_height, conference->video_quality, fps); + } + + switch_core_codec_control(&write_codecs[i]->codec, SCC_VIDEO_BANDWIDTH, SCCT_INT, &bw, SCCT_NONE, NULL, NULL, NULL); + } switch_set_flag((&write_codecs[i]->frame), SFF_RAW_RTP); } @@ -2984,6 +3017,7 @@ pp conference_video_set_incoming_bitrate(imember, kps, SWITCH_TRUE); for (i = 0; i < MAX_MUX_CODECS; i++) { if (write_codecs[i] && switch_core_codec_ready(&write_codecs[i]->codec)) { switch_core_codec_destroy(&write_codecs[i]->codec); + switch_img_free(&(write_codecs[i]->scaled_img)); } } diff --git a/src/mod/applications/mod_conference/mod_conference.c b/src/mod/applications/mod_conference/mod_conference.c index 1e9902a048..791b722ce4 100644 --- a/src/mod/applications/mod_conference/mod_conference.c +++ b/src/mod/applications/mod_conference/mod_conference.c @@ -2411,7 +2411,12 @@ conference_obj_t *conference_new(char *name, conference_xml_cfg_t cfg, switch_co const char *force_rate = NULL, *force_interval = NULL, *force_channels = NULL, *presence_id = NULL; uint32_t force_rate_i = 0, force_interval_i = 0, force_channels_i = 0, video_auto_floor_msec = 0; switch_event_t *event; - + + int scale_h264_canvas_width = 0; + int scale_h264_canvas_height = 0; + int scale_h264_canvas_fps_divisor = 0; + char *scale_h264_canvas_bandwidth = NULL; + /* Validate the conference name */ if (zstr(name)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid Record! no name.\n"); @@ -2720,6 +2725,28 @@ conference_obj_t *conference_new(char *name, conference_xml_cfg_t cfg, switch_co } else { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "video-mode invalid, valid settings are 'passthrough', 'transcode' and 'mux'\n"); } + } else if (!strcasecmp(var, "scale-h264-canvas-size") && !zstr(val)) { + char *p; + + if ((scale_h264_canvas_width = atoi(val))) { + if ((p = strchr(val, 'x'))) { + p++; + if (*p) { + scale_h264_canvas_height = atoi(p); + } + } + } + + if (scale_h264_canvas_width < 320 || scale_h264_canvas_width < 180) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Invalid scale-h264-canvas-size, falling back to 320x180\n"); + scale_h264_canvas_width = 320; + scale_h264_canvas_width = 180; + } + } else if (!strcasecmp(var, "scale-h264-canvas-fps-divisor") && !zstr(val)) { + scale_h264_canvas_fps_divisor = atoi(val); + if (scale_h264_canvas_fps_divisor < 0) scale_h264_canvas_fps_divisor = 0; + } else if (!strcasecmp(var, "scale-h264-canvas-bandwidth") && !zstr(val)) { + scale_h264_canvas_bandwidth = val; } } @@ -2783,6 +2810,11 @@ conference_obj_t *conference_new(char *name, conference_xml_cfg_t cfg, switch_co conference->conference_video_mode = conference_video_mode; + conference->scale_h264_canvas_width = scale_h264_canvas_width; + conference->scale_h264_canvas_height = scale_h264_canvas_height; + conference->scale_h264_canvas_fps_divisor = scale_h264_canvas_fps_divisor; + conference->scale_h264_canvas_bandwidth = switch_core_strdup(conference->pool, scale_h264_canvas_bandwidth); + if (!switch_core_has_video() && (conference->conference_video_mode == CONF_VIDEO_MODE_MUX || conference->conference_video_mode == CONF_VIDEO_MODE_TRANSCODE)) { conference->conference_video_mode = CONF_VIDEO_MODE_PASSTHROUGH; switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "video-mode invalid, only valid setting is 'passthrough' due to no video capabilities\n"); diff --git a/src/mod/applications/mod_conference/mod_conference.h b/src/mod/applications/mod_conference/mod_conference.h index a32510254a..c7407391fd 100644 --- a/src/mod/applications/mod_conference/mod_conference.h +++ b/src/mod/applications/mod_conference/mod_conference.h @@ -663,6 +663,12 @@ typedef struct conference_obj { video_layout_t *new_personal_vlayout; int max_bw_in; int force_bw_in; + + /* special use case, scalling shared h264 canvas*/ + int scale_h264_canvas_width; + int scale_h264_canvas_height; + int scale_h264_canvas_fps_divisor; + char *scale_h264_canvas_bandwidth; } conference_obj_t; /* Relationship with another member */ @@ -794,6 +800,9 @@ typedef struct codec_set_s { switch_codec_t codec; switch_frame_t frame; uint8_t *packet; + switch_image_t *scaled_img; + uint8_t fps_divisor; + uint32_t frame_count; } codec_set_t; typedef void (*conference_key_callback_t) (conference_member_t *, struct caller_control_actions *); From 3d90d752fca16016d4a3556cb02bf9db837feffa Mon Sep 17 00:00:00 2001 From: Seven Du Date: Thu, 4 Feb 2016 08:32:06 +0800 Subject: [PATCH 2/2] FS-8406 #resolve #comment improvement to drop video packets on slow rtmp link --- src/mod/endpoints/mod_rtmp/mod_rtmp.c | 1 - src/mod/endpoints/mod_rtmp/mod_rtmp.h | 1 + src/mod/endpoints/mod_rtmp/rtmp.c | 188 ++++++++++++++++++------ src/mod/endpoints/mod_rtmp/rtmp_tcp.c | 4 + src/mod/endpoints/mod_rtmp/rtmp_video.c | 7 +- 5 files changed, 152 insertions(+), 49 deletions(-) diff --git a/src/mod/endpoints/mod_rtmp/mod_rtmp.c b/src/mod/endpoints/mod_rtmp/mod_rtmp.c index a838fa3423..715e13dc95 100644 --- a/src/mod/endpoints/mod_rtmp/mod_rtmp.c +++ b/src/mod/endpoints/mod_rtmp/mod_rtmp.c @@ -911,7 +911,6 @@ switch_status_t rtmp_session_request(rtmp_profile_t *profile, rtmp_session_t **n { char buf[1024]; #ifndef _WIN32 -#else snprintf(buf, sizeof(buf), "/tmp/rtmp-%s-in.txt", (*newsession)->uuid); (*newsession)->io_debug_in = fopen(buf, "w"); snprintf(buf, sizeof(buf), "/tmp/rtmp-%s-out.txt", (*newsession)->uuid); diff --git a/src/mod/endpoints/mod_rtmp/mod_rtmp.h b/src/mod/endpoints/mod_rtmp/mod_rtmp.h index d75410c315..4edfb8dc45 100644 --- a/src/mod/endpoints/mod_rtmp/mod_rtmp.h +++ b/src/mod/endpoints/mod_rtmp/mod_rtmp.h @@ -515,6 +515,7 @@ struct rtmp_session { uint32_t media_streamid; /* < The stream id that was used for the last "play" command, where we should send media */ switch_size_t dropped_video_frame; + switch_queue_t *video_send_queue; uint8_t media_debug; }; diff --git a/src/mod/endpoints/mod_rtmp/rtmp.c b/src/mod/endpoints/mod_rtmp/rtmp.c index 8c132634e7..285bc81103 100644 --- a/src/mod/endpoints/mod_rtmp/rtmp.c +++ b/src/mod/endpoints/mod_rtmp/rtmp.c @@ -41,6 +41,17 @@ typedef struct { size_t len; } buffer_helper_t; +typedef struct { + uint8_t amfnumber; + uint32_t timestamp; + uint8_t type; + uint32_t stream_id; + switch_size_t len; + uint32_t flags; + unsigned char *message; +} video_send_buffer_t; + + size_t my_buffer_read(void * out_buffer, size_t size, void * user_data) { buffer_helper_t *helper = (buffer_helper_t*)user_data; @@ -561,8 +572,62 @@ switch_status_t rtmp_send_invoke_v(rtmp_session_t *rsession, uint8_t amfnumber, return rtmp_send_message(rsession, amfnumber, timestamp, type, stream_id, buf, helper.pos, 0); } +static int flush_video_send_queue(rtmp_session_t *rsession, switch_bool_t lock) +{ + video_send_buffer_t *b; + void *pop; + switch_queue_t *q = rsession->video_send_queue; + int x = 0; + + if (!q) return 0; + + if (lock) switch_mutex_lock(rsession->socket_mutex); + while (switch_queue_size(q) > 0 && switch_queue_trypop(q, &pop) == SWITCH_STATUS_SUCCESS && pop) { + b = (video_send_buffer_t *)pop; + free(b->message); + free(b); + x++; + } + if (lock) switch_mutex_unlock(rsession->socket_mutex); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Dropped %d Video Frames\n", x); + + return x; +} + +static void buffer_video_send(rtmp_session_t *rsession, uint8_t amfnumber, uint32_t timestamp, uint8_t type, uint32_t stream_id, const unsigned char *message, switch_size_t len, uint32_t flags) +{ + video_send_buffer_t *vbuf; + + switch_mutex_lock(rsession->socket_mutex); + + if (!rsession->video_send_queue) { + switch_queue_create(&rsession->video_send_queue, 1000, rsession->pool); + } + + if (*message == 0x17) { + flush_video_send_queue(rsession, SWITCH_FALSE); + } + + vbuf = malloc(sizeof(video_send_buffer_t)); + switch_assert(vbuf); + + vbuf->amfnumber = amfnumber; + vbuf->timestamp = timestamp; + vbuf->type = type; + vbuf->stream_id = stream_id; + vbuf->len = len; + vbuf->flags = flags; + vbuf->message = malloc(len); + switch_assert(vbuf->message); + + memcpy(vbuf->message, message, len); + + switch_queue_push(rsession->video_send_queue, (void *)vbuf); + switch_mutex_unlock(rsession->socket_mutex); +} + /* Break message down into 128 bytes chunks, add the appropriate headers and send it out */ -switch_status_t rtmp_send_message(rtmp_session_t *rsession, uint8_t amfnumber, uint32_t timestamp, uint8_t type, uint32_t stream_id, const unsigned char *message, switch_size_t len, uint32_t flags) +switch_status_t _rtmp_send_message(rtmp_session_t *rsession, uint8_t amfnumber, uint32_t timestamp, uint8_t type, uint32_t stream_id, const unsigned char *message, switch_size_t len, uint32_t flags) { switch_size_t pos = 0; uint8_t header[12] = { amfnumber & 0x3F, INT24(0), INT24(len), type, INT32_LE(stream_id) }; @@ -575,52 +640,6 @@ switch_status_t rtmp_send_message(rtmp_session_t *rsession, uint8_t amfnumber, u // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%d send_ack=%d send=%d window=%d wait_ack=%d\n", // type, rsession->send_ack, rsession->send, rsession->send_ack_window, rsession->send + 3073 - rsession->send_ack); - if (type == RTMP_TYPE_VIDEO) { - uint32_t window = rsession->send_ack_window; - - if (rsession->media_debug & RTMP_MD_VIDEO_WRITE) { - switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "W V ts:%u data:0x%02x len:%" SWITCH_SIZE_T_FMT "\n", timestamp, *message, len); - } - - /* start to drop video frame on window/2 if the frame is a non-IDR video frame - start to drop video frame on window * 3/4 if the frame is a IDR frame - start to drop audio frame on widnow full - */ - - if (*message == 0x17) { - window = window / 4 * 3; - } else { - window /= 2; - } - - if ((rsession->send_ack + window) < (rsession->send + 3073)) { - /* We're sending too fast, drop the frame */ - rsession->dropped_video_frame++; - switch_log_printf(SWITCH_CHANNEL_UUID_LOG(rsession->uuid), SWITCH_LOG_DEBUG, - "DROP VIDEO FRAME [amfnumber=%d type=0x%x stream_id=0x%x ftype=0x%x] len=%"SWITCH_SIZE_T_FMT - " dropped=%"SWITCH_SIZE_T_FMT"\n", - amfnumber, type, stream_id, *message, len, rsession->dropped_video_frame); - return SWITCH_STATUS_SUCCESS; - } - - if (rsession->dropped_video_frame) { - if (*message != 0x17) { - rsession->dropped_video_frame++; - switch_log_printf(SWITCH_CHANNEL_UUID_LOG(rsession->uuid), SWITCH_LOG_DEBUG, - "DROP VIDEO FRAME [amfnumber=%d type=0x%x stream_id=0x%x ftype=0x%x] len=%"SWITCH_SIZE_T_FMT - " dropped=%"SWITCH_SIZE_T_FMT" waiting for the next IDR\n", - amfnumber, type, stream_id, *message, len, rsession->dropped_video_frame); - - return SWITCH_STATUS_SUCCESS; - } else { - switch_log_printf(SWITCH_CHANNEL_UUID_LOG(rsession->uuid), SWITCH_LOG_INFO, - "Got IDR frame after %"SWITCH_SIZE_T_FMT" frame(s) dropped\n", - rsession->dropped_video_frame); - rsession->dropped_video_frame = 0; - } - } - } - if (type == RTMP_TYPE_AUDIO && (rsession->media_debug & RTMP_MD_AUDIO_WRITE)) { switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "W A ts:%u data:0x%02x len:%" SWITCH_SIZE_T_FMT "\n", timestamp, *message, len); } @@ -696,6 +715,8 @@ switch_status_t rtmp_send_message(rtmp_session_t *rsession, uint8_t amfnumber, u header[3] = timestamp & 0xFF; } + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "=== send type: %d ts: %d bytes: %zu\n", type, timestamp, len); + state->ts = timestamp; state->type = type; state->origlen = len; @@ -740,6 +761,79 @@ end: return status; } +switch_status_t rtmp_send_message(rtmp_session_t *rsession, uint8_t amfnumber, uint32_t timestamp, uint8_t type, uint32_t stream_id, const unsigned char *message, switch_size_t len, uint32_t flags) +{ + switch_status_t status = SWITCH_STATUS_SUCCESS; + int window = rsession->send_ack_window; + + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%d send_ack=%d send=%d window=%d wait_ack=%d\n", + // type, rsession->send_ack, rsession->send, rsession->send_ack_window, rsession->send + 3073 - rsession->send_ack); + + if (type != RTMP_TYPE_VIDEO) { + return _rtmp_send_message(rsession, amfnumber, timestamp, type, stream_id, message, len, flags); + } + + if (rsession->media_debug & RTMP_MD_VIDEO_WRITE) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "W V ts:%u data:0x%02x len:%" SWITCH_SIZE_T_FMT "\n", timestamp, *message, len); + } + + window = window / 4 * 3; + // window = 65000; + + if ((rsession->send_ack + window) < (rsession->send + 3073)) { + buffer_video_send(rsession, amfnumber, timestamp, type, stream_id, message, len, flags); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "queued %zu bytes, ts: %d, queue size:%d\n", len, timestamp, switch_queue_size(rsession->video_send_queue)); + return SWITCH_STATUS_SUCCESS; + } + + if (rsession->video_send_queue && switch_queue_size(rsession->video_send_queue)) { + if (*message == 0x17) { // key frame + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Got a key frame, flush video queue %d\n", switch_queue_size(rsession->video_send_queue)); + flush_video_send_queue(rsession, SWITCH_TRUE); + return _rtmp_send_message(rsession, amfnumber, timestamp, type, stream_id, message, len, flags); + } else { + int x = 0; + void *pop = NULL; + + buffer_video_send(rsession, amfnumber, timestamp, type, stream_id, message, len, flags); + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "queued %zu bytes, ts: %d, queue size:%d\n", len, timestamp, switch_queue_size(rsession->video_send_queue)); + + again: + switch_mutex_lock(rsession->socket_mutex); + switch_queue_trypop(rsession->video_send_queue, &pop); + switch_mutex_unlock(rsession->socket_mutex); + + if (pop) { + video_send_buffer_t *vbuf = (video_send_buffer_t *)pop; + + amfnumber = vbuf->amfnumber; + // timestamp = vbuf->timestamp; + type = vbuf->type; + stream_id = vbuf->stream_id; + len = vbuf->len; + flags = vbuf->flags; + message = vbuf->message; + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "pop len: %zu, ts: %d, queue size: %d\n", len, timestamp, switch_queue_size(rsession->video_send_queue)); + + status = _rtmp_send_message(rsession, amfnumber, timestamp, type, stream_id, message, len, flags); + + free(vbuf->message); + free(vbuf); + + if (status == SWITCH_STATUS_SUCCESS && ((rsession->send_ack + window) >= (rsession->send + 3073) && (++x < 3))) { + pop = NULL; + goto again; + } + } + } + } else { + return _rtmp_send_message(rsession, amfnumber, timestamp, type, stream_id, message, len, flags); + } + + return status; +} + /* Returns SWITCH_STATUS_SUCCESS of the connection is still active or SWITCH_STATUS_FALSE to tear it down */ switch_status_t rtmp_handle_data(rtmp_session_t *rsession) { diff --git a/src/mod/endpoints/mod_rtmp/rtmp_tcp.c b/src/mod/endpoints/mod_rtmp/rtmp_tcp.c index 4389cef0ac..f0156e406d 100644 --- a/src/mod/endpoints/mod_rtmp/rtmp_tcp.c +++ b/src/mod/endpoints/mod_rtmp/rtmp_tcp.c @@ -301,6 +301,10 @@ switch_status_t rtmp_tcp_init(rtmp_profile_t *profile, const char *bindaddr, rtm if (switch_socket_opt_set(io_tcp->listen_socket, SWITCH_SO_TCP_NODELAY, 1)) { goto fail; } + if (1) { + switch_socket_opt_set(io_tcp->listen_socket, SWITCH_SO_RCVBUF, 1572864); + switch_socket_opt_set(io_tcp->listen_socket, SWITCH_SO_SNDBUF, 1572864); + } if (switch_socket_bind(io_tcp->listen_socket, sa)) { goto fail; } diff --git a/src/mod/endpoints/mod_rtmp/rtmp_video.c b/src/mod/endpoints/mod_rtmp/rtmp_video.c index 04b2d3c912..6b9ca84b96 100644 --- a/src/mod/endpoints/mod_rtmp/rtmp_video.c +++ b/src/mod/endpoints/mod_rtmp/rtmp_video.c @@ -583,7 +583,7 @@ switch_status_t rtmp_write_video_frame(switch_core_session_t *session, switch_fr rtmp_rtp2rtmpH264(helper, frame); if (helper->send) { - uint16_t used = switch_buffer_inuse(helper->rtmp_buf); + uint32_t used = switch_buffer_inuse(helper->rtmp_buf); const void *rtmp_data = NULL; switch_buffer_peek_zerocopy(helper->rtmp_buf, &rtmp_data); @@ -633,6 +633,11 @@ switch_status_t rtmp_write_video_frame(switch_core_session_t *session, switch_fr switch_core_session_rwunlock(other_session); } } + + if (rsession->video_send_queue && switch_queue_size(rsession->video_send_queue) > 30) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Need a key frame\n"); + switch_channel_set_flag(channel, CF_VIDEO_REFRESH_REQ); + } skip: switch_buffer_zero(helper->rtmp_buf); switch_buffer_zero(helper->fua_buf);