From f1d8685566bab20beabe82e27af6f895868d9d2f Mon Sep 17 00:00:00 2001 From: Anthony Minessale Date: Thu, 16 Mar 2017 14:55:10 -0500 Subject: [PATCH] FS-10138: [freeswitch-core,mod_conference] Add alpha video to conference --- src/include/switch_core_video.h | 3 +- .../mod_conference/conference_api.c | 12 +++ .../mod_conference/conference_file.c | 22 +++-- .../mod_conference/conference_video.c | 90 +++++++++++++++++-- .../mod_conference/mod_conference.h | 17 +++- src/switch_core_video.c | 44 +++++++-- 6 files changed, 163 insertions(+), 25 deletions(-) diff --git a/src/include/switch_core_video.h b/src/include/switch_core_video.h index b93bf20844..014dd95f36 100644 --- a/src/include/switch_core_video.h +++ b/src/include/switch_core_video.h @@ -315,7 +315,8 @@ SWITCH_DECLARE(switch_image_t *) switch_img_copy_rect(switch_image_t *img, uint3 */ SWITCH_DECLARE(void) switch_img_fill(switch_image_t *img, int x, int y, int w, int h, switch_rgb_color_t *color); -SWITCH_DECLARE(void) switch_img_grey(switch_image_t *img, int x, int y, int w, int h); +SWITCH_DECLARE(void) switch_img_gray(switch_image_t *img, int x, int y, int w, int h); +SWITCH_DECLARE(void) switch_img_sepia(switch_image_t *img, int x, int y, int w, int h); SWITCH_DECLARE(void) switch_img_fill_noalpha(switch_image_t *img, int x, int y, int w, int h, switch_rgb_color_t *color); diff --git a/src/mod/applications/mod_conference/conference_api.c b/src/mod/applications/mod_conference/conference_api.c index ec129e5bb9..cd283eb9d3 100644 --- a/src/mod/applications/mod_conference/conference_api.c +++ b/src/mod/applications/mod_conference/conference_api.c @@ -78,6 +78,7 @@ api_command_t conference_api_sub_commands[] = { {"unvmute", (void_fn_t) & conference_api_sub_unvmute, CONF_API_SUB_MEMBER_TARGET, "unvmute", "<[member_id|all]|last|non_moderator> []"}, {"deaf", (void_fn_t) & conference_api_sub_deaf, CONF_API_SUB_MEMBER_TARGET, "deaf", "<[member_id|all]|last|non_moderator>"}, {"undeaf", (void_fn_t) & conference_api_sub_undeaf, CONF_API_SUB_MEMBER_TARGET, "undeaf", "<[member_id|all]|last|non_moderator>"}, + {"vid-filter", (void_fn_t) & conference_api_sub_video_filter, CONF_API_SUB_MEMBER_TARGET, "vid-filter", "<[member_id|all]|last|non_moderator> "}, {"relate", (void_fn_t) & conference_api_sub_relate, CONF_API_SUB_ARGS_SPLIT, "relate", "[,] [,] [nospeak|nohear|clear]"}, {"lock", (void_fn_t) & conference_api_sub_lock, CONF_API_SUB_ARGS_SPLIT, "lock", ""}, {"unlock", (void_fn_t) & conference_api_sub_unlock, CONF_API_SUB_ARGS_SPLIT, "unlock", ""}, @@ -546,6 +547,17 @@ switch_status_t conference_api_sub_deaf(conference_member_t *member, switch_stre return SWITCH_STATUS_SUCCESS; } +switch_status_t conference_api_sub_video_filter(conference_member_t *member, switch_stream_handle_t *stream, void *data) +{ + char *filter_str = (char *) data; + + conference_video_parse_filter_string(&member->video_filters, filter_str); + + stream->write_function(stream, "+OK\n"); + + return SWITCH_STATUS_SUCCESS; +} + switch_status_t conference_api_sub_undeaf(conference_member_t *member, switch_stream_handle_t *stream, void *data) { switch_event_t *event; diff --git a/src/mod/applications/mod_conference/conference_file.c b/src/mod/applications/mod_conference/conference_file.c index b9e66f7ba3..b0bdbf1e90 100644 --- a/src/mod/applications/mod_conference/conference_file.c +++ b/src/mod/applications/mod_conference/conference_file.c @@ -46,6 +46,7 @@ switch_status_t conference_file_close(conference_obj_t *conference, conference_f { switch_event_t *event; conference_member_t *member = NULL; + mcu_canvas_t *canvas = NULL; if (test_eflag(conference, EFLAG_PLAY_FILE_DONE) && switch_event_create_subclass(&event, SWITCH_EVENT_CUSTOM, CONF_EVENT_MAINT) == SWITCH_STATUS_SUCCESS) { @@ -91,14 +92,17 @@ switch_status_t conference_file_close(conference_obj_t *conference, conference_f conference_al_close(node->al); } #endif - if (conference->playing_video_file) { - conference->canvases[node->canvas_id]->send_keyframe = 1; - conference->playing_video_file = 0; + + canvas = conference->canvases[node->canvas_id]; + + if (canvas->playing_video_file) { + canvas->send_keyframe = 1; + canvas->playing_video_file = 0; } - if (conference->overlay_video_file) { - conference->canvases[node->canvas_id]->send_keyframe = 1; - conference->overlay_video_file = 0; + if (canvas->overlay_video_file) { + canvas->send_keyframe = 1; + canvas->overlay_video_file = 0; } return switch_core_file_close(&node->fh); @@ -282,9 +286,15 @@ switch_status_t conference_file_play(conference_obj_t *conference, char *file, u const char *overlay_layer = switch_event_get_header(fnode->fh.params, "overlay_layer"); const char *overlay_member = switch_event_get_header(fnode->fh.params, "overlay_member"); const char *overlay_role = switch_event_get_header(fnode->fh.params, "overlay_role"); + const char *file_filters = switch_event_get_header(fnode->fh.params, "file_filters"); int canvas_id = -1; int layer_id = -1; + + if (!zstr(file_filters)) { + conference_video_parse_filter_string(&fnode->filters, file_filters); + } + if (loopsstr) { fnode->loops = atoi(loopsstr); diff --git a/src/mod/applications/mod_conference/conference_video.c b/src/mod/applications/mod_conference/conference_video.c index 6c708450fe..d6339c18ec 100644 --- a/src/mod/applications/mod_conference/conference_video.c +++ b/src/mod/applications/mod_conference/conference_video.c @@ -500,7 +500,29 @@ static void set_bounds(int *x, int *y, int img_w, int img_h, int crop_w, int cro } - + +void conference_video_parse_filter_string(conference_file_filter_t *filters, const char *filter_str) +{ + *filters = 0; + + if (!filter_str) return; + + if (switch_stristr("fg-gray", filter_str)) { + *filters |= FILTER_GRAY_FG; + } + + if (switch_stristr("bg-gray", filter_str)) { + *filters |= FILTER_GRAY_BG; + } + + if (switch_stristr("fg-sepia", filter_str)) { + *filters |= FILTER_SEPIA_FG; + } + + if (switch_stristr("bg-sepia", filter_str)) { + *filters |= FILTER_SEPIA_BG; + } +} void conference_video_scale_and_patch(mcu_layer_t *layer, switch_image_t *ximg, switch_bool_t freeze) { @@ -870,6 +892,23 @@ void conference_video_scale_and_patch(mcu_layer_t *layer, switch_image_t *ximg, switch_mutex_lock(layer->overlay_mutex); if (layer->overlay_img) { switch_img_fit(&layer->overlay_img, layer->img->d_w, layer->img->d_h, SWITCH_FIT_SCALE); + + if (layer->overlay_filters & FILTER_GRAY_FG) { + switch_img_gray(layer->img, 0, 0, layer->img->d_w, layer->img->d_h); + } + + if (layer->overlay_filters & FILTER_SEPIA_FG) { + switch_img_sepia(layer->img, 0, 0, layer->img->d_w, layer->img->d_h); + } + + if (layer->overlay_filters & FILTER_GRAY_BG) { + switch_img_gray(layer->overlay_img, 0, 0, layer->overlay_img->d_w, layer->overlay_img->d_h); + } + + if (layer->overlay_filters & FILTER_SEPIA_BG) { + switch_img_sepia(layer->overlay_img, 0, 0, layer->overlay_img->d_w, layer->overlay_img->d_h); + } + switch_img_patch(layer->img, layer->overlay_img, 0, 0); } switch_mutex_unlock(layer->overlay_mutex); @@ -1883,9 +1922,13 @@ void conference_video_canvas_del_fnode_layer(conference_obj_t *conference, confe fnode->layer_id = -1; fnode->canvas_id = -1; xlayer->fnode = NULL; + + switch_mutex_lock(xlayer->overlay_mutex); + switch_img_free(&xlayer->overlay_img); if (fnode->layer_lock < 0) { conference_video_reset_layer(xlayer); } + switch_mutex_unlock(xlayer->overlay_mutex); } switch_mutex_unlock(canvas->mutex); } @@ -2338,6 +2381,7 @@ void conference_video_patch_fnode(mcu_canvas_t *canvas, conference_file_node_t * switch_mutex_lock(layer->overlay_mutex); switch_img_free(&layer->overlay_img); layer->overlay_img = file_frame.img; + layer->overlay_filters = fnode->filters; switch_mutex_unlock(layer->overlay_mutex); } else { switch_img_free(&layer->cur_img); @@ -2388,9 +2432,9 @@ void conference_video_fnode_check(conference_file_node_t *fnode, int canvas_id) if (full_screen) { canvas->play_file = 1; if (fnode->fh.mm.fmt == SWITCH_IMG_FMT_ARGB) { - canvas->conference->overlay_video_file = 1; + canvas->overlay_video_file = 1; } else { - canvas->conference->playing_video_file = 1; + canvas->playing_video_file = 1; } } else { conference_video_canvas_set_fnode_layer(canvas, fnode, -1); @@ -2568,6 +2612,16 @@ void conference_video_pop_next_image(conference_member_t *member, switch_image_t conference_video_check_flush(member, SWITCH_FALSE); } + if (img) { + if (member->video_filters & FILTER_GRAY_FG) { + switch_img_gray(img, 0, 0, img->d_w, img->d_h); + } + + if (member->video_filters & FILTER_SEPIA_FG) { + switch_img_sepia(img, 0, 0, img->d_w, img->d_h); + } + } + *imgP = img; } @@ -3098,7 +3152,7 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr conference_video_attach_video_layer(imember, canvas, canvas->layout_floor_id); } - if (conference->playing_video_file) { + if (canvas->playing_video_file) { switch_img_free(&img); switch_core_session_rwunlock(imember->session); continue; @@ -3576,7 +3630,7 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr } switch_mutex_unlock(conference->file_mutex); - if (!conference->playing_video_file) { + if (!canvas->playing_video_file) { for (i = 0; i < canvas->total_layers; i++) { mcu_layer_t *layer = &canvas->layers[i]; @@ -3647,7 +3701,7 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr switch_mutex_lock(conference->file_mutex); if (conference->fnode && switch_test_flag(&conference->fnode->fh, SWITCH_FILE_OPEN)) { - if (conference->overlay_video_file) { + if (canvas->overlay_video_file) { if (switch_core_file_read_video(&conference->fnode->fh, &write_frame, SVR_FLUSH) == SWITCH_STATUS_SUCCESS) { if (canvas->play_file) { @@ -3663,6 +3717,22 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr switch_image_t *overlay_img = NULL; switch_img_copy(canvas->img, &overlay_img); + if (conference->fnode->filters & FILTER_GRAY_BG) { + switch_img_gray(overlay_img, 0, 0, overlay_img->d_w, overlay_img->d_h); + } + + if (conference->fnode->filters & FILTER_SEPIA_BG) { + switch_img_sepia(overlay_img, 0, 0, overlay_img->d_w, overlay_img->d_h); + } + + if (conference->fnode->filters & FILTER_GRAY_FG) { + switch_img_gray(file_img, 0, 0, file_img->d_w, file_img->d_h); + } + + if (conference->fnode->filters & FILTER_SEPIA_FG) { + switch_img_sepia(file_img, 0, 0, file_img->d_w, file_img->d_h); + } + write_img = overlay_img; switch_img_patch(write_img, file_img, 0, 0); switch_img_free(&file_img); @@ -3676,7 +3746,7 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr } else if (file_img) { write_img = file_img; } - } else if (conference->playing_video_file) { + } else if (canvas->playing_video_file) { if (switch_core_file_read_video(&conference->fnode->fh, &write_frame, SVR_FLUSH) == SWITCH_STATUS_SUCCESS) { if (canvas->play_file) { @@ -4503,10 +4573,12 @@ switch_status_t conference_video_thread_callback(switch_core_session_t *session, if (conference_utils_test_flag(member->conference, CFLAG_VIDEO_MUXING)) { switch_image_t *img_copy = NULL; - if (frame->img && (member->video_layer_id > -1 || member->canvas) && + int canvas_id = member->canvas_id; + + if (frame->img && (member->video_layer_id > -1) && canvas_id > -1 && conference_utils_member_test_flag(member, MFLAG_CAN_BE_SEEN) && switch_queue_size(member->video_queue) < member->conference->video_fps.fps * 2 && - !member->conference->playing_video_file) { + !member->conference->canvases[canvas_id]->playing_video_file) { if (conference_utils_member_test_flag(member, MFLAG_FLIP_VIDEO) || conference_utils_member_test_flag(member, MFLAG_ROTATE_VIDEO)) { if (conference_utils_member_test_flag(member, MFLAG_ROTATE_VIDEO)) { diff --git a/src/mod/applications/mod_conference/mod_conference.h b/src/mod/applications/mod_conference/mod_conference.h index cd5feebeea..b920e00bc4 100644 --- a/src/mod/applications/mod_conference/mod_conference.h +++ b/src/mod/applications/mod_conference/mod_conference.h @@ -369,6 +369,13 @@ typedef struct al_handle_s { #endif struct conference_obj; +typedef enum { + FILTER_GRAY_FG = (1 << 0), + FILTER_GRAY_BG = (1 << 1), + FILTER_SEPIA_FG = (1 << 2), + FILTER_SEPIA_BG = (1 << 3) +} conference_file_filter_t; + typedef struct conference_file_node { switch_file_handle_t fh; switch_speech_handle_t *sh; @@ -390,6 +397,7 @@ typedef struct conference_file_node { int loops; int new_fnode; int layer_lock; + conference_file_filter_t filters; } conference_file_node_t; typedef enum { @@ -496,6 +504,7 @@ typedef struct mcu_layer_s { switch_frame_geometry_t manual_geometry; mcu_layer_cam_opts_t cam_opts; switch_mutex_t *overlay_mutex; + conference_file_filter_t overlay_filters; } mcu_layer_t; typedef struct video_layout_s { @@ -551,6 +560,8 @@ typedef struct mcu_canvas_s { switch_image_t *bgimg; switch_image_t *fgimg; switch_thread_rwlock_t *video_rwlock; + int playing_video_file; + int overlay_video_file; } mcu_canvas_t; /* Record Node */ @@ -718,8 +729,6 @@ typedef struct conference_obj { switch_hash_t *layout_hash; switch_hash_t *layout_group_hash; struct conference_fps video_fps; - int playing_video_file; - int overlay_video_file; int recording_members; uint32_t video_floor_packets; video_layout_t *new_personal_vlayout; @@ -873,7 +882,7 @@ struct conference_member { uint32_t text_framesize; mcu_layer_cam_opts_t cam_opts; - + conference_file_filter_t video_filters; }; typedef enum { @@ -1043,6 +1052,7 @@ void conference_video_check_used_layers(mcu_canvas_t *canvas); void conference_video_check_flush(conference_member_t *member, switch_bool_t force); void conference_video_set_canvas_letterbox_bgcolor(mcu_canvas_t *canvas, char *color); void conference_video_set_canvas_bgcolor(mcu_canvas_t *canvas, char *color); +void conference_video_parse_filter_string(conference_file_filter_t *filters, const char *filter_str); void conference_video_scale_and_patch(mcu_layer_t *layer, switch_image_t *ximg, switch_bool_t freeze); void conference_video_reset_layer(mcu_layer_t *layer); void conference_video_reset_layer_cam(mcu_layer_t *layer); @@ -1159,6 +1169,7 @@ switch_status_t conference_api_sub_tvmute(conference_member_t *member, switch_st switch_status_t conference_api_sub_unvmute(conference_member_t *member, switch_stream_handle_t *stream, void *data); switch_status_t conference_api_sub_deaf(conference_member_t *member, switch_stream_handle_t *stream, void *data); switch_status_t conference_api_sub_undeaf(conference_member_t *member, switch_stream_handle_t *stream, void *data); +switch_status_t conference_api_sub_video_filter(conference_member_t *member, switch_stream_handle_t *stream, void *data); switch_status_t conference_api_sub_floor(conference_member_t *member, switch_stream_handle_t *stream, void *data); switch_status_t conference_api_sub_vid_floor(conference_member_t *member, switch_stream_handle_t *stream, void *data); switch_status_t conference_api_sub_clear_vid_floor(conference_obj_t *conference, switch_stream_handle_t *stream, void *data); diff --git a/src/switch_core_video.c b/src/switch_core_video.c index 445382ca95..f1fbfd957a 100644 --- a/src/switch_core_video.c +++ b/src/switch_core_video.c @@ -1410,14 +1410,16 @@ SWITCH_DECLARE(void) switch_img_fill_noalpha(switch_image_t *img, int x, int y, #endif } -SWITCH_DECLARE(void) switch_img_grey(switch_image_t *img, int x, int y, int w, int h) +SWITCH_DECLARE(void) switch_img_sepia(switch_image_t *img, int x, int y, int w, int h) { #ifdef SWITCH_HAVE_YUV - int len, i, max_h; - if (x < 0 || y < 0 || x >= img->d_w || y >= img->d_h) return; - if (img->fmt == SWITCH_IMG_FMT_I420) { + if (img->fmt == SWITCH_IMG_FMT_ARGB) { + ARGBSepia(img->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED], x, y, w, h); + } else if (img->fmt == SWITCH_IMG_FMT_I420) { + int len, i, max_h; + max_h = MIN(y + h, img->d_h); len = MIN(w, img->d_w - x); @@ -1430,8 +1432,38 @@ SWITCH_DECLARE(void) switch_img_grey(switch_image_t *img, int x, int y, int w, i len /= 2; for (i = y; i < max_h; i += 2) { - memset(img->planes[SWITCH_PLANE_U] + img->stride[SWITCH_PLANE_U] * (i / 2) + x / 2, 0, len); - memset(img->planes[SWITCH_PLANE_V] + img->stride[SWITCH_PLANE_V] * (i / 2) + x / 2, 0, len); + memset(img->planes[SWITCH_PLANE_U] + img->stride[SWITCH_PLANE_U] * (i / 2) + x / 2, 108, len); + memset(img->planes[SWITCH_PLANE_V] + img->stride[SWITCH_PLANE_V] * (i / 2) + x / 2, 137, len); + } + } +#endif +} + +SWITCH_DECLARE(void) switch_img_gray(switch_image_t *img, int x, int y, int w, int h) +{ +#ifdef SWITCH_HAVE_YUV + + if (x < 0 || y < 0 || x >= img->d_w || y >= img->d_h) return; + + if (img->fmt == SWITCH_IMG_FMT_ARGB) { + ARGBGray(img->planes[SWITCH_PLANE_PACKED], img->stride[SWITCH_PLANE_PACKED], x, y, w, h); + } else if (img->fmt == SWITCH_IMG_FMT_I420) { + int len, i, max_h; + + max_h = MIN(y + h, img->d_h); + len = MIN(w, img->d_w - x); + + if (x & 1) { x++; len--; } + if (y & 1) y++; + if (len <= 0) return; + + if ((len & 1) && (x + len) < img->d_w - 1) len++; + + len /= 2; + + for (i = y; i < max_h; i += 2) { + memset(img->planes[SWITCH_PLANE_U] + img->stride[SWITCH_PLANE_U] * (i / 2) + x / 2, 128, len); + memset(img->planes[SWITCH_PLANE_V] + img->stride[SWITCH_PLANE_V] * (i / 2) + x / 2, 128, len); } } #endif