From c60ae0f0e11e761dd43d75bf9979a47721ab1f64 Mon Sep 17 00:00:00 2001 From: Anthony Minessale Date: Tue, 21 Feb 2017 15:52:53 -0600 Subject: [PATCH] FS-10050 cont --- src/include/switch_core_video.h | 1 + src/include/switch_module_interfaces.h | 2 + src/mod/applications/mod_av/avformat.c | 11 ++ .../mod_video_filter/mod_video_filter.c | 129 +++++++++++++++++- src/switch_core_video.c | 75 +++++++++- 5 files changed, 208 insertions(+), 10 deletions(-) diff --git a/src/include/switch_core_video.h b/src/include/switch_core_video.h index 198d5f1720..9101975ad0 100644 --- a/src/include/switch_core_video.h +++ b/src/include/switch_core_video.h @@ -391,6 +391,7 @@ SWITCH_DECLARE(switch_status_t) switch_I420_copy2(uint8_t *src_planes[], int src /*!\brief chromakey an img, img must be RGBA and return modified img */ SWITCH_DECLARE(void) switch_img_chromakey(switch_image_t *img, switch_rgb_color_t *mask, int threshold); +SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_rgb_color_t *mask, int *thresholds, int count); SWITCH_END_EXTERN_C diff --git a/src/include/switch_module_interfaces.h b/src/include/switch_module_interfaces.h index 79758ccf33..90c2a9052d 100644 --- a/src/include/switch_module_interfaces.h +++ b/src/include/switch_module_interfaces.h @@ -336,6 +336,8 @@ typedef struct switch_mm_s { switch_video_profile_t vprofile; switch_video_encode_speed_t vencspd; uint8_t try_hardware_encoder; + int scale_w; + int scale_h; } switch_mm_t; /*! an abstract representation of a file handle (some parameters based on compat with libsndfile) */ diff --git a/src/mod/applications/mod_av/avformat.c b/src/mod/applications/mod_av/avformat.c index 4d87a50738..f96b7b2f1b 100644 --- a/src/mod/applications/mod_av/avformat.c +++ b/src/mod/applications/mod_av/avformat.c @@ -1286,6 +1286,7 @@ struct av_file_context { int64_t seek_ts; switch_bool_t read_paused; int errs; + switch_file_handle_t *handle; }; typedef struct av_file_context av_file_context_t; @@ -1574,6 +1575,7 @@ again: } img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, vframe->width, vframe->height, 1); + if (img) { int64_t *pts = malloc(sizeof(int64_t)); @@ -1706,6 +1708,8 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa context->pool = handle->memory_pool; context->seek_ts = -1; context->offset = DFT_RECORD_OFFSET; + context->handle = handle; + if (handle->params && (tmp = switch_event_get_header(handle->params, "av_video_offset"))) { context->offset = atoi(tmp); } @@ -2180,6 +2184,13 @@ static switch_status_t av_file_read_video(switch_file_handle_t *handle, switch_f context->vid_ready = 1; frame->img = (switch_image_t *) pop; + + if (frame->img && context->handle->mm.scale_w && context->handle->mm.scale_h) { + if (frame->img->d_w != context->handle->mm.scale_w || frame->img->d_h != context->handle->mm.scale_h) { + switch_img_fit(&frame->img, context->handle->mm.scale_w, context->handle->mm.scale_h, SWITCH_FIT_SIZE); + } + } + return SWITCH_STATUS_SUCCESS; } diff --git a/src/mod/applications/mod_video_filter/mod_video_filter.c b/src/mod/applications/mod_video_filter/mod_video_filter.c index aae716149f..2e0e573724 100644 --- a/src/mod/applications/mod_video_filter/mod_video_filter.c +++ b/src/mod/applications/mod_video_filter/mod_video_filter.c @@ -38,26 +38,36 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_video_filter_load); SWITCH_MODULE_SHUTDOWN_FUNCTION(mod_video_filter_shutdown); SWITCH_MODULE_DEFINITION(mod_video_filter, mod_video_filter_load, mod_video_filter_shutdown, NULL); +#define MAX_MASK 25 + typedef struct chromakey_context_s { int threshold; switch_image_t *bgimg; switch_image_t *bgimg_scaled; + switch_file_handle_t vfh; switch_rgb_color_t bgcolor; - switch_rgb_color_t mask; + switch_rgb_color_t mask[MAX_MASK]; + int thresholds[MAX_MASK]; + int mask_len; switch_core_session_t *session; + switch_mutex_t *command_mutex; } chromakey_context_t; static void init_context(chromakey_context_t *context) { switch_color_set_rgb(&context->bgcolor, "#000000"); - switch_color_set_rgb(&context->mask, "#FFFFFF"); context->threshold = 300; + switch_mutex_init(&context->command_mutex, SWITCH_MUTEX_NESTED, switch_core_session_get_pool(context->session)); } static void uninit_context(chromakey_context_t *context) { switch_img_free(&context->bgimg); switch_img_free(&context->bgimg_scaled); + if (switch_test_flag(&context->vfh, SWITCH_FILE_OPEN)) { + switch_core_file_close(&context->vfh); + memset(&context->vfh, 0, sizeof(context->vfh)); + } } static void parse_params(chromakey_context_t *context, int start, int argc, char **argv, const char **function, switch_media_bug_flag_t *flags) @@ -65,32 +75,89 @@ static void parse_params(chromakey_context_t *context, int start, int argc, char int n = argc - start; int i = start; + switch_mutex_lock(context->command_mutex); + if (n > 0 && argv[i]) { // color - switch_color_set_rgb(&context->mask, argv[i]); + int j = 0; + char *list[MAX_MASK]; + int list_argc; + + list_argc = switch_split(argv[i], ':', list); + + context->mask_len = 0; + memset(context->thresholds, 0, sizeof(context->thresholds[0]) * MAX_MASK); + + for (j = 0; j < list_argc; j++) { + char *p; + int thresh = 0; + + if ((p = strchr(list[j], '+'))) { + *p++ = '\0'; + thresh = atoi(p); + if (thresh < 0) thresh = 0; + } + + switch_color_set_rgb(&context->mask[j], list[j]); + if (thresh) { + context->thresholds[j] = thresh; + } + context->mask_len++; + } } i++; if (n > 1 && argv[i]) { // thresh int thresh = atoi(argv[i]); + int j; - if (thresh > 0) context->threshold = thresh; + if (thresh > 0) { + context->threshold = thresh; + + for (j = 0; j < context->mask_len; j++) { + if (!context->thresholds[j]) context->thresholds[j] = context->threshold; + } + } } i++; if (n > 2 && argv[i]) { + + if (switch_test_flag(&context->vfh, SWITCH_FILE_OPEN)) { + switch_core_file_close(&context->vfh); + memset(&context->vfh, 0, sizeof(context->vfh)); + } + if (context->bgimg) { switch_img_free(&context->bgimg); } + if (context->bgimg_scaled) { switch_img_free(&context->bgimg_scaled); } + if (argv[i][0] == '#') { // bgcolor switch_color_set_rgb(&context->bgcolor, argv[i]); + } else if (switch_stristr(".png", argv[i])) { + if (!(context->bgimg = switch_img_read_png(argv[i], SWITCH_IMG_FMT_I420))) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening png\n"); + } } else { - context->bgimg = switch_img_read_png(argv[i], SWITCH_IMG_FMT_I420); + + if (switch_core_file_open(&context->vfh, argv[i], 1, 8000, + SWITCH_FILE_FLAG_READ | SWITCH_FILE_DATA_SHORT | SWITCH_FILE_FLAG_VIDEO, + switch_core_session_get_pool(context->session)) != SWITCH_STATUS_SUCCESS) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error opening video file\n"); + } else { + switch_vid_params_t vp = { 0 }; + + switch_core_media_get_vid_params(context->session, &vp); + context->vfh.mm.scale_w = vp.width; + context->vfh.mm.scale_h = vp.height; + context->vfh.mm.fps = vp.fps; + } } } @@ -102,6 +169,12 @@ static void parse_params(chromakey_context_t *context, int start, int argc, char } i++; + + switch_mutex_unlock(context->command_mutex); + + switch_core_session_request_video_refresh(context->session); + switch_core_media_gen_key_frame(context->session); + } static switch_status_t video_thread_callback(switch_core_session_t *session, switch_frame_t *frame, void *user_data) @@ -119,13 +192,15 @@ static switch_status_t video_thread_callback(switch_core_session_t *session, swi return SWITCH_STATUS_SUCCESS; } + switch_mutex_lock(context->command_mutex); + data = malloc(frame->img->d_w * frame->img->d_h * 4); switch_assert(data); switch_img_to_raw(frame->img, data, frame->img->d_w * 4, SWITCH_IMG_FMT_ARGB); img = switch_img_wrap(NULL, SWITCH_IMG_FMT_ARGB, frame->img->d_w, frame->img->d_h, 1, data); switch_assert(img); - switch_img_chromakey(img, &context->mask, context->threshold); + switch_img_chromakey_multi(img, context->mask, context->thresholds, context->mask_len); if (context->bgimg) { if (context->bgimg_scaled && (context->bgimg_scaled->d_w != frame->img->d_w || context->bgimg_scaled->d_h != frame->img->d_h)) { @@ -137,6 +212,46 @@ static switch_status_t video_thread_callback(switch_core_session_t *session, swi } switch_img_patch(frame->img, context->bgimg_scaled, 0, 0); + } else if (switch_test_flag(&context->vfh, SWITCH_FILE_OPEN)) { + switch_image_t *use_img = NULL; + switch_frame_t file_frame = { 0 }; + switch_status_t status; + + context->vfh.mm.scale_w = frame->img->d_w; + context->vfh.mm.scale_h = frame->img->d_h; + + status = switch_core_file_read_video(&context->vfh, &file_frame, SVR_FLUSH); + switch_core_file_command(&context->vfh, SCFC_FLUSH_AUDIO); + + if (file_frame.img) { + switch_img_free(&context->bgimg_scaled); + use_img = context->bgimg_scaled = file_frame.img; + } else { + use_img = context->bgimg_scaled; + } + + if (use_img) { + switch_img_patch(frame->img, use_img, 0, 0); + } + + if (status != SWITCH_STATUS_SUCCESS && status != SWITCH_STATUS_BREAK) { + int close = 1; + + if (context->vfh.params) { + const char *loopstr = switch_event_get_header(context->vfh.params, "loop"); + if (switch_true(loopstr)) { + uint32_t pos = 0; + switch_core_file_seek(&context->vfh, &pos, 0, SEEK_SET); + close = 0; + } + } + + if (close) { + switch_core_file_close(&context->vfh); + } + } + + } else { switch_img_fill(frame->img, 0, 0, img->d_w, img->d_h, &context->bgcolor); } @@ -145,6 +260,8 @@ static switch_status_t video_thread_callback(switch_core_session_t *session, swi switch_img_free(&img); free(data); + switch_mutex_unlock(context->command_mutex); + return SWITCH_STATUS_SUCCESS; } diff --git a/src/switch_core_video.c b/src/switch_core_video.c index 9994af9134..bb18cf4171 100644 --- a/src/switch_core_video.c +++ b/src/switch_core_video.c @@ -80,6 +80,15 @@ static inline void switch_color_yuv2rgb(switch_yuv_color_t *yuv, switch_rgb_colo */ static inline int switch_color_distance(switch_rgb_color_t *c1, switch_rgb_color_t *c2); +/*!\brief compute distance between a color and a list of colors +* +* \param[in] c1 RGB color1 +* \param[in] clist RGB color list +* \param[in] count number of colors in list +* \param[in] threshold hint of target threshold to stop processing list +*/ +static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, int *thresholds); + /*!\brief Draw a pixel on an image * * \param[in] img Image descriptor @@ -534,9 +543,10 @@ SWITCH_DECLARE(switch_image_t *) switch_img_copy_rect(switch_image_t *img, uint3 #endif } -SWITCH_DECLARE(void) switch_img_chromakey(switch_image_t *img, switch_rgb_color_t *mask, int threshold) +SWITCH_DECLARE(void) switch_img_chromakey_multi(switch_image_t *img, switch_rgb_color_t *mask, int *thresholds, int count) { - uint8_t *pixel; + uint8_t *pixel, *last_pixel = NULL; + int last_hits = 0; switch_assert(img); if (img->fmt != SWITCH_IMG_FMT_ARGB) return; @@ -545,9 +555,49 @@ SWITCH_DECLARE(void) switch_img_chromakey(switch_image_t *img, switch_rgb_color_ for (; pixel < (img->planes[SWITCH_PLANE_PACKED] + img->d_w * img->d_h * 4); pixel += 4) { switch_rgb_color_t *color = (switch_rgb_color_t *)pixel; - int distance = switch_color_distance(color, mask); + int hits = 0; - if (distance <= threshold) { + if (last_pixel && (*(uint32_t *)pixel & 0xFFFFFF) == (*(uint32_t *)last_pixel & 0xFFFFFF)) { + hits = last_hits; + } else { + hits = switch_color_distance_multi(color, mask, count, thresholds); + } + + last_hits = hits; + last_pixel = pixel; + + if (hits) { + *pixel = 0; + } + } + + return; +} + +SWITCH_DECLARE(void) switch_img_chromakey(switch_image_t *img, switch_rgb_color_t *mask, int threshold) +{ + uint8_t *pixel, *last_pixel = NULL; + int last_threshold = 0; + switch_assert(img); + + if (img->fmt != SWITCH_IMG_FMT_ARGB) return; + + pixel = img->planes[SWITCH_PLANE_PACKED]; + + for (; pixel < (img->planes[SWITCH_PLANE_PACKED] + img->d_w * img->d_h * 4); pixel += 4) { + switch_rgb_color_t *color = (switch_rgb_color_t *)pixel; + int threshold = 0; + + if (last_pixel && (*(uint32_t *)pixel & 0xFFFFFF) == (*(uint32_t *)last_pixel & 0xFFFFFF)) { + threshold = last_threshold; + } else { + threshold = switch_color_distance(color, mask); + } + + last_threshold = threshold; + last_pixel = pixel; + + if (threshold) { *pixel = 0; } } @@ -805,6 +855,23 @@ static inline int switch_color_distance(switch_rgb_color_t *c1, switch_rgb_color return sqrt((((512+rmean)*r*r)>>8) + 4*g*g + (((767-rmean)*b*b)>>8)); } +static inline int switch_color_distance_multi(switch_rgb_color_t *c1, switch_rgb_color_t *clist, int count, int *thresholds) +{ + int x = 0, hits = 0; + + for (x = 0; x < count; x++) { + int distance = switch_color_distance(c1, &clist[x]); + + if (distance < thresholds[x]) { + hits++; + } + } + + return hits; +} + + + #define CLAMP(val) MAX(0, MIN(val, 255)) #ifdef SWITCH_HAVE_YUV