FS-10138: [freeswitch-core,mod_conference] Add alpha video to conference #resolve

src/include/switch_module_interfaces.h

@@ -339,6 +339,7 @@ typedef struct switch_mm_s {
 	uint8_t try_hardware_encoder;
 	int scale_w;
 	int scale_h;
+	switch_img_fmt_t fmt;
 } switch_mm_t;
 
 /*! an abstract representation of a file handle (some parameters based on compat with libsndfile) */

src/mod/applications/mod_av/avformat.c

@@ -1330,7 +1330,6 @@ struct av_file_context {
 	switch_bool_t read_paused;
 	int errs;
 	switch_file_handle_t *handle;
-	switch_bool_t alpha_mode;
 };
 
 typedef struct av_file_context av_file_context_t;
@@ -1389,7 +1388,12 @@ static switch_status_t open_input_file(av_file_context_t *context, switch_file_h
 				context->has_video = 1;
 				handle->duration = av_rescale_q(context->video_st.st->duration, context->video_st.st->time_base, AV_TIME_BASE_Q);
 			}
-			handle->mm.source_fps = ceil(av_q2d(context->video_st.st->avg_frame_rate));
+			if (context->video_st.st->avg_frame_rate.num) {
+				handle->mm.source_fps = ceil(av_q2d(context->video_st.st->avg_frame_rate));
+			} else {
+				handle->mm.source_fps = 25;
+			}
+
 			context->read_fps = (int)handle->mm.source_fps;
 		}
 	}
@@ -1585,7 +1589,7 @@ again:
 
 			if (got_data && error >= 0) {
 				switch_img_fmt_t fmt = SWITCH_IMG_FMT_I420;
-				if (context->alpha_mode && (
+				if ((
 						vframe->format == AV_PIX_FMT_YUVA420P ||
 						vframe->format == AV_PIX_FMT_RGBA ||
 						vframe->format == AV_PIX_FMT_ARGB ||
@@ -1632,6 +1636,8 @@ again:
 						continue;
 					}
 				}
+				
+				context->handle->mm.fmt = fmt;
 
 				img = switch_img_alloc(NULL, fmt, vframe->width, vframe->height, 1);
 
@@ -1773,8 +1779,6 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa
 	if (handle->params) {
 		if ((tmp = switch_event_get_header(handle->params, "av_video_offset"))) {
 			context->offset = atoi(tmp);
-		} else if ((tmp = switch_event_get_header(handle->params, "alpha_mode"))) {
-			context->alpha_mode = switch_true(tmp);
 		}
 	}
 

src/mod/applications/mod_conference/conference_file.c

@@ -270,12 +270,18 @@ switch_status_t conference_file_play(conference_obj_t *conference, char *file, u
 		goto done;
 	}
 
+	fnode->layer_lock = -1;
+
 	if (fnode->fh.params) {
 		const char *vol = switch_event_get_header(fnode->fh.params, "vol");
 		const char *position = switch_event_get_header(fnode->fh.params, "position");
 		const char *canvasstr = switch_event_get_header(fnode->fh.params, "canvas");
 		const char *loopsstr = switch_event_get_header(fnode->fh.params, "loops");
+		const char *overlay_layer = switch_event_get_header(fnode->fh.params, "overlay_layer");
+		const char *overlay_member = switch_event_get_header(fnode->fh.params, "overlay_member");
+		const char *overlay_role = switch_event_get_header(fnode->fh.params, "overlay_role");
 		int canvas_id = -1;
+		int layer_id = -1;
 
 		if (loopsstr) {
 			fnode->loops = atoi(loopsstr);
@@ -285,6 +291,35 @@ switch_status_t conference_file_play(conference_obj_t *conference, char *file, u
 			}
 		}
 
+		if (overlay_role) {
+			conference_member_t *member;
+
+			if ((member = conference_member_get_by_role(conference, overlay_role))) {
+				layer_id = member->video_layer_id;
+				switch_thread_rwlock_unlock(member->rwlock);
+			}
+		} else if (overlay_member) {
+			int id = atoi(overlay_member);
+
+			if (id > 0) {
+				conference_member_t *member;
+
+				if ((member = conference_member_get(conference, id))) {
+					layer_id = member->video_layer_id;
+					switch_thread_rwlock_unlock(member->rwlock);
+				}
+			}
+		}
+
+		if (layer_id < 0 && overlay_layer) {
+			layer_id = atoi(overlay_layer);
+		}
+
+		if (layer_id > -1) {
+			fnode->layer_lock = layer_id;
+			fnode->layer_id = layer_id;
+		}
+
 		if (canvasstr) {
 			canvas_id = atoi(canvasstr) - 1;
 		}

src/mod/applications/mod_conference/conference_member.c

@@ -435,6 +435,51 @@ conference_member_t *conference_member_get_by_var(conference_obj_t *conference,
 	return member;
 }
 
+
+/* traverse the conference member list for the specified member with role  and return it's pointer */
+conference_member_t *conference_member_get_by_role(conference_obj_t *conference, const char *role_id)
+{
+	conference_member_t *member = NULL;
+
+	switch_assert(conference != NULL);
+	if (zstr(role_id)) {
+		return NULL;
+	}
+
+	switch_mutex_lock(conference->member_mutex);
+	for (member = conference->members; member; member = member->next) {
+
+		if (conference_utils_member_test_flag(member, MFLAG_NOCHANNEL)) {
+			continue;
+		}
+		
+		if (!zstr(member->video_role_id) && !strcmp(role_id, member->video_role_id)) {
+			break;
+		}
+	}
+
+	if (member) {
+		if (!conference_utils_member_test_flag(member, MFLAG_INTREE) ||
+			conference_utils_member_test_flag(member, MFLAG_KICKED) ||
+			(member->session && !switch_channel_up(switch_core_session_get_channel(member->session)))) {
+
+			/* member is kicked or hanging up so forget it */
+			member = NULL;
+		}
+	}
+
+	if (member) {
+		if (switch_thread_rwlock_tryrdlock(member->rwlock) != SWITCH_STATUS_SUCCESS) {
+			/* if you cant readlock it's way to late to do anything */
+			member = NULL;
+		}
+	}
+
+	switch_mutex_unlock(conference->member_mutex);
+
+	return member;
+}
+
 void conference_member_check_channels(switch_frame_t *frame, conference_member_t *member, switch_bool_t in)
 {
 	if (member->conference->channels != member->read_impl.number_of_channels || conference_utils_member_test_flag(member, MFLAG_POSITIONAL)) {

src/mod/applications/mod_conference/conference_video.c

@@ -415,7 +415,6 @@ void conference_video_reset_layer(mcu_layer_t *layer)
 {
 	switch_img_free(&layer->banner_img);
 	switch_img_free(&layer->logo_img);
-	switch_img_free(&layer->logo_text_img);
 
 	layer->bugged = 0;
 	layer->mute_patched = 0;
@@ -435,6 +434,9 @@ void conference_video_reset_layer(mcu_layer_t *layer)
 
 	conference_video_clear_layer(layer);
 	switch_img_free(&layer->cur_img);
+	switch_mutex_lock(layer->overlay_mutex);
+	switch_img_free(&layer->overlay_img);
+	switch_mutex_unlock(layer->overlay_mutex);
 }
 
 static void set_pan(int crop_point, int *target_point, int accel_speed, int accel_min, int speed)
@@ -848,13 +850,8 @@ void conference_video_scale_and_patch(mcu_layer_t *layer, switch_image_t *ximg,
 		//printf("SCALE %d,%d %dx%d\n", x_pos, y_pos, img_w, img_h);
 
 		switch_img_scale(img, &layer->img, img_w, img_h);
-
+		
-		if (layer->img) {
+		
-			//switch_img_copy(img, &layer->img);
-			switch_img_patch(IMG, layer->img, x_pos + layer->geometry.border, y_pos + layer->geometry.border);
-		}
-
-
 		if (layer->logo_img) {
 			int ew = layer->screen_w - (layer->geometry.border * 2), eh = layer->screen_h - (layer->banner_img ? layer->banner_img->d_h : 0) - (layer->geometry.border * 2);
 			int ex = 0, ey = 0;
@@ -863,18 +860,26 @@ void conference_video_scale_and_patch(mcu_layer_t *layer, switch_image_t *ximg,
 
 			switch_img_find_position(layer->logo_pos, ew, eh, layer->logo_img->d_w, layer->logo_img->d_h, &ex, &ey);
 
-			switch_img_patch(IMG, layer->logo_img, layer->x_pos + ex + layer->geometry.border, layer->y_pos + ey + layer->geometry.border);
+			switch_img_patch(layer->img, layer->logo_img, ex, ey);
-			if (layer->logo_text_img) {
+			//switch_img_patch(IMG, layer->logo_img, layer->x_pos + ex + layer->geometry.border, layer->y_pos + ey + layer->geometry.border);
-				int tx = 0, ty = 0;
+		}
 
-				switch_img_fit(&layer->logo_text_img, (ew / 2) + 1, (eh / 2) + 1, SWITCH_FIT_SIZE);
+		if (layer->img) {
-				switch_img_find_position(POS_LEFT_BOT,
+			//switch_img_copy(img, &layer->img);
-										 layer->logo_img->d_w, layer->logo_img->d_h, layer->logo_text_img->d_w, layer->logo_text_img->d_h, &tx, &ty);
+
-				switch_img_patch(IMG, layer->logo_text_img, layer->x_pos + ex + tx + layer->geometry.border, layer->y_pos + ey + ty + layer->geometry.border);
+			switch_mutex_lock(layer->overlay_mutex);
+			if (layer->overlay_img) {
+				switch_img_fit(&layer->overlay_img, layer->img->d_w, layer->img->d_h, SWITCH_FIT_SCALE);
+				switch_img_patch(layer->img, layer->overlay_img, 0, 0);
 			}
+			switch_mutex_unlock(layer->overlay_mutex);
+
+			switch_img_patch(IMG, layer->img, x_pos + layer->geometry.border, y_pos + layer->geometry.border);
 
 		}
 
+
+
 		layer->last_img_addr = img_addr;
 
 	} else {
@@ -883,6 +888,7 @@ void conference_video_scale_and_patch(mcu_layer_t *layer, switch_image_t *ximg,
 	}
 
 	switch_mutex_unlock(layer->canvas->mutex);
+
 }
 
 void conference_video_set_canvas_bgcolor(mcu_canvas_t *canvas, char *color)
@@ -1060,7 +1066,6 @@ void conference_video_layer_set_logo(conference_member_t *member, mcu_layer_t *l
 
 
 	switch_img_free(&layer->logo_img);
-	switch_img_free(&layer->logo_text_img);
 
 	if (member->video_logo) {
 		switch_img_copy(member->video_logo, &layer->logo_img);
@@ -1260,7 +1265,6 @@ void conference_video_layer_set_banner(conference_member_t *member, mcu_layer_t
 
 	switch_img_free(&layer->banner_img);
 	//switch_img_free(&layer->logo_img);
-	//switch_img_free(&layer->logo_text_img);
 	layer->banner_img = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, layer->screen_w, font_size * 2, 1);
 	conference_video_reset_image(layer->banner_img, &bgcolor);
 	switch_img_txt_handle_render(layer->txthandle, layer->banner_img, font_size / 2, font_size / 2, text, NULL, fg, bg, 0, 0);
@@ -1397,6 +1401,13 @@ void conference_video_init_canvas_layers(conference_obj_t *conference, mcu_canva
 	switch_mutex_lock(canvas->mutex);
 	canvas->layout_floor_id = -1;
 
+	for (i = 0; i < MCU_MAX_LAYERS; i++) {
+		mcu_layer_t *layer = &canvas->layers[i];
+		if (!layer->overlay_mutex) {
+			switch_mutex_init(&layer->overlay_mutex, SWITCH_MUTEX_NESTED, canvas->pool);
+		}
+	}
+
 	if (canvas->vlayout && canvas->vlayout->transition_out) {
 		switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Play transition out [%s]\n", canvas->vlayout->transition_out);
 		conference_file_play(conference, canvas->vlayout->transition_out, 0, NULL, 0);
@@ -1473,6 +1484,7 @@ void conference_video_init_canvas_layers(conference_obj_t *conference, mcu_canva
 
 	for (i = 0; i < MCU_MAX_LAYERS; i++) {
 		mcu_layer_t *layer = &canvas->layers[i];
+
 		if (layer->member) {
 			//conference_video_detach_video_layer(layer->member);
 			conference_video_clear_managed_kps(layer->member);
@@ -1871,7 +1883,9 @@ void conference_video_canvas_del_fnode_layer(conference_obj_t *conference, confe
 		fnode->layer_id = -1;
 		fnode->canvas_id = -1;
 		xlayer->fnode = NULL;
-		conference_video_reset_layer(xlayer);
+		if (fnode->layer_lock < 0) {
+			conference_video_reset_layer(xlayer);
+		}
 	}
 	switch_mutex_unlock(canvas->mutex);
 }
@@ -1883,6 +1897,14 @@ void conference_video_canvas_set_fnode_layer(mcu_canvas_t *canvas, conference_fi
 
 	switch_mutex_lock(canvas->mutex);
 
+	if (fnode->layer_lock > -1) {
+		layer = &canvas->layers[fnode->layer_lock];
+		layer->fnode = fnode;
+		fnode->layer_id = fnode->layer_lock;
+		fnode->canvas_id = canvas->canvas_id;
+		goto end;
+	}
+
 	if (idx == -1) {
 		int i;
 
@@ -2312,8 +2334,22 @@ void conference_video_patch_fnode(mcu_canvas_t *canvas, conference_file_node_t *
 		switch_status_t status = switch_core_file_read_video(&fnode->fh, &file_frame, SVR_FLUSH);
 
 		if (status == SWITCH_STATUS_SUCCESS) {
-			switch_img_free(&layer->cur_img);
+			if (fnode->layer_lock > -1 && layer->member_id > 0) {
-			layer->cur_img = file_frame.img;
+				switch_mutex_lock(layer->overlay_mutex);
+				switch_img_free(&layer->overlay_img);
+				layer->overlay_img = file_frame.img;
+				switch_mutex_unlock(layer->overlay_mutex);
+			} else {
+				switch_img_free(&layer->cur_img);
+				if (file_frame.img && file_frame.img->fmt != SWITCH_IMG_FMT_I420) {
+					switch_image_t *tmp = switch_img_alloc(NULL, SWITCH_IMG_FMT_I420, file_frame.img->d_w, file_frame.img->d_h, 1);
+					switch_img_copy(file_frame.img, &tmp);
+					switch_img_free(&file_frame.img);
+					file_frame.img = tmp;
+				}
+				layer->cur_img = file_frame.img;
+			}
+
 			layer->tagged = 1;
 		} else if (status == SWITCH_STATUS_IGNORE) {
 			if (canvas && fnode->layer_id > -1 ) {
@@ -2351,7 +2387,11 @@ void conference_video_fnode_check(conference_file_node_t *fnode, int canvas_id)
 
 		if (full_screen) {
 			canvas->play_file = 1;
-			canvas->conference->playing_video_file = 1;
+			if (fnode->fh.mm.fmt == SWITCH_IMG_FMT_ARGB) {
+				canvas->conference->overlay_video_file = 1;
+			} else {
+				canvas->conference->playing_video_file = 1;
+			}
 		} else {
 			conference_video_canvas_set_fnode_layer(canvas, fnode, -1);
 
@@ -3605,27 +3645,55 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr
 			write_img = canvas->img;
 			timestamp = canvas->timer.samplecount;
 
-			if (conference->playing_video_file) {
+			switch_mutex_lock(conference->file_mutex);
-				if (switch_core_file_read_video(&conference->fnode->fh, &write_frame, SVR_FLUSH) == SWITCH_STATUS_SUCCESS) {
+			if (conference->fnode && switch_test_flag(&conference->fnode->fh, SWITCH_FILE_OPEN)) {
-					switch_img_free(&file_img);
+				if (conference->overlay_video_file) {
-
+					if (switch_core_file_read_video(&conference->fnode->fh, &write_frame, SVR_FLUSH) == SWITCH_STATUS_SUCCESS) {
-					if (canvas->play_file) {
-						canvas->send_keyframe = 1;
-						canvas->play_file = 0;
-					}
 					
-					switch_img_free(&file_img);
+						if (canvas->play_file) {
-					switch_img_fit(&write_frame.img, canvas->img->d_w, canvas->img->d_h, SWITCH_FIT_SIZE);
+							canvas->send_keyframe = 1;
-					file_img = write_img = write_frame.img;
+							canvas->play_file = 0;
+						}
+					
+						switch_img_free(&file_img);
+						switch_img_fit(&write_frame.img, canvas->img->d_w, canvas->img->d_h, SWITCH_FIT_SIZE);
+						file_img = write_frame.img;
+					
+						if (file_img->fmt == SWITCH_IMG_FMT_ARGB) {
+							switch_img_patch(write_img, file_img, 0, 0);
+							switch_img_free(&file_img);
+							switch_img_copy(write_img, &file_img);
+						} else {
+							write_img = file_img;
+						}
+					
+						//switch_core_timer_sync(&canvas->timer);
+						timestamp = canvas->timer.samplecount;
+					} else if (file_img) {
+						write_img = file_img;
+					}
+				} else if (conference->playing_video_file) {
+					if (switch_core_file_read_video(&conference->fnode->fh, &write_frame, SVR_FLUSH) == SWITCH_STATUS_SUCCESS) {
+					
+						if (canvas->play_file) {
+							canvas->send_keyframe = 1;
+							canvas->play_file = 0;
+						}
+					
+						switch_img_free(&file_img);
+						switch_img_fit(&write_frame.img, canvas->img->d_w, canvas->img->d_h, SWITCH_FIT_SIZE);
+						file_img = write_img = write_frame.img;
 
-					//switch_core_timer_sync(&canvas->timer);
+						//switch_core_timer_sync(&canvas->timer);
-					timestamp = canvas->timer.samplecount;
+						timestamp = canvas->timer.samplecount;
+					} else if (file_img) {
+						write_img = file_img;
+					}
 				} else if (file_img) {
-					write_img = file_img;
+					switch_img_free(&file_img);
 				}
-			} else if (file_img) {
-				switch_img_free(&file_img);
 			}
+			switch_mutex_unlock(conference->file_mutex);
 
 			write_frame.img = write_img;
 
@@ -3734,11 +3802,11 @@ void *SWITCH_THREAD_FUNC conference_video_muxing_thread_run(switch_thread_t *thr
 
 		switch_mutex_lock(canvas->mutex);
 		switch_img_free(&layer->cur_img);
+		switch_img_free(&layer->overlay_img);
 		switch_img_free(&layer->img);
 		layer->banner_patched = 0;
 		switch_img_free(&layer->banner_img);
 		switch_img_free(&layer->logo_img);
-		switch_img_free(&layer->logo_text_img);
 		switch_img_free(&layer->mute_img);
 		switch_mutex_unlock(canvas->mutex);
 
@@ -4081,11 +4149,11 @@ void *SWITCH_THREAD_FUNC conference_video_super_muxing_thread_run(switch_thread_
 
 		switch_mutex_lock(canvas->mutex);
 		switch_img_free(&layer->cur_img);
+		switch_img_free(&layer->overlay_img);
 		switch_img_free(&layer->img);
 		layer->banner_patched = 0;
 		switch_img_free(&layer->banner_img);
 		switch_img_free(&layer->logo_img);
-		switch_img_free(&layer->logo_text_img);
 		switch_img_free(&layer->mute_img);
 		switch_mutex_unlock(canvas->mutex);
 

src/mod/applications/mod_conference/mod_conference.h

@@ -389,6 +389,7 @@ typedef struct conference_file_node {
 	char *res_id;
 	int loops;
 	int new_fnode;
+	int layer_lock;
 } conference_file_node_t;
 
 typedef enum {
@@ -476,9 +477,9 @@ typedef struct mcu_layer_s {
 	switch_size_t last_img_addr;
 	switch_image_t *img;
 	switch_image_t *cur_img;
+	switch_image_t *overlay_img;
 	switch_image_t *banner_img;
 	switch_image_t *logo_img;
-	switch_image_t *logo_text_img;
 	switch_image_t *mute_img;
 	switch_img_txt_handle_t *txthandle;
 	conference_file_node_t *fnode;
@@ -494,6 +495,7 @@ typedef struct mcu_layer_s {
 	switch_frame_geometry_t pan_geometry;
 	switch_frame_geometry_t manual_geometry;
 	mcu_layer_cam_opts_t cam_opts;
+	switch_mutex_t *overlay_mutex;
 } mcu_layer_t;
 
 typedef struct video_layout_s {
@@ -717,6 +719,7 @@ typedef struct conference_obj {
 	switch_hash_t *layout_group_hash;
 	struct conference_fps video_fps;
 	int playing_video_file;
+	int overlay_video_file;
 	int recording_members;
 	uint32_t video_floor_packets;
 	video_layout_t *new_personal_vlayout;
@@ -1073,7 +1076,7 @@ void conference_fnode_check_status(conference_file_node_t *fnode, switch_stream_
 conference_relationship_t *conference_member_add_relationship(conference_member_t *member, uint32_t id);
 conference_member_t *conference_member_get(conference_obj_t *conference, uint32_t id);
 conference_member_t *conference_member_get_by_var(conference_obj_t *conference, const char *var, const char *val);
-
+conference_member_t *conference_member_get_by_role(conference_obj_t *conference, const char *role_id);
 switch_status_t conference_member_del_relationship(conference_member_t *member, uint32_t id);
 switch_status_t conference_member_add(conference_obj_t *conference, conference_member_t *member);
 switch_status_t conference_member_del(conference_obj_t *conference, conference_member_t *member);