diff --git a/src/mod/codecs/mod_openh264/Makefile.am b/src/mod/codecs/mod_openh264/Makefile.am index c541e9563f..058f5bebb7 100644 --- a/src/mod/codecs/mod_openh264/Makefile.am +++ b/src/mod/codecs/mod_openh264/Makefile.am @@ -8,3 +8,4 @@ mod_openh264_la_SOURCES = mod_openh264.cpp mod_openh264_la_CXXFLAGS = $(AM_CXXFLAGS) -I$(OPENH264_DIR)/include/wels mod_openh264_la_LIBADD = $(switch_builddir)/libfreeswitch.la mod_openh264_la_LDFLAGS = -L$(OPENH264_DIR)/lib/ -lopenh264 -avoid-version -module -no-undefined -shared + diff --git a/src/mod/codecs/mod_openh264/mod_openh264.cpp b/src/mod/codecs/mod_openh264/mod_openh264.cpp new file mode 100644 index 0000000000..9a98776d4f --- /dev/null +++ b/src/mod/codecs/mod_openh264/mod_openh264.cpp @@ -0,0 +1,582 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2005-2013, Anthony Minessale II + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Seven Du + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Seven Du + * + * mod_openh264 -- H264 Coded Using Cisco OpenH264 + * + */ + +// #define DEBUG_H264 +#include + +#define EPSN (0.000001f) // (1e-6) // desired float precision +#define PESN (0.000001f) // (1e-6) // desired float precision +#define MT_ENABLED + +#include "codec_api.h" +//#include "inc/logging.h" // for debug + +#define FPS 15.0f // frame rate +#define H264_NALU_BUFFER_SIZE 65536 +#define MAX_NALUS 100 +#define SLICE_SIZE 1200 //NALU Slice Size + +SWITCH_MODULE_LOAD_FUNCTION(mod_openh264_load); +SWITCH_MODULE_DEFINITION(mod_openh264, mod_openh264_load, NULL, NULL); + +typedef struct h264_codec_context_s { + ISVCEncoder *encoder; + switch_bool_t encoder_initialized; + SEncParamExt encoder_params; + SFrameBSInfo bit_stream_info; + EVideoFrameType last_frame_type; + int cur_layer; + int cur_nalu_index; + uint8_t last_nalu_type; + uint8_t last_nri; + int last_nalu_data_pos; + int nalu_eat; + + ISVCDecoder *decoder; + SDecodingParam decoder_params; + switch_buffer_t *nalu_buffer; + switch_image_t *img; + int got_sps; + int64_t pts; + switch_size_t last_received_timestamp; + switch_bool_t last_received_complete_picture; +} h264_codec_context_t; + +int FillSpecificParameters(SEncParamExt& param) { + /* Test for temporal, spatial, SNR scalability */ + param.iPicWidth = 352; // width of picture in samples + param.iPicHeight = 288; // height of picture in samples + param.iTargetBitrate = 384000; // target bitrate desired + param.iRCMode = RC_QUALITY_MODE; // rc mode control + param.uiMaxNalSize = SLICE_SIZE * 20; + param.iTemporalLayerNum = 1; // layer number at temporal level + param.iSpatialLayerNum = 1; // layer number at spatial level + param.bEnableDenoise = 0; // denoise control + param.bEnableBackgroundDetection = 1; // background detection control + param.bEnableAdaptiveQuant = 1; // adaptive quantization control + param.bEnableLongTermReference = 0; // long term reference control + param.iLtrMarkPeriod = 30; + + param.iComplexityMode = MEDIUM_COMPLEXITY; + param.uiIntraPeriod = FPS * 3; // period of Intra frame + param.bEnableSpsPpsIdAddition = 0; + param.bPrefixNalAddingCtrl = 0; + + int iIndexLayer = 0; + param.sSpatialLayers[iIndexLayer].iVideoWidth = 352; + param.sSpatialLayers[iIndexLayer].iVideoHeight = 288; + param.sSpatialLayers[iIndexLayer].fFrameRate = 15.0f; + // param.sSpatialLayers[iIndexLayer].iQualityLayerNum = 1; + param.sSpatialLayers[iIndexLayer].iSpatialBitrate = 384000; + +#ifdef MT_ENABLED + param.sSpatialLayers[iIndexLayer].sSliceCfg.uiSliceMode = SM_DYN_SLICE; + param.sSpatialLayers[iIndexLayer].sSliceCfg.sSliceArgument.uiSliceSizeConstraint = SLICE_SIZE; + +#endif + + float fMaxFr = param.sSpatialLayers[param.iSpatialLayerNum - 1].fFrameRate; + for (int32_t i = param.iSpatialLayerNum - 2; i >= 0; --i) { + if (param.sSpatialLayers[i].fFrameRate > fMaxFr + EPSN) { + fMaxFr = param.sSpatialLayers[i].fFrameRate; + } + } + param.fMaxFrameRate = fMaxFr; + + return 0; +} + +// return none-zero on error +long set_decoder_options(ISVCDecoder *decoder) +{ + int32_t iColorFormat = videoFormatI420; + // EBufferProperty eOutputProperty = BUFFER_HOST; + long ret = 0; + + ret += decoder->SetOption(DECODER_OPTION_DATAFORMAT, &iColorFormat); + // ret += decoder->SetOption(DECODER_OPTION_OUTPUT_PROPERTY, &eOutputProperty); + + return ret; +} + +static switch_size_t buffer_h264_nalu(h264_codec_context_t *context, switch_frame_t *frame) +{ + uint8_t nalu_idc = 0; + uint8_t nalu_type = 0; + uint8_t *data = (uint8_t *)frame->data; + uint8_t nalu_hdr = *data; + uint8_t sync_bytes[] = {0, 0, 0, 1}; + switch_buffer_t *buffer = context->nalu_buffer; + switch_size_t size = 0; + + if (!frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "No frame in codec!!\n"); + return size; + } + + nalu_idc = (nalu_hdr & 0x60) >> 5; + nalu_type = nalu_hdr & 0x1f; + + if (!context->got_sps && nalu_type != 7) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Waiting SPS/PPS\n"); + switch_set_flag(frame, SFF_WAIT_KEY_FRAME); + return size; + } + + if (!context->got_sps) context->got_sps = 1; + + size = switch_buffer_write(buffer, sync_bytes, sizeof(sync_bytes)); + if (size == 0 ) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Buffer Memory Error!\n"); + + size = switch_buffer_write(buffer, frame->data, frame->datalen); + if (size == 0 ) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Buffer Memory Error!\n"); + +#ifdef DEBUG_H264 + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "ts: %ld len: %4d %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x mark=%d size=%d\n", + (frame)->timestamp, (frame)->datalen, + *((uint8_t *)(frame)->data), *((uint8_t *)(frame)->data + 1), + *((uint8_t *)(frame)->data + 2), *((uint8_t *)(frame)->data + 3), + *((uint8_t *)(frame)->data + 4), *((uint8_t *)(frame)->data + 5), + *((uint8_t *)(frame)->data + 6), *((uint8_t *)(frame)->data + 7), + *((uint8_t *)(frame)->data + 8), *((uint8_t *)(frame)->data + 9), + *((uint8_t *)(frame)->data + 10), (frame)->m, size); +#endif + + return size; +} + +static switch_status_t nalu_slice(h264_codec_context_t *context, void *data, uint32_t *len, uint32_t *flag) +{ + int nalu_len; + uint8_t *buffer; + switch_status_t status = SWITCH_STATUS_SUCCESS; + + *flag &= ~SFF_MARKER; + + if (context->cur_nalu_index >= context->bit_stream_info.sLayerInfo[context->cur_layer].iNalCount) { + context->cur_nalu_index = 0; + context->cur_layer++; + context->last_nalu_data_pos = 0; + } + +#if 0 + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "uiTemporalId:%d uiSpatialId:%d uiQualityId:%d uiLayerType:%d FrameType: %d\n", + context->bit_stream_info.sLayerInfo[context->cur_layer].uiTemporalId, + context->bit_stream_info.sLayerInfo[context->cur_layer].uiSpatialId, + context->bit_stream_info.sLayerInfo[context->cur_layer].uiQualityId, + context->bit_stream_info.sLayerInfo[context->cur_layer].uiLayerType, + context->last_frame_type); +#endif + + if (context->last_frame_type == videoFrameTypeSkip || + context->cur_layer >= context->bit_stream_info.iLayerNum) { + *len = 0; + *flag |= SFF_MARKER; + context->cur_layer = 0; + context->cur_nalu_index = 0; + return status; + } + + nalu_len = context->bit_stream_info.sLayerInfo[context->cur_layer].pNalLengthInByte[context->cur_nalu_index] - 4; // NALU w/o sync bits + buffer = context->bit_stream_info.sLayerInfo[context->cur_layer].pBsBuf; + +#if 0 + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "layer: %d/%d nalu:%d/%d nalu_len:%d/%d\n", + context->cur_layer, context->bit_stream_info.iLayerNum, + context->cur_nalu_index, context->bit_stream_info.sLayerInfo[context->cur_layer].iNalCount, + nalu_len, context->last_nalu_data_pos); +#endif + + switch_assert(nalu_len > 0); + + if (nalu_len <= SLICE_SIZE) { + uint8_t nalu_type; + + context->last_nalu_data_pos += 4; + nalu_type = *(buffer + context->last_nalu_data_pos) & 0x1f; + + // if (nalu_type == 7) switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_CRIT, "Got SPS\n"); + + memcpy(data, (buffer + context->last_nalu_data_pos), nalu_len); + *len = nalu_len; + // *flag |= (nalu_type == 6 || nalu_type == 7 || nalu_type == 8 || (nalu_type == 0xe && context->last_nalu_type == 8)) ? 0 : SFF_MARKER; + if ((context->cur_nalu_index == context->bit_stream_info.sLayerInfo[context->cur_layer].iNalCount - 1) && + (context->cur_layer == context->bit_stream_info.iLayerNum - 1)) { + *flag |= SFF_MARKER; + } else { + status = SWITCH_STATUS_MORE_DATA; + } + context->cur_nalu_index++; + context->last_nalu_data_pos += nalu_len; + context->last_nalu_type = nalu_type; + goto end; + } else { + int left = nalu_len; + uint8_t *p = (uint8_t *)data; + + if (context->nalu_eat) { + left = nalu_len + 4 - context->nalu_eat; + switch_assert(left > 0); + } + + if (left > (SLICE_SIZE - 2)) { + uint8_t start_bit; + + if (context->nalu_eat) { + start_bit = 0; + } else { + start_bit = 0x80; + context->last_nalu_data_pos += 4; + context->last_nalu_type = *(buffer + context->last_nalu_data_pos) & 0x1f; + context->last_nri = context->last_nalu_type & 0x60; + context->last_nalu_data_pos++; + context->nalu_eat = 5; + } + + p[0] = context->last_nri | 28; // FU-A + p[1] = start_bit | context->last_nalu_type; + + memcpy(p + 2, buffer + context->last_nalu_data_pos, SLICE_SIZE - 2); + context->last_nalu_data_pos += (SLICE_SIZE - 2); + context->nalu_eat += (SLICE_SIZE - 2); + *len = SLICE_SIZE; + status = SWITCH_STATUS_MORE_DATA; + goto end; + } else { + p[0] = context->last_nri | 28; // FU-A + p[1] = 0x40 | context->last_nalu_type; + memcpy(p + 2, buffer + context->last_nalu_data_pos, left); + context->last_nalu_data_pos += left; + *len = left + 2; + *flag |= SFF_MARKER; + context->nalu_eat = 0; + context->cur_nalu_index++; + status = SWITCH_STATUS_MORE_DATA; + goto end; + } + } + +end: + return status; +} + +static switch_status_t switch_h264_init(switch_codec_t *codec, switch_codec_flag_t flags, const switch_codec_settings_t *codec_settings) +{ + h264_codec_context_t *context = NULL; + int encoding, decoding; + int ret; + + encoding = (flags & SWITCH_CODEC_FLAG_ENCODE); + decoding = (flags & SWITCH_CODEC_FLAG_DECODE); + + if (!(encoding || decoding)) { + return SWITCH_STATUS_FALSE; + } + + if (codec->fmtp_in) { + codec->fmtp_out = switch_core_strdup(codec->memory_pool, codec->fmtp_in); + } + + context = (h264_codec_context_t*)switch_core_alloc(codec->memory_pool, sizeof(h264_codec_context_t)); + memset(context, 0, sizeof(*context)); + + if (decoding) { + WelsCreateDecoder(&context->decoder); + + if (!context->decoder) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "CreateDecoder Error\n"); + return SWITCH_STATUS_FALSE; + } + + context->decoder_params.eOutputColorFormat = videoFormatI420; + context->decoder_params.uiTargetDqLayer = (uint8_t) -1; + context->decoder_params.eEcActiveIdc = ERROR_CON_SLICE_COPY; + context->decoder_params.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC; + context->decoder_params.sVideoProperty.size = sizeof(context->decoder_params.sVideoProperty); + + if (context->decoder->Initialize(&context->decoder_params)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Decoder Initialize failed\n"); + goto error; + } + + if (set_decoder_options(context->decoder)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Set Decoder Option Error\n"); + } + } + + if (encoding) { + ret = WelsCreateSVCEncoder(&context->encoder); + if (ret) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Cannot create encoder, error: %d\n", ret); + goto error; + } + + FillSpecificParameters(context->encoder_params); + } + + //if (encoding | decoding) WelsStderrSetTraceLevel(10); + + switch_buffer_create_dynamic(&(context->nalu_buffer), H264_NALU_BUFFER_SIZE, H264_NALU_BUFFER_SIZE * 8, 0); + codec->private_info = context; + + return SWITCH_STATUS_SUCCESS; + +error: + // TODO, do some proper clean up + return SWITCH_STATUS_FALSE; +} + + +static switch_status_t init_encoder(h264_codec_context_t *context, uint32_t width, uint32_t height) +{ + int i; + + context->encoder_params.iPicWidth = width; + context->encoder_params.iPicHeight = height; + for (int i=0; iencoder_params.iSpatialLayerNum; i++) { + context->encoder_params.sSpatialLayers[i].iVideoWidth = width; + context->encoder_params.sSpatialLayers[i].iVideoHeight = height; + } + + /* just do it, the encoder will Uninitialize first by itself if already initialized */ + if (cmResultSuccess != context->encoder->InitializeExt(&context->encoder_params)) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoder Init Error\n"); + return SWITCH_STATUS_FALSE; + } + context->encoder_initialized = SWITCH_TRUE; + return SWITCH_STATUS_SUCCESS; +} + +static switch_status_t switch_h264_encode(switch_codec_t *codec, + switch_image_t *img, + void *encoded_data, uint32_t *encoded_data_len, + unsigned int *flag) +{ + h264_codec_context_t *context = (h264_codec_context_t *)codec->private_info; + int width = 0; + int height = 0; + long enc_ret; + SSourcePicture* pic = NULL; + long result; + + if (*flag & SFF_WAIT_KEY_FRAME) { + context->encoder->ForceIntraFrame(1); + } + + if (img == NULL) { + return nalu_slice(context, encoded_data, encoded_data_len, flag); + } + + width = img->d_w; + height = img->d_h; + + //switch_assert(width > 0 && (width % 2 == 0)); + //switch_assert(height > 0 && (height % 2 == 0)); + + if (!context->encoder_initialized) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "initializing encoder %dx%d\n", width, height); + init_encoder(context, width, height); + } + + if (width != context->encoder_params.iPicWidth || height != context->encoder_params.iPicHeight ) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "picture size changed from %dx%d to %dx%d, reinitializing encoder", + context->encoder_params.iPicWidth, context->encoder_params.iPicHeight, width, height); + init_encoder(context, width, height); + } + + pic = new SSourcePicture; + if (pic == NULL) goto error; + + pic->iColorFormat = videoFormatI420; + pic->iPicHeight = height; + pic->iPicWidth = width; + pic->iStride[0] = img->stride[0]; + pic->iStride[1] = img->stride[1]; // = img->stride[2]; + + pic->pData[0] = img->planes[0]; + pic->pData[1] = img->planes[1]; + pic->pData[2] = img->planes[2]; + + result = (EVideoFrameType)context->encoder->EncodeFrame(pic, &context->bit_stream_info); + if (result != cmResultSuccess ) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "EncodeFrame() failed, result = %ld\n", result); + goto error; + } + + context->cur_layer = 0; + context->cur_nalu_index = 0; + context->last_nalu_data_pos = 0; + + if(pic){ + delete pic; + pic = NULL; + } + return nalu_slice(context, encoded_data, encoded_data_len, flag); + +error: + if(pic){ + delete pic; + pic = NULL; + } + + *encoded_data_len = 0; + *flag |= SFF_MARKER; + return SWITCH_STATUS_FALSE; +} + +static switch_status_t switch_h264_decode(switch_codec_t *codec, + switch_frame_t *frame, + switch_image_t **img, + unsigned int *flag) +{ + h264_codec_context_t *context = (h264_codec_context_t *)codec->private_info; + switch_size_t size = 0; + uint32_t error_code; + + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "len: %d ts: %u mark:%d\n", frame->datalen, ntohl(frame->timestamp), frame->m); + + if (context->last_received_timestamp && context->last_received_timestamp != frame->timestamp && + (!frame->m) && (!context->last_received_complete_picture)) { + // possible packet loss + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Packet Loss, skip privousely received packets\n"); + switch_buffer_zero(context->nalu_buffer); + } + + context->last_received_timestamp = frame->timestamp; + context->last_received_complete_picture = frame->m ? SWITCH_TRUE : SWITCH_FALSE; + + size = buffer_h264_nalu(context, frame); + + if (frame->m && size) { + int got_picture = 0; + int decoded_len; + int i; + const void *nalu = NULL; + int width, height; + SBufferInfo dest_buffer_info; + switch_buffer_peek_zerocopy(context->nalu_buffer, &nalu); + uint8_t* pData[3] = { 0 }; + + pData[0] = NULL; + pData[1] = NULL; + pData[2] = NULL; + memset(&dest_buffer_info, 0, sizeof(dest_buffer_info)); + + error_code = context->decoder->DecodeFrame2((uint8_t *)nalu, size, (uint8_t **)pData, &dest_buffer_info); + + if (error_code == dsErrorFree && dest_buffer_info.iBufferStatus == 1) { + width = dest_buffer_info.UsrData.sSystemBuffer.iWidth; + height = dest_buffer_info.UsrData.sSystemBuffer.iHeight; + + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "got pic: [%dx%d]\n", width, height); + + if (!context->img) { + context->img = switch_img_wrap(NULL, SWITCH_IMG_FMT_I420, width, height, 0, pData[0]); + assert(context->img); + } + + context->img->w = dest_buffer_info.UsrData.sSystemBuffer.iStride[0]; + context->img->h = dest_buffer_info.UsrData.sSystemBuffer.iStride[1]; + context->img->d_w = width; + context->img->d_h = height; + context->img->planes[0] = pData[0]; + context->img->planes[1] = pData[1]; + context->img->planes[2] = pData[2]; + context->img->stride[0] = dest_buffer_info.UsrData.sSystemBuffer.iStride[0]; + context->img->stride[1] = dest_buffer_info.UsrData.sSystemBuffer.iStride[1]; + context->img->stride[2] = dest_buffer_info.UsrData.sSystemBuffer.iStride[1]; + + *img = context->img; + // TODO: keep going and see if more picture available + // pDecoder->DecodeFrame (NULL, 0, pData, &sDstBufInfo); + } else { + if (error_code) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Decode error: 0x%x\n", error_code); + context->got_sps = 0; + } + } + + switch_buffer_zero(context->nalu_buffer); + return SWITCH_STATUS_SUCCESS; + } + +end: + return SWITCH_STATUS_SUCCESS; +} + +static switch_status_t switch_h264_destroy(switch_codec_t *codec) +{ + h264_codec_context_t *context = (h264_codec_context_t *)codec->private_info; + + if (!context) return SWITCH_STATUS_SUCCESS; + + if (context->nalu_buffer) switch_buffer_destroy(&context->nalu_buffer); + + if (context->encoder) { + context->encoder->Uninitialize(); + WelsDestroySVCEncoder(context->encoder); + } + + if (context->decoder) { + if (context->img) switch_img_free(context->img); + context->decoder->Uninitialize(); + WelsDestroyDecoder(context->decoder); + } + + return SWITCH_STATUS_SUCCESS; +} + + +SWITCH_MODULE_LOAD_FUNCTION(mod_openh264_load) +{ + switch_codec_interface_t *codec_interface; + + /* connect my internal structure to the blank pointer passed to me */ + *module_interface = switch_loadable_module_create_module_interface(pool, modname); + + SWITCH_ADD_CODEC(codec_interface, "H264 Video (with Cisco OpenH264)"); + switch_core_codec_add_video_implementation(pool, codec_interface, 99, "H264", NULL, + switch_h264_init, switch_h264_encode, switch_h264_decode, switch_h264_destroy); + + /* indicate that the module should continue to be loaded */ + return SWITCH_STATUS_SUCCESS; +} + +/* For Emacs: + * Local Variables: + * mode:c + * indent-tabs-mode:t + * tab-width:4 + * c-basic-offset:4 + * End: + * For VIM: + * vim:set softtabstop=4 shiftwidth=4 tabstop=4: + */ diff --git a/src/mod/codecs/mod_vpx/Makefile.am b/src/mod/codecs/mod_vpx/Makefile.am index 05d91a1712..becb0c5641 100644 --- a/src/mod/codecs/mod_vpx/Makefile.am +++ b/src/mod/codecs/mod_vpx/Makefile.am @@ -4,7 +4,7 @@ MODNAME=mod_vpx if HAVE_VPX mod_LTLIBRARIES = mod_vpx.la mod_vpx_la_SOURCES = mod_vpx.c -mod_vpx_la_LIBADD = $(switch_builddir)/libfreeswitch.la +mod_vpx_la_LIBADD = $(switch_builddir)/libfreeswitch.la -L/usr/local/lib/ mod_vpx_la_LDFLAGS = -lvpx -avoid-version -module -no-undefined -shared else install: error diff --git a/src/mod/codecs/mod_vpx/README b/src/mod/codecs/mod_vpx/README new file mode 100644 index 0000000000..d2ee1f88a8 --- /dev/null +++ b/src/mod/codecs/mod_vpx/README @@ -0,0 +1,9 @@ +To build this module I used the following steps: + +apt-get install yasm +git /usr/local/src/ +git clone https://chromium.googlesource.com/webm/libvpx +cd libvpx/build/ +../configure --enable-pic --enable-shared +make +make install diff --git a/src/mod/codecs/mod_vpx/mod_vpx.c b/src/mod/codecs/mod_vpx/mod_vpx.c new file mode 100644 index 0000000000..9e96f6f98f --- /dev/null +++ b/src/mod/codecs/mod_vpx/mod_vpx.c @@ -0,0 +1,556 @@ +/* + * FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * Copyright (C) 2005-2012, Anthony Minessale II + * + * Version: MPL 1.1 + * + * The contents of this file are subject to the Mozilla Public License Version + * 1.1 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License + * for the specific language governing rights and limitations under the + * License. + * + * The Original Code is FreeSWITCH Modular Media Switching Software Library / Soft-Switch Application + * + * The Initial Developer of the Original Code is + * Seven Du + * Portions created by the Initial Developer are Copyright (C) + * the Initial Developer. All Rights Reserved. + * + * Contributor(s): + * + * Anthony Minessale II + * Seven Du + * Sam Russell + * + * mod_vpx.c -- VP8/9 Video Codec, with transcoding + * + */ + +#include +#include +#include +#include +#include +#include + +#define FPS 15 +#define SLICE_SIZE 1200 + +SWITCH_MODULE_LOAD_FUNCTION(mod_vpx_load); +SWITCH_MODULE_DEFINITION(mod_vpx, mod_vpx_load, NULL, NULL); + +struct vpx_context { + switch_codec_t *codec; + unsigned int flags; + + vpx_codec_enc_cfg_t config; + + vpx_codec_ctx_t encoder; + vpx_image_t *pic; + switch_bool_t force_key_frame; + int width; + int height; + int bitrate; + int fps; + int format; + int intra_period; + int pts; + int num; + int partition_index; + const vpx_codec_cx_pkt_t *pkt; + int pkt_pos; + vpx_codec_iter_t iter; + switch_time_t last_ts; + + vpx_codec_ctx_t decoder; + switch_buffer_t *vpx_packet_buffer; + int got_key_frame; + switch_size_t last_received_timestamp; + switch_bool_t last_received_complete_picture; + int need_key_frame; +}; +typedef struct vpx_context vpx_context_t; + +static switch_status_t switch_vpx_init(switch_codec_t *codec, switch_codec_flag_t flags, const switch_codec_settings_t *codec_settings) +{ + vpx_context_t *context = NULL; + int encoding, decoding; + vpx_codec_ctx_t *encoder = NULL; + vpx_codec_ctx_t *decoder = NULL; + vpx_codec_enc_cfg_t *config; + const vpx_codec_iface_t* encoder_interface = vpx_codec_vp8_cx(); + const vpx_codec_iface_t* decoder_interface = vpx_codec_vp8_dx(); + + + encoding = (flags & SWITCH_CODEC_FLAG_ENCODE); + decoding = (flags & SWITCH_CODEC_FLAG_DECODE); + + if (!(encoding || decoding) || ((context = switch_core_alloc(codec->memory_pool, sizeof(*context))) == 0)) { + return SWITCH_STATUS_FALSE; + } + memset(context, 0, sizeof(*context)); + context->flags = flags; + codec->private_info = context; + + if (codec->fmtp_in) { + codec->fmtp_out = switch_core_strdup(codec->memory_pool, codec->fmtp_in); + } + + config = &context->config; + + if (vpx_codec_enc_config_default(encoder_interface, config, 0) != VPX_CODEC_OK) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Encoder config Error\n"); + return SWITCH_STATUS_FALSE; + } + + context->width = 352; + context->height = 288; + context->bitrate = 38400; + + // settings + config->g_profile = 1; + config->g_w = context->width; + config->g_h = context->height; + config->rc_target_bitrate = context->bitrate; + config->g_timebase.num = 1; + config->g_timebase.den = 1000; + config->g_error_resilient = VPX_ERROR_RESILIENT_PARTITIONS; + config->g_lag_in_frames = 0; // 0- no frame lagging + config->g_threads = 1; + // rate control settings + config->rc_dropframe_thresh = 0; + config->rc_end_usage = VPX_CBR; + config->g_pass = VPX_RC_ONE_PASS; + // config->kf_mode = VPX_KF_DISABLED; + config->kf_mode = VPX_KF_AUTO; + // config->kf_min_dist = FPS;// Intra Period 3 seconds; + // config->kf_max_dist = FPS * 3; + config->rc_resize_allowed = 0; + config->rc_min_quantizer = 2; + config->rc_max_quantizer = 56; + //Rate control adaptation undershoot control. + // This value, expressed as a percentage of the target bitrate, + // controls the maximum allowed adaptation speed of the codec. + // This factor controls the maximum amount of bits that can be + // subtracted from the target bitrate in order to compensate for + // prior overshoot. + // Valid values in the range 0-1000. + config->rc_undershoot_pct = 100; + //Rate control adaptation overshoot control. + // This value, expressed as a percentage of the target bitrate, + // controls the maximum allowed adaptation speed of the codec. + // This factor controls the maximum amount of bits that can be + // added to the target bitrate in order to compensate for prior + // undershoot. + // Valid values in the range 0-1000. + config->rc_overshoot_pct = 15; + //Decoder Buffer Size. + // This value indicates the amount of data that may be buffered + // by the decoding application. Note that this value is expressed + // in units of time (milliseconds). For example, a value of 5000 + // indicates that the client will buffer (at least) 5000ms worth + // of encoded data. Use the target bitrate (rc_target_bitrate) to + // convert to bits/bytes, if necessary. + config->rc_buf_sz = 1000; + //Decoder Buffer Initial Size. + // This value indicates the amount of data that will be buffered + // by the decoding application prior to beginning playback. + // This value is expressed in units of time (milliseconds). + // Use the target bitrate (rc_target_bitrate) to convert to + // bits/bytes, if necessary. + config->rc_buf_initial_sz = 500; + //Decoder Buffer Optimal Size. + // This value indicates the amount of data that the encoder should + // try to maintain in the decoder's buffer. This value is expressed + // in units of time (milliseconds). + // Use the target bitrate (rc_target_bitrate) to convert to + // bits/bytes, if necessary. + config->rc_buf_optimal_sz = 600; + + if (encoding) { + if (vpx_codec_enc_init(&context->encoder, encoder_interface, config, 0 & VPX_CODEC_USE_OUTPUT_PARTITION) != VPX_CODEC_OK) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", encoder->err, encoder->err_detail); + return SWITCH_STATUS_FALSE; + } + + // The static threshold imposes a change threshold on blocks below which they will be skipped by the encoder. + vpx_codec_control(encoder, VP8E_SET_STATIC_THRESHOLD, 100); + //Set cpu usage, a bit lower than normal (-6) but higher than android (-12) + vpx_codec_control(encoder, VP8E_SET_CPUUSED, -8); + // Only one partition + // vpx_codec_control(encoder, VP8E_SET_TOKEN_PARTITIONS, VP8_ONE_TOKENPARTITION); + // Enable noise reduction + vpx_codec_control(encoder, VP8E_SET_NOISE_SENSITIVITY, 0); + //Set max data rate for Intra frames. + // This value controls additional clamping on the maximum size of a keyframe. + // It is expressed as a percentage of the average per-frame bitrate, with the + // special (and default) value 0 meaning unlimited, or no additional clamping + // beyond the codec's built-in algorithm. + // For example, to allocate no more than 4.5 frames worth of bitrate to a keyframe, set this to 450. + vpx_codec_control(encoder, VP8E_SET_MAX_INTRA_BITRATE_PCT, 0); + } + + if (decoding) { + vp8_postproc_cfg_t ppcfg; + + if (vpx_codec_dec_init(&context->decoder, decoder_interface, NULL, VPX_CODEC_USE_POSTPROC) != VPX_CODEC_OK) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Codec init error: [%d:%s]\n", encoder->err, encoder->err_detail); + return SWITCH_STATUS_FALSE; + } + + // the types of post processing to be done, should be combination of "vp8_postproc_level" + ppcfg.post_proc_flag = VP8_DEMACROBLOCK | VP8_DEBLOCK; + // the strength of deblocking, valid range [0, 16] + ppcfg.deblocking_level = 3; + // Set deblocking settings + vpx_codec_control(decoder, VP8_SET_POSTPROC, &ppcfg); + + switch_buffer_create_dynamic(&context->vpx_packet_buffer, 512, 512, 1024000); + } + + return SWITCH_STATUS_SUCCESS; +} + +/* http://tools.ietf.org/html/draft-ietf-payload-vp8-10 + + The first octets after the RTP header are the VP8 payload descriptor, with the following structure. + + 0 1 2 3 4 5 6 7 + +-+-+-+-+-+-+-+-+ + |X|R|N|S|R| PID | (REQUIRED) + +-+-+-+-+-+-+-+-+ + X: |I|L|T|K| RSV | (OPTIONAL) + +-+-+-+-+-+-+-+-+ + I: |M| PictureID | (OPTIONAL) + +-+-+-+-+-+-+-+-+ + L: | TL0PICIDX | (OPTIONAL) + +-+-+-+-+-+-+-+-+ + T/K:|TID|Y| KEYIDX | (OPTIONAL) + +-+-+-+-+-+-+-+-+ + + + VP8 Payload Header + + 0 1 2 3 4 5 6 7 + +-+-+-+-+-+-+-+-+ + |Size0|H| VER |P| + +-+-+-+-+-+-+-+-+ + | Size1 | + +-+-+-+-+-+-+-+-+ + | Size2 | + +-+-+-+-+-+-+-+-+ + | Bytes 4..N of | + | VP8 payload | + : : + +-+-+-+-+-+-+-+-+ + | OPTIONAL RTP | + | padding | + : : + +-+-+-+-+-+-+-+-+ +*/ + +static switch_status_t consume_partition(vpx_context_t *context, void *data, uint32_t *len, uint32_t *flag) +{ + if (!context->pkt) context->pkt = vpx_codec_get_cx_data(&context->encoder, &context->iter); + + *flag &= ~SFF_MARKER; + + if (context->pkt) { + // if (context->pkt->kind == VPX_CODEC_CX_FRAME_PKT && (context->pkt->data.frame.flags & VPX_FRAME_IS_KEY) && context->pkt_pos == 0) { + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "============================Got a VP8 Key Frame size:[%d]===================================\n", (int)context->pkt->data.frame.sz); + // } + + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "size:%d flag: %x part_id: %d pts: %lld duration:%ld\n", + // (int)context->pkt->data.frame.sz, context->pkt->data.frame.flags, context->pkt->data.frame.partition_id, context->pkt->data.frame.pts, context->pkt->data.frame.duration); + } + + if (!context->pkt || context->pkt_pos >= context->pkt->data.frame.sz - 1 || context->pkt->kind != VPX_CODEC_CX_FRAME_PKT) { + *len = 0; + *flag |= SFF_MARKER; + context->pkt_pos = 0; + context->pkt = NULL; + return SWITCH_STATUS_SUCCESS; + } + + if (context->pkt->data.frame.sz < SLICE_SIZE) { + uint8_t hdr = 0x10; + + memcpy(data, &hdr, 1); + memcpy((uint8_t *)data + 1, context->pkt->data.frame.buf, context->pkt->data.frame.sz); + *len = context->pkt->data.frame.sz + 1; + *flag |= SFF_MARKER; + context->pkt = NULL; + context->pkt_pos = 0; + return SWITCH_STATUS_SUCCESS; + } else { + int left = context->pkt->data.frame.sz - context->pkt_pos; + uint8_t *p = data; + + if (left < SLICE_SIZE) { + p[0] = 0; + memcpy(p+1, (uint8_t *)context->pkt->data.frame.buf + context->pkt_pos, left); + context->pkt_pos = 0; + context->pkt = NULL; + *len = left + 1; + *flag |= SFF_MARKER; + return SWITCH_STATUS_SUCCESS; + } else { + uint8_t hdr = context->pkt_pos == 0 ? 0x10 : 0; + + p[0] = hdr; + memcpy(p+1, (uint8_t *)context->pkt->data.frame.buf + context->pkt_pos, SLICE_SIZE - 1); + context->pkt_pos += (SLICE_SIZE - 1); + *len = SLICE_SIZE; + return SWITCH_STATUS_MORE_DATA; + } + } +} + +static switch_status_t switch_vpx_encode(switch_codec_t *codec, switch_image_t *img, + void *encoded_data, uint32_t *encoded_data_len, + unsigned int *flag) +{ + vpx_context_t *context = (vpx_context_t *)codec->private_info; + uint32_t duration = 90000 / FPS; + int width = 0; + int height = 0; + vpx_enc_frame_flags_t vpx_flags = 0; + + if (*flag & SFF_WAIT_KEY_FRAME) { + context->need_key_frame = 1; + } + + if (img == NULL) { + return consume_partition(context, encoded_data, encoded_data_len, flag); + } + + width = img->d_w; + height = img->d_h; + + //switch_assert(width > 0 && (width % 4 == 0)); + //switch_assert(height > 0 && (height % 4 == 0)); + + if (context->config.g_w != width || context->config.g_h != height) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "VPX reset encoder picture from %dx%d to %dx%d\n", context->config.g_w, context->config.g_h, width, height); + context->config.g_w = width; + context->config.g_h = height; + if (vpx_codec_enc_config_set(&context->encoder, &context->config) != VPX_CODEC_OK) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "VPX reset config error!"); + } + } + + if (context->last_ts == 0) context->last_ts = switch_micro_time_now(); + + if ((switch_micro_time_now() - context->last_ts) > 2 * 1000000) { + // the config params doesn't seems work for generate regular key frames, + // so we do some trick here to force a key frame every 2 sec + // vpx_flags = VPX_EFLAG_FORCE_KF; + context->last_ts = switch_micro_time_now(); + } + + if (context->need_key_frame > 0) { + // force generate a key frame + vpx_flags |= VPX_EFLAG_FORCE_KF; + context->last_ts = switch_micro_time_now(); + context->need_key_frame--; + } + + if (vpx_codec_encode(&context->encoder, (vpx_image_t *)img, context->pts, duration, vpx_flags, VPX_DL_REALTIME) != VPX_CODEC_OK) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "VP8 encode error %d:%s\n", + context->encoder.err, context->encoder.err_detail); + return SWITCH_STATUS_FALSE; + } + + context->pts += duration; + context->iter = NULL; + + return consume_partition(context, encoded_data, encoded_data_len, flag); +} + +static void buffer_vpx_packets(vpx_context_t *context, switch_frame_t *frame) +{ + uint8_t *data = frame->data; + uint8_t S; + uint8_t DES; + uint8_t PID; + int len; + + if (!frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "no frame in codec!!\n"); + return; + } + + DES = *data; + data++; + S = DES & 0x10; + PID = DES & 0x07; + + if (DES & 0x80) { // X + uint8_t X = *data; + data++; + if (X & 0x80) { // I + uint8_t M = (*data) & 0x80; + data++; + if (M) data++; + } + if (X & 0x40) data++; // L + if (X & 0x30) data++; // T/K + } + + len = frame->datalen - (data - (uint8_t *)frame->data); + if (len <= 0) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Invalid packet %d\n", len); + switch_buffer_zero(context->vpx_packet_buffer); + return; + } + + if (S && (PID == 0)) { + uint8_t keyframe; + + keyframe = ((*data) & 0x01) ? 0 : 1; + + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "[%d] PID: %d K:%d P:%d Inv:%d len: %d size:%d\n", frame->datalen, PID, keyframe, profile, invisible, len, size); + + if (keyframe) { + if (!context->got_key_frame) context->got_key_frame = 1; + } + } + + if (!context->got_key_frame) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Waiting for key frame\n"); + switch_set_flag(frame, SFF_WAIT_KEY_FRAME); + return; + } + + switch_buffer_write(context->vpx_packet_buffer, data, len); +} + +static switch_status_t switch_vpx_decode(switch_codec_t *codec, switch_frame_t *frame, switch_image_t **img, unsigned int *flag) +{ + vpx_context_t *context = (vpx_context_t *)codec->private_info; + vpx_codec_ctx_t *decoder = &context->decoder; + switch_size_t len; + + if (!decoder) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "VPX decoder is not initialized!\n"); + return SWITCH_STATUS_FALSE; + } + + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "len: %d ts: %" SWITCH_SIZE_T_FMT " mark:%d\n", frame->datalen, frame->timestamp, frame->m); + + if (context->last_received_timestamp && context->last_received_timestamp != frame->timestamp && + (!frame->m) && (!context->last_received_complete_picture)) { + // possible packet loss + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Packet Loss, skip previouse received frame (to avoid crash?)\n"); + switch_buffer_zero(context->vpx_packet_buffer); + } + + context->last_received_timestamp = frame->timestamp; + context->last_received_complete_picture = frame->m ? SWITCH_TRUE : SWITCH_FALSE; + + buffer_vpx_packets(context, frame); + + len = switch_buffer_inuse(context->vpx_packet_buffer); + + if (frame->m && len) { + uint8_t *data; + vpx_codec_iter_t iter = NULL; + int corrupted = 0; + int err; + // int keyframe = 0; + + switch_buffer_peek_zerocopy(context->vpx_packet_buffer, (void *)&data); + // keyframe = (*data & 0x01) ? 0 : 1; + + // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "buffered: %" SWITCH_SIZE_T_FMT ", key: %d\n", len, keyframe); + + err = vpx_codec_decode(decoder, data, (unsigned int)len, NULL, 0); + + if (err != VPX_CODEC_OK) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Error decoding %" SWITCH_SIZE_T_FMT " bytes, [%d:%d:%s]\n", len, err, decoder->err, decoder->err_detail); + switch_set_flag(frame, SFF_WAIT_KEY_FRAME); + context->got_key_frame = 0; + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "require key frame %d\n", context->got_key_frame); + goto error; + } + + if (vpx_codec_control(decoder, VP8D_GET_FRAME_CORRUPTED, &corrupted) != VPX_CODEC_OK) { + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "VPX control error!\n"); + goto error; + } + + *img = (switch_image_t *)vpx_codec_get_frame(decoder, &iter); + + if (!(*img) || corrupted) { + switch_buffer_zero(context->vpx_packet_buffer); + goto ok; + } + + switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "%dx%d %dx%d\n", (*img)->w,(*img)->h, (*img)->d_w, (*img)->d_h); + + switch_buffer_zero(context->vpx_packet_buffer); + } + +ok: + return SWITCH_STATUS_SUCCESS; + +error: + switch_buffer_zero(context->vpx_packet_buffer); + + return SWITCH_STATUS_FALSE; +} + +static switch_status_t switch_vpx_destroy(switch_codec_t *codec) +{ + vpx_context_t *context = (vpx_context_t *)codec->private_info; + + if (context) { + if ((codec->flags & SWITCH_CODEC_FLAG_ENCODE)) { + vpx_codec_destroy(&context->encoder); // TODO fix crash + } + + if ((codec->flags & SWITCH_CODEC_FLAG_DECODE)) { + vpx_codec_destroy(&context->decoder); + } + + if (context->pic) { + vpx_img_free(context->pic); + context->pic = NULL; + } + if (context->vpx_packet_buffer) { + switch_buffer_destroy(&context->vpx_packet_buffer); + context->vpx_packet_buffer = NULL; + } + } + return SWITCH_STATUS_SUCCESS; +} + +SWITCH_MODULE_LOAD_FUNCTION(mod_vpx_load) +{ + switch_codec_interface_t *codec_interface; + + /* connect my internal structure to the blank pointer passed to me */ + *module_interface = switch_loadable_module_create_module_interface(pool, modname); + SWITCH_ADD_CODEC(codec_interface, "VP8 Video"); + switch_core_codec_add_video_implementation(pool, codec_interface, 99, "VP8", NULL, + switch_vpx_init, switch_vpx_encode, switch_vpx_decode, switch_vpx_destroy); + + /* indicate that the module should continue to be loaded */ + return SWITCH_STATUS_SUCCESS; +} + +/* For Emacs: + * Local Variables: + * mode:c + * indent-tabs-mode:t + * tab-width:4 + * c-basic-offset:4 + * End: + * For VIM: + * vim:set softtabstop=4 shiftwidth=4 tabstop=4 noet: + */