From 5e9a4992e69271a9beb222242bb5945232b3f58f Mon Sep 17 00:00:00 2001
From: Seven Du <dujinfang@gmail.com>
Date: Thu, 18 Jun 2015 09:36:20 +0800
Subject: [PATCH 1/2] FS-7519 FS-7677 trying to add H263 support FS-7519
FS-7677 fix key frame parsing and add some debug logs be verbose about
invalid dimensions cleanup and refactor encoder params
---
src/mod/applications/mod_av/avcodec.c | 615 ++++++++++++++++++++++++--
1 file changed, 575 insertions(+), 40 deletions(-)
diff --git a/src/mod/applications/mod_av/avcodec.c b/src/mod/applications/mod_av/avcodec.c
index dc44cf423b..4fc3cb20b7 100644
--- a/src/mod/applications/mod_av/avcodec.c
+++ b/src/mod/applications/mod_av/avcodec.c
@@ -37,7 +37,9 @@
#include <libavutil/imgutils.h>
#define SLICE_SIZE SWITCH_DEFAULT_VIDEO_SIZE
-#define FPS 15 // frame rate
+#define H264_NALU_BUFFER_SIZE 65536
+#define MAX_NALUS 128
+#define H263_MODE_B // else Mode A only
SWITCH_MODULE_LOAD_FUNCTION(mod_avcodec_load);
@@ -92,17 +94,68 @@ const uint8_t *fs_avc_find_startcode(const uint8_t *p, const uint8_t *end){
return out;
}
+/* RFC 2190 MODE A
+ 0 1 2 3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ |F|P|SBIT |EBIT | SRC |I|U|S|A|R |DBQ| TRB | TR |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*/
-/* codec interface */
+#if SWITCH_BYTE_ORDER == __BIG_ENDIAN
-#define H264_NALU_BUFFER_SIZE 65536
-#define MAX_NALUS 100
+typedef struct h263_payload_header_s{
+ unsigned f:1;
+ unsigned p:1;
+ unsigned sbit:3;
+ unsigned ebit:3;
+ unsigned src:3;
+ unsigned i:1;
+ unsigned u:1;
+ unsigned s:1;
+ unsigned a:1;
+ unsigned r1:1;
+ unsigned r3:3;
+ unsigned dbq:2;
+ unsigned trb:3;
+ unsigned tr:8;
+} h263_payload_header_t;
+
+#else // LITTLE
+
+typedef struct h263_payload_header_s {
+ unsigned ebit:3;
+ unsigned sbit:3;
+ unsigned p:1;
+ unsigned f:1;
+ unsigned r1:1;
+ unsigned a:1;
+ unsigned s:1;
+ unsigned u:1;
+ unsigned i:1;
+ unsigned src:3;
+ unsigned trb:3;
+ unsigned dbq:2;
+ unsigned r3:3;
+ unsigned tr:8;
+} h263_payload_header_t;
+
+#endif
+
+typedef struct h263_state_s {
+ int gobn;
+ int mba;
+ uint8_t hmv1, vmv1, hmv2, vmv2;
+ int quant;
+} h263_state_t;
typedef struct our_h264_nalu_s
{
const uint8_t *start;
const uint8_t *eat;
uint32_t len;
+ h263_payload_header_t h263_header;
+ h263_state_t h263_state;
} our_h264_nalu_t;
typedef struct h264_codec_context_s {
@@ -128,6 +181,7 @@ typedef struct h264_codec_context_s {
AVFrame *encoder_avframe;
AVPacket encoder_avpacket;
our_h264_nalu_t nalus[MAX_NALUS];
+ enum AVCodecID av_codec_id;
} h264_codec_context_t;
static uint8_t ff_input_buffer_padding[FF_INPUT_BUFFER_PADDING_SIZE] = { 0 };
@@ -224,28 +278,450 @@ static switch_status_t buffer_h264_nalu(h264_codec_context_t *context, switch_fr
}
if (frame->m) {
- switch_buffer_write(buffer, ff_input_buffer_padding, sizeof(ff_input_buffer_padding));
context->nalu_28_start = 0;
}
return SWITCH_STATUS_SUCCESS;
}
+static inline int is_valid_h263_dimension(int width, int height)
+{
+ return ((width == 128 && height == 96) ||
+ (width == 176 && height == 144) ||
+ (width == 352 && height == 288) ||
+ (width == 704 && height == 576) ||
+ (width == 1408 && height == 1152));
+}
+static switch_status_t buffer_h263_packets(h264_codec_context_t *context, switch_frame_t *frame)
+{
+ uint8_t *data = frame->data;
+ int header_len = 4; // Mode A, default
+ h263_payload_header_t *h = (h263_payload_header_t *)data;
+
+ if (h->f) {
+ if (h->p) {
+ header_len = 12; // Mode C
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "H263 Mode C is unspported\n");
+ } else {
+ header_len = 8; // Mode B
+ }
+ }
+
+#if 0
+ if (h->i == 0) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Got a H263 Key Frame\n");
+ }
+#endif
+
+ if (0) {
+ static char *h263_src[] = {
+ "NONE",
+ "subQCIF",
+ "QCIF",
+ "CIF",
+ "4CIF",
+ "16CIF",
+ "Reserved-110",
+ "Reserved-111"
+ };
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "SRC: %s\n", h263_src[h->src]);
+ }
+
+ if (frame->datalen <= header_len) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "INVALID PACKET\n");
+ return SWITCH_STATUS_FALSE;
+ }
+
+ if (h->sbit) {
+ int inuse = switch_buffer_inuse(context->nalu_buffer);
+ const void *old_data = NULL;
+ if (!inuse) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "Ignore incomplete packet\n");
+ return SWITCH_STATUS_RESTART;
+ }
+
+ switch_buffer_peek_zerocopy(context->nalu_buffer, &old_data);
+ *((uint8_t *)old_data + inuse - 1) |= (((*(data + header_len) << h->sbit) & 0xFF) >> h->sbit);
+ switch_buffer_write(context->nalu_buffer, data + header_len + 1, frame->datalen - header_len - 1);
+ } else {
+ switch_buffer_write(context->nalu_buffer, data + header_len, frame->datalen - header_len);
+ }
+
+ return SWITCH_STATUS_SUCCESS;
+}
+
+static switch_status_t buffer_h263_rfc4629_packets(h264_codec_context_t *context, switch_frame_t *frame)
+{
+ uint8_t *data = frame->data;
+ uint16_t header = ntohs(*((uint16_t *)frame->data));
+ int startcode, vrc, picture_header;
+ int len = frame->datalen;
+
+ if (frame->datalen < 2) {
+ return SWITCH_STATUS_FALSE;
+ }
+
+ startcode = (header & 0x0400) >> 9;
+ vrc = header & 0x0200;
+ picture_header = (header & 0x01f8) >> 3;
+ data += 2;
+ len -= 2;
+
+ if (vrc) {
+ data += 1;
+ len -= 1;
+ }
+
+ if (picture_header) {
+ data += picture_header;
+ len -= picture_header;
+ }
+
+ if (len < 0) return SWITCH_STATUS_FALSE;
+
+ if (startcode) {
+ uint8_t zeros[2] = { 0 };
+ switch_buffer_write(context->nalu_buffer, zeros, 2);
+ }
+
+ switch_buffer_write(context->nalu_buffer, data, len);
+
+ return SWITCH_STATUS_SUCCESS;
+}
+
+void rtp_callback(struct AVCodecContext *avctx, void *data, int size, int mb_nb)
+{
+#ifndef H263_MODE_B
+ uint8_t *d = data;
+ uint32_t code = (ntohl(*(uint32_t *)data) & 0xFFFFFC00) >> 10;
+ h264_codec_context_t *context = (h264_codec_context_t *)avctx->opaque;
+
+ switch_assert(context);
+
+ if (code == 0x20) { // start
+ context->nalu_current_index = 0;
+ context->nalus[context->nalu_current_index].h263_header.src = (*(d + 4) & 0x0B) >> 2;
+ context->nalus[context->nalu_current_index].h263_header.i = (*(d + 4) & 0x02) >> 1;
+ } else {
+ context->nalus[context->nalu_current_index].h263_header.src = context->nalus[0].h263_header.src;
+ context->nalus[context->nalu_current_index].h263_header.i = context->nalus[0].h263_header.i;
+ }
+
+ context->nalus[context->nalu_current_index].start = data;
+ context->nalus[context->nalu_current_index].len = size;
+ context->nalu_current_index++;
+
+#if 0
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR,
+ "size: %d mb_nb: %d [%02x %02x %02x %02x] %x index: %d %s\n",
+ size, mb_nb, *d, *(d+1), *(d+2), *(d+3), code, context->nalu_current_index,
+ size > 1500 ? "===============Exceedding MTU===============" : "");
+#endif
+
+#endif
+}
+
+const uint8_t *fs_h263_find_resync_marker_reverse(const uint8_t *restrict start,
+ const uint8_t *restrict end)
+{
+ const uint8_t *p = end - 1;
+ start += 1; /* Make sure we never return the original start. */
+ for (; p > start; p -= 2) {
+ if (!*p) {
+ if (!p[ 1] && p[2]) return p;
+ else if (!p[-1] && p[1]) return p - 1;
+ }
+ }
+ return end;
+}
+
+#ifdef H263_MODE_B
+
+static void fs_rtp_parse_h263_rfc2190(h264_codec_context_t *context, AVPacket *pkt)
+{
+ int len, sbits = 0, ebits = 0;
+ h263_payload_header_t h = { 0 };
+ h263_state_t state = { 0 };
+ int size = pkt->size;
+ const uint8_t *buf = pkt->data;
+ const uint8_t *p = buf;
+ const uint8_t *buf_base = buf;
+ uint32_t code = (ntohl(*(uint32_t *)buf) & 0xFFFFFC00) >> 10;
+ int mb_info_size = 0;
+ int mb_info_pos = 0, mb_info_count = 0;
+ const uint8_t *mb_info;
+
+ mb_info = av_packet_get_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, &mb_info_size);
+ mb_info_count = mb_info_size / 12;
+
+ if (size < 4) return;
+
+ if (code == 0x20) { /* Picture Start Code */
+ h.tr = (ntohl(*(uint32_t *)buf) & 0x000003FC) >> 2;
+ p += 4;
+ h.src = ((*p) & 0x1C) >> 2;
+ h.i = ((*p) & 0x02) >> 1;
+ h.u = ((*p) & 0x01);
+ p++;
+ h.s = ((*p) & 0x80) >> 7;
+ h.a = ((*p) & 0x40) >> 6;
+ }
+
+ while (size > 0) {
+ h263_state_t packet_start_state = state;
+ len = (SLICE_SIZE - 8) < size ? (SLICE_SIZE - 8) : size;
+
+ /* Look for a better place to split the frame into packets. */
+ if (len < size) {
+ const uint8_t *end = fs_h263_find_resync_marker_reverse(buf,
+ buf + len);
+ len = end - buf;
+ if (len == SLICE_SIZE - 8) {
+ /* Skip mb info prior to the start of the current ptr */
+ while (mb_info_pos < mb_info_count) {
+ uint32_t pos = *(uint32_t *)&mb_info[12 * mb_info_pos] / 8;
+ if (pos >= buf - buf_base) break;
+ mb_info_pos++;
+ }
+ /* Find the first mb info past the end pointer */
+ while (mb_info_pos + 1 < mb_info_count) {
+ uint32_t pos = *(uint32_t *)&mb_info[12 * (mb_info_pos + 1)] / 8;
+ if (pos >= end - buf_base) break;
+ mb_info_pos++;
+ }
+
+ if (mb_info_pos < mb_info_count) {
+ const uint8_t *ptr = &mb_info[12 * mb_info_pos];
+ uint32_t bit_pos = *(uint32_t *)ptr;
+ uint32_t pos = (bit_pos + 7) / 8;
+
+ if (pos <= end - buf_base) {
+ state.quant = ptr[4];
+ state.gobn = ptr[5];
+ state.mba = *(uint16_t *)&ptr[6];
+ state.hmv1 = (int8_t) ptr[8];
+ state.vmv1 = (int8_t) ptr[9];
+ state.hmv2 = (int8_t) ptr[10];
+ state.vmv2 = (int8_t) ptr[11];
+ ebits = 8 * pos - bit_pos;
+ len = pos - (buf - buf_base);
+ mb_info_pos++;
+ } else {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING,
+ "Unable to split H263 packet! mb_info_pos=%d mb_info_count=%d pos=%d max=%ld\n", mb_info_pos, mb_info_count, pos, end - buf_base);
+ }
+ } else {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Should Not Happen!!! mb_info_pos=%d mb_info_count=%d mb_info_size=%d\n", mb_info_pos, mb_info_count, mb_info_size);
+ }
+ }
+ }
+
+ if (size > 2 && !buf[0] && !buf[1]) {
+ our_h264_nalu_t *nalu = &context->nalus[context->nalu_current_index];
+
+ h.f = 0; // Mode A
+ h.sbit = sbits;
+ h.ebit = ebits;
+ nalu->start = buf;
+ nalu->len = len;
+ nalu->h263_header = h;
+ context->nalu_current_index++;
+ } else {
+ our_h264_nalu_t *nalu = &context->nalus[context->nalu_current_index];
+
+ h.f = 1; // Mode B
+ h.ebit = ebits;
+ h.sbit = sbits;
+ nalu->start = buf;
+ nalu->len = len;
+ nalu->h263_header = h;
+ nalu->h263_state = packet_start_state;
+ context->nalu_current_index++;
+ }
+
+ if (ebits) {
+ sbits = 8 - ebits;
+ len--;
+ } else {
+ sbits = 0;
+ }
+ buf += len;
+ size -= len;
+ ebits = 0;
+ }
+}
+#endif
+
+static void fs_rtp_parse_h263_rfc4629(h264_codec_context_t *context, AVPacket *pkt)
+{
+ int len;
+ uint8_t *buf = pkt->data;
+ our_h264_nalu_t *nalu;
+ int size = pkt->size;
+
+ while (size > 0) {
+ nalu = &context->nalus[context->nalu_current_index];
+ len = (SLICE_SIZE - 2) > size ? size : (SLICE_SIZE - 2);
+
+ /* Look for a better place to split the frame into packets. */
+ if (len < size) {
+ const uint8_t *end = fs_h263_find_resync_marker_reverse(buf, buf + len);
+ len = end - buf;
+ }
+
+ nalu->start = buf;
+ nalu->len = len;
+
+ context->nalu_current_index++;
+
+ buf += len;
+ size -= len;
+ }
+}
+
+static switch_status_t consume_h263_bitstream(h264_codec_context_t *context, switch_frame_t *frame)
+{
+ our_h264_nalu_t *nalu = &context->nalus[context->nalu_current_index];
+
+ if (!nalu->h263_header.f) { // Mode A
+ h263_payload_header_t *h = frame->data;
+ *h = nalu->h263_header;
+ memcpy(((uint8_t *)frame->data) + sizeof(*h), nalu->start, nalu->len);
+ frame->datalen = nalu->len + sizeof(*h);
+ context->nalu_current_index++;
+
+#ifdef H263_MODE_B
+ } else { //Mode B
+
+/*
+ 0 1 2 3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ |F|P|SBIT |EBIT | SRC | QUANT | GOBN | MBA |R |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ |I|U|S|A| HMV1 | VMV1 | HMV2 | VMV2 |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+*/
+ h263_payload_header_t *h = frame->data;
+ uint8_t *p = frame->data;
+
+#if 0
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "header: f:%d p:%d src:%d sbit:%d ebit:%d\n",
+ nalu->h263_header.f,
+ nalu->h263_header.p,
+ nalu->h263_header.src,
+ nalu->h263_header.sbit,
+ nalu->h263_header.ebit);
+
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "state: gobn:%x mba:%x quant:%x hmv1:%x vmv1:%x hmv2:%x vmv2:%x\n",
+ nalu->h263_state.gobn,
+ nalu->h263_state.mba,
+ nalu->h263_state.quant,
+ nalu->h263_state.hmv1,
+ nalu->h263_state.vmv1,
+ nalu->h263_state.hmv2,
+ nalu->h263_state.vmv2);
+#endif
+ *h = nalu->h263_header;
+ p++;
+ *p &= 0xE0;
+ *p |= (nalu->h263_state.quant & 0x1F);
+ p++;
+ *p = ((nalu->h263_state.gobn << 3) & 0xF8);
+ *p |= ((nalu->h263_state.mba >> 6) & 0x07);
+ p++;
+ *p = ((nalu->h263_state.mba & 0x1F) << 2);
+ p++;
+ *p = (nalu->h263_header.i << 7);
+ *p |= (nalu->h263_header.u << 6);
+ *p |= (nalu->h263_header.s << 5);
+ *p |= (nalu->h263_header.a << 4);
+ *p |= ((nalu->h263_state.hmv1 >> 3) & 0x0F);
+ p++;
+ *p = ((nalu->h263_state.hmv1 & 0x07) << 5);
+ *p |= ((nalu->h263_state.vmv1 >> 2) & 0x1F);
+ p++;
+ *p = ((nalu->h263_state.vmv1 & 0x03) << 6);
+ *p |= ((nalu->h263_state.hmv2 >> 1) & 0x3F);
+ p++;
+ *p = ((nalu->h263_state.hmv2 & 0x01) << 7);
+ *p |= (nalu->h263_state.vmv2);
+ p++;
+
+ memcpy(p, nalu->start, nalu->len);
+ frame->datalen = nalu->len + 8;
+ context->nalu_current_index++;
+#endif
+ }
+
+ if (!context->nalus[context->nalu_current_index].len) frame->m = 1;
+
+#if 0
+ {
+ uint8_t *p = frame->data;
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "len: %d, mark:%d %02x %02x %02x %02x\n", frame->datalen, frame->m, *p, *(p+1), *(p+2), *(p+3));
+ if (frame->m && (nalu->h263_header.i == 0)) {
+ // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "Key frame generated!!\n");
+ }
+ }
+#endif
+
+ return frame->m ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_MORE_DATA;
+}
+
+
+static switch_status_t consume_h263p_bitstream(h264_codec_context_t *context, switch_frame_t *frame)
+{
+ our_h264_nalu_t *nalu = &context->nalus[context->nalu_current_index];
+ uint8_t *data = frame->data;
+ const uint8_t *p = nalu->start;
+ int len = nalu->len;
+
+ if (*p == 0 && *(p+1) == 0) {
+ *data++ = 0x04;
+ p += 2;
+ len -= 2;
+ } else {
+ *data++ = 0;
+ }
+
+ *data++ = 0;
+ memcpy(data, p, len);
+ frame->datalen = len + 2;
+ context->nalu_current_index++;
+
+ if (!context->nalus[context->nalu_current_index].len) frame->m = 1;
+
+ {
+ uint8_t *p = frame->data;
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "len: %d, mark:%d %02x %02x %02x %02x\n", frame->datalen, frame->m, *p, *(p+1), *(p+2), *(p+3));
+ }
+
+ if (frame->m) av_free_packet(&context->encoder_avpacket);
+
+ return frame->m ? SWITCH_STATUS_SUCCESS : SWITCH_STATUS_MORE_DATA;
+}
static switch_status_t consume_nalu(h264_codec_context_t *context, switch_frame_t *frame)
{
our_h264_nalu_t *nalu = &context->nalus[context->nalu_current_index];
- if (!nalu->start) {
+ if (!nalu->len) {
frame->datalen = 0;
frame->m = 0;
if (context->encoder_avpacket.size > 0) av_free_packet(&context->encoder_avpacket);
- if (context->encoder_avframe->data[0]) av_freep(&context->encoder_avframe->data[0]);
+ // if (context->encoder_avframe->data[0]) av_freep(&context->encoder_avframe->data[0]);
context->nalu_current_index = 0;
return SWITCH_STATUS_NOTFOUND;
}
+ if (context->av_codec_id == AV_CODEC_ID_H263) {
+ return consume_h263_bitstream(context, frame);
+ } else if (context->av_codec_id == AV_CODEC_ID_H263P) {
+ return consume_h263p_bitstream(context, frame);
+ }
+
assert(nalu->len);
if (nalu->len <= SLICE_SIZE) {
@@ -295,10 +771,15 @@ static switch_status_t consume_nalu(h264_codec_context_t *context, switch_frame_
static switch_status_t open_encoder(h264_codec_context_t *context, uint32_t width, uint32_t height)
{
- if (!context->encoder) context->encoder = avcodec_find_encoder(AV_CODEC_ID_H264);
+ if (!context->encoder) context->encoder = avcodec_find_encoder(context->av_codec_id);
if (!context->encoder) {
- switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Cannot find AV_CODEC_ID_H264 encoder\n");
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Cannot find encoder id: %d\n", context->av_codec_id);
+ return SWITCH_STATUS_FALSE;
+ }
+
+ if (context->av_codec_id == AV_CODEC_ID_H263 && (!is_valid_h263_dimension(width, height))) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "You want %dx%d, but valid sizes are 128x96, 176x144, 352x288, 704x576, and 1408x1152. Try H.263+\n", width, height);
return SWITCH_STATUS_FALSE;
}
@@ -341,30 +822,40 @@ static switch_status_t open_encoder(h264_codec_context_t *context, uint32_t widt
context->encoder_ctx->height = context->codec_settings.video.height;
/* frames per second */
context->encoder_ctx->time_base = (AVRational){1, 90};
- //context->encoder_ctx->gop_size = FPS * 10; /* emit one intra frame every 3 seconds */
- //context->encoder_ctx->max_b_frames = 0;
+ context->encoder_ctx->max_b_frames = 0;
context->encoder_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
context->encoder_ctx->thread_count = 1;//switch_core_cpu_count() > 4 ? 4 : 1;
-
context->encoder_ctx->bit_rate = context->bandwidth * 1024;
context->encoder_ctx->rc_max_rate = context->bandwidth * 1024;
context->encoder_ctx->rc_buffer_size = context->bandwidth * 1024 * 4;
-
context->encoder_ctx->rtp_payload_size = SLICE_SIZE;
- context->encoder_ctx->profile = FF_PROFILE_H264_BASELINE;
- context->encoder_ctx->level = 30;
- av_opt_set(context->encoder_ctx->priv_data, "preset", "veryfast", 0);
- av_opt_set(context->encoder_ctx->priv_data, "tune", "zerolatency", 0);
- av_opt_set(context->encoder_ctx->priv_data, "profile", "baseline", 0);
+ if (context->av_codec_id == AV_CODEC_ID_H263 || context->av_codec_id == AV_CODEC_ID_H263P) {
+ context->encoder_ctx->rtp_callback = rtp_callback;
+ context->encoder_ctx->rc_min_rate = context->encoder_ctx->rc_max_rate;
+ context->encoder_ctx->opaque = context;
+ av_opt_set_int(context->encoder_ctx->priv_data, "mb_info", SLICE_SIZE - 8, 0);
+ } else if (context->av_codec_id == AV_CODEC_ID_H264) {
+ context->encoder_ctx->profile = FF_PROFILE_H264_BASELINE;
+ context->encoder_ctx->level = 30;
+ av_opt_set(context->encoder_ctx->priv_data, "preset", "veryfast", 0);
+ av_opt_set(context->encoder_ctx->priv_data, "tune", "zerolatency", 0);
+ av_opt_set(context->encoder_ctx->priv_data, "profile", "baseline", 0);
+ //av_opt_set_int(context->encoder_ctx->priv_data, "slice-max-size", SLICE_SIZE, 0);
+
+ // libx264-medium.ffpreset preset
+ context->encoder_ctx->coder_type = 1; // coder = 1
+ context->encoder_ctx->flags|=CODEC_FLAG_LOOP_FILTER; // flags=+loop
+ context->encoder_ctx->me_cmp|= 1; // cmp=+chroma, where CHROMA = 1
+ context->encoder_ctx->me_method=ME_HEX; // me_method=hex
+ context->encoder_ctx->me_subpel_quality = 7; // subq=7
+ context->encoder_ctx->me_range = 16; // me_range=16
+ context->encoder_ctx->max_b_frames = 3; // bf=3
+ context->encoder_ctx->refs = 3; // refs=3
+ context->encoder_ctx->trellis = 1; // trellis=1
+ }
// libx264-medium.ffpreset preset
- context->encoder_ctx->coder_type = 1; // coder = 1
- context->encoder_ctx->flags|=CODEC_FLAG_LOOP_FILTER; // flags=+loop
- context->encoder_ctx->me_cmp|= 1; // cmp=+chroma, where CHROMA = 1
- context->encoder_ctx->me_method=ME_HEX; // me_method=hex
- context->encoder_ctx->me_subpel_quality = 7; // subq=7
- context->encoder_ctx->me_range = 16; // me_range=16
context->encoder_ctx->gop_size = 250; // g=250
context->encoder_ctx->keyint_min = 25; // keyint_min=25
context->encoder_ctx->scenechange_threshold = 40; // sc_threshold=40
@@ -374,12 +865,7 @@ static switch_status_t open_encoder(h264_codec_context_t *context, uint32_t widt
context->encoder_ctx->qmin = 10; // qmin=10
context->encoder_ctx->qmax = 51; // qmax=51
context->encoder_ctx->max_qdiff = 4; // qdiff=4
- context->encoder_ctx->max_b_frames = 3; // bf=3
- //context->encoder_ctx->refs = 3; // refs=3
- context->encoder_ctx->trellis = 1; // trellis=1
-
- //av_opt_set_int(context->encoder_ctx->priv_data, "slice-max-size", SLICE_SIZE, 0);
if (avcodec_open2(context->encoder_ctx, context->encoder, NULL) < 0) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Could not open codec\n");
@@ -412,11 +898,24 @@ static switch_status_t switch_h264_init(switch_codec_t *codec, switch_codec_flag
context->codec_settings = *codec_settings;
}
+ if (!strcmp(codec->implementation->iananame, "H263")) {
+ context->av_codec_id = AV_CODEC_ID_H263;
+ } else if (!strcmp(codec->implementation->iananame, "H263-1998")) {
+ context->av_codec_id = AV_CODEC_ID_H263P;
+ } else {
+ context->av_codec_id = AV_CODEC_ID_H264;
+ }
+
if (decoding) {
- context->decoder = avcodec_find_decoder(AV_CODEC_ID_H264);
+ context->decoder = avcodec_find_decoder(context->av_codec_id);
+
+ if (!context->decoder && context->av_codec_id == AV_CODEC_ID_H263P) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "Cannot find AV_CODEC_ID_H263P decoder, trying AV_CODEC_ID_H263 instead\n");
+ context->decoder = avcodec_find_decoder(AV_CODEC_ID_H263);
+ }
if (!context->decoder) {
- switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Cannot find codec AV_CODEC_ID_H264\n");
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Cannot find codec id %d\n", context->av_codec_id);
goto error;
}
@@ -482,14 +981,19 @@ static switch_status_t switch_h264_encode(switch_codec_t *codec, switch_frame_t
if (frame->datalen < SWITCH_DEFAULT_VIDEO_SIZE) return SWITCH_STATUS_FALSE;
+ width = img->d_w;
+ height = img->d_h;
+
+ if (context->av_codec_id == AV_CODEC_ID_H263 && (!is_valid_h263_dimension(width, height))) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "You want %dx%d, but valid H263 sizes are 128x96, 176x144, 352x288, 704x576, and 1408x1152. Try H.263+\n", width, height);
+ goto error;
+ }
+
if (frame->flags & SFF_SAME_IMAGE) {
// read from nalu buffer
return consume_nalu(context, frame);
}
- width = img->d_w;
- height = img->d_h;
-
if (!avctx || !avcodec_is_open(avctx)) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_INFO, "initializing encoder %dx%d\n", width, height);
if (open_encoder(context, width, height) != SWITCH_STATUS_SUCCESS) {
@@ -567,7 +1071,7 @@ static switch_status_t switch_h264_encode(switch_codec_t *codec, switch_frame_t
fill_avframe(avframe, img);
- //avframe->pts = context->pts++;
+ avframe->pts = context->pts++;
if (context->need_key_frame) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Send AV KEYFRAME\n");
@@ -576,6 +1080,8 @@ static switch_status_t switch_h264_encode(switch_codec_t *codec, switch_frame_t
}
/* encode the image */
+ memset(context->nalus, 0, sizeof(context->nalus));
+ context->nalu_current_index = 0;
ret = avcodec_encode_video2(avctx, pkt, avframe, got_output);
if (ret < 0) {
@@ -594,8 +1100,23 @@ process:
const uint8_t *p = pkt->data;
int i = 0;
- //switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Encoded frame %" SWITCH_INT64_T_FMT " (size=%5d) nalu_type=0x%x %d\n", context->pts, pkt->size, *((uint8_t *)pkt->data +4), *got_output);
+ if (context->av_codec_id == AV_CODEC_ID_H263) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Encoded frame %" SWITCH_INT64_T_FMT " (size=%5d) [0x%02x 0x%02x 0x%02x 0x%02x] got_output: %d slices: %d\n", context->pts, pkt->size, *((uint8_t *)pkt->data), *((uint8_t *)(pkt->data + 1)), *((uint8_t *)(pkt->data + 2)), *((uint8_t *)(pkt->data + 3)), *got_output, avctx->slices);
+#ifdef H263_MODE_B
+ fs_rtp_parse_h263_rfc2190(context, pkt);
+#endif
+
+ context->nalu_current_index = 0;
+ return consume_nalu(context, frame);
+ } else if (context->av_codec_id == AV_CODEC_ID_H263P){
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Encoded frame %" SWITCH_INT64_T_FMT " (size=%5d) [0x%02x 0x%02x 0x%02x 0x%02x] got_output: %d slices: %d\n", context->pts, pkt->size, *((uint8_t *)pkt->data), *((uint8_t *)(pkt->data + 1)), *((uint8_t *)(pkt->data + 2)), *((uint8_t *)(pkt->data + 3)), *got_output, avctx->slices);
+ fs_rtp_parse_h263_rfc4629(context, pkt);
+ context->nalu_current_index = 0;
+ return consume_nalu(context, frame);
+ } else {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG1, "Encoded frame %" SWITCH_INT64_T_FMT " (size=%5d) nalu_type=0x%x %d\n", context->pts, pkt->size, *((uint8_t *)pkt->data +4), *got_output);
+ }
/* split into nalus */
memset(context->nalus, 0, sizeof(context->nalus));
@@ -644,7 +1165,13 @@ static switch_status_t switch_h264_decode(switch_codec_t *codec, switch_frame_t
context->last_received_timestamp = frame->timestamp;
context->last_received_complete_picture = frame->m ? SWITCH_TRUE : SWITCH_FALSE;
- status = buffer_h264_nalu(context, frame);
+ if (context->av_codec_id == AV_CODEC_ID_H263) {
+ status = buffer_h263_packets(context, frame);
+ } else if (context->av_codec_id == AV_CODEC_ID_H263P) {
+ status = buffer_h263_rfc4629_packets(context, frame);
+ } else {
+ status = buffer_h264_nalu(context, frame);
+ }
if (status == SWITCH_STATUS_RESTART) {
switch_set_flag(frame, SFF_WAIT_KEY_FRAME);
@@ -660,13 +1187,13 @@ static switch_status_t switch_h264_decode(switch_codec_t *codec, switch_frame_t
int got_picture = 0;
int decoded_len;
- if (size > FF_INPUT_BUFFER_PADDING_SIZE) {
+ if (size > 0) {
av_init_packet(&pkt);
pkt.data = NULL;
pkt.size = 0;
+ switch_buffer_write(context->nalu_buffer, ff_input_buffer_padding, sizeof(ff_input_buffer_padding));
switch_buffer_peek_zerocopy(context->nalu_buffer, (const void **)&pkt.data);
-
- pkt.size = size - FF_INPUT_BUFFER_PADDING_SIZE;
+ pkt.size = size;
picture = av_frame_alloc();
assert(picture);
decoded_len = avcodec_decode_video2(avctx, picture, &got_picture, &pkt);
@@ -930,6 +1457,14 @@ SWITCH_MODULE_LOAD_FUNCTION(mod_avcodec_load)
switch_core_codec_add_video_implementation(pool, codec_interface, 99, "H264", NULL,
switch_h264_init, switch_h264_encode, switch_h264_decode, switch_h264_control, switch_h264_destroy);
+ SWITCH_ADD_CODEC(codec_interface, "H263 Video");
+ switch_core_codec_add_video_implementation(pool, codec_interface, 34, "H263", NULL,
+ switch_h264_init, switch_h264_encode, switch_h264_decode, switch_h264_control, switch_h264_destroy);
+
+ SWITCH_ADD_CODEC(codec_interface, "H263+ Video");
+ switch_core_codec_add_video_implementation(pool, codec_interface, 34, "H263-1998", NULL,
+ switch_h264_init, switch_h264_encode, switch_h264_decode, switch_h264_control, switch_h264_destroy);
+
SWITCH_ADD_API(api_interface, "av_codec", "av_codec information", av_codec_api_function, "");
/* indicate that the module should continue to be loaded */
From f55ac43672d6cdaeb412b14964b7701b3d7de4b5 Mon Sep 17 00:00:00 2001
From: Seven Du <dujinfang@gmail.com>
Date: Thu, 23 Jul 2015 17:15:13 +0800
Subject: [PATCH 2/2] FS-7677 handle packet loss
---
src/mod/applications/mod_av/avcodec.c | 31 ++++++++++++++++++++++++++-
1 file changed, 30 insertions(+), 1 deletion(-)
diff --git a/src/mod/applications/mod_av/avcodec.c b/src/mod/applications/mod_av/avcodec.c
index 4fc3cb20b7..44967bf347 100644
--- a/src/mod/applications/mod_av/avcodec.c
+++ b/src/mod/applications/mod_av/avcodec.c
@@ -182,6 +182,7 @@ typedef struct h264_codec_context_s {
AVPacket encoder_avpacket;
our_h264_nalu_t nalus[MAX_NALUS];
enum AVCodecID av_codec_id;
+ uint16_t last_seq; // last received frame->seq
} h264_codec_context_t;
static uint8_t ff_input_buffer_padding[FF_INPUT_BUFFER_PADDING_SIZE] = { 0 };
@@ -298,6 +299,7 @@ static switch_status_t buffer_h263_packets(h264_codec_context_t *context, switch
uint8_t *data = frame->data;
int header_len = 4; // Mode A, default
h263_payload_header_t *h = (h263_payload_header_t *)data;
+ int delta = 0;
if (h->f) {
if (h->p) {
@@ -309,11 +311,38 @@ static switch_status_t buffer_h263_packets(h264_codec_context_t *context, switch
}
#if 0
- if (h->i == 0) {
+ //emulate packet loss
+ static int z = 0;
+ if ((z++ % 200 == 0) && h->i) return SWITCH_STATUS_RESTART;
+#endif
+
+#if 0
+ if (h->i == 0 && frame->m) {
switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_ERROR, "Got a H263 Key Frame\n");
}
#endif
+ delta = frame->seq - context->last_seq;
+
+ if (delta > 1) {
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_WARNING, "packet loss? frame seq: %d last seq: %d, delta = %d\n", frame->seq, context->last_seq, delta);
+ if (delta > 2) { // wait for key frame
+ if (h->i) {
+ switch_set_flag(frame, SFF_WAIT_KEY_FRAME);
+ return SWITCH_STATUS_RESTART;
+ } else { // key frame
+ context->last_seq = frame->seq;
+ switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_NOTICE, "Received a H263 key frame after delta %d\n", delta);
+ }
+ }
+ // return SWITCH_STATUS_RESTART;
+ } else if (delta < 1) {
+ // probabaly stream changed
+ return SWITCH_STATUS_RESTART;
+ } else { // delta == 1
+ context->last_seq = frame->seq;
+ }
+
if (0) {
static char *h263_src[] = {
"NONE",