From ebb20b848e5585664e978be21fde2243cce4459b Mon Sep 17 00:00:00 2001 From: Roman Arutyunyan Date: Fri, 12 Oct 2012 20:45:47 +0400 Subject: [PATCH] implemented MPEG-TS sample re-arranging in HLS stream to fix crackles because of low-resolution timestamp in RTMP vs MPEG-TS --- hls/ngx_rtmp_hls_module.c | 39 +++++++++++++++++++++++++++++++++++++++ ngx_rtmp_codec_module.c | 25 ++++++++++++++++++++++--- ngx_rtmp_codec_module.h | 1 + 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/hls/ngx_rtmp_hls_module.c b/hls/ngx_rtmp_hls_module.c index f7ecbfb..5073800 100644 --- a/hls/ngx_rtmp_hls_module.c +++ b/hls/ngx_rtmp_hls_module.c @@ -73,6 +73,9 @@ typedef struct { ngx_int_t out_astream; int8_t nal_bytes; + int64_t aframe_base; + int64_t aframe_num; + AVFormatContext *out_format; } ngx_rtmp_hls_ctx_t; @@ -82,6 +85,7 @@ typedef struct { ngx_flag_t hls; ngx_msec_t fraglen; ngx_msec_t muxdelay; + ngx_msec_t sync; ngx_msec_t playlen; size_t nfrags; ngx_rtmp_hls_ctx_t **ctx; @@ -127,6 +131,13 @@ static ngx_command_t ngx_rtmp_hls_commands[] = { offsetof(ngx_rtmp_hls_app_conf_t, muxdelay), NULL }, + { ngx_string("hls_sync"), + NGX_RTMP_MAIN_CONF|NGX_RTMP_SRV_CONF|NGX_RTMP_APP_CONF|NGX_CONF_TAKE1, + ngx_conf_set_msec_slot, + NGX_RTMP_APP_CONF_OFFSET, + offsetof(ngx_rtmp_hls_app_conf_t, sync), + NULL }, + ngx_null_command }; @@ -909,6 +920,7 @@ ngx_rtmp_hls_audio(ngx_rtmp_session_t *s, ngx_rtmp_header_t *h, ngx_rtmp_hls_ctx_t *ctx; ngx_rtmp_codec_ctx_t *codec_ctx; AVPacket packet; + int64_t dts, ddts; static u_char buffer[NGX_RTMP_HLS_BUFSIZE]; hacf = ngx_rtmp_get_module_app_conf(s, ngx_rtmp_hls_module); @@ -940,6 +952,31 @@ ngx_rtmp_hls_audio(ngx_rtmp_session_t *s, ngx_rtmp_header_t *h, packet.data = buffer; packet.size = ngx_rtmp_hls_chain2buffer(buffer, sizeof(buffer), in, 1); + if (hacf->sync && codec_ctx->sample_rate) { + + /* TODO: We assume here AAC frame size is 1024 + * Need to handle AAC frames with frame size of 960 */ + + dts = ctx->aframe_base + ctx->aframe_num * 90000 * 1024 / + codec_ctx->sample_rate; + ddts = dts - packet.dts; + + if (ddts > (int64_t) hacf->sync * 90 || + ddts < (int64_t) hacf->sync * -90) + { + ctx->aframe_base = packet.dts; + ctx->aframe_num = 0; + + ngx_log_debug2(NGX_LOG_DEBUG_RTMP, s->connection->log, 0, + "hls: sync breakup ddts=%L (%.5fs)", + ddts, ddts / 90000.); + } else { + packet.dts = dts; + } + + ctx->aframe_num++; + } + if (codec_ctx->audio_codec_id == NGX_RTMP_AUDIO_AAC) { if (packet.size == 0) { ngx_log_error(NGX_LOG_ERR, s->connection->log, 0, @@ -1119,6 +1156,7 @@ ngx_rtmp_hls_create_app_conf(ngx_conf_t *cf) conf->hls = NGX_CONF_UNSET; conf->fraglen = NGX_CONF_UNSET; conf->muxdelay = NGX_CONF_UNSET; + conf->sync = NGX_CONF_UNSET; conf->playlen = NGX_CONF_UNSET; conf->nbuckets = 1024; @@ -1135,6 +1173,7 @@ ngx_rtmp_hls_merge_app_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_value(conf->hls, prev->hls, 0); ngx_conf_merge_msec_value(conf->fraglen, prev->fraglen, 5000); ngx_conf_merge_msec_value(conf->muxdelay, prev->muxdelay, 700); + ngx_conf_merge_msec_value(conf->sync, prev->sync, 0); ngx_conf_merge_msec_value(conf->playlen, prev->playlen, 30000); ngx_conf_merge_str_value(conf->path, prev->path, ""); conf->ctx = ngx_pcalloc(cf->pool, diff --git a/ngx_rtmp_codec_module.c b/ngx_rtmp_codec_module.c index 25ef0a9..778d23f 100644 --- a/ngx_rtmp_codec_module.c +++ b/ngx_rtmp_codec_module.c @@ -162,6 +162,11 @@ ngx_rtmp_codec_av(ngx_rtmp_session_t *s, ngx_rtmp_header_t *h, static ngx_uint_t sample_rates[] = { 5512, 11025, 22050, 44100 }; + static ngx_uint_t aac_sample_rates[] = + { 96000, 88200, 64000, 48000, + 44100, 32000, 24000, 22050, + 16000, 12000, 11025, 8000, + 7350, 0, 0, 0 }; if (h->type != NGX_RTMP_MSG_AUDIO && h->type != NGX_RTMP_MSG_VIDEO) { return NGX_OK; @@ -183,7 +188,10 @@ ngx_rtmp_codec_av(ngx_rtmp_session_t *s, ngx_rtmp_header_t *h, ctx->audio_codec_id = (fmt & 0xf0) >> 4; ctx->audio_channels = (fmt & 0x01) + 1; ctx->sample_size = (fmt & 0x02) ? 2 : 1; - ctx->sample_rate = sample_rates[(fmt & 0x0c) >> 2]; + + if (ctx->aac_sample_rate == 0) { + ctx->sample_rate = sample_rates[(fmt & 0x0c) >> 2]; + } } else { ctx->video_codec_id = (fmt & 0x0f); } @@ -207,8 +215,19 @@ ngx_rtmp_codec_av(ngx_rtmp_session_t *s, ngx_rtmp_header_t *h, header = &ctx->aac_header; pheader = &ctx->aac_pheader; version = &ctx->aac_version; - ngx_log_debug0(NGX_LOG_DEBUG_RTMP, s->connection->log, 0, - "codec: AAC header arrived"); + + /* 1 byte RTMP audio header + * 1 byte AAC conf/data + * 3 bytes until 18-22 bits of AAC header */ + if (in->buf->last - in->buf->pos >= 5) { + ctx->aac_sample_rate = aac_sample_rates[ + (in->buf->pos[4] >> 2) & 0xff]; + ctx->sample_rate = ctx->aac_sample_rate; + } + + ngx_log_debug1(NGX_LOG_DEBUG_RTMP, s->connection->log, 0, + "codec: AAC header arrived, sample_rate=%u", + ctx->aac_sample_rate); } } else { if (ctx->video_codec_id == NGX_RTMP_VIDEO_H264) { diff --git a/ngx_rtmp_codec_module.h b/ngx_rtmp_codec_module.h index 4a95e32..d02f100 100644 --- a/ngx_rtmp_codec_module.h +++ b/ngx_rtmp_codec_module.h @@ -55,6 +55,7 @@ typedef struct { ngx_uint_t video_codec_id; ngx_uint_t audio_data_rate; ngx_uint_t audio_codec_id; + ngx_uint_t aac_sample_rate; ngx_uint_t sample_rate; /* 5512, 11025, 22050, 44100 */ ngx_uint_t sample_size; /* 1=8bit, 2=16bit */ ngx_uint_t audio_channels; /* 1, 2 */