#include #include #include #include #include #include #include #include static AVOutputFormat *fmt; static AVFormatContext *ofmt_ctx; int output_service_number; char output_service_name[128]; char output_service_provider_name[128]; enum ScreenStatus {UNKNOWN, WORKING, WARNINNG, ERROR}; struct ScreenInfo{ int v_pid; int v_stream_index; int a_pid; int a_stream_index; int service_number; char service_name[128]; char service_provider_name[128]; enum ScreenStatus screenStatus; char errorDisc[512]; char warningDisc[512]; AVCodecContext *v_dec_ctx; AVCodecContext *v_enc_ctx; AVCodecContext *a_dec_ctx; AVCodecContext *a_enc_ctx; AVFilterContext *afifo_ctx; AVFilterContext *fifo_ctx; AVFilterContext *setpts_ctx; AVFilterContext *abuffersrc_ctx; AVFilterContext *aformat_ctx; AVFilterContext *showvolume_ctx; AVFilterContext *buffersrc_ctx; AVFilterContext *scale_ctx; AVFilterContext *buffersink_ctx; AVFilterContext *nullsrc_ctx; AVFilterContext *overlay1_ctx; AVFilterContext *overlay2_ctx; struct ScreenInfo *next; }; struct ThreadInfo { int threadId; AVFormatContext *ifmt_ctx; AVCodecContext *enc_ctx; AVCodecContext *volume_ctx; struct ScreenInfo *screenInfo; }; const char *graph_desc = "[0:a]showvolume=w=240:h=20:o=1:f=0.50:r=25[vol0];" "nullsrc=size=320x240[base1];" "[0:v]fifo,setpts=PTS-STARTPTS,scale=320x240[w0h0];" "[w0h0][vol0]overlay=eval=0:x=280[w0h0a];" "[base1][w0h0a]overlay=shortest=1:eval=0[vout]"; AVFilterInOut *outputs; AVFilterInOut *inputs; AVFilterGraph *filter_graph; static int open_input_file(const char *filename, struct ThreadInfo *threadInfo){ int ret; unsigned int i, j, k; struct ScreenInfo *screenInfo = NULL, *temp = NULL; bool isAudiofound = false, isVideoFound = false; threadInfo->ifmt_ctx = NULL; if ((ret = avformat_open_input(&(threadInfo->ifmt_ctx), filename, NULL, NULL)) < 0) { av_log(NULL, AV_LOG_ERROR, "Cannot open input file\n"); return ret; } if ((ret = avformat_find_stream_info(threadInfo->ifmt_ctx, NULL)) < 0) { av_log(NULL, AV_LOG_ERROR, "Cannot find stream information\n"); return ret; } for(i = 0; i < threadInfo->ifmt_ctx->nb_programs; i++){ AVDictionaryEntry *tag = NULL; screenInfo = (struct ScreenInfo *)malloc(sizeof(struct ScreenInfo)); memset(screenInfo, 0, sizeof(struct ScreenInfo)); screenInfo->next = NULL; screenInfo->service_number = threadInfo->ifmt_ctx->programs[i]->id; tag = NULL; if((tag = av_dict_get(threadInfo->ifmt_ctx->programs[i]->metadata, "service_name", tag, AV_DICT_IGNORE_SUFFIX))){ sprintf(screenInfo->service_name, "%s", tag->value); } else { sprintf(screenInfo->service_name, "Program %d", screenInfo->service_number); } tag = NULL; if((tag = av_dict_get(threadInfo->ifmt_ctx->programs[i]->metadata, "service_provider", tag, AV_DICT_IGNORE_SUFFIX))){ if(strlen(tag->value) > 3) { sprintf(screenInfo->service_provider_name, "%s", tag->value); } else { strcpy(screenInfo->service_provider_name, "Not Available"); } } isAudiofound = false; for(j = 0; j < threadInfo->ifmt_ctx->programs[i]->nb_stream_indexes; j++) { k = threadInfo->ifmt_ctx->programs[i]->stream_index[j]; AVStream *stream = threadInfo->ifmt_ctx->streams[k]; if(stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO || stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO){ if(stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && isAudiofound){ continue; } AVCodec *dec = avcodec_find_decoder(stream->codecpar->codec_id); if (!dec) { av_log(NULL, AV_LOG_ERROR, "Failed to find decoder for stream #%u\n", i); return AVERROR_DECODER_NOT_FOUND; } AVCodecContext *codec_ctx; codec_ctx = avcodec_alloc_context3(dec); if (!codec_ctx) { av_log(NULL, AV_LOG_ERROR, "Failed to allocate the decoder context for stream #%u\n", i); return AVERROR(ENOMEM); } ret = avcodec_parameters_to_context(codec_ctx, stream->codecpar); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Failed to copy decoder parameters to input decoder context " "for stream #%u\n", i); return ret; } if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO){ codec_ctx->framerate = av_guess_frame_rate(threadInfo->ifmt_ctx, stream, NULL); } ret = avcodec_open2(codec_ctx, dec, NULL); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Failed to open decoder for stream #%u\n", i); return ret; } if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO){ screenInfo->v_dec_ctx = codec_ctx; screenInfo->v_pid = stream->id; screenInfo->v_stream_index = stream->index; } else { screenInfo->a_dec_ctx = codec_ctx; screenInfo->a_pid = stream->id; screenInfo->a_stream_index = stream->index; isAudiofound = true; } } } temp = threadInfo->screenInfo; if(!temp){ threadInfo->screenInfo = screenInfo; } else { while(temp->next){ temp = temp->next; } temp->next = screenInfo; } } if(!threadInfo->ifmt_ctx->nb_programs){ isAudiofound = false, isVideoFound = false; screenInfo = (struct ScreenInfo *)malloc(sizeof(struct ScreenInfo)); memset(screenInfo, 0, sizeof(struct ScreenInfo)); screenInfo->next = NULL; screenInfo->service_number = 10; strcpy(screenInfo->service_name, "???"); strcpy(screenInfo->service_provider_name, "???"); for (i = 0; i < threadInfo->ifmt_ctx->nb_streams; i++) { AVStream *stream = threadInfo->ifmt_ctx->streams[i]; if(stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO || stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO){ if(isAudiofound && isVideoFound){ continue; } else if(stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO && isVideoFound){ continue; } else if(stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO && isAudiofound){ continue; } AVCodec *dec = avcodec_find_decoder(stream->codecpar->codec_id); if (!dec) { av_log(NULL, AV_LOG_ERROR, "Failed to find decoder for stream #%u\n", i); return AVERROR_DECODER_NOT_FOUND; } AVCodecContext *codec_ctx; codec_ctx = avcodec_alloc_context3(dec); if (!codec_ctx) { av_log(NULL, AV_LOG_ERROR, "Failed to allocate the decoder context for stream #%u\n", i); return AVERROR(ENOMEM); } ret = avcodec_parameters_to_context(codec_ctx, stream->codecpar); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Failed to copy decoder parameters to input decoder context " "for stream #%u\n", i); return ret; } if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO){ codec_ctx->framerate = av_guess_frame_rate(threadInfo->ifmt_ctx, stream, NULL); } ret = avcodec_open2(codec_ctx, dec, NULL); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Failed to open decoder for stream #%u\n", i); return ret; } if (codec_ctx->codec_type == AVMEDIA_TYPE_VIDEO){ screenInfo->v_dec_ctx = codec_ctx; screenInfo->v_pid = stream->id; screenInfo->v_stream_index = stream->index; isVideoFound = true; } else { screenInfo->a_dec_ctx = codec_ctx; screenInfo->a_pid = stream->id; screenInfo->a_stream_index = stream->index; isAudiofound = true; } } } temp = threadInfo->screenInfo; if(!temp){ threadInfo->screenInfo = screenInfo; } else { while(temp->next){ temp = temp->next; } temp->next = screenInfo; } } av_dump_format(threadInfo->ifmt_ctx, 0, filename, 0); return 0; } static int open_output_file(const char *filename, struct ThreadInfo *threadInfo){ AVStream *out_stream; AVCodecContext *enc_ctx; AVCodec *encoder; int ret; fmt = av_guess_format("mpegts", NULL, NULL); if(!fmt){ printf("Could not deduce output format from file extension: using MPEGTS.\n"); } ofmt_ctx = avformat_alloc_context(); //avformat_alloc_output_context2(&ofmt_ctx, NULL, NULL, filename); if (!ofmt_ctx) { av_log(NULL, AV_LOG_ERROR, "Could not create output context\n"); return AVERROR_UNKNOWN; } ofmt_ctx->oformat = fmt; AVProgram *program = av_new_program(ofmt_ctx, output_service_number); av_dict_set(&(program->metadata), "service_name", output_service_name, 0); av_dict_set(&(program->metadata), "service_provider", output_service_provider_name, 0); out_stream = avformat_new_stream(ofmt_ctx, NULL); if (!out_stream) { av_log(NULL, AV_LOG_ERROR, "Failed allocating output stream\n"); return AVERROR_UNKNOWN; } out_stream->id = 256; encoder = avcodec_find_encoder_by_name("h264_nvenc"); if (!encoder) { av_log(NULL, AV_LOG_FATAL, "h264_nvenc encoder not found\n"); encoder = avcodec_find_encoder_by_name("libx264"); if (!encoder) { av_log(NULL, AV_LOG_FATAL, "libx264 encoder not found\n"); return AVERROR_INVALIDDATA; } } av_program_add_stream_index(ofmt_ctx, output_service_number, out_stream->index); enc_ctx = avcodec_alloc_context3(encoder); if (!enc_ctx) { av_log(NULL, AV_LOG_FATAL, "Failed to allocate the encoder context\n"); return AVERROR(ENOMEM); } enc_ctx->height = 240; enc_ctx->width = 320; enc_ctx->sample_aspect_ratio = threadInfo->screenInfo->v_dec_ctx->sample_aspect_ratio; enc_ctx->pix_fmt = AV_PIX_FMT_YUV420P; enc_ctx->framerate = (AVRational){25, 1}; enc_ctx->time_base = av_inv_q(enc_ctx->framerate); //(AVRational){1, 25}; // Third parameter can be used to pass settings to encoder ret = avcodec_open2(enc_ctx, encoder, NULL); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Cannot open video encoder for stream #%u\n", out_stream->index); enc_ctx = avcodec_alloc_context3(encoder); if (!enc_ctx) { av_log(NULL, AV_LOG_ERROR, "Cannot open video encoder for stream #%u\n", out_stream->index); return ret; } } ret = avcodec_parameters_from_context(out_stream->codecpar, enc_ctx); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Failed to copy encoder parameters to output stream #%u\n", out_stream->index); return ret; } if (ofmt_ctx->oformat->flags & AVFMT_GLOBALHEADER) enc_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; ofmt_ctx->oformat->flags |= AVFMT_NOTIMESTAMPS; ofmt_ctx->oformat->flags |= AVFMT_TS_NONSTRICT; threadInfo->enc_ctx = enc_ctx; av_dump_format(ofmt_ctx, 0, filename, 1); if (!(ofmt_ctx->oformat->flags & AVFMT_NOFILE)) { ret = avio_open(&(ofmt_ctx->pb), filename, AVIO_FLAG_WRITE); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Could not open output file '%s'", filename); return ret; } } /* init muxer, write output file header */ ret = avformat_write_header(ofmt_ctx, NULL); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Error occurred when opening output file\n"); return ret; } return 0; } int init_filtersTest(struct ThreadInfo *threadInfo){ char strbuf[512]; int ret = 0; const AVFilter *abuffer = avfilter_get_by_name("abuffer"); const AVFilter *buffer = avfilter_get_by_name("buffer"); const AVFilter *buffersink = avfilter_get_by_name("buffersink"); outputs = avfilter_inout_alloc(); inputs = avfilter_inout_alloc(); filter_graph = avfilter_graph_alloc(); if (!outputs || !inputs || !filter_graph) { return AVERROR(ENOMEM); } if ((ret = avfilter_graph_parse2(filter_graph, graph_desc, &inputs, &outputs)) < 0) { printf("avfilger_graph_parse2 failed\n"); return 0; } struct ScreenInfo *screenInfo = threadInfo->screenInfo; while(screenInfo){ //abuffer filter for source audio frame buffer AVCodecContext *avctx = threadInfo->ifmt_ctx->streams[screenInfo->a_stream_index]->codec; AVRational time_base = threadInfo->ifmt_ctx->streams[screenInfo->a_stream_index]->time_base; snprintf(strbuf, sizeof(strbuf), "time_base=%d/%d:sample_rate=%d:sample_fmt=%s:channel_layout=0x%"PRIx64, time_base.num, time_base.den, avctx->sample_rate, av_get_sample_fmt_name(avctx->sample_fmt), avctx->channel_layout); printf("abuffer=%s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->abuffersrc_ctx), abuffer, "in", strbuf, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing abuffer filter\n"); return ret; } //buffer filter for source video frame buffer snprintf(strbuf, sizeof(strbuf), "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d:sws_param=1", screenInfo->v_dec_ctx->width, screenInfo->v_dec_ctx->height, screenInfo->v_dec_ctx->pix_fmt, screenInfo->v_dec_ctx->time_base.num, screenInfo->v_dec_ctx->time_base.den, screenInfo->v_dec_ctx->sample_aspect_ratio.num, screenInfo->v_dec_ctx->sample_aspect_ratio.den); printf("buffer=%s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->buffersrc_ctx), buffer, "in1", strbuf, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing buffer filter\n"); return ret; } //buffersink filter for sink video frame for output ret = avfilter_graph_create_filter(&(screenInfo->buffersink_ctx), buffersink, "out", NULL, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing buffersink filter\n"); return ret; } enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE }; ret = av_opt_set_int_list(screenInfo->buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN); AVFilterInOut *cur, *in, *out; int i; for (i = 0; i < threadInfo->ifmt_ctx->nb_streams; i++) { in = NULL; out = NULL; for (cur = inputs; cur; cur = cur->next) { if(threadInfo->ifmt_ctx->streams[i]->codecpar->codec_type == avfilter_pad_get_type(cur->filter_ctx->input_pads, cur->pad_idx)){ in = cur; } } for (cur = outputs; cur; cur = cur->next) { if (threadInfo->ifmt_ctx->streams[i]->codecpar->codec_type == avfilter_pad_get_type(cur->filter_ctx->output_pads, cur->pad_idx)) { out = cur; } } if(threadInfo->ifmt_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO){ printf("Video in: %s, out: %s[%d]\n", in->name, out->name, out->pad_idx); if ((ret = avfilter_link(screenInfo->buffersrc_ctx, 0, in->filter_ctx, in->pad_idx)) < 0) { return ret; } if ((ret = avfilter_link(out->filter_ctx, out->pad_idx, screenInfo->buffersink_ctx, 0)) < 0) { return ret; } } else { if ((ret = avfilter_link(screenInfo->abuffersrc_ctx, 0, in->filter_ctx, in->pad_idx)) < 0) { return ret; } //printf("Audio in: %s, out: %s[%d]\n", in->name, out->name, out->pad_idx); /*if ((ret = avfilter_link(out->filter_ctx, out->pad_idx, screenInfo->buffersink_ctx, 0)) < 0) { return ret; } printf("Audio in: %s, out: %s[%d]\n", in->name, out->name, out->pad_idx);*/ } } screenInfo = screenInfo->next; break; } printf("before av!\n"); if ((ret = avfilter_graph_config(filter_graph, NULL)) < 0){ printf("avfilter_graph_config failed\n"); return 0; } printf("after avfilter_graph_config\n"); printf("graph:\n\n %s\n", avfilter_graph_dump(filter_graph, NULL)); return 0; } static int init_filters(struct ThreadInfo *threadInfo){ int ret; struct ScreenInfo *screenInfo = NULL; char strbuf[512]; /*"[0:a]showvolume=w=240:h=20:o=1:f=0.50:r=25[vol0];" "nullsrc=size=320x240[base1];" "[0:v]fifo,setpts=PTS-STARTPTS,scale=320x240[w0h0];" "[w0h0][vol0]overlay=eval=0:x=280[w0h0a];" "[base1][w0h0a]overlay=shortest=1:eval=0";*/ // create new graph filter_graph = avfilter_graph_alloc(); if (!filter_graph) { av_log(NULL, AV_LOG_ERROR, "unable to create filter graph: out of memory\n"); return -1; } if(threadInfo){ //filter for audio const AVFilter *abuffer = avfilter_get_by_name("abuffer"); const AVFilter *showvolume = avfilter_get_by_name("showvolume"); const AVFilter *aformat = avfilter_get_by_name("aformat"); const AVFilter *afifo = avfilter_get_by_name("afifo"); //filter for video const AVFilter *buffer = avfilter_get_by_name("buffer"); const AVFilter *fifo = avfilter_get_by_name("fifo"); const AVFilter *setpts = avfilter_get_by_name("setpts"); const AVFilter *scale = avfilter_get_by_name("scale"); const AVFilter *buffersink = avfilter_get_by_name("buffersink"); //nullsrc filter const AVFilter *nullsrc = avfilter_get_by_name("nullsrc"); //overlay filters const AVFilter *overlay1 = avfilter_get_by_name("overlay"); const AVFilter *overlay2 = avfilter_get_by_name("overlay"); screenInfo = threadInfo->screenInfo; while(screenInfo){ if(!(screenInfo->v_pid && screenInfo->a_pid)){ screenInfo = screenInfo->next; printf("no audio/video stream found!\n"); continue; } //abuffer filter for source audio frame buffer AVCodecContext *avctx = threadInfo->ifmt_ctx->streams[screenInfo->a_stream_index]->codec; AVRational time_base = threadInfo->ifmt_ctx->streams[screenInfo->a_stream_index]->time_base; snprintf(strbuf, sizeof(strbuf), "time_base=%d/%d:sample_rate=%d:sample_fmt=%d:channel_layout=0x%"PRIx64, time_base.num, time_base.den, avctx->sample_rate, 6, //avctx->sample_fmt, //av_get_sample_fmt_name(avctx->sample_fmt), avctx->channel_layout); fprintf(stderr, "abuffer=%s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->abuffersrc_ctx), abuffer, "abuffer", strbuf, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing abuffer filter\n"); return ret; } //afifo filter to store audio frames ret = avfilter_graph_create_filter(&(screenInfo->afifo_ctx), afifo, "afifo", NULL, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing afifo filter\n"); return ret; } //aformat filter snprintf(strbuf, sizeof(strbuf), "sample_fmts=%s:sample_rates=%d:channel_layouts=0x%"PRIx64, av_get_sample_fmt_name(AV_SAMPLE_FMT_FLTP), avctx->sample_rate, (uint64_t)AV_CH_LAYOUT_STEREO); fprintf(stderr, "aformat: %s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->aformat_ctx), aformat, "aformat", strbuf, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing abuffer filter\n"); return ret; } //showvolume filter int w = 240; int h = 20; int o = 1; double f = 0.50; int r = 25; snprintf(strbuf, sizeof(strbuf), "w=%d:h=%d:o=%d:f=%f:r=%d", w, h, o, f, r); fprintf(stderr, "showvolume=%s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->showvolume_ctx), showvolume, "showvolume", strbuf, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing showvolume filter\n"); return ret; } //buffer filter for source video frame buffer snprintf(strbuf, sizeof(strbuf), "video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d:sws_param=2", screenInfo->v_dec_ctx->width, screenInfo->v_dec_ctx->height, screenInfo->v_dec_ctx->pix_fmt, screenInfo->v_dec_ctx->time_base.num, screenInfo->v_dec_ctx->time_base.den, screenInfo->v_dec_ctx->sample_aspect_ratio.num, screenInfo->v_dec_ctx->sample_aspect_ratio.den); fprintf(stderr, "buffer=%s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->buffersrc_ctx), buffer, "buffer", strbuf, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing buffer filter\n"); return ret; } //buffersink filter for sink video frame for output ret = avfilter_graph_create_filter(&(screenInfo->buffersink_ctx), buffersink, "buffersink", NULL, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing buffersink filter\n"); return ret; } enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE }; ret = av_opt_set_int_list(screenInfo->buffersink_ctx, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE, AV_OPT_SEARCH_CHILDREN); //fifo filter ret = avfilter_graph_create_filter(&(screenInfo->fifo_ctx), fifo, "fifo", NULL, NULL, filter_graph); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "errpr initializing fifo filter"); return ret; } //setpts filter snprintf(strbuf, sizeof(strbuf), "PTS-STARTPTS"); fprintf(stderr, "setpts=%s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->setpts_ctx), setpts, "setpts", strbuf, NULL, filter_graph); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "errpr initializing setpts filter"); return ret; } //scale filter snprintf(strbuf, sizeof(strbuf), "320x240"); fprintf(stderr, "scale=%s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->scale_ctx), scale, "scale", strbuf, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing scale filter\n"); return ret; } //nullsrc filter snprintf(strbuf, sizeof(strbuf), "size=320x240"); fprintf(stderr, "nullsrc=%s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->nullsrc_ctx), nullsrc, "nullsrc", strbuf, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing nullsrc filter\n"); return ret; } //overlay filter 1 snprintf(strbuf, sizeof(strbuf), "eval=0:x=280"); fprintf(stderr, "overlay=%s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->overlay1_ctx), overlay1, "overlay1", strbuf, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing overlay1 filter\n"); return ret; } //overlay filter 2 snprintf(strbuf, sizeof(strbuf), "shortest=1:eval=0"); fprintf(stderr, "overlay=%s\n", strbuf); ret = avfilter_graph_create_filter(&(screenInfo->overlay2_ctx), overlay2, "overlay2", strbuf, NULL, filter_graph); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error initializing overlay2 filter\n"); return ret; } // connect inputs and outputs filters //connecting abuffer to afifo ret = avfilter_link(screenInfo->abuffersrc_ctx, 0, screenInfo->afifo_ctx, 0); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link abuffer filter to afifo filter\n"); break; } //connecting afifo to aformat ret = avfilter_link(screenInfo->afifo_ctx, 0, screenInfo->aformat_ctx, 0); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link afifo filter to aformat filter\n"); break; } //connecting aformat to showvolume filter ret = avfilter_link(screenInfo->aformat_ctx, 0, screenInfo->showvolume_ctx, 0); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link aformat filter to showvolume filter\n"); break; } //connecting buffer to fifo filter ret = avfilter_link(screenInfo->buffersrc_ctx, 0, screenInfo->fifo_ctx, 0); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link buffer filter to fifo filter\n"); break; } //connecting fifo filter to setspt filter ret = avfilter_link(screenInfo->fifo_ctx, 0, screenInfo->setpts_ctx, 0); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link fifo filter to setpts filter\n"); break; } //connecting setpts filter to scale filter ret = avfilter_link(screenInfo->setpts_ctx, 0, screenInfo->scale_ctx, 0); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link setpts filter to scale filter\n"); break; } //connecting scale to overlay1 ret = avfilter_link(screenInfo->scale_ctx, 0, screenInfo->overlay1_ctx, 0); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link scale filter to overlay1 filter\n"); break; } //connecting showvolume to overlay1 ret = avfilter_link(screenInfo->showvolume_ctx, 0, screenInfo->overlay1_ctx, 1); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link showvolume filter to overlay1 filter\n"); break; } //connecting nullsrc to overlay2 ret = avfilter_link(screenInfo->nullsrc_ctx, 0, screenInfo->overlay2_ctx, 0); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link nullsrc filter to overlay2 filter\n"); break; } //connecting overlay1 to overlay2 ret = avfilter_link(screenInfo->overlay1_ctx, 0, screenInfo->overlay2_ctx, 1); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link overlay1 filter to overlay2 filter\n"); break; } //connecting overlay2 to buffersink for final video output ret = avfilter_link(screenInfo->overlay2_ctx, 0, screenInfo->buffersink_ctx, 0); if(ret < 0){ av_log(NULL, AV_LOG_FATAL, "failed to link overlay2 filter to buffersink filter\n"); break; } screenInfo = screenInfo->next; } } ret = avfilter_graph_config(filter_graph, NULL); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "error configuring the filter graph\n"); return ret; } printf("graph:\n\n %s\n", avfilter_graph_dump(filter_graph, NULL)); return 0; } int save_frame_as_jpeg(AVCodecContext *pCodecCtx, AVFrame *pFrame, int FrameNo) { int ret; char strbuf[512]; AVCodec *jpegCodec = avcodec_find_encoder(AV_CODEC_ID_MJPEG); if (!jpegCodec) { printf("cant fined mjpeg encoder\n"); return -1; } AVCodecContext *jpegContext = avcodec_alloc_context3(jpegCodec); if (!jpegContext) { printf("cant alloc mjpeg context\n"); return -1; } jpegContext->thread_count = 3; jpegContext->thread_type = FF_THREAD_SLICE; jpegContext->codec_id = AV_CODEC_ID_MJPEG; jpegContext->codec_type = AVMEDIA_TYPE_VIDEO; jpegContext->bit_rate = pCodecCtx->bit_rate; jpegContext->pix_fmt = AV_PIX_FMT_YUVJ420P; jpegContext->height = pFrame->height; jpegContext->width = pFrame->width; jpegContext->time_base.num = pCodecCtx->time_base.num; jpegContext->time_base.den = pCodecCtx->time_base.den; ret = avcodec_open2(jpegContext, jpegCodec, NULL); if (ret < 0) { printf("failed to open mjepg encoder. Ret: %d[%s]\n", ret, av_make_error_string(strbuf, sizeof(strbuf), ret)); return -1; } FILE *JPEGFile; char JPEGFName[256]; AVPacket packet = {.data = NULL, .size = 0}; av_init_packet(&packet); ret = avcodec_send_frame(jpegContext, pFrame); if(ret != 0){ printf("avcodec_send_frame failed while encoding frame to jpg. Error: %s[%d]\n", av_make_error_string(strbuf, sizeof(strbuf), ret), ret); return -1; } ret = avcodec_receive_packet(jpegContext, &packet); if(ret != 0){ printf("avcodec_receive_packet failed while encoding frame to jpg\n"); return -1; } sprintf(JPEGFName, "data/dvr-%06d.jpg", FrameNo); JPEGFile = fopen(JPEGFName, "wb"); if(!JPEGFile){ printf("jpeg file open failed\n"); } fwrite(packet.data, 1, packet.size, JPEGFile); fclose(JPEGFile); av_packet_unref(&packet); avcodec_close(jpegContext); return 0; } static int encode_write_frame(AVFrame *filt_frame, unsigned int stream_index, int *got_frame, struct ThreadInfo *threadInfo, struct ScreenInfo *screenInfo) { int ret; char buff[256]; AVPacket enc_pkt; static int count = 0; av_log(NULL, AV_LOG_INFO, "Encoding frame\n"); /* encode filtered frame */ enc_pkt.data = NULL; enc_pkt.size = 0; av_init_packet(&enc_pkt); enum AVMediaType type; type = threadInfo->ifmt_ctx->streams[stream_index]->codecpar->codec_type; if(type == AVMEDIA_TYPE_VIDEO){ printf("encoding video frame\n"); ret = avcodec_send_frame(threadInfo->enc_ctx, filt_frame); if(ret != 0){ av_make_error_string(buff, sizeof(buff), ret); printf("Error: %s\n", buff); fprintf(stderr, "avcodec_send_frame failed from stream id: %d[0x%X]\n", screenInfo->v_pid, screenInfo->v_pid); av_frame_free(&filt_frame); return 0; } printf("after avcodec send frame\n"); ret = avcodec_receive_packet(threadInfo->enc_ctx, &enc_pkt); if(ret != 0){ fprintf(stderr, "avcodec_receive_packet failed from stream id: %d[0x%X]\n", screenInfo->v_pid, screenInfo->v_pid); return 0; } printf("after avcodec receive packet\n"); av_packet_rescale_ts(&enc_pkt, threadInfo->ifmt_ctx->streams[stream_index]->time_base, screenInfo->v_dec_ctx->time_base); } else if(type == AVMEDIA_TYPE_AUDIO){ printf("encoding audio frame\n"); //there will be no audio frames ret = avcodec_send_frame(threadInfo->enc_ctx, filt_frame); if(ret != 0){ fprintf(stderr, "avcodec_send_frame failed from stream id: %d[0x%X]\n", screenInfo->a_pid, screenInfo->a_pid); av_frame_free(&filt_frame); return 0; } ret = avcodec_receive_packet(threadInfo->enc_ctx, &enc_pkt); if(ret != 0){ fprintf(stderr, "avcodec_receive_packet failed from stream id: %d[0x%X]\n", screenInfo->a_pid, screenInfo->a_pid); return 0; } av_packet_rescale_ts(&enc_pkt, threadInfo->ifmt_ctx->streams[stream_index]->time_base, screenInfo->a_dec_ctx->time_base); } save_frame_as_jpeg(threadInfo->enc_ctx, filt_frame, count); count++; av_frame_free(&filt_frame); if (ret < 0) return ret; /* prepare packet for muxing */ enc_pkt.stream_index = stream_index; /*av_packet_rescale_ts(&enc_pkt, stream_ctx[stream_index].enc_ctx->time_base, ofmt_ctx->streams[stream_index]->time_base);*/ av_packet_rescale_ts(&enc_pkt, threadInfo->ifmt_ctx->streams[stream_index]->time_base, screenInfo->a_dec_ctx->time_base); //enc_pkt.pts = av_rescale_q(1, ofmt_ctx->streams[0]->codec->time_base, ofmt_ctx->streams[0]->time_base); av_log(NULL, AV_LOG_DEBUG, "Muxing frame\n"); /* mux encoded frame */ ret = av_interleaved_write_frame(ofmt_ctx, &enc_pkt); av_log(NULL, AV_LOG_DEBUG, "after Muxing frame\n"); return 1; } static int filter_encode_write_frame(AVFrame *frame, unsigned int stream_index, AVFilterContext *buffersrc_ctx, AVFilterContext *buffersink_ctx, struct ThreadInfo *threadInfo, struct ScreenInfo *screenInfo){ int ret; AVFrame *filt_frame; char tempStr[512]; av_log(NULL, AV_LOG_INFO, "Pushing decoded frame to filters. Stream Index: %d\n", stream_index); /* push the decoded frame into the filtergraph */ ret = av_buffersrc_add_frame_flags(buffersrc_ctx, frame, AV_BUFFERSRC_FLAG_PUSH | AV_BUFFERSRC_FLAG_KEEP_REF); // AV_BUFFERSRC_FLAG_NO_CHECK_FORMAT); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Error while feeding the filtergraph\n"); return ret; } /* pull filtered frames from the filtergraph */ while (1) { filt_frame = av_frame_alloc(); if (!filt_frame) { ret = AVERROR(ENOMEM); break; } av_log(NULL, AV_LOG_INFO, "Pulling filtered frame from filters\n"); //ret = av_buffersink_get_samples(buffersink_ctx, filt_frame, 0); ret = av_buffersink_get_frame(buffersink_ctx, filt_frame); if (ret < 0) { /* if no more frames for output - returns AVERROR(EAGAIN) * if flushed and no more frames for output - returns AVERROR_EOF * rewrite retcode to 0 to show it as normal procedure completion */ if (ret == AVERROR(EAGAIN)){ av_log(NULL, AV_LOG_INFO, "no frame in buff!\n"); } else if(ret == AVERROR_EOF){ av_log(NULL, AV_LOG_INFO, "End of FILE!!!\n"); } else { av_strerror(ret, tempStr, sizeof(tempStr)); av_log(NULL, AV_LOG_INFO, "av_buffersink_get_frame failed with return code: %d[%s]\n", ret, tempStr); } ret = 0; av_frame_free(&filt_frame); break; } //filt_frame->pict_type = AV_PICTURE_TYPE_NONE; ret = encode_write_frame(filt_frame, stream_index, NULL, threadInfo, screenInfo); if (ret < 0){ av_log(NULL, AV_LOG_INFO, "encode_write_frame failed! Error: %d[%s]\n", ret, av_make_error_string(tempStr, sizeof(tempStr), ret)); break; } } return ret; } static int ifilter_send_frame(AVFilterContext *filterCtx, AVFrame *frame, struct ThreadInfo *threadInfo){ int need_reinit = 0, ret, i; need_reinit = filterCtx->inputs[0]->format != frame->format; printf("\n\tfilter format: %d\n", filterCtx->inputs[0]->format); printf("\tframe format: %d\n", frame->format); printf("\t1) format change flag: %d\n", need_reinit); switch(filterCtx->inputs[0]->type){ case AVMEDIA_TYPE_AUDIO: need_reinit |= filterCtx->inputs[0]->sample_rate != frame->sample_rate || filterCtx->inputs[0]->channels != frame->channels || filterCtx->inputs[0]->channel_layout != frame->channel_layout; break; case AVMEDIA_TYPE_VIDEO: need_reinit |= filterCtx->inputs[0]->w != frame->width || filterCtx->inputs[0]->h != frame->height; break; default: break; } printf("\t2) format change flag: %d\n", need_reinit); if (need_reinit) { //av_buffer_unref(&(filterCtx->inputs[0]->hw_frames_ctx)); filterCtx->inputs[0]->format = frame->format; filterCtx->inputs[0]->w = frame->width; filterCtx->inputs[0]->h = frame->height; filterCtx->inputs[0]->sample_aspect_ratio = frame->sample_aspect_ratio; filterCtx->inputs[0]->sample_rate = frame->sample_rate; filterCtx->inputs[0]->channels = frame->channels; filterCtx->inputs[0]->channel_layout = frame->channel_layout; /*if(frame->hw_frames_ctx){ filterCtx->inputs[0]->hw_frames_ctx = av_buffer_ref(frame->hw_frames_ctx); }*/ } else { return 0; } printf("\tneed_reinit: %d, graph: %d\n", need_reinit, filterCtx->graph?1:0); int j; AVFilterContext *tempCtx; for(i = 0; i < filter_graph->nb_filters; i++){ tempCtx = filter_graph->filters[i]; for(j = 0; j < tempCtx->nb_outputs; j++){ tempCtx->outputs[j] = NULL; } for(j = 0; j < tempCtx->nb_inputs; j++){ tempCtx->inputs[j] = NULL; } } init_filters(threadInfo); printf("New graph:\n\n%s\n", avfilter_graph_dump(filter_graph, NULL)); return need_reinit; } int main(int argc, char **argv){ int ret; AVPacket packet = { .data = NULL, .size = 0 }; AVFrame *frame = NULL; enum AVMediaType type; unsigned int stream_index; int got_frame; struct ThreadInfo *threadInfo = NULL; threadInfo = (struct ThreadInfo *)malloc(sizeof(struct ThreadInfo)); memset(threadInfo, 0, sizeof(struct ThreadInfo)); struct ScreenInfo *temp = NULL; output_service_number = 1; sprintf(output_service_name, "TestScreen"); sprintf(output_service_provider_name, "TestScreenProvider"); if (argc != 3) { av_log(NULL, AV_LOG_ERROR, "Usage: %s \n", argv[0]); return 1; } av_log_set_level(AV_LOG_DEBUG); if ((ret = open_input_file(argv[1], threadInfo)) < 0) goto end; if ((ret = open_output_file(argv[2], threadInfo)) < 0) goto end; if ((ret = init_filters(threadInfo)) < 0) goto end; while (1) { if ((ret = av_read_frame(threadInfo->ifmt_ctx, &packet)) < 0) break; stream_index = packet.stream_index; type = threadInfo->ifmt_ctx->streams[packet.stream_index]->codecpar->codec_type; av_log(NULL, AV_LOG_DEBUG, "Demuxer gave frame of stream_index %u\n", stream_index); got_frame = 0; temp = threadInfo->screenInfo; while(temp){ if(temp->a_stream_index == stream_index || temp->v_stream_index == stream_index){ got_frame = 1; break; } temp = temp->next; } if(!got_frame){ av_packet_unref(&packet); continue; } if (filter_graph) { av_log(NULL, AV_LOG_DEBUG, "Going to decode & filter the frame\n"); frame = av_frame_alloc(); if (!frame) { ret = AVERROR(ENOMEM); break; } if (type == AVMEDIA_TYPE_VIDEO){ av_packet_rescale_ts(&packet, threadInfo->ifmt_ctx->streams[stream_index]->time_base, temp->v_dec_ctx->time_base); ret = avcodec_send_packet(temp->v_dec_ctx, &packet); if(ret != 0){ fprintf(stderr, "avcodec_send_packet failed from stream id: %d[0x%X]\n", temp->v_pid, temp->v_pid); continue; } ret = avcodec_receive_frame(temp->v_dec_ctx, frame); if(ret != 0){ fprintf(stderr, "avcodec_receive_frame failed from stream id: %d[0x%X]\n", temp->v_pid, temp->v_pid); av_frame_free(&frame); continue; } frame->pts = frame->best_effort_timestamp; ret = filter_encode_write_frame(frame, stream_index, temp->buffersrc_ctx, temp->buffersink_ctx, threadInfo, temp); av_frame_free(&frame); if (ret < 0) goto end; } else if (type == AVMEDIA_TYPE_AUDIO){ av_packet_rescale_ts(&packet, threadInfo->ifmt_ctx->streams[stream_index]->time_base, temp->a_dec_ctx->time_base); ret = avcodec_send_packet(temp->a_dec_ctx, &packet); if(ret != 0){ fprintf(stderr, "avcodec_send_packet failed from stream id: %d[0x%X]\n", temp->a_pid, temp->a_pid); continue; } ret = avcodec_receive_frame(temp->a_dec_ctx, frame); if(ret != 0){ fprintf(stderr, "avcodec_receive_frame failed from stream id: %d[0x%X]\n", temp->a_pid, temp->a_pid); av_frame_free(&frame); continue; } //TODO: //ifilter_send_frame(temp->afifo_ctx, frame, threadInfo); frame->pts = frame->best_effort_timestamp; ret = filter_encode_write_frame(frame, stream_index, temp->abuffersrc_ctx, temp->buffersink_ctx, threadInfo, temp); av_frame_free(&frame); if (ret < 0) goto end; } } av_packet_unref(&packet); } /* flush filters and encoders */ ret = filter_encode_write_frame(NULL, temp->v_stream_index, temp->buffersrc_ctx, temp->buffersink_ctx, threadInfo, temp); if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Flushing filter failed\n"); goto end; } /*ret = filter_encode_write_frame(NULL, temp->a_stream_index, temp->abuffersrc_ctx, temp->buffersink_ctx, threadInfo, temp);*/ if (ret < 0) { av_log(NULL, AV_LOG_ERROR, "Flushing filter failed\n"); goto end; } av_write_trailer(ofmt_ctx); end: avfilter_graph_free(&filter_graph); av_packet_unref(&packet); av_frame_free(&frame); temp = threadInfo->screenInfo; while(temp){ avcodec_free_context(&(temp->a_dec_ctx)); avcodec_free_context(&(temp->v_dec_ctx)); } avformat_close_input(&(threadInfo->ifmt_ctx)); if (ofmt_ctx && !(ofmt_ctx->oformat->flags & AVFMT_NOFILE)) avio_closep(&ofmt_ctx->pb); avformat_free_context(ofmt_ctx); if (ret < 0) av_log(NULL, AV_LOG_ERROR, "Error occurred: %s\n", av_err2str(ret)); return 1; }