feat(ffmpeg): speed optimization and export memory buffer

ffmpeg/ffmpeg.c

@@ -214,36 +214,6 @@ AVFrame *convert_frame_to_rgb(AVFrame *frame, int alpha) {
     return output_frame;
 }
 
-int encode_frame_to_image(AVFormatContext *fmt_ctx, AVFrame *frame, AVPacket *pkt) {
-    AVCodec *enc = avcodec_find_encoder(AV_CODEC_ID_PNG);
-    if (!enc) {
-        return AVERROR_ENCODER_NOT_FOUND;
-    }
-    AVCodecContext *enc_ctx = avcodec_alloc_context3(enc);
-    if (!enc_ctx) {
-        return AVERROR(ENOMEM);
-    }
-    enc_ctx->width = frame->width;
-    enc_ctx->height = frame->height;
-    enc_ctx->pix_fmt = frame->format;
-    enc_ctx->codec_type = AVMEDIA_TYPE_VIDEO;
-    enc_ctx->time_base = (AVRational) {1, 1};
-    enc_ctx->compression_level = INT_MAX;
-    int err = open_codec(enc_ctx, enc);
-    if (err < 0) {
-        avcodec_free_context(&enc_ctx);
-        return err;
-    }
-    err = avcodec_send_frame(enc_ctx, frame);
-    if (err < 0) {
-        avcodec_free_context(&enc_ctx);
-        return err;
-    }
-    err = avcodec_receive_packet(enc_ctx, pkt);
-    avcodec_free_context(&enc_ctx);
-    return err;
-}
-
 AVPacket create_packet() {
     AVPacket *pkt = av_packet_alloc();
     pkt->data = NULL;
@@ -462,8 +432,10 @@ static AVFrame *get_best_frame(ThumbContext *thumb_ctx) {
             n = i;
         }
     }
-    thumb_ctx->alpha = alpha_check(thumb_ctx->frames[n].frame, thumb_ctx->desc->flags,
+    thumb_ctx->alpha = alpha_check(
-                                   thumb_ctx->frames[n].hist[thumb_ctx->hist_size - 1]);
+      thumb_ctx->frames[n].frame,
+      thumb_ctx->desc->flags,
+      thumb_ctx->frames[n].hist[thumb_ctx->hist_size - 1]);
     return thumb_ctx->frames[n].frame;
 }
 

ffmpeg/ffmpeg.go

@@ -80,11 +80,8 @@ func LoadAVContext(ctx context.Context, reader io.Reader, size int64) (*AVContex
 	return av, nil
 }
 
-func (av *AVContext) ExportImage() ([]byte, error) {
+func (av *AVContext) Export() (buf []byte, err error) {
-	if !av.hasFrame {
+	return exportBuffer(av)
-		return nil, nil
-	}
-	return encodeFrameImage(av)
 }
 
 func (av *AVContext) Close() {
@@ -279,15 +276,16 @@ func convertFrameToRGB(av *AVContext) error {
 	return nil
 }
 
-func encodeFrameImage(av *AVContext) ([]byte, error) {
+func exportBuffer(av *AVContext) ([]byte, error) {
-	pkt := C.create_packet()
+	if !av.hasFrame {
-	err := C.encode_frame_to_image(av.formatContext, av.frame, &pkt)
+		return nil, ErrDecoderNotFound
-	if err < 0 {
-		return nil, avError(err)
 	}
-	p := C.GoBytes(unsafe.Pointer(pkt.data), pkt.size)
+	size := av.height * av.width
-	if pkt.buf != nil {
+	if av.hasAlpha {
-		C.av_packet_unref(&pkt)
+		size *= 4
+	} else {
+		size *= 3
 	}
-	return p, nil
+	buf := C.GoBytes(unsafe.Pointer(av.frame.data[0]), C.int(size))
+	return buf, nil
 }

ffmpeg/ffmpeg.h

@@ -48,8 +48,6 @@ int create_codec_context(AVStream *video_stream, AVCodecContext **dec_ctx);
 
 AVFrame *convert_frame_to_rgb(AVFrame *frame, int alpha);
 
-int encode_frame_to_image(AVFormatContext *fmt_ctx, AVFrame *frame, AVPacket *pkt);
-
 AVPacket create_packet();
 
 int

go.mod

@@ -3,7 +3,7 @@ module github.com/cshum/imagorvideo
 go 1.18
 
 require (
-	github.com/cshum/imagor v1.1.0
+	github.com/cshum/imagor v1.1.1-0.20220913124751-09512f7d5162
 	github.com/gabriel-vasile/mimetype v1.4.1
 	github.com/stretchr/testify v1.8.0
 	go.uber.org/zap v1.23.0

go.sum

@@ -85,8 +85,10 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH
 github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
 github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cshum/imagor v1.1.0 h1:iE/iajwH2vgVbAGMRQ2fVnIN43uafgKXUptSb6YPmus=
+github.com/cshum/imagor v1.1.1-0.20220913065018-821f28372e5d h1:IBCJ+FL9bCEyWbyrZas4HxG7yJsj0wt7zn8GQa9Abas=
-github.com/cshum/imagor v1.1.0/go.mod h1:nhOPIyQwjrbcopfe7ODwLQ7hbb+Ww5Kt6Jo29iJ+7HY=
+github.com/cshum/imagor v1.1.1-0.20220913065018-821f28372e5d/go.mod h1:nhOPIyQwjrbcopfe7ODwLQ7hbb+Ww5Kt6Jo29iJ+7HY=
+github.com/cshum/imagor v1.1.1-0.20220913124751-09512f7d5162 h1:OwyoZXwwcWDDdcjqZKSP1qAzvkv3txTmbhFPRuBU3p0=
+github.com/cshum/imagor v1.1.1-0.20220913124751-09512f7d5162/go.mod h1:nhOPIyQwjrbcopfe7ODwLQ7hbb+Ww5Kt6Jo29iJ+7HY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

processor.go

@@ -130,14 +130,19 @@ func (p *Processor) Process(ctx context.Context, in *imagor.Blob, params imagorp
 	case 8:
 		filters = append(filters, imagorpath.Filter{Name: "rotate", Args: "90"})
 	}
-	buf, err := av.ExportImage()
+	buf, err := av.Export()
 	if err != nil || len(buf) == 0 {
 		if err == nil {
 			err = imagor.ErrUnsupportedFormat
 		}
 		return
 	}
-	out = imagor.NewBlobFromBytes(buf)
+	bands := 3
+	if meta.HasAlpha {
+		bands = 4
+	}
+	out = imagor.NewBlobFromMemory(buf, meta.Width, meta.Height, bands)
+
 	if len(filters) > 0 {
 		params.Filters = append(filters, params.Filters...)
 		params.Path = imagorpath.GeneratePath(params)