From ee9b5e4739bf78b39dc34c9fcc76fc731b09b788 Mon Sep 17 00:00:00 2001 From: Dipesh Pandit Date: Thu, 31 Mar 2022 12:19:31 +0530 Subject: [PATCH] fix crash with lz4 We cannot fix the size of output buffer at client during lz4 compression. Client receives chunks from the server and the size of these chunks varies depending upon the data sent by the server. The output buffer capacity at client should be adjusted based on the size of the chunk received from the server. --- src/bin/pg_basebackup/bbstreamer_lz4.c | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/bin/pg_basebackup/bbstreamer_lz4.c b/src/bin/pg_basebackup/bbstreamer_lz4.c index 67f841d..2ffe224 100644 --- a/src/bin/pg_basebackup/bbstreamer_lz4.c +++ b/src/bin/pg_basebackup/bbstreamer_lz4.c @@ -73,7 +73,6 @@ bbstreamer_lz4_compressor_new(bbstreamer *next, bc_specification *compress) bbstreamer_lz4_frame *streamer; LZ4F_errorCode_t ctxError; LZ4F_preferences_t *prefs; - size_t compressed_bound; Assert(next != NULL); @@ -92,17 +91,6 @@ bbstreamer_lz4_compressor_new(bbstreamer *next, bc_specification *compress) if ((compress->options & BACKUP_COMPRESSION_OPTION_LEVEL) != 0) prefs->compressionLevel = compress->level; - /* - * Find out the compression bound, it specifies the minimum destination - * capacity required in worst case for the success of compression operation - * (LZ4F_compressUpdate) based on a given source size and preferences. - */ - compressed_bound = LZ4F_compressBound(streamer->base.bbs_buffer.maxlen, prefs); - - /* Enlarge buffer if it falls short of compression bound. */ - if (streamer->base.bbs_buffer.maxlen < compressed_bound) - enlargeStringInfo(&streamer->base.bbs_buffer, compressed_bound); - ctxError = LZ4F_createCompressionContext(&streamer->cctx, LZ4F_VERSION); if (LZ4F_isError(ctxError)) pg_log_error("could not create lz4 compression context: %s", @@ -170,7 +158,6 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer, * forward the content to next streamer and empty the buffer. */ out_bound = LZ4F_compressBound(len, &mystreamer->prefs); - Assert(mystreamer->base.bbs_buffer.maxlen >= out_bound); if (avail_out < out_bound) { bbstreamer_content(mystreamer->base.bbs_next, member, @@ -178,6 +165,10 @@ bbstreamer_lz4_compressor_content(bbstreamer *streamer, mystreamer->bytes_written, context); + /* Enlarge buffer if it falls short of out bound. */ + if (mystreamer->base.bbs_buffer.maxlen < out_bound) + enlargeStringInfo(&mystreamer->base.bbs_buffer, out_bound); + avail_out = mystreamer->base.bbs_buffer.maxlen; mystreamer->bytes_written = 0; next_out = (uint8 *) mystreamer->base.bbs_buffer.data; @@ -218,7 +209,6 @@ bbstreamer_lz4_compressor_finalize(bbstreamer *streamer) /* Find out the footer bound and update the output buffer. */ footer_bound = LZ4F_compressBound(0, &mystreamer->prefs); - Assert(mystreamer->base.bbs_buffer.maxlen >= footer_bound); if ((mystreamer->base.bbs_buffer.maxlen - mystreamer->bytes_written) < footer_bound) { @@ -227,6 +217,10 @@ bbstreamer_lz4_compressor_finalize(bbstreamer *streamer) mystreamer->bytes_written, BBSTREAMER_UNKNOWN); + /* Enlarge buffer if it falls short of footer bound. */ + if (mystreamer->base.bbs_buffer.maxlen < footer_bound) + enlargeStringInfo(&mystreamer->base.bbs_buffer, footer_bound); + avail_out = mystreamer->base.bbs_buffer.maxlen; mystreamer->bytes_written = 0; next_out = (uint8 *) mystreamer->base.bbs_buffer.data; -- 1.8.3.1