Re: [REVIEW] Re: Compression of full-page-writes - Mailing list pgsql-hackers

On 2014-07-04 19:27:10 +0530, Rahila Syed wrote:
> +    /* Allocates memory for compressed backup blocks according to the compression
> +     * algorithm used.Once per session at the time of insertion of first XLOG
> +     * record.
> +     * This memory stays till the end of session. OOM is handled by making the
> +     * code proceed without FPW compression*/
> +    static char *compressed_pages[XLR_MAX_BKP_BLOCKS];
> +    static bool compressed_pages_allocated = false;
> +    if (compress_backup_block != BACKUP_BLOCK_COMPRESSION_OFF &&
> +        compressed_pages_allocated!= true)
> +    {
> +        size_t buffer_size = VARHDRSZ;
> +        int j;
> +        if (compress_backup_block == BACKUP_BLOCK_COMPRESSION_SNAPPY)
> +            buffer_size += snappy_max_compressed_length(BLCKSZ);
> +        else if (compress_backup_block == BACKUP_BLOCK_COMPRESSION_LZ4)
> +            buffer_size += LZ4_compressBound(BLCKSZ);
> +        else if (compress_backup_block == BACKUP_BLOCK_COMPRESSION_PGLZ)
> +            buffer_size += PGLZ_MAX_OUTPUT(BLCKSZ);
> +        for (j = 0; j < XLR_MAX_BKP_BLOCKS; j++)
> +        {    compressed_pages[j] = (char *) malloc(buffer_size);
> +            if(compressed_pages[j] == NULL)
> +            {
> +                compress_backup_block=BACKUP_BLOCK_COMPRESSION_OFF;
> +                break;
> +            }
> +        }
> +        compressed_pages_allocated = true;
> +    }

Why not do this in InitXLOGAccess() or similar?

>      /*
>       * Make additional rdata chain entries for the backup blocks, so that we
>       * don't need to special-case them in the write loop.  This modifies the
> @@ -1015,11 +1048,32 @@ begin:;
>          rdt->next = &(dtbuf_rdt2[i]);
>          rdt = rdt->next;
>  
> +        if (compress_backup_block != BACKUP_BLOCK_COMPRESSION_OFF)
> +        {
> +        /* Compress the backup block before including it in rdata chain */
> +            rdt->data = CompressBackupBlock(page, BLCKSZ - bkpb->hole_length,
> +                                            compressed_pages[i], &(rdt->len));
> +            if (rdt->data != NULL)
> +            {
> +                /*
> +                 * write_len is the length of compressed block and its varlena
> +                 * header
> +                 */
> +                write_len += rdt->len;
> +                bkpb->hole_length = BLCKSZ - rdt->len;
> +                /*Adding information about compression in the backup block header*/
> +                bkpb->block_compression=compress_backup_block;
> +                rdt->next = NULL;
> +                continue;
> +            }
> +        }
> +

So, you're compressing backup blocks one by one. I wonder if that's the
right idea and if we shouldn't instead compress all of them in one run to
increase the compression ratio.


> +/*
>   * Get a pointer to the right location in the WAL buffer containing the
>   * given XLogRecPtr.
>   *
> @@ -4061,6 +4174,50 @@ RestoreBackupBlockContents(XLogRecPtr lsn, BkpBlock bkpb, char *blk,
>      {
>          memcpy((char *) page, blk, BLCKSZ);
>      }
> +    /* Decompress if backup block is compressed*/
> +    else if (VARATT_IS_COMPRESSED((struct varlena *) blk)
> +                && bkpb.block_compression!=BACKUP_BLOCK_COMPRESSION_OFF)
> +    {
> +        if (bkpb.block_compression == BACKUP_BLOCK_COMPRESSION_SNAPPY)
> +        {
> +            int ret;
> +            size_t compressed_length = VARSIZE((struct varlena *) blk) - VARHDRSZ;
> +            char *compressed_data = (char *)VARDATA((struct varlena *) blk);
> +            size_t s_uncompressed_length;
> +
> +            ret = snappy_uncompressed_length(compressed_data,
> +                    compressed_length,
> +                    &s_uncompressed_length);
> +            if (!ret)
> +                elog(ERROR, "snappy: failed to determine compression length");
> +            if (BLCKSZ != s_uncompressed_length)
> +                elog(ERROR, "snappy: compression size mismatch %d != %zu",
> +                        BLCKSZ, s_uncompressed_length);
> +
> +            ret = snappy_uncompress(compressed_data,
> +                    compressed_length,
> +                    page);
> +            if (ret != 0)
> +                elog(ERROR, "snappy: decompression failed: %d", ret);
> +        }
> +        else if (bkpb.block_compression == BACKUP_BLOCK_COMPRESSION_LZ4)
> +        {
> +            int ret;
> +            size_t compressed_length = VARSIZE((struct varlena *) blk) - VARHDRSZ;
> +            char *compressed_data = (char *)VARDATA((struct varlena *) blk);
> +            ret = LZ4_decompress_fast(compressed_data, page,
> +                    BLCKSZ);
> +            if (ret != compressed_length)
> +                elog(ERROR, "lz4: decompression size mismatch: %d vs %zu", ret,
> +                        compressed_length);
> +        }
> +        else if (bkpb.block_compression == BACKUP_BLOCK_COMPRESSION_PGLZ)
> +        {
> +            pglz_decompress((PGLZ_Header *) blk, (char *) page);
> +        }
> +        else
> +            elog(ERROR, "Wrong value for compress_backup_block GUC");
> +    }
>      else
>      {
>          memcpy((char *) page, blk, bkpb.hole_offset);

So why aren't we compressing the hole here instead of compressing the
parts that the current logic deems to be filled with important information?

>  /*
>   * Options for enum values stored in other modules
>   */
> @@ -3498,6 +3512,16 @@ static struct config_enum ConfigureNamesEnum[] =
>          NULL, NULL, NULL
>      },
>  
> +    {
> +        {"compress_backup_block", PGC_SIGHUP, WAL_SETTINGS,
> +            gettext_noop("Compress backup block in WAL using specified compression algorithm."),
> +            NULL
> +        },
> +        &compress_backup_block,
> +        BACKUP_BLOCK_COMPRESSION_OFF, backup_block_compression_options,
> +        NULL, NULL, NULL
> +    },
> +

This should be named 'compress_full_page_writes' or so, even if a
temporary guc. There's the 'full_page_writes' guc and I see little
reaason to deviate from its name.

Greetings,

Andres Freund