diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 66aa0fc..c6bb74d 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -54,6 +54,20 @@ typedef struct bool sendtblspcmapfile; } basebackup_options; +/* Checksum algorithm option for manifest */ +enum manifestCheckSum +{ + MC_NONE = 0, + MC_SHA256, + MC_CRC +}; + +/* checksum algorithm context */ +typedef union checksumCtx +{ + pg_sha256_ctx sha256_ctx; + pg_crc32c crc_ctx; +} ChecksumCtx; static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, @@ -72,7 +86,7 @@ static void SendBackupHeader(List *tablespaces); static void InitializeManifest(StringInfo manifest); static void AddFileToManifest(StringInfo manifest, const char *tsoid, const char *filename, size_t size, time_t mtime, - uint8 *shabuf); + ChecksumCtx *cCtx); static void SendBackupManifest(StringInfo manifest); static char *escape_field_for_manifest(const char *s); static void base_backup_cleanup(int code, Datum arg); @@ -82,6 +96,9 @@ static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli); static int compareWalFileNames(const ListCell *a, const ListCell *b); static void throttle(size_t increment); static bool is_checksummed_file(const char *fullpath, const char *filename); +static void initilize_manifest_checksum(ChecksumCtx *cCtx); +static void update_manifest_checksum(ChecksumCtx *cCtx, const char *buf, off_t cnt); +static int final_manifest_checksum(ChecksumCtx *cCtx, char *checksumbuf); /* Was the backup currently in-progress initiated in recovery mode? */ static bool backup_started_in_recovery = false; @@ -132,8 +149,8 @@ static long long int total_checksum_failures; /* Do not verify checksums. */ static bool noverify_checksums = false; -/* Add file entry in to manifest with checksums. */ -static bool manifest_with_checksums = false; + +static enum manifestCheckSum manifest_with_checksums = MC_NONE; /* * The contents of these directories are removed or recreated during server @@ -677,6 +694,7 @@ parse_basebackup_options(List *options, basebackup_options *opt) bool o_maxrate = false; bool o_tablespace_map = false; bool o_noverify_checksums = false; + bool o_manifest_with_checksums = false; MemSet(opt, 0, sizeof(*opt)); foreach(lopt, options) @@ -767,11 +785,23 @@ parse_basebackup_options(List *options, basebackup_options *opt) } else if (strcmp(defel->defname, "manifest_with_checksums") == 0) { - if (manifest_with_checksums) + char *manifest_with_checksum_algo = NULL; + if (o_manifest_with_checksums) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate option \"%s\"", defel->defname))); - manifest_with_checksums = true; + manifest_with_checksum_algo = strVal(defel->arg); + + if (pg_strcasecmp(manifest_with_checksum_algo, "SHA256") == 0) + manifest_with_checksums = MC_SHA256; + else if (pg_strcasecmp(manifest_with_checksum_algo, "CRC") == 0) + manifest_with_checksums = MC_CRC; + else if (pg_strcasecmp(manifest_with_checksum_algo, "NONE") == 0) + manifest_with_checksums = MC_NONE; + else + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid manifest_with_checksums option \"%s\"", manifest_with_checksum_algo))); } else @@ -907,14 +937,16 @@ InitializeManifest(StringInfo manifest) static void AddFileToManifest(StringInfo manifest, const char *tsoid, const char *filename, size_t size, time_t mtime, - uint8 *shabuf) + ChecksumCtx *cCtx) { char pathbuf[MAXPGPATH]; char *escaped_filename; static char timebuf[128]; - static char shatextbuf[PG_SHA256_DIGEST_LENGTH * 2 + 1]; - int shatextlen; + static char checksumbuf[256]; + char encode_checksumbuf[256]; struct pg_tm *tm; + char *checksumlabel = NULL; + int checksumbuflen; /* * If this file is part of a tablespace, the filename passed to this @@ -941,19 +973,32 @@ AddFileToManifest(StringInfo manifest, const char *tsoid, elog(ERROR, "could not convert epoch to timestamp: %m"); pg_strftime(timebuf, sizeof(timebuf), "%Y-%m-%d %H:%M:%S %Z", tm); - /* Convert checksum to hexadecimal. */ - if (manifest_with_checksums) + /* Generate final checksum and Convert it to hexadecimal. */ + if (manifest_with_checksums != MC_NONE) { - shatextlen = - hex_encode((char *) shabuf, PG_SHA256_DIGEST_LENGTH, shatextbuf); - Assert(shatextlen + 1 == sizeof(shatextbuf)); - shatextbuf[shatextlen] = '\0'; + checksumbuflen = final_manifest_checksum(cCtx, checksumbuf); + switch (manifest_with_checksums) + { + case MC_SHA256: + checksumlabel = "SHA256:"; + break; + case MC_CRC: + checksumlabel = "CRC:"; + break; + case MC_NONE: + break; + } + checksumbuflen = hex_encode(checksumbuf, + checksumbuflen, + encode_checksumbuf); + encode_checksumbuf[checksumbuflen] = '\0'; } /* Add to manifest. */ - appendStringInfo(manifest, "File\t%s\t%zu\t%s\t%s\n", + appendStringInfo(manifest, "File\t%s\t%zu\t%s\t%s%s\n", escaped_filename == NULL ? filename : escaped_filename, - size, timebuf, manifest_with_checksums ? shatextbuf : "-"); + size, timebuf, checksumlabel ? checksumlabel : "", + manifest_with_checksums != MC_NONE ? encode_checksumbuf : "-"); /* Avoid leaking memory. */ if (escaped_filename != NULL) @@ -966,24 +1011,34 @@ AddFileToManifest(StringInfo manifest, const char *tsoid, static void SendBackupManifest(StringInfo manifest) { - pg_sha256_ctx sha256_ctx; - uint8 shabuf[PG_SHA256_DIGEST_LENGTH]; + char checksumbuf[256]; StringInfoData protobuf; - int shastringlen; + int checksumbuflen; + ChecksumCtx cCtx; + /* Checksum the manifest. */ - if (manifest_with_checksums) + if (manifest_with_checksums != MC_NONE) { - pg_sha256_init(&sha256_ctx); - pg_sha256_update(&sha256_ctx, (uint8 *) manifest->data, manifest->len); - pg_sha256_final(&sha256_ctx, shabuf); + initilize_manifest_checksum(&cCtx); + update_manifest_checksum(&cCtx, manifest->data, manifest->len); + checksumbuflen = final_manifest_checksum(&cCtx, (char *) checksumbuf); appendStringInfoString(manifest, "Manifest-Checksum\t"); - shastringlen = PG_SHA256_DIGEST_LENGTH * 2; - enlargeStringInfo(manifest, shastringlen); - shastringlen = hex_encode((char *) shabuf, PG_SHA256_DIGEST_LENGTH, + switch (manifest_with_checksums) + { + case MC_SHA256: + appendStringInfoString(manifest, "SHA256:"); + break; + case MC_CRC: + appendStringInfoString(manifest, "CRC:"); + break; + case MC_NONE: + break; + } + enlargeStringInfo(manifest, checksumbuflen * 2); + checksumbuflen = hex_encode(checksumbuf, checksumbuflen, manifest->data + manifest->len); - Assert(shastringlen == PG_SHA256_DIGEST_LENGTH * 2); - manifest->len += shastringlen; + manifest->len += checksumbuflen; appendStringInfoChar(manifest, '\n'); } @@ -1115,11 +1170,7 @@ sendFileWithContent(const char *filename, const char *content, struct stat statbuf; int pad, len; - pg_sha256_ctx sha256_ctx; - uint8 shabuf[PG_SHA256_DIGEST_LENGTH]; - - if (manifest_with_checksums) - pg_sha256_init(&sha256_ctx); + ChecksumCtx cCtx; len = strlen(content); @@ -1153,14 +1204,14 @@ sendFileWithContent(const char *filename, const char *content, pq_putmessage('d', buf, pad); } - if (manifest_with_checksums) + if (manifest_with_checksums != MC_NONE) { - pg_sha256_update(&sha256_ctx, (uint8 *) content, len); - pg_sha256_final(&sha256_ctx, shabuf); + initilize_manifest_checksum(&cCtx); + update_manifest_checksum(&cCtx, content, len); } AddFileToManifest(manifest, NULL, filename, len, statbuf.st_mtime, - shabuf); + &cCtx); } /* @@ -1559,7 +1610,6 @@ is_checksummed_file(const char *fullpath, const char *filename) * Copied from pg_dump, but modified to work with libpq for sending */ - /* * Given the member, write the TAR header & send the file. * @@ -1591,11 +1641,9 @@ sendFile(const char *readfilename, const char *tarfilename, int segmentno = 0; char *segmentpath; bool verify_checksum = false; - pg_sha256_ctx sha256_ctx; - uint8 shabuf[PG_SHA256_DIGEST_LENGTH]; + ChecksumCtx cCtx; - if (manifest_with_checksums) - pg_sha256_init(&sha256_ctx); + initilize_manifest_checksum(&cCtx); fp = AllocateFile(readfilename, "rb"); if (fp == NULL) @@ -1766,8 +1814,7 @@ sendFile(const char *readfilename, const char *tarfilename, (errmsg("base backup could not send data, aborting backup"))); /* Also feed it to the checksum machinery. */ - if (manifest_with_checksums) - pg_sha256_update(&sha256_ctx, (uint8 *) buf, cnt); + update_manifest_checksum(&cCtx, buf, cnt); len += cnt; throttle(cnt); @@ -1793,8 +1840,7 @@ sendFile(const char *readfilename, const char *tarfilename, { cnt = Min(sizeof(buf), statbuf->st_size - len); pq_putmessage('d', buf, cnt); - if (manifest_with_checksums) - pg_sha256_update(&sha256_ctx, (uint8 *) buf, cnt); + update_manifest_checksum(&cCtx, buf, cnt); len += cnt; throttle(cnt); } @@ -1826,11 +1872,8 @@ sendFile(const char *readfilename, const char *tarfilename, } total_checksum_failures += checksum_failures; - - if (manifest_with_checksums) - pg_sha256_final(&sha256_ctx, shabuf); AddFileToManifest(manifest, tsoid, tarfilename, statbuf->st_size, - statbuf->st_mtime, shabuf); + statbuf->st_mtime, &cCtx); return true; } @@ -1966,3 +2009,63 @@ throttle(size_t increment) */ throttled_last = GetCurrentTimestamp(); } + +/* + * Initilize the manifest checksum context according to the provided algorithm. + */ +static void +initilize_manifest_checksum(ChecksumCtx *cCtx) +{ + switch (manifest_with_checksums) + { + case MC_SHA256: + pg_sha256_init(&cCtx->sha256_ctx); + break; + case MC_CRC: + INIT_CRC32C(cCtx->crc_ctx); + break; + case MC_NONE: + break; + } +} + +static void +update_manifest_checksum(ChecksumCtx *cCtx, const char *buf, off_t cnt) +{ + switch (manifest_with_checksums) + { + case MC_SHA256: + pg_sha256_update(&cCtx->sha256_ctx, (uint8 *) buf, cnt); + break; + case MC_CRC: + COMP_CRC32C(cCtx->crc_ctx, buf, cnt); + break; + case MC_NONE: + break; + } +} + +/* + * Function calculate the final checksum for the provided context and returns + * the length of checksum. + */ +static int +final_manifest_checksum(ChecksumCtx *cCtx, char *checksumbuf) +{ + int checksumlen = 0; + switch (manifest_with_checksums) + { + case MC_SHA256: + pg_sha256_final(&cCtx->sha256_ctx, (uint8 *)checksumbuf); + checksumlen = PG_SHA256_DIGEST_LENGTH; + break; + case MC_CRC: + FIN_CRC32C(cCtx->crc_ctx); + pg_ltoa(cCtx->crc_ctx, checksumbuf); + checksumlen = strlen(checksumbuf); + break; + case MC_NONE: + break; + } + return checksumlen; +} diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index 542a3f7..409bd23 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -215,10 +215,10 @@ base_backup_opt: $$ = makeDefElem("noverify_checksums", (Node *)makeInteger(true), -1); } - | K_MANIFEST_WITH_CHECKSUMS + | K_MANIFEST_WITH_CHECKSUMS SCONST { $$ = makeDefElem("manifest_with_checksums", - (Node *)makeInteger(true), -1); + (Node *)makeString($2), -1); } ; diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index f4f8ffe..f7b16ab 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -141,7 +141,7 @@ static bool temp_replication_slot = true; static bool create_slot = false; static bool no_slot = false; static bool verify_checksums = true; -static bool manifest_with_checksums = false; +static char *manifest_with_checksums = NULL; static bool success = false; static bool made_new_pgdata = false; @@ -400,7 +400,7 @@ usage(void) printf(_(" --no-verify-checksums\n" " do not verify checksums\n")); printf(_(" --manifest-with-checksums\n" - " do calculate checksums for manifest files\n")); + " calculate checksums for manifest files using provided algorithm\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\nConnection options:\n")); printf(_(" -d, --dbname=CONNSTR connection string\n")); @@ -1824,7 +1824,7 @@ BaseBackup(void) } basebkp = - psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s %s", + psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s MANIFEST_WITH_CHECKSUMS '%s'", escaped_label, showprogress ? "PROGRESS" : "", includewal == FETCH_WAL ? "WAL" : "", @@ -1833,7 +1833,7 @@ BaseBackup(void) maxrate_clause ? maxrate_clause : "", format == 't' ? "TABLESPACE_MAP" : "", verify_checksums ? "" : "NOVERIFY_CHECKSUMS", - manifest_with_checksums ? "MANIFEST_WITH_CHECKSUMS" : ""); + manifest_with_checksums ? manifest_with_checksums : "NONE"); if (PQsendQuery(conn, basebkp) == 0) { @@ -2166,7 +2166,7 @@ main(int argc, char **argv) {"waldir", required_argument, NULL, 1}, {"no-slot", no_argument, NULL, 2}, {"no-verify-checksums", no_argument, NULL, 3}, - {"manifest-with-checksums", no_argument, NULL, 4}, + {"manifest-with-checksums", required_argument, NULL, 'm'}, {NULL, 0, NULL, 0} }; int c; @@ -2194,7 +2194,7 @@ main(int argc, char **argv) atexit(cleanup_directories_atexit); - while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP", + while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvPm:", long_options, &option_index)) != -1) { switch (c) @@ -2335,8 +2335,8 @@ main(int argc, char **argv) case 3: verify_checksums = false; break; - case 4: - manifest_with_checksums = true; + case 'm': + manifest_with_checksums = pg_strdup(optarg); break; default: