From 38a95ed1d76101162d2bb62d1c943fe41b4ce84d Mon Sep 17 00:00:00 2001 From: Masahiko Sawada Date: Fri, 6 Jan 2023 13:26:09 +0900 Subject: [PATCH v2] Add progress reporting to pg_verifybackup This adds a new option to pg_verifybackup called -P/--progress, showing every second some information about the state of checksum verification. Similarly to what is done for pg_rewind and pg_basebackup, the information printed in the progress report consists of the current amount of data computed and the total amount of data to compute. This could be extended later on. --- doc/src/sgml/ref/pg_verifybackup.sgml | 15 ++++ src/bin/pg_verifybackup/pg_verifybackup.c | 85 +++++++++++++++++++++-- src/bin/pg_verifybackup/t/004_options.pl | 6 ++ 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/ref/pg_verifybackup.sgml b/doc/src/sgml/ref/pg_verifybackup.sgml index 5f83c98706..36335e0a18 100644 --- a/doc/src/sgml/ref/pg_verifybackup.sgml +++ b/doc/src/sgml/ref/pg_verifybackup.sgml @@ -178,6 +178,21 @@ PostgreSQL documentation + + + + + + Enable progress reporting. Turning this on will deliver a progress + report while verifying checksums. + + + This option cannot be used together with the option + . + + + + diff --git a/src/bin/pg_verifybackup/pg_verifybackup.c b/src/bin/pg_verifybackup/pg_verifybackup.c index 7634dfc285..ca0bcabb6d 100644 --- a/src/bin/pg_verifybackup/pg_verifybackup.c +++ b/src/bin/pg_verifybackup/pg_verifybackup.c @@ -16,12 +16,14 @@ #include #include #include +#include #include "common/hashfn.h" #include "common/logging.h" #include "fe_utils/simple_list.h" #include "getopt_long.h" #include "parse_manifest.h" +#include "pgtime.h" /* * For efficiency, we'd like our hash table containing information about the @@ -57,6 +59,8 @@ typedef struct manifest_file bool matched; bool bad; } manifest_file; +#define should_check_checksums(m) \ + (((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE)) /* * Define a hash table which we can use to store information about the files @@ -147,10 +151,19 @@ static void report_fatal_error(const char *pg_restrict fmt,...) pg_attribute_printf(1, 2) pg_attribute_noreturn(); static bool should_ignore_relpath(verifier_context *context, char *relpath); +static void progress_report(bool finished); static void usage(void); static const char *progname; +/* options */ +static bool show_progress = false; +static bool skip_checksums = false; + +/* Progress indicators */ +static uint64 total_size = 0; +static uint64 done_size = 0; + /* * Main entry point. */ @@ -162,6 +175,7 @@ main(int argc, char **argv) {"ignore", required_argument, NULL, 'i'}, {"manifest-path", required_argument, NULL, 'm'}, {"no-parse-wal", no_argument, NULL, 'n'}, + {"progress", no_argument, NULL, 'P'}, {"quiet", no_argument, NULL, 'q'}, {"skip-checksums", no_argument, NULL, 's'}, {"wal-directory", required_argument, NULL, 'w'}, @@ -174,7 +188,6 @@ main(int argc, char **argv) char *manifest_path = NULL; bool no_parse_wal = false; bool quiet = false; - bool skip_checksums = false; char *wal_directory = NULL; char *pg_waldump_path = NULL; @@ -219,7 +232,7 @@ main(int argc, char **argv) simple_string_list_append(&context.ignore_list, "recovery.signal"); simple_string_list_append(&context.ignore_list, "standby.signal"); - while ((c = getopt_long(argc, argv, "ei:m:nqsw:", long_options, NULL)) != -1) + while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1) { switch (c) { @@ -241,6 +254,9 @@ main(int argc, char **argv) case 'n': no_parse_wal = true; break; + case 'P': + show_progress = true; + break; case 'q': quiet = true; break; @@ -277,6 +293,10 @@ main(int argc, char **argv) exit(1); } + /* Complain if the specified arguments conflict */ + if (show_progress && quiet) + pg_fatal("cannot specify both %s and %s", "-P/--progress", "-q/--quiet"); + /* Unless --no-parse-wal was specified, we will need pg_waldump. */ if (!no_parse_wal) { @@ -638,6 +658,10 @@ verify_backup_file(verifier_context *context, char *relpath, char *fullpath) m->bad = true; } + /* Update statistics for progress report, if necessary */ + if (show_progress && !skip_checksums && should_check_checksums(m)) + total_size += m->size; + /* * We don't verify checksums at this stage. We first finish verifying that * we have the expected set of files with the expected sizes, and only @@ -675,11 +699,12 @@ verify_backup_checksums(verifier_context *context) manifest_files_iterator it; manifest_file *m; + progress_report(false); + manifest_files_start_iterate(context->ht, &it); while ((m = manifest_files_iterate(context->ht, &it)) != NULL) { - if (m->matched && !m->bad && m->checksum_type != CHECKSUM_TYPE_NONE && - !should_ignore_relpath(context, m->pathname)) + if (should_check_checksums(m) && !should_ignore_relpath(context, m->pathname)) { char *fullpath; @@ -694,6 +719,8 @@ verify_backup_checksums(verifier_context *context) pfree(fullpath); } } + + progress_report(true); } /* @@ -740,6 +767,10 @@ verify_file_checksum(verifier_context *context, manifest_file *m, close(fd); return; } + + /* Report progress */ + done_size += rc; + progress_report(false); } if (rc < 0) report_backup_error(context, "could not read file \"%s\": %m", @@ -894,6 +925,51 @@ hash_string_pointer(char *s) return hash_bytes(ss, strlen(s)); } +/* + * Print a progress report based on the global variables. + * + * Progress report is written at maximum once per second, unless the finished + * parameter is set to true. + * + * If finished is set to true, this is the last progress report. The cursor + * is moved to the next line. + */ +static void +progress_report(bool finished) +{ + static pg_time_t last_progress_report = 0; + pg_time_t now; + int percent_size = 0; + char totalsize_str[32]; + char donesize_str[32]; + + if (!show_progress) + return; + + now = time(NULL); + if (now == last_progress_report && !finished) + return; /* Max once per second */ + + last_progress_report = now; + percent_size = total_size ? (int) ((done_size * 100 / total_size)) : 0; + + snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT, + total_size / 1024); + snprintf(donesize_str, sizeof(donesize_str), UINT64_FORMAT, + done_size / 1024); + + fprintf(stderr, + _("%*s/%s kB (%d%%) verified"), + (int) strlen(totalsize_str), + donesize_str, totalsize_str, percent_size); + + /* + * Stay on the same line if reporting to a terminal and we're not done + * yet. + */ + fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr); +} + /* * Print out usage information and exit. */ @@ -907,6 +983,7 @@ usage(void) printf(_(" -i, --ignore=RELATIVE_PATH ignore indicated path\n")); printf(_(" -m, --manifest-path=PATH use specified path for manifest\n")); printf(_(" -n, --no-parse-wal do not try to parse WAL files\n")); + printf(_(" -P, --progress show progress information\n")); printf(_(" -q, --quiet do not print any output, except for errors\n")); printf(_(" -s, --skip-checksums skip checksum verification\n")); printf(_(" -w, --wal-directory=PATH use specified path for WAL files\n")); diff --git a/src/bin/pg_verifybackup/t/004_options.pl b/src/bin/pg_verifybackup/t/004_options.pl index 25c485e0ee..466d79c967 100644 --- a/src/bin/pg_verifybackup/t/004_options.pl +++ b/src/bin/pg_verifybackup/t/004_options.pl @@ -28,6 +28,12 @@ ok($result, "-q succeeds: exit code 0"); is($stdout, '', "-q succeeds: no stdout"); is($stderr, '', "-q succeeds: no stderr"); +# Test invalid options +command_fails_like( + [ 'pg_verifybackup', '-P', '-q', $backup_path ], + qr{cannot specify both -P/--progress and -q/--quiet}, + 'cannot use options -P and -q at the same time'); + # Corrupt the PG_VERSION file. my $version_pathname = "$backup_path/PG_VERSION"; my $version_contents = slurp_file($version_pathname); -- 2.31.1