From 4ca55a929e49d194d533df8f68114ef815502af2 Mon Sep 17 00:00:00 2001 From: Peter Geoghegan Date: Thu, 24 Nov 2022 18:20:36 -0800 Subject: [PATCH v9 3/5] Add eager and lazy freezing strategies to VACUUM. Avoid large build-ups of all-visible pages by making non-aggressive VACUUMs freeze pages proactively for VACUUMs/tables where eager vacuuming is deemed appropriate. VACUUM determines its freezing strategy based on the value of the new vacuum_freeze_strategy_threshold GUC (or reloption) in most cases: Tables that exceeds the size threshold use the eager freezing strategy. Otherwise VACUUM uses the lazy freezing strategy, which is essentially the same approach that VACUUM has always taken to freezing (though not quite, due to the influence of page level freezing following recent work). When the eager strategy is in use, lazy_scan_prune will trigger freezing a page's tuples at the point that it notices that it will at least become all-visible -- it can be made all-frozen instead. We still respect FreezeLimit, though: the presence of any XID < FreezeLimit also triggers page-level freezing (just as it would with the lazy strategy). Eager freezing is generally only applied when vacuuming larger tables, where freezing most individual heap pages at the first opportunity (in the first VACUUM operation where they can definitely be set all-visible) will improve performance stability. A later commit will add skipping strategies, which are designed to work in tandem with the freezing strategy work from this commit. Note that the vacuum_freeze_strategy_threshold GUC will also influence VACUUM's choice of skipping strategy. There will be lazy and eager skipping strategies to complement VACUUM's lazy and eager freezing strategies. Author: Peter Geoghegan Reviewed-By: Jeff Davis Reviewed-By: Andres Freund Discussion: https://postgr.es/m/CAH2-WzkFok_6EAHuK39GaW4FjEFQsY=3J0AAd6FXk93u-Xq3Fg@mail.gmail.com --- src/include/commands/vacuum.h | 10 +++++ src/include/utils/rel.h | 1 + src/backend/access/common/reloptions.c | 11 ++++++ src/backend/access/heap/vacuumlazy.c | 37 ++++++++++++++++++- src/backend/commands/vacuum.c | 17 ++++++++- src/backend/postmaster/autovacuum.c | 10 +++++ src/backend/utils/misc/guc_tables.c | 11 ++++++ src/backend/utils/misc/postgresql.conf.sample | 1 + doc/src/sgml/config.sgml | 19 +++++++++- doc/src/sgml/maintenance.sgml | 6 +-- doc/src/sgml/ref/create_table.sgml | 14 +++++++ doc/src/sgml/ref/vacuum.sgml | 16 ++++---- 12 files changed, 139 insertions(+), 14 deletions(-) diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 896d1b1ac..de28d581a 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -220,6 +220,9 @@ typedef struct VacuumParams * use default */ int multixact_freeze_table_age; /* multixact age at which to scan * whole table */ + int freeze_strategy_threshold; /* threshold to use eager + * freezing, in total heap blocks, + * -1 to use default */ bool is_wraparound; /* force a for-wraparound vacuum */ int log_min_duration; /* minimum execution threshold in ms at * which autovacuum is logged, -1 to use @@ -272,6 +275,12 @@ struct VacuumCutoffs */ TransactionId FreezeLimit; MultiXactId MultiXactCutoff; + + /* + * Threshold cutoff point (expressed in # of physical heap rel blocks in + * rel's main fork) for triggering eager/all-visible freezing strategy + */ + BlockNumber freeze_strategy_threshold; }; /* @@ -295,6 +304,7 @@ extern PGDLLIMPORT int vacuum_freeze_min_age; extern PGDLLIMPORT int vacuum_freeze_table_age; extern PGDLLIMPORT int vacuum_multixact_freeze_min_age; extern PGDLLIMPORT int vacuum_multixact_freeze_table_age; +extern PGDLLIMPORT int vacuum_freeze_strategy_threshold; extern PGDLLIMPORT int vacuum_failsafe_age; extern PGDLLIMPORT int vacuum_multixact_failsafe_age; diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index f383a2fca..e195b63d7 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -308,6 +308,7 @@ typedef struct AutoVacOpts int vacuum_ins_threshold; int analyze_threshold; int vacuum_cost_limit; + int freeze_strategy_threshold; int freeze_min_age; int freeze_max_age; int freeze_table_age; diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index 75b734489..4b680501c 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -260,6 +260,15 @@ static relopt_int intRelOpts[] = }, -1, 1, 10000 }, + { + { + "autovacuum_freeze_strategy_threshold", + "Table size at which VACUUM freezes using eager strategy.", + RELOPT_KIND_HEAP | RELOPT_KIND_TOAST, + ShareUpdateExclusiveLock + }, + -1, 0, INT_MAX + }, { { "autovacuum_freeze_min_age", @@ -1851,6 +1860,8 @@ default_reloptions(Datum reloptions, bool validate, relopt_kind kind) offsetof(StdRdOptions, autovacuum) + offsetof(AutoVacOpts, analyze_threshold)}, {"autovacuum_vacuum_cost_limit", RELOPT_TYPE_INT, offsetof(StdRdOptions, autovacuum) + offsetof(AutoVacOpts, vacuum_cost_limit)}, + {"autovacuum_freeze_strategy_threshold", RELOPT_TYPE_INT, + offsetof(StdRdOptions, autovacuum) + offsetof(AutoVacOpts, freeze_strategy_threshold)}, {"autovacuum_freeze_min_age", RELOPT_TYPE_INT, offsetof(StdRdOptions, autovacuum) + offsetof(AutoVacOpts, freeze_min_age)}, {"autovacuum_freeze_max_age", RELOPT_TYPE_INT, diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index fe64bd6ed..307842582 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -152,6 +152,8 @@ typedef struct LVRelState bool aggressive; /* Use visibility map to skip? (disabled by DISABLE_PAGE_SKIPPING) */ bool skipwithvm; + /* Eagerly freeze all tuples on pages about to be set all-visible? */ + bool eager_freeze_strategy; /* Wraparound failsafe has been triggered? */ bool failsafe_active; /* Consider index vacuuming bypass optimization? */ @@ -241,6 +243,7 @@ typedef struct LVSavedErrInfo /* non-export function prototypes */ static void lazy_scan_heap(LVRelState *vacrel); +static void lazy_scan_strategy(LVRelState *vacrel); static BlockNumber lazy_scan_skip(LVRelState *vacrel, Buffer *vmbuffer, BlockNumber next_block, bool *next_unskippable_allvis, @@ -469,6 +472,10 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->skipwithvm = skipwithvm; + /* + * Determine freezing strategy used by VACUUM + */ + lazy_scan_strategy(vacrel); if (verbose) { if (vacrel->aggressive) @@ -1252,6 +1259,25 @@ lazy_scan_heap(LVRelState *vacrel) lazy_cleanup_all_indexes(vacrel); } +/* + * lazy_scan_strategy() -- Determine freezing strategy. + * + * Our traditional/lazy freezing strategy is useful when putting off the work + * of freezing totally avoids work that turns out to have been unnecessary. + * On the other hand we eagerly freeze pages when that strategy spreads out + * the burden of freezing over time. + */ +static void +lazy_scan_strategy(LVRelState *vacrel) +{ + BlockNumber rel_pages = vacrel->rel_pages; + + Assert(vacrel->scanned_pages == 0); + + vacrel->eager_freeze_strategy = + rel_pages >= vacrel->cutoffs.freeze_strategy_threshold; +} + /* * lazy_scan_skip() -- set up range of skippable blocks using visibility map. * @@ -1773,9 +1799,18 @@ retry: * one XID/MXID from before FreezeLimit/MultiXactCutoff is present. Also * freeze when pruning generated an FPI, if doing so means that we set the * page all-frozen afterwards (this could happen during second heap pass). + * + * When ongoing VACUUM opted to use the eager freezing strategy, we freeze + * any page that will become all-visible, making it all-frozen instead. + * (Actually, the all-visible/eager freezing strategy doesn't quite work + * that way. It triggers freezing for pages that it sees will thereby be + * set all-frozen in the VM immediately afterwards -- a stricter test. + * Some pages that can be set all-visible cannot also be set all-frozen, + * even after freezing, due to the presence of lock-only MultiXactIds.) */ if (pagefrz.freeze_required || tuples_frozen == 0 || - (prunestate->all_visible && prunestate->all_frozen && prune_fpi)) + (prunestate->all_visible && prunestate->all_frozen && + (vacrel->eager_freeze_strategy || prune_fpi))) { /* * We're freezing the page. Our final NewRelfrozenXid doesn't need to diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index cdc39d17d..420b85be6 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -67,6 +67,7 @@ int vacuum_freeze_min_age; int vacuum_freeze_table_age; int vacuum_multixact_freeze_min_age; int vacuum_multixact_freeze_table_age; +int vacuum_freeze_strategy_threshold; int vacuum_failsafe_age; int vacuum_multixact_failsafe_age; @@ -263,6 +264,9 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) params.multixact_freeze_table_age = -1; } + /* Determine freezing strategy later on using GUC or reloption */ + params.freeze_strategy_threshold = -1; + /* user-invoked vacuum is never "for wraparound" */ params.is_wraparound = false; @@ -931,7 +935,8 @@ vacuum_get_cutoffs(Relation rel, const VacuumParams *params, multixact_freeze_min_age, freeze_table_age, multixact_freeze_table_age, - effective_multixact_freeze_max_age; + effective_multixact_freeze_max_age, + freeze_strategy_threshold; TransactionId nextXID, safeOldestXmin, aggressiveXIDCutoff; @@ -944,6 +949,7 @@ vacuum_get_cutoffs(Relation rel, const VacuumParams *params, multixact_freeze_min_age = params->multixact_freeze_min_age; freeze_table_age = params->freeze_table_age; multixact_freeze_table_age = params->multixact_freeze_table_age; + freeze_strategy_threshold = params->freeze_strategy_threshold; /* Set pg_class fields in cutoffs */ cutoffs->relfrozenxid = rel->rd_rel->relfrozenxid; @@ -1058,6 +1064,15 @@ vacuum_get_cutoffs(Relation rel, const VacuumParams *params, if (MultiXactIdPrecedes(cutoffs->OldestMxact, cutoffs->MultiXactCutoff)) cutoffs->MultiXactCutoff = cutoffs->OldestMxact; + /* + * Determine the freeze_strategy_threshold to use: as specified by the + * caller, or vacuum_freeze_strategy_threshold + */ + if (freeze_strategy_threshold < 0) + freeze_strategy_threshold = vacuum_freeze_strategy_threshold; + Assert(freeze_strategy_threshold >= 0); + cutoffs->freeze_strategy_threshold = freeze_strategy_threshold; + /* * Finally, figure out if caller needs to do an aggressive VACUUM or not. * diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 0746d8022..23e316e59 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -151,6 +151,7 @@ static int default_freeze_min_age; static int default_freeze_table_age; static int default_multixact_freeze_min_age; static int default_multixact_freeze_table_age; +static int default_freeze_strategy_threshold; /* Memory context for long-lived data */ static MemoryContext AutovacMemCxt; @@ -2010,6 +2011,7 @@ do_autovacuum(void) default_freeze_table_age = 0; default_multixact_freeze_min_age = 0; default_multixact_freeze_table_age = 0; + default_freeze_strategy_threshold = 0; } else { @@ -2017,6 +2019,7 @@ do_autovacuum(void) default_freeze_table_age = vacuum_freeze_table_age; default_multixact_freeze_min_age = vacuum_multixact_freeze_min_age; default_multixact_freeze_table_age = vacuum_multixact_freeze_table_age; + default_freeze_strategy_threshold = vacuum_freeze_strategy_threshold; } ReleaseSysCache(tuple); @@ -2801,6 +2804,7 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, int freeze_table_age; int multixact_freeze_min_age; int multixact_freeze_table_age; + int freeze_strategy_threshold; int vac_cost_limit; double vac_cost_delay; int log_min_duration; @@ -2850,6 +2854,11 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, ? avopts->multixact_freeze_table_age : default_multixact_freeze_table_age; + freeze_strategy_threshold = (avopts && + avopts->freeze_strategy_threshold >= 0) + ? avopts->freeze_strategy_threshold + : default_freeze_strategy_threshold; + tab = palloc(sizeof(autovac_table)); tab->at_relid = relid; tab->at_sharedrel = classForm->relisshared; @@ -2872,6 +2881,7 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, tab->at_params.freeze_table_age = freeze_table_age; tab->at_params.multixact_freeze_min_age = multixact_freeze_min_age; tab->at_params.multixact_freeze_table_age = multixact_freeze_table_age; + tab->at_params.freeze_strategy_threshold = freeze_strategy_threshold; tab->at_params.is_wraparound = wraparound; tab->at_params.log_min_duration = log_min_duration; tab->at_vacuum_cost_limit = vac_cost_limit; diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 1bf14eec6..549a2e969 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -2503,6 +2503,17 @@ struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"vacuum_freeze_strategy_threshold", PGC_USERSET, CLIENT_CONN_STATEMENT, + gettext_noop("Table size at which VACUUM freezes using eager strategy."), + NULL, + GUC_UNIT_BLOCKS + }, + &vacuum_freeze_strategy_threshold, + (UINT64CONST(4) * 1024 * 1024 * 1024) / BLCKSZ, 0, INT_MAX, + NULL, NULL, NULL + }, + { {"vacuum_defer_cleanup_age", PGC_SIGHUP, REPLICATION_PRIMARY, gettext_noop("Number of transactions by which VACUUM and HOT cleanup should be deferred, if any."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index 043864597..4763cb6bb 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -692,6 +692,7 @@ #idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled #idle_session_timeout = 0 # in milliseconds, 0 is disabled #vacuum_freeze_table_age = 150000000 +#vacuum_freeze_strategy_threshold = 4GB #vacuum_freeze_min_age = 50000000 #vacuum_failsafe_age = 1600000000 #vacuum_multixact_freeze_table_age = 150000000 diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 9830a0309..094f9a35d 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -9129,6 +9129,21 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; + + vacuum_freeze_strategy_threshold (integer) + + vacuum_freeze_strategy_threshold configuration parameter + + + + + Specifies the cutoff size (in pages) that VACUUM + should use to decide whether to its eager freezing strategy. + The default is 4 gigabytes (4GB). + + + + vacuum_freeze_min_age (integer) @@ -9139,7 +9154,9 @@ COPY postgres_log FROM '/full/path/to/logfile.csv' WITH csv; Specifies the cutoff age (in transactions) that VACUUM should use to decide whether to - trigger freezing of pages that have an older XID. + trigger freezing of pages that have an older XID. When VACUUM + uses its eager freezing strategy, freezing a page can also be + triggered when the page contains only all-visible tuples. The default is 50 million transactions. Although users can set this value anywhere from zero to one billion, VACUUM will silently limit the effective value to half diff --git a/doc/src/sgml/maintenance.sgml b/doc/src/sgml/maintenance.sgml index 759ea5ac9..554b3a75d 100644 --- a/doc/src/sgml/maintenance.sgml +++ b/doc/src/sgml/maintenance.sgml @@ -588,9 +588,9 @@ the relfrozenxid column of a table's pg_class row contains the oldest remaining unfrozen XID at the end of the most recent VACUUM that successfully - advanced relfrozenxid (typically the most recent - aggressive VACUUM). Similarly, the - datfrozenxid column of a database's + advanced relfrozenxid. All rows inserted by + transactions older than this cutoff XID are guaranteed to have been frozen. + Similarly, the datfrozenxid column of a database's pg_database row is a lower bound on the unfrozen XIDs appearing in that database — it is just the minimum of the per-table relfrozenxid values within the database. diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index c98223b2a..eabbf9e65 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -1682,6 +1682,20 @@ WITH ( MODULUS numeric_literal, REM + + autovacuum_freeze_strategy_threshold, toast.autovacuum_freeze_strategy_threshold (integer) + + autovacuum_freeze_strategy_threshold storage parameter + + + + + Per-table value for + parameter. + + + + autovacuum_freeze_min_age, toast.autovacuum_freeze_min_age (integer) diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 9cd880ea3..f61433c7d 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -119,14 +119,14 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ and - parameters - set to zero. Aggressive freezing is always performed when the - table is rewritten, so this option is redundant when FULL - is specified. + Selects eager freezing of tuples. Specifying + FREEZE is equivalent to performing + VACUUM with the and parameters set + to zero. Eager freezing is always performed when the table is + rewritten, so this option is redundant when + FULL is specified. -- 2.38.1