From 5a50a20c346e6e96006d4b92f3b1ec7b0692e66f Mon Sep 17 00:00:00 2001 From: Masahiko Sawada Date: Thu, 7 Mar 2019 09:45:11 +0900 Subject: [PATCH v11 1/2] Add INDEX_CLEANUP option to VACUUM command If this option is false, VACUUM does HOT-pruning for live tuples but doesn't remove dead tuples completely and disables index vacuum. vacrelstats->dead_tuples could have tuples that became dead after checked at a HOT-pruning time, which are not marked as dead. Per discussion on pgsql-hackers We normally records and remove them but with this option we don't process and leave for the next vacuum for simplifing the code. That's okay because it's very rare condition and those tuples will be processed by the next vacuum. --- doc/src/sgml/ref/create_table.sgml | 17 ++++++++ doc/src/sgml/ref/vacuum.sgml | 28 ++++++++++++ src/backend/access/common/reloptions.c | 13 +++++- src/backend/access/heap/vacuumlazy.c | 78 ++++++++++++++++++++++++++-------- src/backend/commands/vacuum.c | 21 ++++++++- src/backend/postmaster/autovacuum.c | 2 +- src/bin/psql/tab-complete.c | 6 ++- src/include/commands/vacuum.h | 3 +- src/include/utils/rel.h | 1 + src/test/regress/expected/vacuum.out | 3 ++ src/test/regress/sql/vacuum.sql | 3 ++ 11 files changed, 151 insertions(+), 24 deletions(-) diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 1660784..a6eba05 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -1345,6 +1345,23 @@ WITH ( MODULUS numeric_literal, REM + vacuum_index_cleanup (boolean) + + + Per table setting to use INDEX_CLEANUP option + of VACUUM command. The default value is true. + If false, autovacuum daemon and VACUUM + never perform index vacuuming and index cleanup, that is, always set + INDEX_CLEANUP option to false. + Note that out of disk space due to index bloat. Setting this parameter to + false makes sense to avoid scanning large indexes when + the table has a few dead tuples. See for more + details on INDEX_CLEANUP option. + + + + + autovacuum_vacuum_threshold, toast.autovacuum_vacuum_threshold (integer) diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index 906d0c2..3f87979 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -32,6 +32,7 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ boolean ] DISABLE_PAGE_SKIPPING [ boolean ] SKIP_LOCKED [ boolean ] + INDEX_CLEANUP [ boolean ] and table_and_columns is: @@ -182,6 +183,26 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ ) but not sufficient for + avoiding index bloat. It defaults to TRUE. + This option is ignored if the table does not have index. This cannot + be used in conjunction with FULL option. + + + + + boolean @@ -275,6 +296,13 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ also enables and disables index cleanup. + The INDEX_CLEANUP options to + VACUUM takes precedence over this option. + + + VACUUM causes a substantial increase in I/O traffic, which might cause poor performance for other active sessions. Therefore, it is sometimes advisable to use the cost-based vacuum delay feature. diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index b58a1f7..e2c0de3 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -138,6 +138,15 @@ static relopt_bool boolRelOpts[] = }, false }, + { + { + "vacuum_index_cleanup", + "Enables index vacuuming and index cleanup", + RELOPT_KIND_HEAP, + ShareUpdateExclusiveLock + }, + true + }, /* list terminator */ {{NULL}} }; @@ -1388,7 +1397,9 @@ default_reloptions(Datum reloptions, bool validate, relopt_kind kind) {"parallel_workers", RELOPT_TYPE_INT, offsetof(StdRdOptions, parallel_workers)}, {"vacuum_cleanup_index_scale_factor", RELOPT_TYPE_REAL, - offsetof(StdRdOptions, vacuum_cleanup_index_scale_factor)} + offsetof(StdRdOptions, vacuum_cleanup_index_scale_factor)}, + {"vacuum_index_cleanup", RELOPT_TYPE_BOOL, + offsetof(StdRdOptions, vacuum_index_cleanup)} }; options = parseRelOptions(reloptions, validate, kind, &numoptions); diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 5c554f9..4911131 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -112,8 +112,8 @@ typedef struct LVRelStats { - /* hasindex = true means two-pass strategy; false means one-pass */ - bool hasindex; + /* useindex = true means two-pass strategy; false means one-pass */ + bool useindex; /* Overall statistics about rel */ BlockNumber old_rel_pages; /* previous value of pg_class.relpages */ BlockNumber rel_pages; /* total number of pages */ @@ -125,6 +125,8 @@ typedef struct LVRelStats double new_rel_tuples; /* new estimated total # of tuples */ double new_live_tuples; /* new estimated total # of live tuples */ double new_dead_tuples; /* new estimated total # of dead tuples */ + double nleft_dead_tuples; /* # of dead tuples we left */ + double nleft_dead_itemids; /* # of dead item pointers we left */ BlockNumber pages_removed; double tuples_deleted; BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */ @@ -258,7 +260,8 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, /* Open all indexes of the relation */ vac_open_indexes(onerel, RowExclusiveLock, &nindexes, &Irel); - vacrelstats->hasindex = (nindexes > 0); + vacrelstats->useindex = (nindexes > 0 && + (params->options & VACOPT_INDEX_CLEANUP) != 0); /* Do the vacuuming */ lazy_scan_heap(onerel, params->options, vacrelstats, Irel, nindexes, aggressive); @@ -332,7 +335,7 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, new_rel_pages, new_live_tuples, new_rel_allvisible, - vacrelstats->hasindex, + nindexes > 0, new_frozen_xid, new_min_multi, false); @@ -404,6 +407,10 @@ heap_vacuum_rel(Relation onerel, VacuumParams *params, vacrelstats->new_dead_tuples, OldestXmin); appendStringInfo(&buf, + _("%.0f tuples and %.0f item identifiers are left as dead.\n"), + vacrelstats->nleft_dead_tuples, + vacrelstats->nleft_dead_itemids); + appendStringInfo(&buf, _("buffer usage: %d hits, %d misses, %d dirtied\n"), VacuumPageHit, VacuumPageMiss, @@ -485,7 +492,10 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats, live_tuples, /* live tuples (reltuples estimate) */ tups_vacuumed, /* tuples cleaned up by vacuum */ nkeep, /* dead-but-not-removable tuples */ - nunused; /* unused item pointers */ + nunused, /* unused item pointers */ + nleft_dead_tuples, /* tuples we left as dead */ + nleft_dead_itemids; /* item pointers we left as dead, + * includes nleft_dead_tuples. */ IndexBulkDeleteResult **indstats; int i; PGRUsage ru0; @@ -518,6 +528,7 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats, empty_pages = vacuumed_pages = 0; next_fsm_block_to_vacuum = (BlockNumber) 0; num_tuples = live_tuples = tups_vacuumed = nkeep = nunused = 0; + nleft_dead_itemids = nleft_dead_tuples = 0; indstats = (IndexBulkDeleteResult **) palloc0(nindexes * sizeof(IndexBulkDeleteResult *)); @@ -1054,7 +1065,16 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats, HeapTupleIsHeapOnly(&tuple)) nkeep += 1; else + { tupgone = true; /* we can delete the tuple */ + + /* + * Since the dead tuples will be not be vacuumed + * and ignored when index cleanup is disabled we + * count them for reporting. + */ + nleft_dead_tuples++; + } all_visible = false; break; case HEAPTUPLE_LIVE: @@ -1206,15 +1226,33 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats, } /* - * If there are no indexes then we can vacuum the page right now - * instead of doing a second scan. + * If there are no indexes we can vacuum the page right now instead of + * doing a second scan. Also we don't do that but forget dead tuples + * when index cleanup is disabled. */ - if (nindexes == 0 && - vacrelstats->num_dead_tuples > 0) + if (!vacrelstats->useindex && vacrelstats->num_dead_tuples > 0) { - /* Remove tuples from heap */ - lazy_vacuum_page(onerel, blkno, buf, 0, vacrelstats, &vmbuffer); - has_dead_tuples = false; + if (nindexes == 0) + { + /* Remove tuples from heap if the table has no index */ + lazy_vacuum_page(onerel, blkno, buf, 0, vacrelstats, &vmbuffer); + vacuumed_pages++; + has_dead_tuples = false; + } + else + { + /* + * Here, we have indexes but index cleanup is disabled. Instead of + * vacuuming the dead tuples on the heap, we just forget them. + * + * Note that vacrelstats->dead_tuples could have tuples which + * became dead after HOT-pruning but are not marked dead yet. + * We do not process them because it's a very rare condition, and + * the next vacuum will process them anyway. + */ + Assert((options & VACOPT_INDEX_CLEANUP) == 0); + nleft_dead_itemids += vacrelstats->num_dead_tuples; + } /* * Forget the now-vacuumed tuples, and press on, but be careful @@ -1222,7 +1260,6 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats, * valid. */ vacrelstats->num_dead_tuples = 0; - vacuumed_pages++; /* * Periodically do incremental FSM vacuuming to make newly-freed @@ -1348,7 +1385,9 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats, /* save stats for use later */ vacrelstats->tuples_deleted = tups_vacuumed; - vacrelstats->new_dead_tuples = nkeep; + vacrelstats->new_dead_tuples = nkeep + nleft_dead_tuples; + vacrelstats->nleft_dead_tuples = nleft_dead_tuples; + vacrelstats->nleft_dead_itemids = nleft_dead_itemids; /* now we can compute the new value for pg_class.reltuples */ vacrelstats->new_live_tuples = vac_estimate_reltuples(onerel, @@ -1417,8 +1456,11 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats, PROGRESS_VACUUM_PHASE_INDEX_CLEANUP); /* Do post-vacuum cleanup and statistics update for each index */ - for (i = 0; i < nindexes; i++) - lazy_cleanup_index(Irel[i], indstats[i], vacrelstats); + if (vacrelstats->useindex) + { + for (i = 0; i < nindexes; i++) + lazy_cleanup_index(Irel[i], indstats[i], vacrelstats); + } /* If no indexes, make log report that lazy_vacuum_heap would've made */ if (vacuumed_pages) @@ -1449,6 +1491,8 @@ lazy_scan_heap(Relation onerel, int options, LVRelStats *vacrelstats, "%u pages are entirely empty.\n", empty_pages), empty_pages); + appendStringInfo(&buf, "%.0f tuples and %.0f item identifiers are left as dead.\n", + nleft_dead_tuples, nleft_dead_itemids); appendStringInfo(&buf, _("%s."), pg_rusage_show(&ru0)); ereport(elevel, @@ -2092,7 +2136,7 @@ lazy_space_alloc(LVRelStats *vacrelstats, BlockNumber relblocks) autovacuum_work_mem != -1 ? autovacuum_work_mem : maintenance_work_mem; - if (vacrelstats->hasindex) + if (vacrelstats->useindex) { maxtuples = (vac_work_mem * 1024L) / sizeof(ItemPointerData); maxtuples = Min(maxtuples, INT_MAX); diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 10df766..1d1b215 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -93,6 +93,7 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) bool freeze = false; bool full = false; bool disable_page_skipping = false; + bool index_cleanup = true; /* by default */ ListCell *lc; /* Parse options list */ @@ -120,6 +121,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) full = defGetBoolean(opt); else if (strcmp(opt->defname, "disable_page_skipping") == 0) disable_page_skipping = defGetBoolean(opt); + else if (strcmp(opt->defname, "index_cleanup") == 0) + index_cleanup = defGetBoolean(opt); else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -135,7 +138,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) (analyze ? VACOPT_ANALYZE : 0) | (freeze ? VACOPT_FREEZE : 0) | (full ? VACOPT_FULL : 0) | - (disable_page_skipping ? VACOPT_DISABLE_PAGE_SKIPPING : 0); + (disable_page_skipping ? VACOPT_DISABLE_PAGE_SKIPPING : 0) | + (index_cleanup ? VACOPT_INDEX_CLEANUP : 0); /* sanity checks on options */ Assert(params.options & (VACOPT_VACUUM | VACOPT_ANALYZE)); @@ -251,7 +255,8 @@ vacuum(List *relations, VacuumParams *params, stmttype))); /* - * Sanity check DISABLE_PAGE_SKIPPING option. + * Sanity check DISABLE_PAGE_SKIPPING option and INDEX_CLEANUP + * option. */ if ((params->options & VACOPT_FULL) != 0 && (params->options & VACOPT_DISABLE_PAGE_SKIPPING) != 0) @@ -259,6 +264,11 @@ vacuum(List *relations, VacuumParams *params, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("VACUUM option DISABLE_PAGE_SKIPPING cannot be used with FULL"))); + if ((params->options & VACOPT_FULL) != 0 && + (params->options & VACOPT_INDEX_CLEANUP) == 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("VACUUM option INDEX_CLEANUP cannot be set to false with FULL"))); /* * Send info about dead objects to the statistics collector, unless we are * in autovacuum --- autovacuum.c does this for itself. @@ -1720,6 +1730,13 @@ vacuum_rel(Oid relid, RangeVar *relation, VacuumParams *params) LockRelationIdForSession(&onerelid, lmode); /* + * Disables index cleanup based on reloptions. + */ + if (onerel->rd_options && + !((StdRdOptions *) onerel->rd_options)->vacuum_index_cleanup) + params->options &= ~(VACOPT_INDEX_CLEANUP); + + /* * Remember the relation's TOAST relation for later, if the caller asked * us to process it. In VACUUM FULL, though, the toast table is * automatically rebuilt by cluster_rel so we shouldn't recurse to it. diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index fa875db..51c60fc 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -2883,7 +2883,7 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, tab->at_relid = relid; tab->at_sharedrel = classForm->relisshared; tab->at_params.options = VACOPT_SKIPTOAST | - (dovacuum ? VACOPT_VACUUM : 0) | + (dovacuum ? VACOPT_VACUUM | VACOPT_INDEX_CLEANUP : 0) | (doanalyze ? VACOPT_ANALYZE : 0) | (!wraparound ? VACOPT_SKIP_LOCKED : 0); tab->at_params.freeze_min_age = freeze_min_age; diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index c18977c..c60efd9 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1039,6 +1039,7 @@ static const char *const table_storage_parameters[] = { "toast.log_autovacuum_min_duration", "toast_tuple_target", "user_catalog_table", + "vacuum_index_cleanup", NULL }; @@ -3431,8 +3432,9 @@ psql_completion(const char *text, int start, int end) */ if (ends_with(prev_wd, '(') || ends_with(prev_wd, ',')) COMPLETE_WITH("FULL", "FREEZE", "ANALYZE", "VERBOSE", - "DISABLE_PAGE_SKIPPING", "SKIP_LOCKED"); - else if (TailMatches("FULL|FREEZE|ANALYZE|VERBOSE|DISABLE_PAGE_SKIPPING|SKIP_LOCKED")) + "DISABLE_PAGE_SKIPPING", "SKIP_LOCKED", + "INDEX_CLEANUP"); + else if (TailMatches("FULL|FREEZE|ANALYZE|VERBOSE|DISABLE_PAGE_SKIPPING|SKIP_LOCKED|INDEX_CLEANUP")) COMPLETE_WITH("ON", "OFF"); } else if (HeadMatches("VACUUM") && TailMatches("(")) diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 77086f3..7bbc18c 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -145,7 +145,8 @@ typedef enum VacuumOption VACOPT_FULL = 1 << 4, /* FULL (non-concurrent) vacuum */ VACOPT_SKIP_LOCKED = 1 << 5, /* skip if cannot get lock */ VACOPT_SKIPTOAST = 1 << 6, /* don't process the TOAST table, if any */ - VACOPT_DISABLE_PAGE_SKIPPING = 1 << 7 /* don't skip any pages */ + VACOPT_DISABLE_PAGE_SKIPPING = 1 << 7, /* don't skip any pages */ + VACOPT_INDEX_CLEANUP = 1 << 8 /* Do index vacuum and cleanup */ } VacuumOption; /* diff --git a/src/include/utils/rel.h b/src/include/utils/rel.h index 5402851..89a7fbf 100644 --- a/src/include/utils/rel.h +++ b/src/include/utils/rel.h @@ -266,6 +266,7 @@ typedef struct StdRdOptions AutoVacOpts autovacuum; /* autovacuum-related options */ bool user_catalog_table; /* use as an additional catalog relation */ int parallel_workers; /* max number of parallel workers */ + bool vacuum_index_cleanup; /* enables index vacuuming and cleanup */ } StdRdOptions; #define HEAP_MIN_FILLFACTOR 10 diff --git a/src/test/regress/expected/vacuum.out b/src/test/regress/expected/vacuum.out index 07d0703..a1de992 100644 --- a/src/test/regress/expected/vacuum.out +++ b/src/test/regress/expected/vacuum.out @@ -80,6 +80,9 @@ CONTEXT: SQL function "do_analyze" statement 1 SQL function "wrap_do_analyze" statement 1 VACUUM FULL vactst; VACUUM (DISABLE_PAGE_SKIPPING) vaccluster; +VACUUM (INDEX_CLEANUP FALSE) vaccluster; +VACUUM (INDEX_CLEANUP FALSE) vactst; -- option is ignored if no indexes +VACUUM (INDEX_CLEANUP FALSE, FREEZE) vaccluster; -- partitioned table CREATE TABLE vacparted (a int, b char) PARTITION BY LIST (a); CREATE TABLE vacparted1 PARTITION OF vacparted FOR VALUES IN (1); diff --git a/src/test/regress/sql/vacuum.sql b/src/test/regress/sql/vacuum.sql index 81f3822..8745e34 100644 --- a/src/test/regress/sql/vacuum.sql +++ b/src/test/regress/sql/vacuum.sql @@ -61,6 +61,9 @@ VACUUM FULL vaccluster; VACUUM FULL vactst; VACUUM (DISABLE_PAGE_SKIPPING) vaccluster; +VACUUM (INDEX_CLEANUP FALSE) vaccluster; +VACUUM (INDEX_CLEANUP FALSE) vactst; -- option is ignored if no indexes +VACUUM (INDEX_CLEANUP FALSE, FREEZE) vaccluster; -- partitioned table CREATE TABLE vacparted (a int, b char) PARTITION BY LIST (a); -- 1.8.3.1