From f7ce64ee76a20929392ead5f3e3a6298219a3e38 Mon Sep 17 00:00:00 2001 From: Melanie Plageman Date: Wed, 8 Nov 2023 17:36:22 -0500 Subject: [PATCH v1 7/9] Track vacuum freeze and page unfreeze stats At the beginning of the vacuum of a relation, set up a new PgStat_Frz entry in the table-level freeze statistics, PgStat_StatTabEntry->frz_buckets and initialize relevant members in the LVRelState. When vacuum freezes a page, increment the freeze-related statistics in the LVRelState. Then, at the end of the vacuum, transfer the stats gathered during vacuum to the current PgStat_Frz in PgStat_StatTabEntry->frz_buckets. When modifying a frozen page, count this as an unfreeze in the PgStat_Frz entry which covers the page freeze LSN. --- src/backend/access/heap/heapam.c | 116 ++++++++++-- src/backend/access/heap/pruneheap.c | 1 + src/backend/access/heap/vacuumlazy.c | 110 +++++++++++- src/backend/utils/activity/pgstat_relation.c | 175 ++++++++++++++++++- src/include/access/heapam.h | 5 + src/include/commands/vacuum.h | 29 +++ src/include/pgstat.h | 11 +- 7 files changed, 423 insertions(+), 24 deletions(-) diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 80828f3efe..c8d0045bef 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -1831,8 +1831,11 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, TransactionId xid = GetCurrentTransactionId(); HeapTuple heaptup; Buffer buffer; + XLogRecPtr insert_lsn = InvalidXLogRecPtr; + XLogRecPtr page_lsn = InvalidXLogRecPtr; Buffer vmbuffer = InvalidBuffer; bool all_visible_cleared = false; + bool all_frozen_cleared = false; /* Cheap, simplistic check that the tuple matches the rel's rowtype. */ Assert(HeapTupleHeaderGetNatts(tup->t_data) <= @@ -1882,9 +1885,11 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, { all_visible_cleared = true; PageClearAllVisible(BufferGetPage(buffer)); - visibilitymap_clear(relation, - ItemPointerGetBlockNumber(&(heaptup->t_self)), - vmbuffer, VISIBILITYMAP_VALID_BITS); + + all_frozen_cleared = visibilitymap_clear(relation, + ItemPointerGetBlockNumber(&(heaptup->t_self)), + vmbuffer, + VISIBILITYMAP_VALID_BITS) & VISIBILITYMAP_ALL_FROZEN; } /* @@ -1976,6 +1981,9 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, recptr = XLogInsert(RM_HEAP_ID, info); + insert_lsn = recptr; + page_lsn = PageGetLSN(page); + PageSetLSN(page, recptr); } @@ -1996,6 +2004,11 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, /* Note: speculative insertions are counted too, even if aborted later */ pgstat_count_heap_insert(relation, 1); + if (all_frozen_cleared) + pgstat_count_page_unfreeze(RelationGetRelid(relation), + relation->rd_rel->relisshared, + page_lsn, insert_lsn); + /* * If heaptup is a private copy, release it. Don't forget to copy t_self * back to the caller's image, too. @@ -2161,6 +2174,9 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, while (ndone < ntuples) { Buffer buffer; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + XLogRecPtr insert_lsn = InvalidXLogRecPtr; + bool all_frozen_cleared = false; bool all_visible_cleared = false; bool all_frozen_set = false; int nthispage; @@ -2248,9 +2264,10 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, { all_visible_cleared = true; PageClearAllVisible(page); - visibilitymap_clear(relation, - BufferGetBlockNumber(buffer), - vmbuffer, VISIBILITYMAP_VALID_BITS); + all_frozen_cleared = visibilitymap_clear(relation, + BufferGetBlockNumber(buffer), + vmbuffer, + VISIBILITYMAP_VALID_BITS) & VISIBILITYMAP_ALL_FROZEN; } else if (all_frozen_set) PageSetAllVisible(page); @@ -2372,14 +2389,22 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, recptr = XLogInsert(RM_HEAP2_ID, info); + insert_lsn = recptr; + page_lsn = PageGetLSN(page); PageSetLSN(page, recptr); } END_CRIT_SECTION(); + if (all_frozen_cleared) + pgstat_count_page_unfreeze(RelationGetRelid(relation), + relation->rd_rel->relisshared, + page_lsn, insert_lsn); + /* * If we've frozen everything on the page, update the visibilitymap. - * We're already holding pin on the vmbuffer. + * We're already holding pin on the vmbuffer. We don't count page + * freezes done as part of COPY FREEZE toward page freeze stats. */ if (all_frozen_set) { @@ -2532,6 +2557,9 @@ heap_delete(Relation relation, ItemPointer tid, bool have_tuple_lock = false; bool iscombo; bool all_visible_cleared = false; + bool all_frozen_cleared = false; + XLogRecPtr insert_lsn = InvalidXLogRecPtr; + XLogRecPtr page_lsn = InvalidXLogRecPtr; HeapTuple old_key_tuple = NULL; /* replica identity of the tuple */ bool old_key_copied = false; @@ -2788,8 +2816,10 @@ l1: { all_visible_cleared = true; PageClearAllVisible(page); - visibilitymap_clear(relation, BufferGetBlockNumber(buffer), - vmbuffer, VISIBILITYMAP_VALID_BITS); + all_frozen_cleared = visibilitymap_clear(relation, + BufferGetBlockNumber(buffer), + vmbuffer, + VISIBILITYMAP_VALID_BITS) & VISIBILITYMAP_ALL_FROZEN; } /* store transaction information of xact deleting the tuple */ @@ -2872,6 +2902,9 @@ l1: recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE); + insert_lsn = recptr; + page_lsn = PageGetLSN(page); + PageSetLSN(page, recptr); } @@ -2915,6 +2948,11 @@ l1: pgstat_count_heap_delete(relation); + if (all_frozen_cleared) + pgstat_count_page_unfreeze(RelationGetRelid(relation), + relation->rd_rel->relisshared, + page_lsn, insert_lsn); + if (old_key_tuple != NULL && old_key_copied) heap_freetuple(old_key_tuple); @@ -3021,6 +3059,12 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, infomask_new_tuple, infomask2_new_tuple; + bool cleared_all_frozen = false; + bool cleared_all_frozen_new = false; + XLogRecPtr old_page_lsn = InvalidXLogRecPtr; + XLogRecPtr new_page_lsn = InvalidXLogRecPtr; + XLogRecPtr insert_lsn = InvalidXLogRecPtr; + Assert(ItemPointerIsValid(otid)); /* Cheap, simplistic check that the tuple matches the rel's rowtype. */ @@ -3503,7 +3547,6 @@ l2: TransactionId xmax_lock_old_tuple; uint16 infomask_lock_old_tuple, infomask2_lock_old_tuple; - bool cleared_all_frozen = false; /* * To prevent concurrent sessions from updating the tuple, we have to @@ -3578,6 +3621,8 @@ l2: cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0; XLogRegisterData((char *) &xlrec, SizeOfHeapLock); recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK); + insert_lsn = recptr; + old_page_lsn = PageGetLSN(page); PageSetLSN(page, recptr); } @@ -3585,6 +3630,16 @@ l2: LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + if (cleared_all_frozen) + { + pgstat_count_page_unfreeze(RelationGetRelid(relation), + relation->rd_rel->relisshared, + old_page_lsn, insert_lsn); + + /* Avoid double counting page unfreezes. */ + cleared_all_frozen = false; + } + /* * Let the toaster do its thing, if needed. * @@ -3786,15 +3841,15 @@ l2: { all_visible_cleared = true; PageClearAllVisible(BufferGetPage(buffer)); - visibilitymap_clear(relation, BufferGetBlockNumber(buffer), - vmbuffer, VISIBILITYMAP_VALID_BITS); + cleared_all_frozen = visibilitymap_clear(relation, BufferGetBlockNumber(buffer), + vmbuffer, VISIBILITYMAP_VALID_BITS) & VISIBILITYMAP_ALL_FROZEN; } if (newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf))) { all_visible_cleared_new = true; PageClearAllVisible(BufferGetPage(newbuf)); - visibilitymap_clear(relation, BufferGetBlockNumber(newbuf), - vmbuffer_new, VISIBILITYMAP_VALID_BITS); + cleared_all_frozen_new = visibilitymap_clear(relation, BufferGetBlockNumber(newbuf), + vmbuffer_new, VISIBILITYMAP_VALID_BITS) & VISIBILITYMAP_ALL_FROZEN; } if (newbuf != buffer) @@ -3823,9 +3878,14 @@ l2: all_visible_cleared_new); if (newbuf != buffer) { + new_page_lsn = PageGetLSN(BufferGetPage(newbuf)); PageSetLSN(BufferGetPage(newbuf), recptr); } + + old_page_lsn = PageGetLSN(BufferGetPage(buffer)); PageSetLSN(BufferGetPage(buffer), recptr); + + insert_lsn = recptr; } END_CRIT_SECTION(); @@ -3861,6 +3921,16 @@ l2: pgstat_count_heap_update(relation, use_hot_update, newbuf != buffer); + if (cleared_all_frozen) + pgstat_count_page_unfreeze(RelationGetRelid(relation), + relation->rd_rel->relisshared, + old_page_lsn, insert_lsn); + + if (newbuf != buffer && cleared_all_frozen_new) + pgstat_count_page_unfreeze(RelationGetRelid(relation), + relation->rd_rel->relisshared, + new_page_lsn, insert_lsn); + /* * If heaptup is a private copy, release it. Don't forget to copy t_self * back to the caller's image, too. @@ -4146,6 +4216,8 @@ heap_lock_tuple(Relation relation, HeapTuple tuple, Page page; Buffer vmbuffer = InvalidBuffer; BlockNumber block; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + XLogRecPtr insert_lsn = InvalidXLogRecPtr; TransactionId xid, xmax; uint16 old_infomask, @@ -4791,6 +4863,8 @@ failed: /* we don't decode row locks atm, so no need to log the origin */ recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK); + insert_lsn = recptr; + page_lsn = PageGetLSN(page); PageSetLSN(page, recptr); } @@ -4818,6 +4892,11 @@ out_unlocked: if (have_tuple_lock) UnlockTupleTuplock(relation, tid, mode); + if (cleared_all_frozen) + pgstat_count_page_unfreeze(RelationGetRelid(relation), + relation->rd_rel->relisshared, + page_lsn, insert_lsn); + return result; } @@ -5270,6 +5349,8 @@ heap_lock_updated_tuple_rec(Relation rel, ItemPointer tid, TransactionId xid, new_xmax; TransactionId priorXmax = InvalidTransactionId; bool cleared_all_frozen = false; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + XLogRecPtr insert_lsn = InvalidXLogRecPtr; bool pinned_desired_page; Buffer vmbuffer = InvalidBuffer; BlockNumber block; @@ -5543,6 +5624,8 @@ l4: XLogRegisterData((char *) &xlrec, SizeOfHeapLockUpdated); recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_LOCK_UPDATED); + insert_lsn = recptr; + page_lsn = PageGetLSN(page); PageSetLSN(page, recptr); } @@ -5575,6 +5658,11 @@ out_unlocked: if (vmbuffer != InvalidBuffer) ReleaseBuffer(vmbuffer); + if (cleared_all_frozen) + pgstat_count_page_unfreeze(RelationGetRelid(rel), + rel->rd_rel->relisshared, + page_lsn, insert_lsn); + return result; } diff --git a/src/backend/access/heap/pruneheap.c b/src/backend/access/heap/pruneheap.c index c5f1abd95a..cfd764c1f3 100644 --- a/src/backend/access/heap/pruneheap.c +++ b/src/backend/access/heap/pruneheap.c @@ -237,6 +237,7 @@ heap_page_prune(Relation relation, Buffer buffer, */ presult->ndeleted = 0; presult->nnewlpdead = 0; + presult->page_lsn = PageGetLSN(page); maxoff = PageGetMaxOffsetNumber(page); tup.t_tableOid = RelationGetRelid(prstate.rel); diff --git a/src/backend/access/heap/vacuumlazy.c b/src/backend/access/heap/vacuumlazy.c index 9409cf6b38..a74516003f 100644 --- a/src/backend/access/heap/vacuumlazy.c +++ b/src/backend/access/heap/vacuumlazy.c @@ -227,7 +227,8 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, minmulti_updated; BlockNumber orig_rel_pages, new_rel_pages, - new_rel_allvisible; + new_rel_allvisible, + new_rel_allfrozen; PGRUsage ru0; TimestampTz starttime = 0; PgStat_Counter startreadtime = 0, @@ -341,6 +342,13 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->nonempty_pages = 0; /* dead_items_alloc allocates vacrel->dead_items later on */ + vacrel->sum_frozen_page_ages = 0; + vacrel->vm_pages_frozen = 0; + vacrel->already_frozen_pages = 0; + vacrel->max_frz_page_age = InvalidXLogRecPtr; + vacrel->min_frz_page_age = InvalidXLogRecPtr; + vacrel->freeze_fpis = 0; + /* Allocate/initialize output statistics state */ vacrel->new_rel_tuples = 0; vacrel->new_live_tuples = 0; @@ -405,6 +413,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, vacrel->relname))); } + pgstat_setup_vacuum_frz_stats(RelationGetRelid(rel), + rel->rd_rel->relisshared); + /* * Allocate dead_items array memory using dead_items_alloc. This handles * parallel VACUUM initialization as part of allocating shared memory @@ -483,10 +494,13 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, * pg_class.relpages to */ new_rel_pages = vacrel->rel_pages; /* After possible rel truncation */ - visibilitymap_count(rel, &new_rel_allvisible, NULL); + visibilitymap_count(rel, &new_rel_allvisible, &new_rel_allfrozen); if (new_rel_allvisible > new_rel_pages) new_rel_allvisible = new_rel_pages; + if (new_rel_allfrozen > new_rel_pages) + new_rel_allfrozen = new_rel_pages; + /* * Now actually update rel's pg_class entry. * @@ -501,7 +515,9 @@ heap_vacuum_rel(Relation rel, VacuumParams *params, /* Report results to the cumulative stats system, too. */ pgstat_report_vacuum(RelationGetRelid(rel), - rel->rd_rel->relisshared, vacrel); + rel->rd_rel->relisshared, vacrel, + orig_rel_pages, new_rel_allfrozen); + pgstat_progress_end_command(); if (instrument) @@ -995,6 +1011,7 @@ lazy_scan_heap(LVRelState *vacrel) if (!all_visible_according_to_vm && prunestate.all_visible) { uint8 flags = VISIBILITYMAP_ALL_VISIBLE; + uint8 previous_flags; if (prunestate.all_frozen) { @@ -1017,9 +1034,18 @@ lazy_scan_heap(LVRelState *vacrel) */ PageSetAllVisible(page); MarkBufferDirty(buf); - visibilitymap_set(vacrel->rel, blkno, buf, InvalidXLogRecPtr, - vmbuffer, prunestate.visibility_cutoff_xid, - flags); + previous_flags = visibilitymap_set(vacrel->rel, blkno, buf, InvalidXLogRecPtr, + vmbuffer, prunestate.visibility_cutoff_xid, + flags); + + /* + * If we newly set all frozen here, count it. Don't worry about + * updating page age statistics since we are not actually + * modifying the tuples on the page. + */ + if (prunestate.all_frozen && + !(previous_flags & VISIBILITYMAP_ALL_FROZEN)) + vacrel->vm_pages_frozen++; } /* @@ -1033,6 +1059,13 @@ lazy_scan_heap(LVRelState *vacrel) { elog(WARNING, "page is not marked all-visible but visibility map bit is set in relation \"%s\" page %u", vacrel->relname, blkno); + + /* + * In the case of data corruption, we don't bother counting the + * page as unfrozen in the stats. We don't have easy access to the + * page LSN from before any modifications were made by vacuum, so + * it is hard to count it properly here anyway. + */ visibilitymap_clear(vacrel->rel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1058,6 +1091,11 @@ lazy_scan_heap(LVRelState *vacrel) vacrel->relname, blkno); PageClearAllVisible(page); MarkBufferDirty(buf); + + /* + * This unfreeze is not counted in stats for the same reason + * detailed above. + */ visibilitymap_clear(vacrel->rel, blkno, vmbuffer, VISIBILITYMAP_VALID_BITS); } @@ -1094,6 +1132,12 @@ lazy_scan_heap(LVRelState *vacrel) vmbuffer, InvalidTransactionId, VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN); + + /* + * Don't worry about updating page age stats since we only updated + * the VM. + */ + vacrel->vm_pages_frozen++; } /* @@ -1212,6 +1256,9 @@ lazy_scan_skip(LVRelState *vacrel, Buffer *vmbuffer, BlockNumber next_block, next_unskippable_block, vmbuffer); + if ((mapbits & VISIBILITYMAP_ALL_FROZEN) != 0) + vacrel->already_frozen_pages++; + if ((mapbits & VISIBILITYMAP_ALL_VISIBLE) == 0) { Assert((mapbits & VISIBILITYMAP_ALL_FROZEN) == 0); @@ -1403,6 +1450,9 @@ lazy_scan_new_or_empty(LVRelState *vacrel, Buffer buf, BlockNumber blkno, vmbuffer, InvalidTransactionId, VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN); END_CRIT_SECTION(); + + /* Count a page freeze since we set it in the VM. */ + vacrel->vm_pages_frozen++; } freespace = PageGetHeapFreeSpace(page); @@ -1452,6 +1502,8 @@ lazy_scan_prune(LVRelState *vacrel, lpdead_items, live_tuples, recently_dead_tuples; + XLogRecPtr insert_lsn; + int64 page_age; HeapPageFreeze pagefrz; int64 fpi_before = pgWalUsage.wal_fpi; OffsetNumber deadoffsets[MaxHeapTuplesPerPage]; @@ -1675,6 +1727,22 @@ lazy_scan_prune(LVRelState *vacrel, */ vacrel->offnum = InvalidOffsetNumber; + insert_lsn = GetInsertRecPtr(); + + /* + * The page may have been modified by pruning, however, we want to know + * how many LSNs since it was last modified by a DML operation. + */ + page_age = insert_lsn - presult.page_lsn; + + /* + * Because GetInsertRecPtr() returns the approximate insert LSN (it may be + * up to a page behind the real insert LSN), there is a small chance for + * it to be behind page lsn. In this case, the page is basically 0, so + * count it as such. + */ + page_age = Max(page_age, 0); + /* * Freeze the page when heap_prepare_freeze_tuple indicates that at least * one XID/MXID from before FreezeLimit/MultiXactCutoff is present. Also @@ -1712,8 +1780,20 @@ lazy_scan_prune(LVRelState *vacrel, { TransactionId snapshotConflictHorizon; + fpi_before = pgWalUsage.wal_fpi; + vacrel->pages_frozen++; + vacrel->sum_frozen_page_ages += page_age; + + if (vacrel->max_frz_page_age == InvalidXLogRecPtr || + page_age > vacrel->max_frz_page_age) + vacrel->max_frz_page_age = page_age; + + if (vacrel->min_frz_page_age == InvalidXLogRecPtr || + page_age < vacrel->min_frz_page_age) + vacrel->min_frz_page_age = page_age; + /* * We can use visibility_cutoff_xid as our cutoff for conflicts * when the whole page is eligible to become all-frozen in the VM @@ -1737,6 +1817,9 @@ lazy_scan_prune(LVRelState *vacrel, heap_freeze_execute_prepared(vacrel->rel, buf, snapshotConflictHorizon, frozen, tuples_frozen); + + if (pgWalUsage.wal_fpi > fpi_before) + vacrel->freeze_fpis++; } } else @@ -2489,6 +2572,7 @@ lazy_vacuum_heap_page(LVRelState *vacrel, BlockNumber blkno, Buffer buffer, if (heap_page_is_all_visible(vacrel, buffer, &visibility_cutoff_xid, &all_frozen)) { + uint8 previous_flags; uint8 flags = VISIBILITYMAP_ALL_VISIBLE; if (all_frozen) @@ -2498,8 +2582,18 @@ lazy_vacuum_heap_page(LVRelState *vacrel, BlockNumber blkno, Buffer buffer, } PageSetAllVisible(page); - visibilitymap_set(vacrel->rel, blkno, buffer, InvalidXLogRecPtr, - vmbuffer, visibility_cutoff_xid, flags); + previous_flags = visibilitymap_set(vacrel->rel, blkno, buffer, InvalidXLogRecPtr, + vmbuffer, visibility_cutoff_xid, flags); + + /* + * If we set the page all frozen in the VM and it was not marked as + * such before, count it here. We don't consider page age for max and + * min page age for the purpose of per-vacuum stats here since we will + * not have newly frozen tuples on the page and only will have marked + * the page frozen in the VM. + */ + if (all_frozen && !(previous_flags & VISIBILITYMAP_ALL_FROZEN)) + vacrel->vm_pages_frozen++; } /* Revert to the previous phase information for error traceback */ diff --git a/src/backend/utils/activity/pgstat_relation.c b/src/backend/utils/activity/pgstat_relation.c index eb387e7eac..e985a6a514 100644 --- a/src/backend/utils/activity/pgstat_relation.c +++ b/src/backend/utils/activity/pgstat_relation.c @@ -368,6 +368,154 @@ pgstat_setup_vacuum_frz_stats(Oid tableoid, bool shared) } +/* + * When a frozen page from a table with oid tableoid is modified, the page LSN + * before modification is passed into this function. This LSN is used to + * identify which bucket contains stats from the freeze period in which this + * page was frozen. Then that bucket's unfreeze counter incremented. If the + * page did not stay frozen for target_page_freeze_duration amount of time, it + * is also counted as an early unfreeze. + * + * MTODO: instead of accessing the table in shared memory, this should be + * cached locally and refetched when counting an unfreeze which is newer than + * any of its local recorded freeze periods. + */ +void +pgstat_count_page_unfreeze(Oid tableoid, bool shared, + XLogRecPtr page_lsn, XLogRecPtr insert_lsn) +{ + Oid dboid = (shared ? InvalidOid : MyDatabaseId); + PgStat_EntryRef *entry_ref; + PgStat_StatTabEntry *tabentry; + TimestampTz current_time; + + if (!pgstat_track_counts) + return; + + entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, + dboid, tableoid, false); + + tabentry = &((PgStatShared_Relation *) entry_ref->shared_stats)->stats; + + current_time = GetCurrentTimestamp(); + + /* + * Loop through the freeze stats stored in the ring, starting with the + * oldest. By starting with the oldest, and since they are in order, we + * know that we will run into the bucket containing the period in which + * our page was frozen. + */ + for (int i = 0; i < tabentry->frz_nbuckets_used; i++) + { + PgStat_Frz *frz; + XLogRecPtr end_lsn; + TimestampTz end_time; + int64 time_elapsed; + int64 lsns_elapsed; + int64 frz_lsn; + int64 page_frz_time; + int64 page_frz_duration; + int64 page_age; + + frz = &tabentry->frz_buckets[ + (tabentry->frz_oldest + i) % VAC_FRZ_STATS_MAX_NBUCKETS + ]; + + /* no entry should have been added without a start lsn */ + Assert(frz->start_lsn != InvalidXLogRecPtr); + + /* + * If the bucket starts after our page LSN, we know we have passed any + * freeze bucket containing the freeze period in which our page could + * have been frozen, so we are done. + */ + if (frz->start_lsn >= page_lsn) + break; + + /* + * If this is a past sample (not a current, unfinished sample) and it + * ended before our page was frozen, we know our page was not frozen + * by this sample. + */ + if (frz->end_lsn != InvalidXLogRecPtr && frz->end_lsn < page_lsn) + continue; + + /* + * We've found the bucket to which this page LSN belongs. If this + * entry isn't over, then let's use the current time as the end time + * for the purpose of calculation. + */ + if (frz->end_lsn == InvalidXLogRecPtr) + { + end_lsn = insert_lsn; + end_time = current_time; + } + else + { + end_lsn = frz->end_lsn; + end_time = frz->end_time; + } + + /* Time in microseconds covered by the freeze bucket */ + time_elapsed = end_time - frz->start_time; + /* LSNs covered by the freeze bucket */ + lsns_elapsed = end_lsn - frz->start_lsn; + + /* How many LSNs into the bucket was the page frozen */ + frz_lsn = page_lsn - frz->start_lsn; + + /* + * Time that corresponds to page LSN at which the page was frozen; + * basically the time at which the page was frozen + */ + page_frz_time = (float) frz_lsn / + lsns_elapsed * time_elapsed + frz->start_time; + + /* amount of time page stayed frozen (in microseconds) */ + page_frz_duration = current_time - page_frz_time; + + /* + * Depending on the LSN generation rate, if the page was frozen close + * to the end of the bucket, page_frz_duration may be negative. + */ + page_frz_duration = Max(page_frz_duration, 0); + + /* + * If the page stayed frozen less than target page freeze duration, it + * is an early unfreeze. Note that target_page_freeze_duration is in + * seconds. + */ + if (page_frz_duration < target_page_freeze_duration * USECS_PER_SEC) + frz->early_unfreezes++; + + frz->unfreezes++; + + page_age = insert_lsn - page_lsn; + + frz->total_frozen_duration_lsns += page_age; + + if (frz->max_frozen_duration_lsns == InvalidXLogRecPtr || + page_age > frz->max_frozen_duration_lsns) + frz->max_frozen_duration_lsns = page_age; + + if (frz->min_frozen_duration_lsns == InvalidXLogRecPtr || + page_age < frz->min_frozen_duration_lsns) + frz->min_frozen_duration_lsns = page_age; + + break; + } + + /* + * If the page is older than any of our currently tracked vacuums, we + * aren't going to count it. We are only concerned with the efficacy of + * our more recent vacuums. If a very old page is being unfrozen, that is + * fine anyway. + */ + + pgstat_unlock_entry(entry_ref); +} + + /* * Report that the table was just vacuumed and flush IO statistics. * @@ -375,13 +523,17 @@ pgstat_setup_vacuum_frz_stats(Oid tableoid, bool shared) * pgstat_report_vacuum(). */ void -pgstat_report_vacuum(Oid tableoid, bool shared, LVRelState *vacrel) +pgstat_report_vacuum(Oid tableoid, bool shared, LVRelState *vacrel, + BlockNumber orig_rel_pages, + BlockNumber new_rel_all_frozen) { PgStat_EntryRef *entry_ref; PgStatShared_Relation *shtabentry; PgStat_StatTabEntry *tabentry; Oid dboid = (shared ? InvalidOid : MyDatabaseId); TimestampTz ts; + XLogRecPtr end_lsn; + PgStat_Frz *vacstat; if (!pgstat_track_counts) return; @@ -389,6 +541,9 @@ pgstat_report_vacuum(Oid tableoid, bool shared, LVRelState *vacrel) /* Store the data in the table's hash table entry. */ ts = GetCurrentTimestamp(); + /* Don't use an approximate insert LSN for vacuum start and end */ + end_lsn = GetXLogInsertRecPtr(); + /* block acquiring lock for the same reason as pgstat_report_autovac() */ entry_ref = pgstat_get_entry_ref_locked(PGSTAT_KIND_RELATION, dboid, tableoid, false); @@ -409,6 +564,24 @@ pgstat_report_vacuum(Oid tableoid, bool shared, LVRelState *vacrel) tabentry->dead_tuples = vacrel->recently_dead_tuples + vacrel->missed_dead_tuples; + vacstat = &tabentry->frz_buckets[tabentry->frz_current]; + vacstat->end_lsn = end_lsn; + vacstat->end_time = ts; + + vacstat->scanned_pages = vacrel->scanned_pages; + vacstat->relsize_end = vacrel->rel_pages; + vacstat->relsize_start = orig_rel_pages; + + vacstat->frozen_pages_end = new_rel_all_frozen; + vacstat->frozen_pages_start = vacrel->already_frozen_pages; + + vacstat->freezes = vacrel->pages_frozen; + vacstat->sum_page_age_lsns = vacrel->sum_frozen_page_ages; + vacstat->vm_page_freezes = vacrel->vm_pages_frozen; + vacstat->max_frz_page_age = vacrel->max_frz_page_age; + vacstat->min_frz_page_age = vacrel->min_frz_page_age; + vacstat->freeze_fpis = vacrel->freeze_fpis; + /* * It is quite possible that a non-aggressive VACUUM ended up skipping * various pages, however, we'll zero the insert counter here regardless. diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index a2d7a0ea72..5b7e7b33f0 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -196,6 +196,11 @@ typedef struct HeapPageFreeze */ typedef struct PruneResult { + /* + * Page LSN prior to pruning, regardless of whether or not the page was + * pruned. This is used by freeze logic. + */ + XLogRecPtr page_lsn; int ndeleted; /* Number of tuples deleted from the page */ int nnewlpdead; /* Number of newly LP_DEAD items */ diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index fa91e1b465..ff7c75ed05 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -361,6 +361,35 @@ typedef struct LVRelState BlockNumber missed_dead_pages; /* # pages with missed dead tuples */ BlockNumber nonempty_pages; /* actually, last nonempty page + 1 */ + /* + * The following members track freeze-related statistics which will be + * accumulated into the PgStat_StatTabEntry at the end of the vacuum. + */ + + /* + * Number of pages marked frozen in the VM before relation is vacuumed. + * This does not include pages marked frozen by this vacuum. + */ + BlockNumber already_frozen_pages; + + /* + * The sum of the age of every page with tuples frozen by this vacuum of + * the relation. + */ + int64 sum_frozen_page_ages; + + /* + * Pages newly marked frozen in the VM. This is inclusive of pages_frozen. + */ + BlockNumber vm_pages_frozen; + + /* oldest and youngest page we froze during this vacuum */ + XLogRecPtr max_frz_page_age; + XLogRecPtr min_frz_page_age; + + /* number of freeze records emitted by this vacuum containing FPIs */ + BlockNumber freeze_fpis; + /* Statistics output by us, for table */ double new_rel_tuples; /* new estimated total # of tuples */ double new_live_tuples; /* new estimated total # of live tuples */ diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 00c67deda4..e273b3aaee 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -11,10 +11,12 @@ #ifndef PGSTAT_H #define PGSTAT_H +#include "access/xlogdefs.h" #include "commands/vacuum.h" #include "datatype/timestamp.h" #include "portability/instr_time.h" #include "postmaster/pgarch.h" /* for MAX_XFN_CHARS */ +#include "storage/block.h" #include "utils/backend_progress.h" /* for backward compatibility */ #include "utils/backend_status.h" /* for backward compatibility */ #include "utils/relcache.h" @@ -718,10 +720,17 @@ extern void pgstat_init_relation(Relation rel); extern void pgstat_assoc_relation(Relation rel); extern void pgstat_unlink_relation(Relation rel); -extern void pgstat_report_vacuum(Oid tableoid, bool shared, LVRelState *vacrel); +extern void pgstat_report_vacuum(Oid tableoid, bool shared, LVRelState *vacrel, + BlockNumber orig_rel_pages, + BlockNumber new_rel_all_frozen); extern void pgstat_setup_vacuum_frz_stats(Oid tableoid, bool shared); +extern void pgstat_count_page_unfreeze(Oid tableoid, bool shared, + XLogRecPtr page_lsn, XLogRecPtr insert_lsn); + +extern void pgstat_count_page_freeze(Oid tableoid, bool shared, int64 page_age); + extern void pgstat_report_analyze(Relation rel, PgStat_Counter livetuples, PgStat_Counter deadtuples, bool resetcounter); -- 2.37.2