From ec09bfcde836cb23f090fd6088b89f0fb9a68000 Mon Sep 17 00:00:00 2001 From: Bharath Rupireddy Date: Mon, 28 Nov 2022 10:07:55 +0000 Subject: [PATCH v3] Introduce a new view for checkpointer related stats pg_stat_bgwriter view currently reports checkpointer stats as well. It is that way because historically checkpointer was part of bgwriter until the commits 806a2ae and bf405ba, that went into PG 9.2, separated them out. It is time for us to separate checkpointer stats to its own view called pg_stat_checkpointer. For now, we deprecate the use of these checkpointer related fields under pg_stat_bgwriter view for smoother transitioning. Eventually, we need to remove them from pg_stat_bgwriter view. Bump catalog version. Author: Bharath Rupireddy Reviewed-by: Andres Freund, Bertrand Drouvot Discussion: https://www.postgresql.org/message-id/CALj2ACVxX2ii=66RypXRweZe2EsBRiPMj0aHfRfHUeXJcC7kHg@mail.gmail.com --- doc/src/sgml/monitoring.sgml | 165 +++++++++++++++++- src/backend/catalog/system_views.sql | 17 +- .../utils/activity/pgstat_checkpointer.c | 1 + src/backend/utils/adt/pgstatfuncs.c | 19 +- src/include/catalog/pg_proc.dat | 22 ++- src/include/pgstat.h | 1 + src/test/recovery/t/029_stats_restart.pl | 6 +- src/test/regress/expected/rules.out | 14 +- src/test/regress/expected/stats.out | 21 ++- src/test/regress/sql/stats.sql | 12 +- 10 files changed, 236 insertions(+), 42 deletions(-) diff --git a/doc/src/sgml/monitoring.sgml b/doc/src/sgml/monitoring.sgml index 5579b8b9e0..c69d3b21c0 100644 --- a/doc/src/sgml/monitoring.sgml +++ b/doc/src/sgml/monitoring.sgml @@ -448,6 +448,15 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser + + pg_stat_checkpointerpg_stat_checkpointer + One row only, showing statistics about the + checkpointer process's activity. See + + pg_stat_checkpointer for details. + + + pg_stat_walpg_stat_wal One row only, showing statistics about WAL activity. See @@ -3654,7 +3663,11 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i checkpoints_timed bigint - Number of scheduled checkpoints that have been performed + Number of scheduled checkpoints that have been performed. Use of this + field under pg_stat_bgwriter view has been + deprecated, because the field actually shows checkpointer process + activity and is moved to a new view called + pg_stat_checkpointer. @@ -3663,7 +3676,11 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i checkpoints_req bigint - Number of requested checkpoints that have been performed + Number of requested checkpoints that have been performed. Use of this + field under pg_stat_bgwriter view has been + deprecated, because the field actually shows checkpointer process + activity and is moved to a new view called + pg_stat_checkpointer. @@ -3673,7 +3690,11 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i Total amount of time that has been spent in the portion of - checkpoint processing where files are written to disk, in milliseconds + checkpoint processing where files are written to disk, in milliseconds. + Use of this field under pg_stat_bgwriter view + has been deprecated, because the field actually shows checkpointer + process activity and is moved to a new view called + pg_stat_checkpointer. @@ -3684,7 +3705,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk, in - milliseconds + milliseconds. Use of this field under + pg_stat_bgwriter view has been deprecated, + because the field actually shows checkpointer process activity and is + moved to a new view called pg_stat_checkpointer. @@ -3693,7 +3717,11 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i buffers_checkpoint bigint - Number of buffers written during checkpoints + Number of buffers written during checkpoints. Use of this field under + pg_stat_bgwriter view has been deprecated, + because the field actually shows checkpointer process activity and is + moved to a new view called + pg_stat_checkpointer. @@ -3721,7 +3749,11 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i buffers_backend bigint - Number of buffers written directly by a backend + Number of buffers written directly by a backend. Use of this field + under pg_stat_bgwriter view has been + deprecated, because the field actually shows checkpointer process + activity and is moved to a new view called + pg_stat_checkpointer. @@ -3732,7 +3764,11 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i Number of times a backend had to execute its own fsync call (normally the background writer handles those - even when the backend does its own write) + even when the backend does its own write). Use of this field under + pg_stat_bgwriter view has been deprecated, + because the field actually shows checkpointer process activity and is + moved to a new view called + pg_stat_checkpointer. @@ -3759,6 +3795,115 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i + + <structname>pg_stat_checkpointer</structname> + + + pg_stat_checkpointer + + + + The pg_stat_checkpointer view will always have a + single row, containing data about the checkpointer process of the cluster. + + + + <structname>pg_stat_checkpointer</structname> View + + + + + Column Type + + + Description + + + + + + + + checkpoints_timed bigint + + + Number of scheduled checkpoints that have been performed + + + + + + checkpoints_req bigint + + + Number of requested checkpoints that have been performed + + + + + + checkpoint_write_time double precision + + + Total amount of time that has been spent in the portion of + checkpoint processing where files are written to disk, in milliseconds + + + + + + checkpoint_sync_time double precision + + + Total amount of time that has been spent in the portion of + checkpoint processing where files are synchronized to disk, in + milliseconds + + + + + + buffers_checkpoint bigint + + + Number of buffers written during checkpoints + + + + + + buffers_backend bigint + + + Number of buffers written directly by a backend + + + + + + buffers_backend_fsync bigint + + + Number of times a backend had to execute its own + fsync call (normally the checkpointer handles those + even when the backend does its own write) + + + + + + stats_reset timestamp with time zone + + + Time at which these statistics were last reset + + + + +
+ + + <structname>pg_stat_wal</structname> @@ -5391,8 +5536,10 @@ SELECT pid, wait_event_type, wait_event FROM pg_stat_activity WHERE wait_event i Resets some cluster-wide statistics counters to zero, depending on the argument. The argument can be bgwriter to reset all the counters shown in - the pg_stat_bgwriter - view, archiver to reset all the counters shown in + the pg_stat_bgwriter view, + checkpointer to reset all the counters shown in + the pg_stat_checkpointer view, + archiver to reset all the counters shown in the pg_stat_archiver view, wal to reset all the counters shown in the pg_stat_wal view or diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index 2d8104b090..e4e49efe76 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -1105,11 +1105,11 @@ CREATE VIEW pg_stat_archiver AS CREATE VIEW pg_stat_bgwriter AS SELECT - pg_stat_get_bgwriter_timed_checkpoints() AS checkpoints_timed, - pg_stat_get_bgwriter_requested_checkpoints() AS checkpoints_req, + pg_stat_get_timed_checkpoints() AS checkpoints_timed, + pg_stat_get_requested_checkpoints() AS checkpoints_req, pg_stat_get_checkpoint_write_time() AS checkpoint_write_time, pg_stat_get_checkpoint_sync_time() AS checkpoint_sync_time, - pg_stat_get_bgwriter_buf_written_checkpoints() AS buffers_checkpoint, + pg_stat_get_buf_written_checkpoints() AS buffers_checkpoint, pg_stat_get_bgwriter_buf_written_clean() AS buffers_clean, pg_stat_get_bgwriter_maxwritten_clean() AS maxwritten_clean, pg_stat_get_buf_written_backend() AS buffers_backend, @@ -1117,6 +1117,17 @@ CREATE VIEW pg_stat_bgwriter AS pg_stat_get_buf_alloc() AS buffers_alloc, pg_stat_get_bgwriter_stat_reset_time() AS stats_reset; +CREATE VIEW pg_stat_checkpointer AS + SELECT + pg_stat_get_timed_checkpoints() AS checkpoints_timed, + pg_stat_get_requested_checkpoints() AS checkpoints_req, + pg_stat_get_checkpoint_write_time() AS checkpoint_write_time, + pg_stat_get_checkpoint_sync_time() AS checkpoint_sync_time, + pg_stat_get_buf_written_checkpoints() AS buffers_checkpoint, + pg_stat_get_buf_written_backend() AS buffers_backend, + pg_stat_get_buf_fsync_backend() AS buffers_backend_fsync, + pg_stat_get_checkpointer_stat_reset_time() AS stats_reset; + CREATE VIEW pg_stat_wal AS SELECT w.wal_records, diff --git a/src/backend/utils/activity/pgstat_checkpointer.c b/src/backend/utils/activity/pgstat_checkpointer.c index af8d513e7b..2757a10b7e 100644 --- a/src/backend/utils/activity/pgstat_checkpointer.c +++ b/src/backend/utils/activity/pgstat_checkpointer.c @@ -89,6 +89,7 @@ pgstat_checkpointer_reset_all_cb(TimestampTz ts) &stats_shmem->stats, sizeof(stats_shmem->stats), &stats_shmem->changecount); + stats_shmem->stats.stat_reset_timestamp = ts; LWLockRelease(&stats_shmem->lock); } diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index ae3365d917..38a2595cdd 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -1662,19 +1662,19 @@ pg_stat_get_db_sessions_killed(PG_FUNCTION_ARGS) } Datum -pg_stat_get_bgwriter_timed_checkpoints(PG_FUNCTION_ARGS) +pg_stat_get_timed_checkpoints(PG_FUNCTION_ARGS) { PG_RETURN_INT64(pgstat_fetch_stat_checkpointer()->timed_checkpoints); } Datum -pg_stat_get_bgwriter_requested_checkpoints(PG_FUNCTION_ARGS) +pg_stat_get_requested_checkpoints(PG_FUNCTION_ARGS) { PG_RETURN_INT64(pgstat_fetch_stat_checkpointer()->requested_checkpoints); } Datum -pg_stat_get_bgwriter_buf_written_checkpoints(PG_FUNCTION_ARGS) +pg_stat_get_buf_written_checkpoints(PG_FUNCTION_ARGS) { PG_RETURN_INT64(pgstat_fetch_stat_checkpointer()->buf_written_checkpoints); } @@ -1707,6 +1707,12 @@ pg_stat_get_checkpoint_sync_time(PG_FUNCTION_ARGS) pgstat_fetch_stat_checkpointer()->checkpoint_sync_time); } +Datum +pg_stat_get_checkpointer_stat_reset_time(PG_FUNCTION_ARGS) +{ + PG_RETURN_TIMESTAMPTZ(pgstat_fetch_stat_checkpointer()->stat_reset_timestamp); +} + Datum pg_stat_get_bgwriter_stat_reset_time(PG_FUNCTION_ARGS) { @@ -2082,14 +2088,9 @@ pg_stat_reset_shared(PG_FUNCTION_ARGS) if (strcmp(target, "archiver") == 0) pgstat_reset_of_kind(PGSTAT_KIND_ARCHIVER); else if (strcmp(target, "bgwriter") == 0) - { - /* - * Historically checkpointer was part of bgwriter, continue to reset - * both for now. - */ pgstat_reset_of_kind(PGSTAT_KIND_BGWRITER); + else if (strcmp(target, "checkpointer") == 0) pgstat_reset_of_kind(PGSTAT_KIND_CHECKPOINTER); - } else if (strcmp(target, "recovery_prefetch") == 0) XLogPrefetchResetStats(); else if (strcmp(target, "wal") == 0) diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index f9301b2627..1b24969a01 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -5628,20 +5628,24 @@ proargnames => '{archived_count,last_archived_wal,last_archived_time,failed_count,last_failed_wal,last_failed_time,stats_reset}', prosrc => 'pg_stat_get_archiver' }, { oid => '2769', - descr => 'statistics: number of timed checkpoints started by the bgwriter', - proname => 'pg_stat_get_bgwriter_timed_checkpoints', provolatile => 's', + descr => 'statistics: number of timed checkpoints started by the checkpointer', + proname => 'pg_stat_get_timed_checkpoints', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => '', - prosrc => 'pg_stat_get_bgwriter_timed_checkpoints' }, + prosrc => 'pg_stat_get_timed_checkpoints' }, { oid => '2770', - descr => 'statistics: number of backend requested checkpoints started by the bgwriter', - proname => 'pg_stat_get_bgwriter_requested_checkpoints', provolatile => 's', + descr => 'statistics: number of backend requested checkpoints started by the checkpointer', + proname => 'pg_stat_get_requested_checkpoints', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => '', - prosrc => 'pg_stat_get_bgwriter_requested_checkpoints' }, + prosrc => 'pg_stat_get_requested_checkpoints' }, { oid => '2771', - descr => 'statistics: number of buffers written by the bgwriter during checkpoints', - proname => 'pg_stat_get_bgwriter_buf_written_checkpoints', provolatile => 's', + descr => 'statistics: number of buffers written by the checkpointer', + proname => 'pg_stat_get_buf_written_checkpoints', provolatile => 's', proparallel => 'r', prorettype => 'int8', proargtypes => '', - prosrc => 'pg_stat_get_bgwriter_buf_written_checkpoints' }, + prosrc => 'pg_stat_get_buf_written_checkpoints' }, +{ oid => '8206', descr => 'statistics: last reset for the checkpointer', + proname => 'pg_stat_get_checkpointer_stat_reset_time', provolatile => 's', + proparallel => 'r', prorettype => 'timestamptz', proargtypes => '', + prosrc => 'pg_stat_get_checkpointer_stat_reset_time' }, { oid => '2772', descr => 'statistics: number of buffers written by the bgwriter for cleaning dirty buffers', proname => 'pg_stat_get_bgwriter_buf_written_clean', provolatile => 's', diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 9e2ce6f011..e23d80bba8 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -274,6 +274,7 @@ typedef struct PgStat_CheckpointerStats PgStat_Counter buf_written_checkpoints; PgStat_Counter buf_written_backend; PgStat_Counter buf_fsync_backend; + TimestampTz stat_reset_timestamp; } PgStat_CheckpointerStats; typedef struct PgStat_StatDBEntry diff --git a/src/test/recovery/t/029_stats_restart.pl b/src/test/recovery/t/029_stats_restart.pl index 1bf7b568cc..f4e262a4bf 100644 --- a/src/test/recovery/t/029_stats_restart.pl +++ b/src/test/recovery/t/029_stats_restart.pl @@ -173,7 +173,7 @@ is($wal_start->{reset}, $wal_restart->{reset}, ## Check that checkpoint stats are reset, WAL stats aren't affected -$node->safe_psql($connect_db, "SELECT pg_stat_reset_shared('bgwriter')"); +$node->safe_psql($connect_db, "SELECT pg_stat_reset_shared('checkpointer')"); $sect = "post ckpt reset"; my $ckpt_reset = checkpoint_stats(); @@ -323,9 +323,9 @@ sub checkpoint_stats my %results; $results{count} = $node->safe_psql($connect_db, - "SELECT checkpoints_timed + checkpoints_req FROM pg_stat_bgwriter"); + "SELECT checkpoints_timed + checkpoints_req FROM pg_stat_checkpointer"); $results{reset} = $node->safe_psql($connect_db, - "SELECT stats_reset FROM pg_stat_bgwriter"); + "SELECT stats_reset FROM pg_stat_checkpointer"); return \%results; } diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 37c1c86473..81bb0adfc3 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1814,17 +1814,25 @@ pg_stat_archiver| SELECT s.archived_count, s.last_failed_time, s.stats_reset FROM pg_stat_get_archiver() s(archived_count, last_archived_wal, last_archived_time, failed_count, last_failed_wal, last_failed_time, stats_reset); -pg_stat_bgwriter| SELECT pg_stat_get_bgwriter_timed_checkpoints() AS checkpoints_timed, - pg_stat_get_bgwriter_requested_checkpoints() AS checkpoints_req, +pg_stat_bgwriter| SELECT pg_stat_get_timed_checkpoints() AS checkpoints_timed, + pg_stat_get_requested_checkpoints() AS checkpoints_req, pg_stat_get_checkpoint_write_time() AS checkpoint_write_time, pg_stat_get_checkpoint_sync_time() AS checkpoint_sync_time, - pg_stat_get_bgwriter_buf_written_checkpoints() AS buffers_checkpoint, + pg_stat_get_buf_written_checkpoints() AS buffers_checkpoint, pg_stat_get_bgwriter_buf_written_clean() AS buffers_clean, pg_stat_get_bgwriter_maxwritten_clean() AS maxwritten_clean, pg_stat_get_buf_written_backend() AS buffers_backend, pg_stat_get_buf_fsync_backend() AS buffers_backend_fsync, pg_stat_get_buf_alloc() AS buffers_alloc, pg_stat_get_bgwriter_stat_reset_time() AS stats_reset; +pg_stat_checkpointer| SELECT pg_stat_get_timed_checkpoints() AS checkpoints_timed, + pg_stat_get_requested_checkpoints() AS checkpoints_req, + pg_stat_get_checkpoint_write_time() AS checkpoint_write_time, + pg_stat_get_checkpoint_sync_time() AS checkpoint_sync_time, + pg_stat_get_buf_written_checkpoints() AS buffers_checkpoint, + pg_stat_get_buf_written_backend() AS buffers_backend, + pg_stat_get_buf_fsync_backend() AS buffers_backend_fsync, + pg_stat_get_checkpointer_stat_reset_time() AS stats_reset; pg_stat_database| SELECT d.oid AS datid, d.datname, CASE diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out index 1d84407a03..8ed7f895b2 100644 --- a/src/test/regress/expected/stats.out +++ b/src/test/regress/expected/stats.out @@ -782,8 +782,8 @@ SELECT sessions > :db_stat_sessions FROM pg_stat_database WHERE datname = (SELEC t (1 row) --- Test pg_stat_bgwriter checkpointer-related stats, together with pg_stat_wal -SELECT checkpoints_req AS rqst_ckpts_before FROM pg_stat_bgwriter \gset +-- Test pg_stat_checkpointer checkpointer-related stats, together with pg_stat_wal +SELECT checkpoints_req AS rqst_ckpts_before FROM pg_stat_checkpointer \gset -- Test pg_stat_wal (and make a temp table so our temp schema exists) SELECT wal_bytes AS wal_bytes_before FROM pg_stat_wal \gset CREATE TEMP TABLE test_stats_temp AS SELECT 17; @@ -793,7 +793,7 @@ DROP TABLE test_stats_temp; -- results of the first. CHECKPOINT; CHECKPOINT; -SELECT checkpoints_req > :rqst_ckpts_before FROM pg_stat_bgwriter; +SELECT checkpoints_req > :rqst_ckpts_before FROM pg_stat_checkpointer; ?column? ---------- t @@ -884,6 +884,21 @@ SELECT stats_reset > :'bgwriter_reset_ts'::timestamptz FROM pg_stat_bgwriter; (1 row) SELECT stats_reset AS bgwriter_reset_ts FROM pg_stat_bgwriter \gset +-- Test that reset_shared with checkpointer specified as the stats type works +SELECT stats_reset AS checkpointer_reset_ts FROM pg_stat_checkpointer \gset +SELECT pg_stat_reset_shared('checkpointer'); + pg_stat_reset_shared +---------------------- + +(1 row) + +SELECT stats_reset > :'checkpointer_reset_ts'::timestamptz FROM pg_stat_checkpointer; + ?column? +---------- + t +(1 row) + +SELECT stats_reset AS checkpointer_reset_ts FROM pg_stat_checkpointer \gset -- Test that reset_shared with wal specified as the stats type works SELECT stats_reset AS wal_reset_ts FROM pg_stat_wal \gset SELECT pg_stat_reset_shared('wal'); diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql index b4d6753c71..c8c24abea8 100644 --- a/src/test/regress/sql/stats.sql +++ b/src/test/regress/sql/stats.sql @@ -387,8 +387,8 @@ SELECT sessions AS db_stat_sessions FROM pg_stat_database WHERE datname = (SELEC SELECT pg_stat_force_next_flush(); SELECT sessions > :db_stat_sessions FROM pg_stat_database WHERE datname = (SELECT current_database()); --- Test pg_stat_bgwriter checkpointer-related stats, together with pg_stat_wal -SELECT checkpoints_req AS rqst_ckpts_before FROM pg_stat_bgwriter \gset +-- Test pg_stat_checkpointer checkpointer-related stats, together with pg_stat_wal +SELECT checkpoints_req AS rqst_ckpts_before FROM pg_stat_checkpointer \gset -- Test pg_stat_wal (and make a temp table so our temp schema exists) SELECT wal_bytes AS wal_bytes_before FROM pg_stat_wal \gset @@ -402,7 +402,7 @@ DROP TABLE test_stats_temp; CHECKPOINT; CHECKPOINT; -SELECT checkpoints_req > :rqst_ckpts_before FROM pg_stat_bgwriter; +SELECT checkpoints_req > :rqst_ckpts_before FROM pg_stat_checkpointer; SELECT wal_bytes > :wal_bytes_before FROM pg_stat_wal; -- Test pg_stat_get_backend_idset() and some allied functions. @@ -440,6 +440,12 @@ SELECT pg_stat_reset_shared('bgwriter'); SELECT stats_reset > :'bgwriter_reset_ts'::timestamptz FROM pg_stat_bgwriter; SELECT stats_reset AS bgwriter_reset_ts FROM pg_stat_bgwriter \gset +-- Test that reset_shared with checkpointer specified as the stats type works +SELECT stats_reset AS checkpointer_reset_ts FROM pg_stat_checkpointer \gset +SELECT pg_stat_reset_shared('checkpointer'); +SELECT stats_reset > :'checkpointer_reset_ts'::timestamptz FROM pg_stat_checkpointer; +SELECT stats_reset AS checkpointer_reset_ts FROM pg_stat_checkpointer \gset + -- Test that reset_shared with wal specified as the stats type works SELECT stats_reset AS wal_reset_ts FROM pg_stat_wal \gset SELECT pg_stat_reset_shared('wal'); -- 2.34.1