From 1b488106f7dfc142f219bb8ba0a0c5d2b6b94ba5 Mon Sep 17 00:00:00 2001 From: Andrey Borodin Date: Mon, 15 Feb 2021 21:51:56 +0500 Subject: [PATCH v9] Make all SLRU buffer sizes configurable --- doc/src/sgml/config.sgml | 101 ++++++++++++++++++ src/backend/access/transam/clog.c | 6 ++ src/backend/access/transam/commit_ts.c | 5 +- src/backend/access/transam/multixact.c | 8 +- src/backend/access/transam/subtrans.c | 5 +- src/backend/commands/async.c | 8 +- src/backend/storage/lmgr/predicate.c | 4 +- src/backend/utils/init/globals.c | 8 ++ src/backend/utils/misc/guc.c | 77 +++++++++++++ src/backend/utils/misc/postgresql.conf.sample | 16 +++ src/include/access/multixact.h | 4 - src/include/access/subtrans.h | 3 - src/include/commands/async.h | 5 - src/include/miscadmin.h | 8 ++ src/include/storage/predicate.h | 4 - 15 files changed, 233 insertions(+), 29 deletions(-) diff --git a/doc/src/sgml/config.sgml b/doc/src/sgml/config.sgml index 4df1405d2e..5b24b435f3 100644 --- a/doc/src/sgml/config.sgml +++ b/doc/src/sgml/config.sgml @@ -1852,6 +1852,107 @@ include_dir 'conf.d' + + + multixact_offsets_slru_buffers (integer) + + multixact_offsets_slru_buffers configuration parameter + + + + + Specifies the amount of memory to be used for MultiXact offsets. MultiXact offsets + are used to store information about offsets of multiple row lockers (caused by SELECT FOR UPDATE and others). + It defaults to 64 kilobytes (64KB). + + + + + + multixact_members_slru_buffers (integer) + + multixact_members_slru_buffers configuration parameter + + + + + Specifies the amount of memory to be used for MultiXact members. MultiXact members + are used to store information about XIDs of multiple row lockers. Tipically multixact_members_slru_buffers + is twice more than multixact_offsets_slru_buffers. + It defaults to 128 kilobytes (128KB). + + + + + + subtrans_slru_buffers (integer) + + subtrans_slru_buffers configuration parameter + + + + + Specifies the amount of memory to be used for subtransactions. + It defaults to 256 kilobytes (256KB). + + + + + + notify_slru_buffers (integer) + + notify_slru_buffers configuration parameter + + + + + Specifies the amount of memory to be used for asincronous notifications (NOTIFY, LISTEN). + It defaults to 64 kilobytes (64KB). + + + + + + serial_slru_buffers (integer) + + serial_slru_buffers configuration parameter + + + + + Specifies the amount of memory to be used for predicate locks. + It defaults to 128 kilobytes (128KB). + + + + + + clog_slru_buffers (integer) + + clog_slru_buffers configuration parameter + + + + + Specifies the amount of memory to be used for CLOG. + It defaults to 0, in this case CLOG size is taken as shared_buffers / 512. + + + + + + commit_ts_slru_buffers (integer) + + commit_ts_slru_buffers configuration parameter + + + + + Specifies the amount of memory to be used for commit timestamps. + It defaults to 0, in this case CLOG size is taken as shared_buffers / 512. + + + max_stack_depth (integer) diff --git a/src/backend/access/transam/clog.c b/src/backend/access/transam/clog.c index 6fa4713fb4..e57106b374 100644 --- a/src/backend/access/transam/clog.c +++ b/src/backend/access/transam/clog.c @@ -659,6 +659,9 @@ TransactionIdGetStatus(TransactionId xid, XLogRecPtr *lsn) /* * Number of shared CLOG buffers. * + * If values is confugured via GUC - just use given value. Otherwise + * apply following euristics. + * * On larger multi-processor systems, it is possible to have many CLOG page * requests in flight at one time which could lead to disk access for CLOG * page if the required page is not found in memory. Testing revealed that we @@ -675,6 +678,9 @@ TransactionIdGetStatus(TransactionId xid, XLogRecPtr *lsn) Size CLOGShmemBuffers(void) { + /* consider 0 and 1 as unset GUC */ + if (clog_slru_buffers > 1) + return clog_slru_buffers; return Min(128, Max(4, NBuffers / 512)); } diff --git a/src/backend/access/transam/commit_ts.c b/src/backend/access/transam/commit_ts.c index 48e8d66286..7de3bca63d 100644 --- a/src/backend/access/transam/commit_ts.c +++ b/src/backend/access/transam/commit_ts.c @@ -530,7 +530,10 @@ pg_xact_commit_timestamp_origin(PG_FUNCTION_ARGS) Size CommitTsShmemBuffers(void) { - return Min(16, Max(4, NBuffers / 1024)); + /* consider 0 and 1 as unset GUC */ + if (commit_ts_slru_buffers > 1) + return commit_ts_slru_buffers; + return Min(16, Max(4, NBuffers / 512)); } /* diff --git a/src/backend/access/transam/multixact.c b/src/backend/access/transam/multixact.c index 7dcfa02323..dd7dd19ff4 100644 --- a/src/backend/access/transam/multixact.c +++ b/src/backend/access/transam/multixact.c @@ -1831,8 +1831,8 @@ MultiXactShmemSize(void) mul_size(sizeof(MultiXactId) * 2, MaxOldestSlot)) size = SHARED_MULTIXACT_STATE_SIZE; - size = add_size(size, SimpleLruShmemSize(NUM_MULTIXACTOFFSET_BUFFERS, 0)); - size = add_size(size, SimpleLruShmemSize(NUM_MULTIXACTMEMBER_BUFFERS, 0)); + size = add_size(size, SimpleLruShmemSize(multixact_offsets_slru_buffers, 0)); + size = add_size(size, SimpleLruShmemSize(multixact_members_slru_buffers, 0)); return size; } @@ -1848,13 +1848,13 @@ MultiXactShmemInit(void) MultiXactMemberCtl->PagePrecedes = MultiXactMemberPagePrecedes; SimpleLruInit(MultiXactOffsetCtl, - "MultiXactOffset", NUM_MULTIXACTOFFSET_BUFFERS, 0, + "MultiXactOffset", multixact_offsets_slru_buffers, 0, MultiXactOffsetSLRULock, "pg_multixact/offsets", LWTRANCHE_MULTIXACTOFFSET_BUFFER, SYNC_HANDLER_MULTIXACT_OFFSET); SlruPagePrecedesUnitTests(MultiXactOffsetCtl, MULTIXACT_OFFSETS_PER_PAGE); SimpleLruInit(MultiXactMemberCtl, - "MultiXactMember", NUM_MULTIXACTMEMBER_BUFFERS, 0, + "MultiXactMember", multixact_offsets_slru_buffers, 0, MultiXactMemberSLRULock, "pg_multixact/members", LWTRANCHE_MULTIXACTMEMBER_BUFFER, SYNC_HANDLER_MULTIXACT_MEMBER); diff --git a/src/backend/access/transam/subtrans.c b/src/backend/access/transam/subtrans.c index 6a8e521f89..0c24353d3a 100644 --- a/src/backend/access/transam/subtrans.c +++ b/src/backend/access/transam/subtrans.c @@ -31,6 +31,7 @@ #include "access/slru.h" #include "access/subtrans.h" #include "access/transam.h" +#include "miscadmin.h" #include "pg_trace.h" #include "utils/snapmgr.h" @@ -184,14 +185,14 @@ SubTransGetTopmostTransaction(TransactionId xid) Size SUBTRANSShmemSize(void) { - return SimpleLruShmemSize(NUM_SUBTRANS_BUFFERS, 0); + return SimpleLruShmemSize(subtrans_slru_buffers, 0); } void SUBTRANSShmemInit(void) { SubTransCtl->PagePrecedes = SubTransPagePrecedes; - SimpleLruInit(SubTransCtl, "Subtrans", NUM_SUBTRANS_BUFFERS, 0, + SimpleLruInit(SubTransCtl, "Subtrans", subtrans_slru_buffers, 0, SubtransSLRULock, "pg_subtrans", LWTRANCHE_SUBTRANS_BUFFER, SYNC_HANDLER_NONE); SlruPagePrecedesUnitTests(SubTransCtl, SUBTRANS_XACTS_PER_PAGE); diff --git a/src/backend/commands/async.c b/src/backend/commands/async.c index 42b232d98b..a7dd92add0 100644 --- a/src/backend/commands/async.c +++ b/src/backend/commands/async.c @@ -107,7 +107,7 @@ * frontend during startup.) The above design guarantees that notifies from * other backends will never be missed by ignoring self-notifies. * - * The amount of shared memory used for notify management (NUM_NOTIFY_BUFFERS) + * The amount of shared memory used for notify management (notify_slru_buffers) * can be varied without affecting anything but performance. The maximum * amount of notification data that can be queued at one time is determined * by slru.c's wraparound limit; see QUEUE_MAX_PAGE below. @@ -225,7 +225,7 @@ typedef struct QueuePosition * * Resist the temptation to make this really large. While that would save * work in some places, it would add cost in others. In particular, this - * should likely be less than NUM_NOTIFY_BUFFERS, to ensure that backends + * should likely be less than notify_slru_buffers, to ensure that backends * catch up before the pages they'll need to read fall out of SLRU cache. */ #define QUEUE_CLEANUP_DELAY 4 @@ -514,7 +514,7 @@ AsyncShmemSize(void) size = mul_size(MaxBackends + 1, sizeof(QueueBackendStatus)); size = add_size(size, offsetof(AsyncQueueControl, backend)); - size = add_size(size, SimpleLruShmemSize(NUM_NOTIFY_BUFFERS, 0)); + size = add_size(size, SimpleLruShmemSize(notify_slru_buffers, 0)); return size; } @@ -562,7 +562,7 @@ AsyncShmemInit(void) * Set up SLRU management of the pg_notify data. */ NotifyCtl->PagePrecedes = asyncQueuePagePrecedes; - SimpleLruInit(NotifyCtl, "Notify", NUM_NOTIFY_BUFFERS, 0, + SimpleLruInit(NotifyCtl, "Notify", notify_slru_buffers, 0, NotifySLRULock, "pg_notify", LWTRANCHE_NOTIFY_BUFFER, SYNC_HANDLER_NONE); diff --git a/src/backend/storage/lmgr/predicate.c b/src/backend/storage/lmgr/predicate.c index 074df5b38c..ad6d800dbd 100644 --- a/src/backend/storage/lmgr/predicate.c +++ b/src/backend/storage/lmgr/predicate.c @@ -872,7 +872,7 @@ SerialInit(void) */ SerialSlruCtl->PagePrecedes = SerialPagePrecedesLogically; SimpleLruInit(SerialSlruCtl, "Serial", - NUM_SERIAL_BUFFERS, 0, SerialSLRULock, "pg_serial", + serial_slru_buffers, 0, SerialSLRULock, "pg_serial", LWTRANCHE_SERIAL_BUFFER, SYNC_HANDLER_NONE); #ifdef USE_ASSERT_CHECKING SerialPagePrecedesLogicallyUnitTests(); @@ -1395,7 +1395,7 @@ PredicateLockShmemSize(void) /* Shared memory structures for SLRU tracking of old committed xids. */ size = add_size(size, sizeof(SerialControlData)); - size = add_size(size, SimpleLruShmemSize(NUM_SERIAL_BUFFERS, 0)); + size = add_size(size, SimpleLruShmemSize(serial_slru_buffers, 0)); return size; } diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c index a5976ad5b1..88e785c246 100644 --- a/src/backend/utils/init/globals.c +++ b/src/backend/utils/init/globals.c @@ -150,3 +150,11 @@ int VacuumCostBalance = 0; /* working state for vacuum */ bool VacuumCostActive = false; double vacuum_cleanup_index_scale_factor; + +int multixact_offsets_slru_buffers = 8; +int multixact_members_slru_buffers = 16; +int subtrans_slru_buffers = 32; +int notify_slru_buffers = 8; +int serial_slru_buffers = 16; +int clog_slru_buffers = 0; +int commit_ts_slru_buffers = 0; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index eafdb1118e..d30f85bff8 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -2269,6 +2269,83 @@ static struct config_int ConfigureNamesInt[] = NULL, NULL, NULL }, + { + {"multixact_offsets_slru_buffers", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets the number of shared memory buffers used for MultiXact offsets SLRU."), + NULL, + GUC_UNIT_BLOCKS + }, + &multixact_offsets_slru_buffers, + 8, 2, INT_MAX / 2, + NULL, NULL, NULL + }, + + { + {"multixact_members_slru_buffers", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets the number of shared memory buffers used for MultiXact members SLRU."), + NULL, + GUC_UNIT_BLOCKS + }, + &multixact_members_slru_buffers, + 16, 2, INT_MAX / 2, + NULL, NULL, NULL + }, + + { + {"subtrans_slru_buffers", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets the number of shared memory buffers used for substrnsactions SLRU."), + NULL, + GUC_UNIT_BLOCKS + }, + &subtrans_slru_buffers, + 32, 2, INT_MAX / 2, + NULL, NULL, NULL + }, + + { + {"notify_slru_buffers", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets the number of shared memory buffers used for asyncronous notifications SLRU."), + NULL, + GUC_UNIT_BLOCKS + }, + ¬ify_slru_buffers, + 8, 2, INT_MAX / 2, + NULL, NULL, NULL + }, + + { + {"serial_slru_buffers", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets the number of shared memory buffers used for predicate locks SLRU."), + NULL, + GUC_UNIT_BLOCKS + }, + &serial_slru_buffers, + 16, 2, INT_MAX / 2, + NULL, NULL, NULL + }, + + { + {"clog_slru_buffers", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets the number of shared memory buffers used for commit log SLRU."), + NULL, + GUC_UNIT_BLOCKS + }, + &serial_slru_buffers, + 0, 0, INT_MAX / 2, + NULL, NULL, NULL + }, + + { + {"commit_ts_slru_buffers", PGC_POSTMASTER, RESOURCES_MEM, + gettext_noop("Sets the number of shared memory buffers used for commit timestamps SLRU."), + NULL, + GUC_UNIT_BLOCKS + }, + &commit_ts_slru_buffers, + 0, 0, INT_MAX / 2, + NULL, NULL, NULL + }, + { {"temp_buffers", PGC_USERSET, RESOURCES_MEM, gettext_noop("Sets the maximum number of temporary buffers used by each session."), diff --git a/src/backend/utils/misc/postgresql.conf.sample b/src/backend/utils/misc/postgresql.conf.sample index db6db376eb..e88271c048 100644 --- a/src/backend/utils/misc/postgresql.conf.sample +++ b/src/backend/utils/misc/postgresql.conf.sample @@ -189,6 +189,22 @@ # (change requires restart) #backend_flush_after = 0 # measured in pages, 0 disables +# - SLRU Buffers - + +#multixact_offsets_slru_buffers = 8 # memory used for MultiXact offsets + # (change requires restart) +#multixact_members_slru_buffers = 16 # memory used for MultiXact members + # (change requires restart) +#subtrans_slru_buffers = 32 # memory used for subtransactions + # (change requires restart) +#notify_slru_buffers = 8 # memory used for asynchronous notifications + # (change requires restart) +#serial_slru_buffers = 16 # memory used for predicate locks + # (change requires restart) +#clog_slru_buffers = 0 # memory used for CLOG + # (change requires restart) +#commit_ts_slru_buffers = 0 # memory used for commit timestamps + # (change requires restart) #------------------------------------------------------------------------------ # WRITE-AHEAD LOG diff --git a/src/include/access/multixact.h b/src/include/access/multixact.h index 4bbb035eae..97c0a46376 100644 --- a/src/include/access/multixact.h +++ b/src/include/access/multixact.h @@ -29,10 +29,6 @@ #define MaxMultiXactOffset ((MultiXactOffset) 0xFFFFFFFF) -/* Number of SLRU buffers to use for multixact */ -#define NUM_MULTIXACTOFFSET_BUFFERS 8 -#define NUM_MULTIXACTMEMBER_BUFFERS 16 - /* * Possible multixact lock modes ("status"). The first four modes are for * tuple locks (FOR KEY SHARE, FOR SHARE, FOR NO KEY UPDATE, FOR UPDATE); the diff --git a/src/include/access/subtrans.h b/src/include/access/subtrans.h index d0ab44ae82..ca0999056e 100644 --- a/src/include/access/subtrans.h +++ b/src/include/access/subtrans.h @@ -11,9 +11,6 @@ #ifndef SUBTRANS_H #define SUBTRANS_H -/* Number of SLRU buffers to use for subtrans */ -#define NUM_SUBTRANS_BUFFERS 32 - extern void SubTransSetParent(TransactionId xid, TransactionId parent); extern TransactionId SubTransGetParent(TransactionId xid); extern TransactionId SubTransGetTopmostTransaction(TransactionId xid); diff --git a/src/include/commands/async.h b/src/include/commands/async.h index 9217f66b91..fa831e3721 100644 --- a/src/include/commands/async.h +++ b/src/include/commands/async.h @@ -15,11 +15,6 @@ #include -/* - * The number of SLRU page buffers we use for the notification queue. - */ -#define NUM_NOTIFY_BUFFERS 8 - extern bool Trace_notify; extern volatile sig_atomic_t notifyInterruptPending; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 1bdc97e308..51c7ec8d68 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -162,6 +162,14 @@ extern PGDLLIMPORT int MaxBackends; extern PGDLLIMPORT int MaxConnections; extern PGDLLIMPORT int max_worker_processes; extern PGDLLIMPORT int max_parallel_workers; +extern PGDLLIMPORT int multixact_offsets_slru_buffers; +extern PGDLLIMPORT int multixact_members_slru_buffers; +extern PGDLLIMPORT int multixact_members_slru_buffers; +extern PGDLLIMPORT int subtrans_slru_buffers; +extern PGDLLIMPORT int notify_slru_buffers; +extern PGDLLIMPORT int serial_slru_buffers; +extern PGDLLIMPORT int clog_slru_buffers; +extern PGDLLIMPORT int commit_ts_slru_buffers; extern PGDLLIMPORT int MyProcPid; extern PGDLLIMPORT pg_time_t MyStartTime; diff --git a/src/include/storage/predicate.h b/src/include/storage/predicate.h index 152b698611..c72779bd88 100644 --- a/src/include/storage/predicate.h +++ b/src/include/storage/predicate.h @@ -26,10 +26,6 @@ extern int max_predicate_locks_per_xact; extern int max_predicate_locks_per_relation; extern int max_predicate_locks_per_page; - -/* Number of SLRU buffers to use for Serial SLRU */ -#define NUM_SERIAL_BUFFERS 16 - /* * A handle used for sharing SERIALIZABLEXACT objects between the participants * in a parallel query. -- 2.24.3 (Apple Git-128)