From 9c76c28fd369e176dbadde13134bfad071e04a30 Mon Sep 17 00:00:00 2001 From: Jelte Fennema-Nio Date: Sun, 19 Oct 2025 00:36:30 +0200 Subject: [PATCH v1 2/3] Add GoAway protocol message for graceful but fast server shutdown/switchover This commit introduces a new GoAway backend-to-frontend protocol message (byte 'g') that the server can send to the client to politely request that client to disconnect/reconnect when convenient. This message is advisory only - the connection remains fully functional and clients may continue executing queries and starting new transactions. "When convenient" is obviously not very well defined, but the primary target clients are clients that maintain a connection pool. Such clients should disconnect/reconnect a connection in the pool when there's no user of that connection. This is similar to how such clients often currently remove a connection from the pool after the connection hits a maximum lifetime of e.g. 1 hour. This new message is used by Postgres during the already existing "smart" shutdown procedure (i.e. when postmaster receives SIGTERM). When Postgres is in "smart" shutdown mode existing clients can continue to run queries as usual but new connection attempts are rejected. This mode is primarily useful when triggering a switchover of a read replica. A load balancer can route new connections only to the new read replica, while the old load balancer keeps serving the existing connections until they disconnect. The problem is that this draining of connections could often take a long time. Even when clients only run very short queries/transactions because the session can be kept open much longer (many connection pools use 1 hour max lifetime of a connection by default). With the introduction of the GoAway message Postgres now sends this message to all connected clients when it enters smart shutdown mode. If these clients respond to the message by reconnecting/disconnecting earlier than their maximum connection lifetime the draining can complete much quicker. Similar benefits to switchover duration can be achieved for other applications or proxies implementing the Postgres protocol, like when switching over a cluster of PgBouncer machines to a newer version. Applications/clients that use libpq can periodically check the result of PQgoAwayReceived() at an inactive time to see whether they are asked to reconnect. --- doc/src/sgml/libpq.sgml | 29 ++++++++++++++++++++ doc/src/sgml/protocol.sgml | 41 ++++++++++++++++++++++++++++ src/backend/postmaster/postmaster.c | 24 ++++++++++++++++ src/backend/storage/ipc/procsignal.c | 3 ++ src/backend/tcop/postgres.c | 37 +++++++++++++++++++++++++ src/include/libpq/protocol.h | 1 + src/include/storage/procsignal.h | 1 + src/include/tcop/tcopprot.h | 1 + src/interfaces/libpq/exports.txt | 1 + src/interfaces/libpq/fe-connect.c | 1 + src/interfaces/libpq/fe-exec.c | 27 ++++++++++++++++++ src/interfaces/libpq/fe-protocol3.c | 11 +++++++- src/interfaces/libpq/libpq-fe.h | 5 ++++ src/interfaces/libpq/libpq-int.h | 1 + 14 files changed, 182 insertions(+), 1 deletion(-) diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml index 5bf59a19855..7554d50a6c3 100644 --- a/doc/src/sgml/libpq.sgml +++ b/doc/src/sgml/libpq.sgml @@ -2992,6 +2992,35 @@ int PQserverVersion(const PGconn *conn); + + PQgoAwayReceivedPQgoAwayReceived + + + Returns true if the server has sent a GoAway message, + requesting the client to disconnect when convenient. + + +int PQgoAwayReceived(const PGconn *conn); + + + + + The GoAway message is sent by the server during a + smart shutdown to politely request that clients disconnect. This is + advisory only - the connection remains fully functional and queries + can continue to be executed. Applications should check this flag + periodically and disconnect gracefully when possible, such as after + completing the current transaction or unit of work. + + + + This message is only sent to clients using protocol version 3.3 or + later. The function returns 1 if the GoAway message + was received, 0 otherwise. + + + + PQerrorMessagePQerrorMessage diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 8234079deba..340514db007 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -5240,6 +5240,47 @@ psql "dbname=postgres replication=database" -c "IDENTIFY_SYSTEM;" + + GoAway (B) + + + + Byte1('g') + + + Identifies the message as a polite shutdown request. + + + + + + Int32(4) + + + Length of message contents in bytes, including self. + + + + + + + The server sends this message during a smart shutdown to politely + request that the client disconnect when convenient. This message is + advisory only - the connection remains fully functional and the client + may continue to execute queries. Applications should check for this + message using PQgoAwayReceived() and disconnect + gracefully when possible, such as after completing the current + transaction. + + + + This message is only sent to clients using protocol version 3.3 or later. + The server will wait for clients to disconnect during a smart shutdown, + but may eventually force termination if shutdown takes too long. + + + + GSSENCRequest (F) diff --git a/src/backend/postmaster/postmaster.c b/src/backend/postmaster/postmaster.c index e1d643b013d..79f116a5a5e 100644 --- a/src/backend/postmaster/postmaster.c +++ b/src/backend/postmaster/postmaster.c @@ -2126,7 +2126,31 @@ process_pm_shutdown_request(void) * later state, do not change it. */ if (pmState == PM_RUN || pmState == PM_HOT_STANDBY) + { + dlist_iter iter; + connsAllowed = false; + + /* + * Signal all backends to send a GoAway message to their + * clients, to politely request that they disconnect. + */ + dlist_foreach(iter, &ActiveChildList) + { + PMChild *bp = dlist_container(PMChild, elem, iter.cur); + + /* + * Only signal regular backends and walsenders. Skip + * auxiliary processes and dead-end backends. + */ + if (bp->bkend_type == B_BACKEND || + bp->bkend_type == B_WAL_SENDER) + { + SendProcSignal(bp->pid, PROCSIG_SMART_SHUTDOWN, + INVALID_PROC_NUMBER); + } + } + } else if (pmState == PM_STARTUP || pmState == PM_RECOVERY) { /* There should be no clients, so proceed to stop children */ diff --git a/src/backend/storage/ipc/procsignal.c b/src/backend/storage/ipc/procsignal.c index 087821311cc..6011c30d520 100644 --- a/src/backend/storage/ipc/procsignal.c +++ b/src/backend/storage/ipc/procsignal.c @@ -691,6 +691,9 @@ procsignal_sigusr1_handler(SIGNAL_ARGS) if (CheckProcSignal(PROCSIG_LOG_MEMORY_CONTEXT)) HandleLogMemoryContextInterrupt(); + if (CheckProcSignal(PROCSIG_SMART_SHUTDOWN)) + HandleSmartShutdownInterrupt(); + if (CheckProcSignal(PROCSIG_PARALLEL_APPLY_MESSAGE)) HandleParallelApplyMessageInterrupt(); diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 7dd75a490aa..ad00d865119 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -42,8 +42,10 @@ #include "common/pg_prng.h" #include "jit/jit.h" #include "libpq/libpq.h" +#include "libpq/pqcomm.h" #include "libpq/pqformat.h" #include "libpq/pqsignal.h" +#include "libpq/protocol.h" #include "mb/pg_wchar.h" #include "mb/stringinfo_mb.h" #include "miscadmin.h" @@ -91,6 +93,14 @@ const char *debug_query_string; /* client-supplied query string */ /* Note: whereToSendOutput is initialized for the bootstrap/standalone case */ CommandDest whereToSendOutput = DestDebug; +/* + * Track whether we've been notified of smart shutdown and sent GoAway. + * SmartShutdownPending is set by the PROCSIG_SMART_SHUTDOWN signal handler. + * GoAwaySent tracks whether we've already sent the GoAway message. + */ +static volatile sig_atomic_t SmartShutdownPending = false; +static bool GoAwaySent = false; + /* flag for logging end of session */ bool Log_disconnections = false; @@ -508,6 +518,20 @@ ProcessClientReadInterrupt(bool blocked) /* Check for general interrupts that arrived before/while reading */ CHECK_FOR_INTERRUPTS(); + /* Send GoAway message if smart shutdown is pending */ + if (SmartShutdownPending && !GoAwaySent && + whereToSendOutput == DestRemote && + MyProcPort && MyProcPort->proto >= PG_PROTOCOL(3, 3)) + { + StringInfoData buf; + + pq_beginmessage(&buf, PqMsg_GoAway); + pq_endmessage(&buf); + pq_flush(); + + GoAwaySent = true; + } + /* Process sinval catchup interrupts, if any */ if (catchupInterruptPending) ProcessCatchupInterrupt(); @@ -3087,6 +3111,18 @@ FloatExceptionHandler(SIGNAL_ARGS) "invalid operation, such as division by zero."))); } +/* + * Tell the next CHECK_FOR_INTERRUPTS() or main loop iteration to send a + * GoAway message to the client. Runs in a SIGUSR1 handler. + */ +void +HandleSmartShutdownInterrupt(void) +{ + SmartShutdownPending = true; + InterruptPending = true; + /* latch will be set by procsignal_sigusr1_handler */ +} + /* * Tell the next CHECK_FOR_INTERRUPTS() to check for a particular type of * recovery conflict. Runs in a SIGUSR1 handler. @@ -3313,6 +3349,7 @@ ProcessInterrupts(void) ProcDiePending = false; QueryCancelPending = false; /* ProcDie trumps QueryCancel */ LockErrorCleanup(); + /* As in quickdie, don't risk sending to client during auth */ if (ClientAuthInProgress && whereToSendOutput == DestRemote) whereToSendOutput = DestNone; diff --git a/src/include/libpq/protocol.h b/src/include/libpq/protocol.h index 7bf90053bcb..24fbc9f2613 100644 --- a/src/include/libpq/protocol.h +++ b/src/include/libpq/protocol.h @@ -53,6 +53,7 @@ #define PqMsg_FunctionCallResponse 'V' #define PqMsg_CopyBothResponse 'W' #define PqMsg_ReadyForQuery 'Z' +#define PqMsg_GoAway 'g' #define PqMsg_NoData 'n' #define PqMsg_PortalSuspended 's' #define PqMsg_ParameterDescription 't' diff --git a/src/include/storage/procsignal.h b/src/include/storage/procsignal.h index afeeb1ca019..b629341a4af 100644 --- a/src/include/storage/procsignal.h +++ b/src/include/storage/procsignal.h @@ -36,6 +36,7 @@ typedef enum PROCSIG_BARRIER, /* global barrier interrupt */ PROCSIG_LOG_MEMORY_CONTEXT, /* ask backend to log the memory contexts */ PROCSIG_PARALLEL_APPLY_MESSAGE, /* Message from parallel apply workers */ + PROCSIG_SMART_SHUTDOWN, /* notify backend of smart shutdown */ /* Recovery conflict reasons */ PROCSIG_RECOVERY_CONFLICT_FIRST, diff --git a/src/include/tcop/tcopprot.h b/src/include/tcop/tcopprot.h index c1bcfdec673..b7fd22c43bb 100644 --- a/src/include/tcop/tcopprot.h +++ b/src/include/tcop/tcopprot.h @@ -74,6 +74,7 @@ extern void die(SIGNAL_ARGS); pg_noreturn extern void quickdie(SIGNAL_ARGS); extern void StatementCancelHandler(SIGNAL_ARGS); pg_noreturn extern void FloatExceptionHandler(SIGNAL_ARGS); +extern void HandleSmartShutdownInterrupt(void); extern void HandleRecoveryConflictInterrupt(ProcSignalReason reason); extern void ProcessClientReadInterrupt(bool blocked); extern void ProcessClientWriteInterrupt(bool blocked); diff --git a/src/interfaces/libpq/exports.txt b/src/interfaces/libpq/exports.txt index dbbae642d76..3385e65c389 100644 --- a/src/interfaces/libpq/exports.txt +++ b/src/interfaces/libpq/exports.txt @@ -210,3 +210,4 @@ PQgetAuthDataHook 207 PQdefaultAuthDataHook 208 PQfullProtocolVersion 209 appendPQExpBufferVA 210 +PQgoAwayReceived 211 diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 76f9ce3a23e..54c79e5d214 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -698,6 +698,7 @@ pqDropServerData(PGconn *conn) conn->password_needed = false; conn->gssapi_used = false; conn->write_failed = false; + conn->goaway_received = false; free(conn->write_err_msg); conn->write_err_msg = NULL; conn->oauth_want_retry = false; diff --git a/src/interfaces/libpq/fe-exec.c b/src/interfaces/libpq/fe-exec.c index 0b1e37ec30b..16bf10280e5 100644 --- a/src/interfaces/libpq/fe-exec.c +++ b/src/interfaces/libpq/fe-exec.c @@ -2696,6 +2696,33 @@ PQnotifies(PGconn *conn) return event; } +/* + * PQgoAwayReceived + * returns 1 if a GoAway message has been received from the server + * returns 0 if not + * + * Note that this function does not read any new data from the socket; + * caller should call PQconsumeInput() first if they want to ensure + * all available data has been read. + */ +int +PQgoAwayReceived(PGconn *conn) +{ + if (!conn) + return 0; + + if (conn->goaway_received) + return 1; + + /* + * Parse any available data to see if a GoAway message has arrived. + */ + pqParseInput3(conn); + + return conn->goaway_received ? 1 : 0; +} + + /* * PQputCopyData - send some data to the backend during COPY IN or COPY BOTH * diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c index da7a8db68c8..9e54e3ff03d 100644 --- a/src/interfaces/libpq/fe-protocol3.c +++ b/src/interfaces/libpq/fe-protocol3.c @@ -157,6 +157,15 @@ pqParseInput3(PGconn *conn) if (pqGetErrorNotice3(conn, false)) return; } + else if (id == PqMsg_GoAway) + { + /* + * GoAway is an asynchronous message sent during smart shutdown. + * Process it immediately regardless of connection state. + */ + conn->goaway_received = true; + conn->inCursor += msgLength; + } else if (conn->asyncStatus != PGASYNC_BUSY) { /* If not IDLE state, just wait ... */ @@ -168,7 +177,7 @@ pqParseInput3(PGconn *conn) * ERROR messages are handled using the notice processor; * ParameterStatus is handled normally; anything else is just * dropped on the floor after displaying a suitable warning - * notice. (An ERROR is very possibly the backend telling us why + * notice. (An ERROR is very possibly the backend telling us why * it is about to close the connection, so we don't want to just * discard it...) */ diff --git a/src/interfaces/libpq/libpq-fe.h b/src/interfaces/libpq/libpq-fe.h index 0852584edae..1f3bd671a4b 100644 --- a/src/interfaces/libpq/libpq-fe.h +++ b/src/interfaces/libpq/libpq-fe.h @@ -63,6 +63,10 @@ extern "C" /* Indicates presence of the PQAUTHDATA_PROMPT_OAUTH_DEVICE authdata hook */ #define LIBPQ_HAS_PROMPT_OAUTH_DEVICE 1 +/* Features added in PostgreSQL v19: */ +/* Indicates presence of PQgoAwayReceived */ +#define LIBPQ_HAS_GOAWAY 1 + /* * Option flags for PQcopyResult */ @@ -411,6 +415,7 @@ extern const char *PQparameterStatus(const PGconn *conn, extern int PQprotocolVersion(const PGconn *conn); extern int PQfullProtocolVersion(const PGconn *conn); extern int PQserverVersion(const PGconn *conn); +extern int PQgoAwayReceived(PGconn *conn); extern char *PQerrorMessage(const PGconn *conn); extern int PQsocket(const PGconn *conn); extern int PQbackendPID(const PGconn *conn); diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h index 02c114f1405..7dc6e858a16 100644 --- a/src/interfaces/libpq/libpq-int.h +++ b/src/interfaces/libpq/libpq-int.h @@ -511,6 +511,7 @@ struct pg_conn bool sigpipe_flag; /* can we mask SIGPIPE via MSG_NOSIGNAL? */ bool write_failed; /* have we had a write failure on sock? */ char *write_err_msg; /* write error message, or NULL if OOM */ + bool goaway_received; /* true if server sent GoAway message */ bool auth_required; /* require an authentication challenge from * the server? */ -- 2.51.1