From d944420eecba98e37026908d92d0d043af5a6407 Mon Sep 17 00:00:00 2001 From: "Krishnakumar R (KK)" Date: Tue, 29 Aug 2023 14:30:24 -0700 Subject: [PATCH v1] Move the pre-processing for tokens in bki file from initdb to bootstrap. With these changes bki file will only be opened once in bootstrap and parsing will be done by the bootstrap parser. The flow of bki file processing will be as follows: - In initdb gather the values used to replace the tokens in bki file. - Pass these values into postgres bootstrap startup using '-i' option as key-value pairs. - In bootstrap open the bki file (the bki file name was received as a parameter). - During the parsing of bki file, replace the tokens received as parameters with their values. Related discussion can be found here: https://www.postgresql.org/message-id/20220216021219.ygzrtb3hd5bn7olz%40alap3.anarazel.de --- src/backend/bootstrap/bootparse.y | 2 + src/backend/bootstrap/bootscanner.l | 62 +++++++++++++++++++ src/backend/bootstrap/bootstrap.c | 71 ++++++++++++++++++++- src/bin/initdb/initdb.c | 89 ++++++--------------------- src/include/bootstrap/bootstrap.h | 16 +++++ src/include/common/initdb_bootstrap.h | 21 +++++++ 6 files changed, 188 insertions(+), 73 deletions(-) create mode 100644 src/include/common/initdb_bootstrap.h diff --git a/src/backend/bootstrap/bootparse.y b/src/backend/bootstrap/bootparse.y index 81a1b7bfec..cc8e8633e6 100644 --- a/src/backend/bootstrap/bootparse.y +++ b/src/backend/bootstrap/bootparse.y @@ -95,6 +95,7 @@ static int num_columns_read = 0; %type oidspec optrowtypeoid %token ID +%token BOOT_PARAM %token COMMA EQUALS LPAREN RPAREN /* NULLVAL is a reserved keyword */ %token NULLVAL @@ -485,5 +486,6 @@ boot_ident: | XFORCE { $$ = pstrdup($1); } | XNOT { $$ = pstrdup($1); } | XNULL { $$ = pstrdup($1); } + | BOOT_PARAM { $$ = $1; } ; %% diff --git a/src/backend/bootstrap/bootscanner.l b/src/backend/bootstrap/bootscanner.l index 6a9d4193f2..619c7d40e3 100644 --- a/src/backend/bootstrap/bootscanner.l +++ b/src/backend/bootstrap/bootscanner.l @@ -108,6 +108,61 @@ FORCE { boot_yylval.kw = "FORCE"; return XFORCE; } NOT { boot_yylval.kw = "NOT"; return XNOT; } NULL { boot_yylval.kw = "NULL"; return XNULL; } +NAMEDATALEN { + boot_yylval.str = pstrdup(yytext); + sprintf(boot_yylval.str, "%d", NAMEDATALEN); + return BOOT_PARAM; + } +SIZEOF_POINTER { + boot_yylval.str = pstrdup(yytext); + sprintf(boot_yylval.str, "%d", (int) sizeof(Pointer)); + return BOOT_PARAM; + } +ALIGNOF_POINTER { + boot_yylval.str = pstrdup(yytext); + sprintf(boot_yylval.str, "%s", sizeof(Pointer) == 4? "i": "d"); + return BOOT_PARAM; + } +FLOAT8PASSBYVAL { + boot_yylval.str = pstrdup(yytext); + sprintf(boot_yylval.str, "%s", FLOAT8PASSBYVAL ? "true": "false"); + return BOOT_PARAM; + } +POSTGRES { + if (bootp_null(ebootp, ebootp->username)) return NULLVAL; + boot_yylval.str = pstrdup(ebootp->username); + return BOOT_PARAM; + } +ENCODING { + if (bootp_null(ebootp, ebootp->encoding_id)) return NULLVAL; + boot_yylval.str = pstrdup(ebootp->encoding_id); + return BOOT_PARAM; + } +LC_COLLATE { + if (bootp_null(ebootp, ebootp->lc_collate)) return NULLVAL; + boot_yylval.str = pstrdup(ebootp->lc_collate); + return BOOT_PARAM; + } +LC_CTYPE { + if (bootp_null(ebootp, ebootp->lc_ctype)) return NULLVAL; + boot_yylval.str = pstrdup(ebootp->lc_ctype); + return BOOT_PARAM; + } +ICU_LOCALE { + if (bootp_null(ebootp, ebootp->icu_locale)) return NULLVAL; + boot_yylval.str = pstrdup(ebootp->icu_locale); + return BOOT_PARAM; + } +ICU_RULES { + if (bootp_null(ebootp, ebootp->icu_rules)) return NULLVAL; + boot_yylval.str = pstrdup(ebootp->icu_rules); + return BOOT_PARAM; + } +LOCALE_PROVIDER { + if (bootp_null(ebootp, ebootp->locale_provider)) return NULLVAL; + boot_yylval.str = pstrdup(ebootp->locale_provider); + return BOOT_PARAM; + } {id} { boot_yylval.str = pstrdup(yytext); return ID; @@ -126,6 +181,13 @@ NULL { boot_yylval.kw = "NULL"; return XNULL; } /* LCOV_EXCL_STOP */ +bool bootp_null(extra_bootstrap_params *e, char *s) +{ + if (e == NULL || s == NULL || !strcmp(s, "_null_")) + return true; + return false; +} + void boot_yyerror(const char *message) { diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 5810f8825e..897e199d62 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -28,6 +28,7 @@ #include "catalog/index.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" +#include "common/initdb_bootstrap.h" #include "common/link-canary.h" #include "libpq/pqsignal.h" #include "miscadmin.h" @@ -65,6 +66,7 @@ Relation boot_reldesc; /* current relation descriptor */ Form_pg_attribute attrtypes[MAXATTR]; /* points to attribute info */ int numattr; /* number of attributes for cur. rel */ +extra_bootstrap_params *ebootp = NULL; /* * Basic information associated with each type. This is used before @@ -206,6 +208,7 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) char *progname = argv[0]; int flag; char *userDoption = NULL; + char *bki_file = NULL; Assert(!IsUnderPostmaster); @@ -221,7 +224,7 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) argv++; argc--; - while ((flag = getopt(argc, argv, "B:c:d:D:Fkr:X:-:")) != -1) + while ((flag = getopt(argc, argv, "B:c:d:D:Fi:kr:X:-:")) != -1) { switch (flag) { @@ -273,6 +276,47 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) case 'F': SetConfigOption("fsync", "false", PGC_POSTMASTER, PGC_S_ARGV); break; + case 'i': + + /* + * Here we extract key value pairs from bootstrap parameters + * string passed from initdb. These values are used to replace + * macros in the bki file. + */ + ebootp = palloc0(sizeof(extra_bootstrap_params)); + // Keep a copy so strtok side effects don't affect optarg. + ebootp->param_str = pstrdup(optarg); + for (char *token = strtok(ebootp->param_str, ","); token; token = strtok(NULL, ",")) + { + char *e = strchr(token, '='); + + if (e) + { + char *k; + char *v; + + *e = '\0'; + k = token; + v = e + 1; + if (strcmp(k, BKI_FILE) == 0) + bki_file = v; + else if (strcmp(k, BOOT_USERNAME) == 0) + ebootp->username = v; + else if (strcmp(k, BOOT_ENCODING_ID) == 0) + ebootp->encoding_id = v; + else if (strcmp(k, BOOT_LC_COLLATE) == 0) + ebootp->lc_collate = v; + else if (strcmp(k, BOOT_LC_CTYPE) == 0) + ebootp->lc_ctype = v; + else if (strcmp(k, BOOT_ICU_LOCALE) == 0) + ebootp->icu_locale = v; + else if (strcmp(k, BOOT_ICU_RULES) == 0) + ebootp->icu_rules = v; + else if (strcmp(k, BOOT_LOCALE_PROVIDER) == 0) + ebootp->locale_provider = v; + } + } + break; case 'k': bootstrap_data_checksum_version = PG_DATA_CHECKSUM_VERSION; break; @@ -355,7 +399,22 @@ BootstrapModeMain(int argc, char *argv[], bool check_only) } /* - * Process bootstrap input. + * Open bki file and redirect the input for boot_yyparse to point to the + * file. + * + */ + elog(INFO, "Open bki file %s\n", bki_file); + boot_yyin = fopen(bki_file, "r"); + if (!boot_yyin) + { + elog(ERROR, "Could not open bki_file=%s.", bki_file); + cleanup(); + proc_exit(1); + } + + /* + * Process bootstrap input. As boot_yyin is pointing to bki file, + * boot_yyparse will read from the bki file. */ StartTransactionCommand(); boot_yyparse(); @@ -686,6 +745,14 @@ InsertOneNull(int i) static void cleanup(void) { + if (ebootp) + { + if (ebootp->param_str) + { + pfree(ebootp->param_str); + } + pfree(ebootp); + } if (boot_reldesc != NULL) closerel(NULL); } diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 905b979947..fc69735789 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -71,6 +71,7 @@ #include "catalog/pg_database_d.h" /* pgrminclude ignore */ #include "common/file_perm.h" #include "common/file_utils.h" +#include "common/initdb_bootstrap.h" #include "common/logging.h" #include "common/pg_prng.h" #include "common/restricted_token.h" @@ -1472,90 +1473,36 @@ bootstrap_template1(void) { PG_CMD_DECL; PQExpBufferData cmd; - char **line; - char **bki_lines; - char headerline[MAXPGPATH]; - char buf[64]; printf(_("running bootstrap script ... ")); fflush(stdout); - bki_lines = readfile(bki_file); - - /* Check that bki file appears to be of the right version */ - - snprintf(headerline, sizeof(headerline), "# PostgreSQL %s\n", - PG_MAJORVERSION); - - if (strcmp(headerline, *bki_lines) != 0) - { - pg_log_error("input file \"%s\" does not belong to PostgreSQL %s", - bki_file, PG_VERSION); - pg_log_error_hint("Specify the correct path using the option -L."); - exit(1); - } - - /* Substitute for various symbols used in the BKI file */ - - sprintf(buf, "%d", NAMEDATALEN); - bki_lines = replace_token(bki_lines, "NAMEDATALEN", buf); - - sprintf(buf, "%d", (int) sizeof(Pointer)); - bki_lines = replace_token(bki_lines, "SIZEOF_POINTER", buf); - - bki_lines = replace_token(bki_lines, "ALIGNOF_POINTER", - (sizeof(Pointer) == 4) ? "i" : "d"); - - bki_lines = replace_token(bki_lines, "FLOAT8PASSBYVAL", - FLOAT8PASSBYVAL ? "true" : "false"); - - bki_lines = replace_token(bki_lines, "POSTGRES", - escape_quotes_bki(username)); - - bki_lines = replace_token(bki_lines, "ENCODING", - encodingid_to_string(encodingid)); - - bki_lines = replace_token(bki_lines, "LC_COLLATE", - escape_quotes_bki(lc_collate)); - - bki_lines = replace_token(bki_lines, "LC_CTYPE", - escape_quotes_bki(lc_ctype)); - - bki_lines = replace_token(bki_lines, "ICU_LOCALE", - icu_locale ? escape_quotes_bki(icu_locale) : "_null_"); - - bki_lines = replace_token(bki_lines, "ICU_RULES", - icu_rules ? escape_quotes_bki(icu_rules) : "_null_"); - - sprintf(buf, "%c", locale_provider); - bki_lines = replace_token(bki_lines, "LOCALE_PROVIDER", buf); - /* Also ensure backend isn't confused by this environment var: */ unsetenv("PGCLIENTENCODING"); initPQExpBuffer(&cmd); - printfPQExpBuffer(&cmd, "\"%s\" --boot %s %s", backend_exec, boot_options, extra_options); - appendPQExpBuffer(&cmd, " -X %d", wal_segment_size_mb * (1024 * 1024)); - if (data_checksums) - appendPQExpBuffer(&cmd, " -k"); - if (debug) - appendPQExpBuffer(&cmd, " -d 5"); - - + /* Pass the options consumed in initdb to the bootstrap startup. */ + printf("lc collate: %s\n", lc_collate); + printfPQExpBuffer(&cmd, "\"%s\" --boot -X %d %s %s %s %s -i %s=%s,%s=%s,%s=%s," + "%s=%s,%s=%s,%s=%s,%s=%s,%s=%c", + backend_exec, + wal_segment_size_mb * (1024 * 1024), + boot_options, extra_options, + data_checksums ? "-k" : "", + debug ? "-d 5" : "", + BKI_FILE, bki_file, + BOOT_USERNAME, escape_quotes_bki(username), + BOOT_ENCODING_ID, encodingid_to_string(encodingid), + BOOT_LC_COLLATE, escape_quotes_bki(lc_collate), + BOOT_LC_CTYPE, escape_quotes_bki(lc_ctype), + BOOT_ICU_LOCALE, icu_locale ? escape_quotes_bki(icu_locale) : "_null_", + BOOT_ICU_RULES, icu_rules ? escape_quotes_bki(icu_rules) : "_null_", + BOOT_LOCALE_PROVIDER, locale_provider); PG_CMD_OPEN(cmd.data); - - for (line = bki_lines; *line != NULL; line++) - { - PG_CMD_PUTS(*line); - free(*line); - } - PG_CMD_CLOSE(); termPQExpBuffer(&cmd); - free(bki_lines); - check_ok(); } diff --git a/src/include/bootstrap/bootstrap.h b/src/include/bootstrap/bootstrap.h index e1cb73c5f2..0ced3519f6 100644 --- a/src/include/bootstrap/bootstrap.h +++ b/src/include/bootstrap/bootstrap.h @@ -55,8 +55,24 @@ extern void boot_get_type_io_data(Oid typid, Oid *typoutput); extern int boot_yyparse(void); +extern FILE *boot_yyin; +extern FILE *boot_yyout; extern int boot_yylex(void); extern void boot_yyerror(const char *message) pg_attribute_noreturn(); +typedef struct +{ + char *param_str; + char *username; + char *encoding_id; + char *lc_collate; + char *lc_ctype; + char *icu_locale; + char *icu_rules; + char *locale_provider; +} extra_bootstrap_params; + +extern extra_bootstrap_params * ebootp; +extern bool bootp_null(extra_bootstrap_params * e, char *s); #endif /* BOOTSTRAP_H */ diff --git a/src/include/common/initdb_bootstrap.h b/src/include/common/initdb_bootstrap.h new file mode 100644 index 0000000000..264aa97f09 --- /dev/null +++ b/src/include/common/initdb_bootstrap.h @@ -0,0 +1,21 @@ +/* + * initdb_boostrap.h + * Shared macros between initdb and bootstrap mode. + * + * Copyright (c) 2003-2023, PostgreSQL Global Development Group + * + * src/include/common/initdb_boostrap.h + */ +#ifndef INITDB_BOOTSTRAP_H +#define INITDB_BOOTSTRAP_H + +#define BKI_FILE "bki_file" +#define BOOT_USERNAME "username" +#define BOOT_ENCODING_ID "encodingid" +#define BOOT_LC_COLLATE "lc_collate" +#define BOOT_LC_CTYPE "lc_ctype" +#define BOOT_ICU_LOCALE "icu_locale" +#define BOOT_ICU_RULES "icu_rules" +#define BOOT_LOCALE_PROVIDER "locale_provider" + +#endif /* INITDB_BOOTSTRAP_H */ -- 2.34.1