From d9ab97b8ca00d03ab370de5b99546d01e0480bd5 Mon Sep 17 00:00:00 2001 From: Corey Huinker Date: Fri, 14 Mar 2025 03:54:26 -0400 Subject: [PATCH v9 4/5] Batching getAttributeStats(). The prepared statement getAttributeStats() is fairly heavyweight and could greatly increase pg_dump/pg_upgrade runtime. To alleviate this, create a result set buffer of all of the attribute stats fetched for a batch of 100 relations that could potentially have stats. The query ensures that the order of results exactly matches the needs of the code walking the TOC to print the stats calls. --- src/bin/pg_dump/pg_dump.c | 556 ++++++++++++++++++++++++++------------ 1 file changed, 385 insertions(+), 171 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 38ba6a90106..0c26dc7a1b4 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -143,6 +143,25 @@ typedef enum OidOptions zeroAsNone = 4, } OidOptions; +typedef enum StatsBufferState +{ + STATSBUF_UNINITIALIZED = 0, + STATSBUF_ACTIVE, + STATSBUF_EXHAUSTED +} StatsBufferState; + +typedef struct +{ + PGresult *res; /* results from most recent + * getAttributeStats() */ + int idx; /* first un-consumed row of results */ + TocEntry *te; /* next TOC entry to search for statsitics + * data */ + + StatsBufferState state; /* current state of the buffer */ +} AttributeStatsBuffer; + + /* global decls */ static bool dosync = true; /* Issue fsync() to make dump durable on disk. */ @@ -209,6 +228,18 @@ static int nbinaryUpgradeClassOids = 0; static SequenceItem *sequences = NULL; static int nsequences = 0; +static AttributeStatsBuffer attrstats = +{ + NULL, 0, NULL, STATSBUF_UNINITIALIZED +}; + +/* + * The maximum number of relations that should be fetched in any one + * getAttributeStats() call. + */ + +#define MAX_ATTR_STATS_RELS 100 + /* * The default number of rows per INSERT when * --inserts is specified without --rows-per-insert @@ -222,6 +253,10 @@ static int nsequences = 0; */ #define MAX_BLOBS_PER_ARCHIVE_ENTRY 1000 + + +/* TODO: fmtId(const char *rawid) */ + /* * Macro for producing quoted, schema-qualified name of a dumpable object. */ @@ -399,6 +434,9 @@ static void setupDumpWorker(Archive *AH); static TableInfo *getRootTableInfo(const TableInfo *tbinfo); static bool forcePartitionRootLoad(const TableInfo *tbinfo); static void read_dump_filters(const char *filename, DumpOptions *dopt); +static void appendNamedArgument(PQExpBuffer out, Archive *fout, + const char *argname, const char *argtype, + const char *argval); int @@ -10477,7 +10515,286 @@ statisticsDumpSection(const RelStatsInfo *rsinfo) } /* - * printDumpRelationStats -- + * Fetch next batch of rows from getAttributeStats() + */ +static void +fetchNextAttributeStats(Archive *fout) +{ + ArchiveHandle *AH = (ArchiveHandle *) fout; + PQExpBufferData schemas; + PQExpBufferData relations; + int numoids = 0; + + Assert(AH != NULL); + + /* free last result set, if any */ + if (attrstats.state == STATSBUF_ACTIVE) + PQclear(attrstats.res); + + /* If we have looped around to the start of the TOC, restart */ + if (attrstats.te == AH->toc) + attrstats.te = AH->toc->next; + + initPQExpBuffer(&schemas); + initPQExpBuffer(&relations); + + /* + * Walk ahead looking for relstats entries that are active in this + * section, adding the names to the schemas and relations lists. + */ + while ((attrstats.te != AH->toc) && (numoids < MAX_ATTR_STATS_RELS)) + { + if (attrstats.te->reqs != 0 && + strcmp(attrstats.te->desc, "STATISTICS DATA") == 0) + { + RelStatsInfo *rsinfo = (RelStatsInfo *) attrstats.te->createDumperArg; + + Assert(rsinfo != NULL); + + if (numoids > 0) + { + appendPQExpBufferStr(&schemas, ","); + appendPQExpBufferStr(&relations, ","); + } + appendPQExpBufferStr(&schemas, fmtId(rsinfo->dobj.namespace->dobj.name)); + appendPQExpBufferStr(&relations, fmtId(rsinfo->dobj.name)); + numoids++; + } + + attrstats.te = attrstats.te->next; + } + + if (numoids > 0) + { + PQExpBufferData query; + + initPQExpBuffer(&query); + appendPQExpBuffer(&query, + "EXECUTE getAttributeStats('{%s}'::pg_catalog.text[],'{%s}'::pg_catalog.text[])", + schemas.data, relations.data); + attrstats.res = ExecuteSqlQuery(fout, query.data, PGRES_TUPLES_OK); + attrstats.idx = 0; + } + else + { + attrstats.state = STATSBUF_EXHAUSTED; + attrstats.res = NULL; + attrstats.idx = -1; + } + + termPQExpBuffer(&schemas); + termPQExpBuffer(&relations); +} + +/* + * Prepare the getAttributeStats() statement + * + * This is done automatically if the user specified dumpStatistics. + */ +static void +initAttributeStats(Archive *fout) +{ + ArchiveHandle *AH = (ArchiveHandle *) fout; + PQExpBufferData query; + + Assert(AH != NULL); + initPQExpBuffer(&query); + + appendPQExpBufferStr(&query, + "PREPARE getAttributeStats(pg_catalog.text[], pg_catalog.text[]) AS\n" + "SELECT s.schemaname, s.tablename, s.attname, s.inherited, " + "s.null_frac, s.avg_width, s.n_distinct, s.most_common_vals, " + "s.most_common_freqs, s.histogram_bounds, s.correlation, " + "s.most_common_elems, s.most_common_elem_freqs, " + "s.elem_count_histogram, "); + + if (fout->remoteVersion >= 170000) + appendPQExpBufferStr(&query, + "s.range_length_histogram, " + "s.range_empty_frac, " + "s.range_bounds_histogram "); + else + appendPQExpBufferStr(&query, + "NULL AS range_length_histogram, " + "NULL AS range_empty_frac, " + " NULL AS range_bounds_histogram "); + + /* + * The results must be in the order of relations supplied in the + * parameters to ensure that they are in sync with a walk of the TOC. + * + * The redundant (and incomplete) filter clause on s.tablename = ANY(...) + * is a way to lead the query into using the index + * pg_class_relname_nsp_index which in turn allows the planner to avoid an + * expensive full scan of pg_stats. + * + * We may need to adjust this query for versions that are not so easily + * led. + */ + appendPQExpBufferStr(&query, + "FROM pg_catalog.pg_stats AS s " + "JOIN unnest($1, $2) WITH ORDINALITY AS u(schemaname, tablename, ord) " + "ON s.schemaname = u.schemaname " + "AND s.tablename = u.tablename " + "WHERE s.tablename = ANY($2) " + "ORDER BY u.ord, s.attname, s.inherited"); + + ExecuteSqlStatement(fout, query.data); + + termPQExpBuffer(&query); + + attrstats.te = AH->toc->next; + + fetchNextAttributeStats(fout); + + attrstats.state = STATSBUF_ACTIVE; +} + + +/* + * append a single attribute stat to the buffer for this relation. + */ +static void +appendAttributeStats(Archive *fout, PQExpBuffer out, + const RelStatsInfo *rsinfo) +{ + PGresult *res = attrstats.res; + int tup_num = attrstats.idx; + + const char *attname; + + static bool indexes_set = false; + static int i_attname, + i_inherited, + i_null_frac, + i_avg_width, + i_n_distinct, + i_most_common_vals, + i_most_common_freqs, + i_histogram_bounds, + i_correlation, + i_most_common_elems, + i_most_common_elem_freqs, + i_elem_count_histogram, + i_range_length_histogram, + i_range_empty_frac, + i_range_bounds_histogram; + + if (!indexes_set) + { + /* + * It's a prepared statement, so the indexes will be the same for all + * result sets, so we only need to set them once. + */ + i_attname = PQfnumber(res, "attname"); + i_inherited = PQfnumber(res, "inherited"); + i_null_frac = PQfnumber(res, "null_frac"); + i_avg_width = PQfnumber(res, "avg_width"); + i_n_distinct = PQfnumber(res, "n_distinct"); + i_most_common_vals = PQfnumber(res, "most_common_vals"); + i_most_common_freqs = PQfnumber(res, "most_common_freqs"); + i_histogram_bounds = PQfnumber(res, "histogram_bounds"); + i_correlation = PQfnumber(res, "correlation"); + i_most_common_elems = PQfnumber(res, "most_common_elems"); + i_most_common_elem_freqs = PQfnumber(res, "most_common_elem_freqs"); + i_elem_count_histogram = PQfnumber(res, "elem_count_histogram"); + i_range_length_histogram = PQfnumber(res, "range_length_histogram"); + i_range_empty_frac = PQfnumber(res, "range_empty_frac"); + i_range_bounds_histogram = PQfnumber(res, "range_bounds_histogram"); + indexes_set = true; + } + + appendPQExpBufferStr(out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n"); + appendPQExpBuffer(out, "\t'version', '%u'::integer,\n", + fout->remoteVersion); + appendPQExpBufferStr(out, "\t'schemaname', "); + appendStringLiteralAH(out, rsinfo->dobj.namespace->dobj.name, fout); + appendPQExpBufferStr(out, ",\n\t'relname', "); + appendStringLiteralAH(out, rsinfo->dobj.name, fout); + + if (PQgetisnull(res, tup_num, i_attname)) + pg_fatal("attname cannot be NULL"); + attname = PQgetvalue(res, tup_num, i_attname); + + /* + * Indexes look up attname in indAttNames to derive attnum, all others use + * attname directly. We must specify attnum for indexes, since their + * attnames are not necessarily stable across dump/reload. + */ + if (rsinfo->nindAttNames == 0) + { + appendPQExpBuffer(out, ",\n\t'attname', "); + appendStringLiteralAH(out, attname, fout); + } + else + { + bool found = false; + + for (int i = 0; i < rsinfo->nindAttNames; i++) + if (strcmp(attname, rsinfo->indAttNames[i]) == 0) + { + appendPQExpBuffer(out, ",\n\t'attnum', '%d'::smallint", + i + 1); + found = true; + break; + } + + if (!found) + pg_fatal("could not find index attname \"%s\"", attname); + } + + if (!PQgetisnull(res, tup_num, i_inherited)) + appendNamedArgument(out, fout, "inherited", "boolean", + PQgetvalue(res, tup_num, i_inherited)); + if (!PQgetisnull(res, tup_num, i_null_frac)) + appendNamedArgument(out, fout, "null_frac", "real", + PQgetvalue(res, tup_num, i_null_frac)); + if (!PQgetisnull(res, tup_num, i_avg_width)) + appendNamedArgument(out, fout, "avg_width", "integer", + PQgetvalue(res, tup_num, i_avg_width)); + if (!PQgetisnull(res, tup_num, i_n_distinct)) + appendNamedArgument(out, fout, "n_distinct", "real", + PQgetvalue(res, tup_num, i_n_distinct)); + if (!PQgetisnull(res, tup_num, i_most_common_vals)) + appendNamedArgument(out, fout, "most_common_vals", "text", + PQgetvalue(res, tup_num, i_most_common_vals)); + if (!PQgetisnull(res, tup_num, i_most_common_freqs)) + appendNamedArgument(out, fout, "most_common_freqs", "real[]", + PQgetvalue(res, tup_num, i_most_common_freqs)); + if (!PQgetisnull(res, tup_num, i_histogram_bounds)) + appendNamedArgument(out, fout, "histogram_bounds", "text", + PQgetvalue(res, tup_num, i_histogram_bounds)); + if (!PQgetisnull(res, tup_num, i_correlation)) + appendNamedArgument(out, fout, "correlation", "real", + PQgetvalue(res, tup_num, i_correlation)); + if (!PQgetisnull(res, tup_num, i_most_common_elems)) + appendNamedArgument(out, fout, "most_common_elems", "text", + PQgetvalue(res, tup_num, i_most_common_elems)); + if (!PQgetisnull(res, tup_num, i_most_common_elem_freqs)) + appendNamedArgument(out, fout, "most_common_elem_freqs", "real[]", + PQgetvalue(res, tup_num, i_most_common_elem_freqs)); + if (!PQgetisnull(res, tup_num, i_elem_count_histogram)) + appendNamedArgument(out, fout, "elem_count_histogram", "real[]", + PQgetvalue(res, tup_num, i_elem_count_histogram)); + if (fout->remoteVersion >= 170000) + { + if (!PQgetisnull(res, tup_num, i_range_length_histogram)) + appendNamedArgument(out, fout, "range_length_histogram", "text", + PQgetvalue(res, tup_num, i_range_length_histogram)); + if (!PQgetisnull(res, tup_num, i_range_empty_frac)) + appendNamedArgument(out, fout, "range_empty_frac", "real", + PQgetvalue(res, tup_num, i_range_empty_frac)); + if (!PQgetisnull(res, tup_num, i_range_bounds_histogram)) + appendNamedArgument(out, fout, "range_bounds_histogram", "text", + PQgetvalue(res, tup_num, i_range_bounds_histogram)); + } + appendPQExpBufferStr(out, "\n);\n"); +} + + + +/* + * printRelationStats -- * * Generate the SQL statements needed to restore a relation's statistics. */ @@ -10485,64 +10802,21 @@ static char * printRelationStats(Archive *fout, const void *userArg) { const RelStatsInfo *rsinfo = (RelStatsInfo *) userArg; - const DumpableObject *dobj = &rsinfo->dobj; + const DumpableObject *dobj; + const char *relschema; + const char *relname; + + ArchiveHandle *AH = (ArchiveHandle *) fout; - PQExpBufferData query; PQExpBufferData out; - PGresult *res; - - static bool first_query = true; - static int i_attname; - static int i_inherited; - static int i_null_frac; - static int i_avg_width; - static int i_n_distinct; - static int i_most_common_vals; - static int i_most_common_freqs; - static int i_histogram_bounds; - static int i_correlation; - static int i_most_common_elems; - static int i_most_common_elem_freqs; - static int i_elem_count_histogram; - static int i_range_length_histogram; - static int i_range_empty_frac; - static int i_range_bounds_histogram; - - initPQExpBuffer(&query); - - if (first_query) - { - appendPQExpBufferStr(&query, - "PREPARE getAttributeStats(pg_catalog.text, pg_catalog.text) AS\n" - "SELECT s.attname, s.inherited, " - "s.null_frac, s.avg_width, s.n_distinct, " - "s.most_common_vals, s.most_common_freqs, " - "s.histogram_bounds, s.correlation, " - "s.most_common_elems, s.most_common_elem_freqs, " - "s.elem_count_histogram, "); - - if (fout->remoteVersion >= 170000) - appendPQExpBufferStr(&query, - "s.range_length_histogram, " - "s.range_empty_frac, " - "s.range_bounds_histogram "); - else - appendPQExpBufferStr(&query, - "NULL AS range_length_histogram," - "NULL AS range_empty_frac," - "NULL AS range_bounds_histogram "); - - appendPQExpBufferStr(&query, - "FROM pg_catalog.pg_stats s " - "WHERE s.schemaname = $1 " - "AND s.tablename = $2 " - "ORDER BY s.attname, s.inherited"); - - ExecuteSqlStatement(fout, query.data); - - resetPQExpBuffer(&query); - } + Assert(rsinfo != NULL); + dobj = &rsinfo->dobj; + Assert(dobj != NULL); + relschema = dobj->namespace->dobj.name; + Assert(relschema != NULL); + relname = dobj->name; + Assert(relname != NULL); initPQExpBuffer(&out); @@ -10561,132 +10835,72 @@ printRelationStats(Archive *fout, const void *userArg) appendPQExpBuffer(&out, "\t'relallvisible', '%d'::integer\n);\n", rsinfo->relallvisible); - /* fetch attribute stats */ - appendPQExpBufferStr(&query, "EXECUTE getAttributeStats("); - appendStringLiteralAH(&query, dobj->namespace->dobj.name, fout); - appendPQExpBufferStr(&query, ", "); - appendStringLiteralAH(&query, dobj->name, fout); - appendPQExpBufferStr(&query, ")"); + AH->txnCount++; - res = ExecuteSqlQuery(fout, query.data, PGRES_TUPLES_OK); + if (attrstats.state == STATSBUF_UNINITIALIZED) + initAttributeStats(fout); - if (first_query) + /* + * Because the query returns rows in the same order as the relations + * requested, and because every relation gets at least one row in the + * result set, the first row for this relation must correspond either to + * the current row of this result set (if one exists) or the first row of + * the next result set (if this one is already consumed). + */ + if (attrstats.state != STATSBUF_ACTIVE) + pg_fatal("Exhausted getAttributeStats() before processing %s.%s", + rsinfo->dobj.namespace->dobj.name, + rsinfo->dobj.name); + + /* + * If the current result set has been fully consumed, then the row(s) we + * need (if any) would be found in the next one. This will update + * attrstats.res and attrstats.idx. + */ + if (PQntuples(attrstats.res) <= attrstats.idx) + fetchNextAttributeStats(fout); + + while (true) { - i_attname = PQfnumber(res, "attname"); - i_inherited = PQfnumber(res, "inherited"); - i_null_frac = PQfnumber(res, "null_frac"); - i_avg_width = PQfnumber(res, "avg_width"); - i_n_distinct = PQfnumber(res, "n_distinct"); - i_most_common_vals = PQfnumber(res, "most_common_vals"); - i_most_common_freqs = PQfnumber(res, "most_common_freqs"); - i_histogram_bounds = PQfnumber(res, "histogram_bounds"); - i_correlation = PQfnumber(res, "correlation"); - i_most_common_elems = PQfnumber(res, "most_common_elems"); - i_most_common_elem_freqs = PQfnumber(res, "most_common_elem_freqs"); - i_elem_count_histogram = PQfnumber(res, "elem_count_histogram"); - i_range_length_histogram = PQfnumber(res, "range_length_histogram"); - i_range_empty_frac = PQfnumber(res, "range_empty_frac"); - i_range_bounds_histogram = PQfnumber(res, "range_bounds_histogram"); - first_query = false; - } - - /* restore attribute stats */ - for (int rownum = 0; rownum < PQntuples(res); rownum++) - { - const char *attname; - - appendPQExpBufferStr(&out, "SELECT * FROM pg_catalog.pg_restore_attribute_stats(\n"); - appendPQExpBuffer(&out, "\t'version', '%u'::integer,\n", - fout->remoteVersion); - appendPQExpBufferStr(&out, "\t'schemaname', "); - appendStringLiteralAH(&out, rsinfo->dobj.namespace->dobj.name, fout); - appendPQExpBufferStr(&out, ",\n\t'relname', "); - appendStringLiteralAH(&out, rsinfo->dobj.name, fout); - - if (PQgetisnull(res, rownum, i_attname)) - pg_fatal("attname cannot be NULL"); - attname = PQgetvalue(res, rownum, i_attname); + int i_schemaname; + int i_tablename; + char *schemaname; + char *tablename; /* misnomer, following pg_stats naming */ /* - * Indexes look up attname in indAttNames to derive attnum, all others - * use attname directly. We must specify attnum for indexes, since - * their attnames are not necessarily stable across dump/reload. + * If we hit the end of the result set, then there are no more records + * for this relation, so we should stop, but first get the next result + * set for the next batch of relations. */ - if (rsinfo->nindAttNames == 0) + if (PQntuples(attrstats.res) <= attrstats.idx) { - appendPQExpBuffer(&out, ",\n\t'attname', "); - appendStringLiteralAH(&out, attname, fout); - } - else - { - bool found = false; - - for (int i = 0; i < rsinfo->nindAttNames; i++) - { - if (strcmp(attname, rsinfo->indAttNames[i]) == 0) - { - appendPQExpBuffer(&out, ",\n\t'attnum', '%d'::smallint", - i + 1); - found = true; - break; - } - } - - if (!found) - pg_fatal("could not find index attname \"%s\"", attname); + fetchNextAttributeStats(fout); + break; } - if (!PQgetisnull(res, rownum, i_inherited)) - appendNamedArgument(&out, fout, "inherited", "boolean", - PQgetvalue(res, rownum, i_inherited)); - if (!PQgetisnull(res, rownum, i_null_frac)) - appendNamedArgument(&out, fout, "null_frac", "real", - PQgetvalue(res, rownum, i_null_frac)); - if (!PQgetisnull(res, rownum, i_avg_width)) - appendNamedArgument(&out, fout, "avg_width", "integer", - PQgetvalue(res, rownum, i_avg_width)); - if (!PQgetisnull(res, rownum, i_n_distinct)) - appendNamedArgument(&out, fout, "n_distinct", "real", - PQgetvalue(res, rownum, i_n_distinct)); - if (!PQgetisnull(res, rownum, i_most_common_vals)) - appendNamedArgument(&out, fout, "most_common_vals", "text", - PQgetvalue(res, rownum, i_most_common_vals)); - if (!PQgetisnull(res, rownum, i_most_common_freqs)) - appendNamedArgument(&out, fout, "most_common_freqs", "real[]", - PQgetvalue(res, rownum, i_most_common_freqs)); - if (!PQgetisnull(res, rownum, i_histogram_bounds)) - appendNamedArgument(&out, fout, "histogram_bounds", "text", - PQgetvalue(res, rownum, i_histogram_bounds)); - if (!PQgetisnull(res, rownum, i_correlation)) - appendNamedArgument(&out, fout, "correlation", "real", - PQgetvalue(res, rownum, i_correlation)); - if (!PQgetisnull(res, rownum, i_most_common_elems)) - appendNamedArgument(&out, fout, "most_common_elems", "text", - PQgetvalue(res, rownum, i_most_common_elems)); - if (!PQgetisnull(res, rownum, i_most_common_elem_freqs)) - appendNamedArgument(&out, fout, "most_common_elem_freqs", "real[]", - PQgetvalue(res, rownum, i_most_common_elem_freqs)); - if (!PQgetisnull(res, rownum, i_elem_count_histogram)) - appendNamedArgument(&out, fout, "elem_count_histogram", "real[]", - PQgetvalue(res, rownum, i_elem_count_histogram)); - if (fout->remoteVersion >= 170000) - { - if (!PQgetisnull(res, rownum, i_range_length_histogram)) - appendNamedArgument(&out, fout, "range_length_histogram", "text", - PQgetvalue(res, rownum, i_range_length_histogram)); - if (!PQgetisnull(res, rownum, i_range_empty_frac)) - appendNamedArgument(&out, fout, "range_empty_frac", "real", - PQgetvalue(res, rownum, i_range_empty_frac)); - if (!PQgetisnull(res, rownum, i_range_bounds_histogram)) - appendNamedArgument(&out, fout, "range_bounds_histogram", "text", - PQgetvalue(res, rownum, i_range_bounds_histogram)); - } - appendPQExpBufferStr(&out, "\n);\n"); + i_schemaname = PQfnumber(attrstats.res, "schemaname"); + Assert(i_schemaname >= 0); + i_tablename = PQfnumber(attrstats.res, "tablename"); + Assert(i_tablename >= 0); + + if (PQgetisnull(attrstats.res, attrstats.idx, i_schemaname)) + pg_fatal("getAttributeStats() schemaname cannot be NULL"); + + if (PQgetisnull(attrstats.res, attrstats.idx, i_tablename)) + pg_fatal("getAttributeStats() tablename cannot be NULL"); + + schemaname = PQgetvalue(attrstats.res, attrstats.idx, i_schemaname); + tablename = PQgetvalue(attrstats.res, attrstats.idx, i_tablename); + + /* stop if current stat row isn't for this relation */ + if (strcmp(relname, tablename) != 0 || strcmp(relschema, schemaname) != 0) + break; + + appendAttributeStats(fout, &out, rsinfo); + AH->txnCount++; + attrstats.idx++; } - PQclear(res); - - termPQExpBuffer(&query); return out.data; } -- 2.48.1