From 77d257ab002a5e1b6a2f65e359cbfd7978e3cff5 Mon Sep 17 00:00:00 2001 From: Pengzhou Tang Date: Wed, 20 Nov 2019 06:42:37 -0500 Subject: [PATCH 1/3] ANALYZE tableam API change Extended three ANALYZE-related tableam APIs so AMs can take more control of ANALYZE progress: - scan_analyze_beginscan() : so AMs can has more flexible sampling strategy - scan_analyze_sample_tuple() : so ANALYZE can get extra info as needed - scan_analyze_endscan() : Also use struct AnalyzeSampleContext to provide more convenience, with it tableam analyze routines can provide extra info except the real data, for example: physical size or compression ratio. --- contrib/file_fdw/file_fdw.c | 35 +++--- contrib/postgres_fdw/postgres_fdw.c | 56 +++++---- src/backend/access/heap/heapam_handler.c | 109 ++++++++++++++-- src/backend/access/table/tableam.c | 209 +++++++++++++++++++++++++++++++ src/backend/commands/analyze.c | 181 ++++++++------------------ src/include/access/tableam.h | 138 +++++++++++++++++--- src/include/foreign/fdwapi.h | 7 +- 7 files changed, 530 insertions(+), 205 deletions(-) diff --git a/contrib/file_fdw/file_fdw.c b/contrib/file_fdw/file_fdw.c index 549821c..2344f01 100644 --- a/contrib/file_fdw/file_fdw.c +++ b/contrib/file_fdw/file_fdw.c @@ -19,6 +19,7 @@ #include "access/reloptions.h" #include "access/sysattr.h" #include "access/table.h" +#include "access/tableam.h" #include "catalog/pg_authid.h" #include "catalog/pg_foreign_table.h" #include "commands/copy.h" @@ -157,10 +158,8 @@ static void estimate_size(PlannerInfo *root, RelOptInfo *baserel, static void estimate_costs(PlannerInfo *root, RelOptInfo *baserel, FileFdwPlanState *fdw_private, Cost *startup_cost, Cost *total_cost); -static int file_acquire_sample_rows(Relation onerel, int elevel, - HeapTuple *rows, int targrows, - double *totalrows, double *totaldeadrows); - +static void file_acquire_sample_rows(Relation onerel, int elevel, + AnalyzeSampleContext *context); /* * Foreign-data wrapper handler function: return a struct with pointers @@ -1091,14 +1090,16 @@ estimate_costs(PlannerInfo *root, RelOptInfo *baserel, * may be meaningless, but it's OK because we don't use the estimates * currently (the planner only pays attention to correlation for indexscans). */ -static int +static void file_acquire_sample_rows(Relation onerel, int elevel, - HeapTuple *rows, int targrows, - double *totalrows, double *totaldeadrows) + AnalyzeSampleContext *context) { int numrows = 0; + int targrows = 0; + double totalrows = 0; double rowstoskip = -1; /* -1 means not set yet */ ReservoirStateData rstate; + HeapTuple tuple; TupleDesc tupDesc; Datum *values; bool *nulls; @@ -1111,6 +1112,8 @@ file_acquire_sample_rows(Relation onerel, int elevel, MemoryContext oldcontext = CurrentMemoryContext; MemoryContext tupcontext; + targrows = context->targrows; + Assert(onerel); Assert(targrows > 0); @@ -1144,8 +1147,6 @@ file_acquire_sample_rows(Relation onerel, int elevel, errcallback.previous = error_context_stack; error_context_stack = &errcallback; - *totalrows = 0; - *totaldeadrows = 0; for (;;) { /* Check for user-requested abort or sleep */ @@ -1170,7 +1171,8 @@ file_acquire_sample_rows(Relation onerel, int elevel, */ if (numrows < targrows) { - rows[numrows++] = heap_form_tuple(tupDesc, values, nulls); + tuple = heap_form_tuple(tupDesc, values, nulls); + AnalyzeRecordSampleRow(context, NULL, tuple, ANALYZE_SAMPLE_DATA, numrows++, false /* replace */, false); } else { @@ -1180,7 +1182,7 @@ file_acquire_sample_rows(Relation onerel, int elevel, * not-yet-incremented value of totalrows as t. */ if (rowstoskip < 0) - rowstoskip = reservoir_get_next_S(&rstate, *totalrows, targrows); + rowstoskip = reservoir_get_next_S(&rstate, totalrows, targrows); if (rowstoskip <= 0) { @@ -1191,14 +1193,14 @@ file_acquire_sample_rows(Relation onerel, int elevel, int k = (int) (targrows * sampler_random_fract(rstate.randstate)); Assert(k >= 0 && k < targrows); - heap_freetuple(rows[k]); - rows[k] = heap_form_tuple(tupDesc, values, nulls); + tuple = heap_form_tuple(tupDesc, values, nulls); + AnalyzeRecordSampleRow(context, NULL, tuple, ANALYZE_SAMPLE_DATA, k, true /* replace */, false); } rowstoskip -= 1; } - *totalrows += 1; + totalrows += 1; } /* Remove error callback. */ @@ -1219,7 +1221,8 @@ file_acquire_sample_rows(Relation onerel, int elevel, (errmsg("\"%s\": file contains %.0f rows; " "%d rows in sample", RelationGetRelationName(onerel), - *totalrows, numrows))); + totalrows, numrows))); - return numrows; + context->totalrows += totalrows; + context->totalsampledrows += numrows; } diff --git a/contrib/postgres_fdw/postgres_fdw.c b/contrib/postgres_fdw/postgres_fdw.c index bdc21b3..f0789cc 100644 --- a/contrib/postgres_fdw/postgres_fdw.c +++ b/contrib/postgres_fdw/postgres_fdw.c @@ -17,6 +17,7 @@ #include "access/htup_details.h" #include "access/sysattr.h" #include "access/table.h" +#include "access/tableam.h" #include "catalog/pg_class.h" #include "commands/defrem.h" #include "commands/explain.h" @@ -237,7 +238,6 @@ typedef struct PgFdwAnalyzeState List *retrieved_attrs; /* attr numbers retrieved by query */ /* collected sample rows */ - HeapTuple *rows; /* array of size targrows */ int targrows; /* target # of sample rows */ int numrows; /* # of sample rows collected */ @@ -463,12 +463,11 @@ static void process_query_params(ExprContext *econtext, FmgrInfo *param_flinfo, List *param_exprs, const char **param_values); -static int postgresAcquireSampleRowsFunc(Relation relation, int elevel, - HeapTuple *rows, int targrows, - double *totalrows, - double *totaldeadrows); +static void postgresAcquireSampleRowsFunc(Relation relation, int elevel, + AnalyzeSampleContext *context); static void analyze_row_processor(PGresult *res, int row, - PgFdwAnalyzeState *astate); + PgFdwAnalyzeState *astate, + AnalyzeSampleContext *context); static HeapTuple make_tuple_from_result_row(PGresult *res, int row, Relation rel, @@ -4488,11 +4487,9 @@ postgresAnalyzeForeignTable(Relation relation, * may be meaningless, but it's OK because we don't use the estimates * currently (the planner only pays attention to correlation for indexscans). */ -static int +static void postgresAcquireSampleRowsFunc(Relation relation, int elevel, - HeapTuple *rows, int targrows, - double *totalrows, - double *totaldeadrows) + AnalyzeSampleContext *context) { PgFdwAnalyzeState astate; ForeignTable *table; @@ -4506,13 +4503,11 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel, /* Initialize workspace state */ astate.rel = relation; astate.attinmeta = TupleDescGetAttInMetadata(RelationGetDescr(relation)); - - astate.rows = rows; - astate.targrows = targrows; + astate.targrows = context->targrows; astate.numrows = 0; astate.samplerows = 0; astate.rowstoskip = -1; /* -1 means not set yet */ - reservoir_init_selection_state(&astate.rstate, targrows); + reservoir_init_selection_state(&astate.rstate, astate.targrows); /* Remember ANALYZE context, and create a per-tuple temp context */ astate.anl_cxt = CurrentMemoryContext; @@ -4604,7 +4599,7 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel, /* Process whatever we got. */ numrows = PQntuples(res); for (i = 0; i < numrows; i++) - analyze_row_processor(res, i, &astate); + analyze_row_processor(res, i, &astate, context); PQclear(res); res = NULL; @@ -4628,10 +4623,13 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel, ReleaseConnection(conn); /* We assume that we have no dead tuple. */ - *totaldeadrows = 0.0; + context->totaldeadrows = 0.0; /* We've retrieved all living tuples from foreign server. */ - *totalrows = astate.samplerows; + context->totalrows += astate.samplerows; + + /* Increase the number of sample rows stored in the context */ + context->totalsampledrows += astate.numrows; /* * Emit some interesting relation info @@ -4640,8 +4638,6 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel, (errmsg("\"%s\": table contains %.0f rows, %d rows in sample", RelationGetRelationName(relation), astate.samplerows, astate.numrows))); - - return astate.numrows; } /* @@ -4650,10 +4646,11 @@ postgresAcquireSampleRowsFunc(Relation relation, int elevel, * - Subsequently, replace already-sampled tuples randomly. */ static void -analyze_row_processor(PGresult *res, int row, PgFdwAnalyzeState *astate) +analyze_row_processor(PGresult *res, int row, PgFdwAnalyzeState *astate, AnalyzeSampleContext *context) { int targrows = astate->targrows; int pos; /* array index to store tuple in */ + bool replace; MemoryContext oldcontext; /* Always increment sample row counter. */ @@ -4667,6 +4664,7 @@ analyze_row_processor(PGresult *res, int row, PgFdwAnalyzeState *astate) { /* First targrows rows are always included into the sample */ pos = astate->numrows++; + replace = false; } else { @@ -4683,7 +4681,7 @@ analyze_row_processor(PGresult *res, int row, PgFdwAnalyzeState *astate) /* Choose a random reservoir element to replace. */ pos = (int) (targrows * sampler_random_fract(astate->rstate.randstate)); Assert(pos >= 0 && pos < targrows); - heap_freetuple(astate->rows[pos]); + replace = true; } else { @@ -4696,18 +4694,22 @@ analyze_row_processor(PGresult *res, int row, PgFdwAnalyzeState *astate) if (pos >= 0) { + HeapTuple tuple; /* * Create sample tuple from current result row, and store it in the * position determined above. The tuple has to be created in anl_cxt. */ oldcontext = MemoryContextSwitchTo(astate->anl_cxt); - astate->rows[pos] = make_tuple_from_result_row(res, row, - astate->rel, - astate->attinmeta, - astate->retrieved_attrs, - NULL, - astate->temp_cxt); + tuple = make_tuple_from_result_row(res, row, + astate->rel, + astate->attinmeta, + astate->retrieved_attrs, + NULL, + astate->temp_cxt); + + /* Tuple is already created in anl_cxt, we can record it directly */ + AnalyzeRecordSampleRow(context, NULL, tuple, ANALYZE_SAMPLE_DATA, pos, replace, false); MemoryContextSwitchTo(oldcontext); } diff --git a/src/backend/access/heap/heapam_handler.c b/src/backend/access/heap/heapam_handler.c index 253849e..c57c670 100644 --- a/src/backend/access/heap/heapam_handler.c +++ b/src/backend/access/heap/heapam_handler.c @@ -35,6 +35,7 @@ #include "executor/executor.h" #include "miscadmin.h" #include "pgstat.h" +#include "parser/analyze.h" #include "storage/bufmgr.h" #include "storage/bufpage.h" #include "storage/lmgr.h" @@ -44,6 +45,7 @@ #include "utils/builtins.h" #include "utils/rel.h" +static int compare_rows(const void *a, const void *b); static void reform_and_rewrite_tuple(HeapTuple tuple, Relation OldHeap, Relation NewHeap, Datum *values, bool *isnull, RewriteState rwstate); @@ -974,10 +976,25 @@ heapam_relation_copy_for_cluster(Relation OldHeap, Relation NewHeap, pfree(isnull); } +static void +heapam_scan_analyze_beginscan(Relation onerel, AnalyzeSampleContext *context) +{ + context->scan = table_beginscan_analyze(onerel); + + /* initialize the totalblocks analyze can scan */ + context->totalblocks = RelationGetNumberOfBlocks(onerel); + + /* reset the statistic */ + context->liverows = 0; + context->deadrows = 0; + context->ordered = true; +} + static bool -heapam_scan_analyze_next_block(TableScanDesc scan, BlockNumber blockno, - BufferAccessStrategy bstrategy) +heapam_scan_analyze_next_block(BlockNumber blockno, + AnalyzeSampleContext *context) { + TableScanDesc scan = context->scan; HeapScanDesc hscan = (HeapScanDesc) scan; /* @@ -992,7 +1009,7 @@ heapam_scan_analyze_next_block(TableScanDesc scan, BlockNumber blockno, hscan->rs_cblock = blockno; hscan->rs_cindex = FirstOffsetNumber; hscan->rs_cbuf = ReadBufferExtended(scan->rs_rd, MAIN_FORKNUM, - blockno, RBM_NORMAL, bstrategy); + blockno, RBM_NORMAL, context->bstrategy); LockBuffer(hscan->rs_cbuf, BUFFER_LOCK_SHARE); /* in heap all blocks can contain tuples, so always return true */ @@ -1000,14 +1017,14 @@ heapam_scan_analyze_next_block(TableScanDesc scan, BlockNumber blockno, } static bool -heapam_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, - double *liverows, double *deadrows, - TupleTableSlot *slot) +heapam_scan_analyze_next_tuple(TransactionId OldestXmin, AnalyzeSampleContext *context) { + TableScanDesc scan = context->scan; HeapScanDesc hscan = (HeapScanDesc) scan; Page targpage; OffsetNumber maxoffset; BufferHeapTupleTableSlot *hslot; + TupleTableSlot *slot = AnalyzeGetSampleSlot(context, scan->rs_rd, ANALYZE_SAMPLE_DATA); Assert(TTS_IS_BUFFERTUPLE(slot)); @@ -1033,7 +1050,7 @@ heapam_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, if (!ItemIdIsNormal(itemid)) { if (ItemIdIsDead(itemid)) - *deadrows += 1; + context->deadrows += 1; continue; } @@ -1048,13 +1065,13 @@ heapam_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, { case HEAPTUPLE_LIVE: sample_it = true; - *liverows += 1; + context->liverows += 1; break; case HEAPTUPLE_DEAD: case HEAPTUPLE_RECENTLY_DEAD: /* Count dead and recently-dead rows */ - *deadrows += 1; + context->deadrows += 1; break; case HEAPTUPLE_INSERT_IN_PROGRESS: @@ -1080,7 +1097,7 @@ heapam_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, if (TransactionIdIsCurrentTransactionId(HeapTupleHeaderGetXmin(targtuple->t_data))) { sample_it = true; - *liverows += 1; + context->liverows += 1; } break; @@ -1109,11 +1126,11 @@ heapam_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, * concurrent transaction never commits. */ if (TransactionIdIsCurrentTransactionId(HeapTupleHeaderGetUpdateXid(targtuple->t_data))) - *deadrows += 1; + context->deadrows += 1; else { sample_it = true; - *liverows += 1; + context->liverows += 1; } break; @@ -1142,6 +1159,71 @@ heapam_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, return false; } +static void +heapam_scan_analyze_sample_tuple(int pos, bool replace, AnalyzeSampleContext *context) +{ + TupleTableSlot *slot; + Relation onerel = context->scan->rs_rd; + + Assert(pos >= 0); + /* + * heapam_scan_analyze_next_tuple should already put the tuple + * in the sample slot, just record it into the array of sample + * rows. + */ + slot = AnalyzeGetSampleSlot(context, onerel, ANALYZE_SAMPLE_DATA); + AnalyzeRecordSampleRow(context, slot, NULL, ANALYZE_SAMPLE_DATA, pos, replace, true); + + /* + * if replace happens, the sample rows are no longer ordered + * in physical position. + */ + if (replace) + context->ordered = false; +} + +static void +heapam_scan_analyze_endscan(AnalyzeSampleContext *context) +{ + HeapTuple *rows = AnalyzeGetSampleRows(context, ANALYZE_SAMPLE_DATA, context->totalsampledrows); + + /* + * If we didn't find as many tuples as we wanted then we're done. No sort + * is needed, since they're already in order. + * + * Otherwise we need to sort the collected tuples by position + * (itempointer). + */ + if (!context->ordered) + qsort((void *)rows, context->targrows, sizeof(HeapTuple), compare_rows); + + table_endscan(context->scan); +} + +/* + * qsort comparator for sorting rows[] array + */ +static int +compare_rows(const void *a, const void *b) +{ + HeapTuple ha = *(const HeapTuple *) a; + HeapTuple hb = *(const HeapTuple *) b; + BlockNumber ba = ItemPointerGetBlockNumber(&ha->t_self); + OffsetNumber oa = ItemPointerGetOffsetNumber(&ha->t_self); + BlockNumber bb = ItemPointerGetBlockNumber(&hb->t_self); + OffsetNumber ob = ItemPointerGetOffsetNumber(&hb->t_self); + + if (ba < bb) + return -1; + if (ba > bb) + return 1; + if (oa < ob) + return -1; + if (oa > ob) + return 1; + return 0; +} + static double heapam_index_build_range_scan(Relation heapRelation, Relation indexRelation, @@ -2529,8 +2611,11 @@ static const TableAmRoutine heapam_methods = { .relation_copy_data = heapam_relation_copy_data, .relation_copy_for_cluster = heapam_relation_copy_for_cluster, .relation_vacuum = heap_vacuum_rel, + .scan_analyze_beginscan = heapam_scan_analyze_beginscan, .scan_analyze_next_block = heapam_scan_analyze_next_block, .scan_analyze_next_tuple = heapam_scan_analyze_next_tuple, + .scan_analyze_sample_tuple = heapam_scan_analyze_sample_tuple, + .scan_analyze_endscan = heapam_scan_analyze_endscan, .index_build_range_scan = heapam_index_build_range_scan, .index_validate_scan = heapam_index_validate_scan, diff --git a/src/backend/access/table/tableam.c b/src/backend/access/table/tableam.c index b9ed336..d40eff0 100644 --- a/src/backend/access/table/tableam.c +++ b/src/backend/access/table/tableam.c @@ -23,7 +23,9 @@ #include "access/heapam.h" /* for ss_* */ #include "access/tableam.h" +#include "access/tupconvert.h" #include "access/xact.h" +#include "catalog/pg_type.h" #include "optimizer/plancat.h" #include "storage/bufmgr.h" #include "storage/shmem.h" @@ -650,3 +652,210 @@ table_block_relation_estimate_size(Relation rel, int32 *attr_widths, else *allvisfrac = (double) relallvisible / curpages; } + +/* Create the analyze sample context to acquire sample rows */ +AnalyzeSampleContext * +CreateAnalyzeSampleContext(Relation onerel, + List *anl_cols, + int totaltargrows, + BufferAccessStrategy strategy) +{ + AnalyzeSampleContext *context; + + context = (AnalyzeSampleContext *) palloc(sizeof(AnalyzeSampleContext)); + context->parent = onerel; + context->anl_cols = anl_cols; + context->bstrategy = strategy; + context->totaltargrows = totaltargrows; + context->targrows = totaltargrows; + context->scan = NULL; + context->totalblocks = 0; + context->totalrows = 0; + context->totaldeadrows = 0; + context->totalsampledrows = 0; + context->liverows = 0; + context->deadrows = 0; + context->ordered = false; + context->tup_convert_map = NULL; + + /* empty all sample type */ + memset(context->sample_slots, 0, MAX_ANALYZE_SAMPLE * sizeof(TupleTableSlot *)); + memset(context->sample_rows, 0, MAX_ANALYZE_SAMPLE * sizeof(HeapTuple *)); + + return context; +} + +/* Destroy analyze sample context */ +void +DestroyAnalyzeSampleContext(AnalyzeSampleContext *context) +{ + for (int i = 0; i < MAX_ANALYZE_SAMPLE; i++) + { + TupleTableSlot *slot = context->sample_slots[i]; + if (slot) + ExecDropSingleTupleTableSlot(slot); + } +} + +/* + * To acquire sample rows from an inherited table, all child + * relations use the same analyze sample context, this function + * must be called before starting analyze a new child relation. + */ +void +InitAnalyzeSampleContextForChild(AnalyzeSampleContext *context, + Relation child, + int childtargrows) +{ + /* Set targrows to childtargrows */ + context->targrows = childtargrows; + + /* We may need to convert from child's rowtype to parent's */ + if (!equalTupleDescs(RelationGetDescr(child), + RelationGetDescr(context->parent))) + { + if (context->tup_convert_map) + free_conversion_map(context->tup_convert_map); + /* Create a convert map so it can be used when recording sample rows */ + context->tup_convert_map = + convert_tuples_by_name(RelationGetDescr(child), + RelationGetDescr(context->parent)); + + /* We also cannot use previous sample slot anymore */ + if (context->sample_slots[ANALYZE_SAMPLE_DATA]) + { + ExecDropSingleTupleTableSlot(context->sample_slots[ANALYZE_SAMPLE_DATA]); + context->sample_slots[ANALYZE_SAMPLE_DATA] = NULL; + } + } +} + +void +AnalyzeGetSampleStats(AnalyzeSampleContext *context, + int *totalsampledrows, + double *totalrows, + double *totaldeadrows) +{ + if (totalsampledrows) + *totalsampledrows = context->totalsampledrows; + if (totalrows) + *totalrows = context->totalrows; + if (*totaldeadrows) + *totaldeadrows = context->totaldeadrows; +} + + +/* + * Get or initialize a sample slot to hold sample tuple, normally + * the tuple in the slot will be copied to the sample_rows[type] + * by AnalyzeRecordSampleRow(). + */ +TupleTableSlot * +AnalyzeGetSampleSlot(AnalyzeSampleContext *context, + Relation onerel, + AnalyzeSampleType type) +{ + TupleDesc tupdesc; + int attr_cnt = onerel->rd_att->natts; + + if (context->sample_slots[type]) + return context->sample_slots[type]; + + switch (type) + { + case ANALYZE_SAMPLE_DATA: + tupdesc = RelationGetDescr(onerel); + break; + case ANALYZE_SAMPLE_DISKSIZE: + tupdesc = CreateTemplateTupleDesc(attr_cnt); + for (int i = 1; i <= attr_cnt; i++) + TupleDescInitEntry(tupdesc, i, "", FLOAT8OID, -1, 0); + break; + default: + elog(ERROR, "unknown analyze sample type"); + } + + context->sample_slots[type] = + MakeSingleTupleTableSlot(tupdesc, table_slot_callbacks(onerel)); + return context->sample_slots[type]; +} + +HeapTuple * +AnalyzeGetSampleRows(AnalyzeSampleContext *context, + AnalyzeSampleType type, + int offset) +{ + Assert(offset < context->totaltargrows); + if (!context->sample_rows[type]) + context->sample_rows[type] = + (HeapTuple *) palloc(context->totaltargrows * sizeof(HeapTuple)); + + return context->sample_rows[type] + offset; +} + +/* + * Record a sample tuple into sample_rows[type]. + * + * sample_tuple: + * Input sample tuple. Sometimes, callers has already + * formed sample tuple in its memory context, we can + * record it directly. + * sample_slot: + * Slot which contains the sample tuple. We need to copy + * the sample tuple and then record it. + * pos: + * The postion in the sample_rows[type]. + * replace: + * Replace the old sample tuple in the specified position. + * withtid: + * Set the tid of sample tuple, this is only valid when + * sample_slot is set. + * + * We prefer to use sample_slot if both sample_tuple and + * sample_slot are set, sample_slot is the most common case. + */ +void +AnalyzeRecordSampleRow(AnalyzeSampleContext *context, + TupleTableSlot *sample_slot, + HeapTuple sample_tuple, + AnalyzeSampleType type, + int pos, + bool replace, + bool withtid) +{ + HeapTuple tuple; + HeapTuple *rows; + + rows = AnalyzeGetSampleRows(context, type, context->totalsampledrows); + + /* We need to free the old tuple if replace is true */ + if (replace) + heap_freetuple(rows[pos]); + + Assert(sample_slot || sample_tuple); + if (sample_slot) + tuple = ExecCopySlotHeapTuple(sample_slot); + else + tuple = sample_tuple; + + /* We may need to convert from child's rowtype to parent's */ + if (context->tup_convert_map != NULL) + { + HeapTuple newtup; + newtup = execute_attr_map_tuple(tuple, context->tup_convert_map); + heap_freetuple(tuple); + tuple = newtup; + } + + if (withtid && sample_slot) + tuple->t_self = sample_slot->tts_tid; + + /* store the tuple to right position */ + rows[pos] = tuple; +} + +bool +AnalyzeSampleIsValid(AnalyzeSampleContext *context, AnalyzeSampleType type) +{ + return context->sample_rows[type] != NULL; +} diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index e2033f9..cc1649d 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -83,7 +83,6 @@ int default_statistics_target = 100; static MemoryContext anl_context = NULL; static BufferAccessStrategy vac_strategy; - static void do_analyze_rel(Relation onerel, VacuumParams *params, List *va_cols, AcquireSampleRowsFunc acquirefunc, BlockNumber relpages, @@ -94,13 +93,10 @@ static void compute_index_stats(Relation onerel, double totalrows, MemoryContext col_context); static VacAttrStats *examine_attribute(Relation onerel, int attnum, Node *index_expr); -static int acquire_sample_rows(Relation onerel, int elevel, - HeapTuple *rows, int targrows, - double *totalrows, double *totaldeadrows); -static int compare_rows(const void *a, const void *b); -static int acquire_inherited_sample_rows(Relation onerel, int elevel, - HeapTuple *rows, int targrows, - double *totalrows, double *totaldeadrows); +static void acquire_sample_rows(Relation onerel, int elevel, + AnalyzeSampleContext *context); +static void acquire_inherited_sample_rows(Relation onerel, int elevel, + AnalyzeSampleContext *context); static void update_attstats(Oid relid, bool inh, int natts, VacAttrStats **vacattrstats); static Datum std_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull); @@ -318,6 +314,7 @@ do_analyze_rel(Relation onerel, VacuumParams *params, Oid save_userid; int save_sec_context; int save_nestlevel; + AnalyzeSampleContext *sample_context; if (inh) ereport(elevel, @@ -502,18 +499,21 @@ do_analyze_rel(Relation onerel, VacuumParams *params, if (targrows < minrows) targrows = minrows; + /* create context for acquiring sample rows */ + sample_context = CreateAnalyzeSampleContext(onerel, va_cols, targrows, + vac_strategy); + /* * Acquire the sample rows */ - rows = (HeapTuple *) palloc(targrows * sizeof(HeapTuple)); if (inh) - numrows = acquire_inherited_sample_rows(onerel, elevel, - rows, targrows, - &totalrows, &totaldeadrows); + acquire_inherited_sample_rows(onerel, elevel, sample_context); else - numrows = (*acquirefunc) (onerel, elevel, - rows, targrows, - &totalrows, &totaldeadrows); + (*acquirefunc) (onerel, elevel, sample_context); + + /* Get the sample statistics */ + AnalyzeGetSampleStats(sample_context, &numrows, &totalrows, &totaldeadrows); + rows = AnalyzeGetSampleRows(sample_context, ANALYZE_SAMPLE_DATA, 0); /* * Compute the statistics. Temporary results during the calculations for @@ -592,7 +592,8 @@ do_analyze_rel(Relation onerel, VacuumParams *params, * not for relations representing inheritance trees. */ if (!inh) - BuildRelationExtStatistics(onerel, totalrows, numrows, rows, + BuildRelationExtStatistics(onerel, totalrows, numrows, + rows, attr_cnt, vacattrstats); } @@ -690,6 +691,8 @@ do_analyze_rel(Relation onerel, VacuumParams *params, pg_rusage_show(&ru0)))); } + DestroyAnalyzeSampleContext(sample_context); + /* Roll back any GUC changes executed by index functions */ AtEOXact_GUC(false, save_nestlevel); @@ -1018,26 +1021,26 @@ examine_attribute(Relation onerel, int attnum, Node *index_expr) * block. The previous sampling method put too much credence in the row * density near the start of the table. */ -static int +static void acquire_sample_rows(Relation onerel, int elevel, - HeapTuple *rows, int targrows, - double *totalrows, double *totaldeadrows) + AnalyzeSampleContext *context) { int numrows = 0; /* # rows now in reservoir */ + int targrows = context->targrows; double samplerows = 0; /* total # rows collected */ - double liverows = 0; /* # live rows seen */ - double deadrows = 0; /* # dead rows seen */ double rowstoskip = -1; /* -1 means not set yet */ + double totalrows = 0; + double totaldeadrows = 0; BlockNumber totalblocks; TransactionId OldestXmin; BlockSamplerData bs; ReservoirStateData rstate; - TupleTableSlot *slot; - TableScanDesc scan; Assert(targrows > 0); - totalblocks = RelationGetNumberOfBlocks(onerel); + table_scan_analyze_beginscan(onerel, context); + + totalblocks = context->totalblocks; /* Need a cutoff xmin for HeapTupleSatisfiesVacuum */ OldestXmin = GetOldestXmin(onerel, PROCARRAY_FLAGS_VACUUM); @@ -1047,9 +1050,6 @@ acquire_sample_rows(Relation onerel, int elevel, /* Prepare for sampling rows */ reservoir_init_selection_state(&rstate, targrows); - scan = table_beginscan_analyze(onerel); - slot = table_slot_create(onerel, NULL); - /* Outer loop over blocks to sample */ while (BlockSampler_HasMore(&bs)) { @@ -1057,10 +1057,10 @@ acquire_sample_rows(Relation onerel, int elevel, vacuum_delay_point(); - if (!table_scan_analyze_next_block(scan, targblock, vac_strategy)) + if (!table_scan_analyze_next_block(targblock, context)) continue; - while (table_scan_analyze_next_tuple(scan, OldestXmin, &liverows, &deadrows, slot)) + while (table_scan_analyze_next_tuple(OldestXmin, context)) { /* * The first targrows sample rows are simply copied into the @@ -1076,8 +1076,8 @@ acquire_sample_rows(Relation onerel, int elevel, */ if (numrows < targrows) { - rows[numrows] = ExecCopySlotHeapTuple(slot); - rows[numrows]->t_self = slot->tts_tid; + table_scan_analyze_sample_tuple(numrows, false, context); + numrows++; } else @@ -1099,9 +1099,8 @@ acquire_sample_rows(Relation onerel, int elevel, int k = (int) (targrows * sampler_random_fract(rstate.randstate)); Assert(k >= 0 && k < targrows); - heap_freetuple(rows[k]); - rows[k] = ExecCopySlotHeapTuple(slot); - rows[k]->t_self = slot->tts_tid; + + table_scan_analyze_sample_tuple(k, true, context); } rowstoskip -= 1; @@ -1111,19 +1110,7 @@ acquire_sample_rows(Relation onerel, int elevel, } } - ExecDropSingleTupleTableSlot(slot); - table_endscan(scan); - - /* - * If we didn't find as many tuples as we wanted then we're done. No sort - * is needed, since they're already in order. - * - * Otherwise we need to sort the collected tuples by position - * (itempointer). It's not worth worrying about corner cases where the - * tuples are already sorted. - */ - if (numrows == targrows) - qsort((void *) rows, numrows, sizeof(HeapTuple), compare_rows); + table_scan_analyze_endscan(context); /* * Estimate total numbers of live and dead rows in relation, extrapolating @@ -1134,13 +1121,13 @@ acquire_sample_rows(Relation onerel, int elevel, */ if (bs.m > 0) { - *totalrows = floor((liverows / bs.m) * totalblocks + 0.5); - *totaldeadrows = floor((deadrows / bs.m) * totalblocks + 0.5); + totalrows = floor((context->liverows / bs.m) * totalblocks + 0.5); + totaldeadrows = floor((context->deadrows / bs.m) * totalblocks + 0.5); } else { - *totalrows = 0.0; - *totaldeadrows = 0.0; + totalrows = 0.0; + totaldeadrows = 0.0; } /* @@ -1152,34 +1139,13 @@ acquire_sample_rows(Relation onerel, int elevel, "%d rows in sample, %.0f estimated total rows", RelationGetRelationName(onerel), bs.m, totalblocks, - liverows, deadrows, - numrows, *totalrows))); + context->liverows, + context->deadrows, + numrows, totalrows))); - return numrows; -} - -/* - * qsort comparator for sorting rows[] array - */ -static int -compare_rows(const void *a, const void *b) -{ - HeapTuple ha = *(const HeapTuple *) a; - HeapTuple hb = *(const HeapTuple *) b; - BlockNumber ba = ItemPointerGetBlockNumber(&ha->t_self); - OffsetNumber oa = ItemPointerGetOffsetNumber(&ha->t_self); - BlockNumber bb = ItemPointerGetBlockNumber(&hb->t_self); - OffsetNumber ob = ItemPointerGetOffsetNumber(&hb->t_self); - - if (ba < bb) - return -1; - if (ba > bb) - return 1; - if (oa < ob) - return -1; - if (oa > ob) - return 1; - return 0; + context->totalrows += totalrows; + context->totaldeadrows += totaldeadrows; + context->totalsampledrows += numrows; } @@ -1191,18 +1157,16 @@ compare_rows(const void *a, const void *b) * We fail and return zero if there are no inheritance children, or if all * children are foreign tables that don't support ANALYZE. */ -static int +static void acquire_inherited_sample_rows(Relation onerel, int elevel, - HeapTuple *rows, int targrows, - double *totalrows, double *totaldeadrows) + AnalyzeSampleContext *context) { List *tableOIDs; Relation *rels; AcquireSampleRowsFunc *acquirefuncs; double *relblocks; double totalblocks; - int numrows, - nrels, + int nrels, i; ListCell *lc; bool has_child; @@ -1230,7 +1194,7 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, (errmsg("skipping analyze of \"%s.%s\" inheritance tree --- this inheritance tree contains no child tables", get_namespace_name(RelationGetNamespace(onerel)), RelationGetRelationName(onerel)))); - return 0; + return; } /* @@ -1328,7 +1292,7 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, (errmsg("skipping analyze of \"%s.%s\" inheritance tree --- this inheritance tree contains no analyzable child tables", get_namespace_name(RelationGetNamespace(onerel)), RelationGetRelationName(onerel)))); - return 0; + return; } /* @@ -1337,9 +1301,6 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, * rels have radically different free-space percentages, but it's not * clear that it's worth working harder.) */ - numrows = 0; - *totalrows = 0; - *totaldeadrows = 0; for (i = 0; i < nrels; i++) { Relation childrel = rels[i]; @@ -1350,49 +1311,15 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, { int childtargrows; - childtargrows = (int) rint(targrows * childblocks / totalblocks); + childtargrows = (int) rint(context->totaltargrows * childblocks / totalblocks); /* Make sure we don't overrun due to roundoff error */ - childtargrows = Min(childtargrows, targrows - numrows); + childtargrows = Min(childtargrows, context->totaltargrows - context->totalsampledrows); if (childtargrows > 0) { - int childrows; - double trows, - tdrows; + InitAnalyzeSampleContextForChild(context, childrel, childtargrows); /* Fetch a random sample of the child's rows */ - childrows = (*acquirefunc) (childrel, elevel, - rows + numrows, childtargrows, - &trows, &tdrows); - - /* We may need to convert from child's rowtype to parent's */ - if (childrows > 0 && - !equalTupleDescs(RelationGetDescr(childrel), - RelationGetDescr(onerel))) - { - TupleConversionMap *map; - - map = convert_tuples_by_name(RelationGetDescr(childrel), - RelationGetDescr(onerel)); - if (map != NULL) - { - int j; - - for (j = 0; j < childrows; j++) - { - HeapTuple newtup; - - newtup = execute_attr_map_tuple(rows[numrows + j], map); - heap_freetuple(rows[numrows + j]); - rows[numrows + j] = newtup; - } - free_conversion_map(map); - } - } - - /* And add to counts */ - numrows += childrows; - *totalrows += trows; - *totaldeadrows += tdrows; + (*acquirefunc) (childrel, elevel, context); } } @@ -1402,8 +1329,6 @@ acquire_inherited_sample_rows(Relation onerel, int elevel, */ table_close(childrel, NoLock); } - - return numrows; } diff --git a/src/include/access/tableam.h b/src/include/access/tableam.h index 0b882dc..90d2375 100644 --- a/src/include/access/tableam.h +++ b/src/include/access/tableam.h @@ -37,6 +37,66 @@ struct SampleScanState; struct TBMIterateResult; struct VacuumParams; struct ValidateIndexState; +struct TupleConversionMap; + +typedef enum AnalyzeSampleType +{ + ANALYZE_SAMPLE_DATA = 0, /* real data per column */ + ANALYZE_SAMPLE_DISKSIZE, /* physical size per column */ + MAX_ANALYZE_SAMPLE /* must be last */ +} AnalyzeSampleType; + +typedef struct AnalyzeSampleContext +{ + /* Filled when context is created */ + int totaltargrows; + List *anl_cols; + Relation parent; + BufferAccessStrategy bstrategy; + + /* Filled by table AM analyze routines */ + BlockNumber totalblocks; + TableScanDesc scan; + + /* + * Acquiring sample rows from a inherited table will invoke + * multiple sampling iterations for each child relation, so + * bellow filed is the statistic for each iteration. + */ + int targrows; /* target number of sample rows */ + double liverows; + double deadrows; + bool ordered; /* are sample rows ordered physically */ + + /* + * Statistics filed by all sampling iterations. + */ + int totalsampledrows; /* total number of sample rows stored */ + double totalrows; + double totaldeadrows; + + /* + * If childrel has different rowtype with parent, we + * need to convert sample tuple to the same rowtype + * with parent + */ + struct TupleConversionMap *tup_convert_map; + + /* + * Used by table AM analyze routines to store + * the temporary tuple for different types of + * sample rows, the tuple is finally stored to + * sample_rows[] if the tuple is + * randomly selected. + */ + TupleTableSlot* sample_slots[MAX_ANALYZE_SAMPLE]; + + /* + * stores the final sample rows which will be + * used to compute statistics. + */ + HeapTuple* sample_rows[MAX_ANALYZE_SAMPLE]; +} AnalyzeSampleContext; /* * Bitmask values for the flags argument to the scan_begin callback. @@ -532,9 +592,10 @@ typedef struct TableAmRoutine * clear what a good interface for non block based AMs would be, so there * isn't one yet. */ - bool (*scan_analyze_next_block) (TableScanDesc scan, - BlockNumber blockno, - BufferAccessStrategy bstrategy); + void (*scan_analyze_beginscan) (Relation onerel, AnalyzeSampleContext *context); + + bool (*scan_analyze_next_block) (BlockNumber blockno, + AnalyzeSampleContext *context); /* * See table_scan_analyze_next_tuple(). @@ -544,11 +605,13 @@ typedef struct TableAmRoutine * influence autovacuum scheduling (see comment for relation_vacuum * callback). */ - bool (*scan_analyze_next_tuple) (TableScanDesc scan, - TransactionId OldestXmin, - double *liverows, - double *deadrows, - TupleTableSlot *slot); + bool (*scan_analyze_next_tuple) (TransactionId OldestXmin, + AnalyzeSampleContext *context); + + void (*scan_analyze_sample_tuple) (int pos, bool replace, + AnalyzeSampleContext *context); + + void (*scan_analyze_endscan) (AnalyzeSampleContext *context); /* see table_index_build_range_scan for reference about parameters */ double (*index_build_range_scan) (Relation table_rel, @@ -1474,6 +1537,12 @@ table_relation_vacuum(Relation rel, struct VacuumParams *params, rel->rd_tableam->relation_vacuum(rel, params, bstrategy); } +static inline void +table_scan_analyze_beginscan(Relation rel, struct AnalyzeSampleContext *context) +{ + rel->rd_tableam->scan_analyze_beginscan(rel, context); +} + /* * Prepare to analyze block `blockno` of `scan`. The scan needs to have been * started with table_beginscan_analyze(). Note that this routine might @@ -1483,11 +1552,10 @@ table_relation_vacuum(Relation rel, struct VacuumParams *params, * Returns false if block is unsuitable for sampling, true otherwise. */ static inline bool -table_scan_analyze_next_block(TableScanDesc scan, BlockNumber blockno, - BufferAccessStrategy bstrategy) +table_scan_analyze_next_block(BlockNumber blockno, + struct AnalyzeSampleContext *context) { - return scan->rs_rd->rd_tableam->scan_analyze_next_block(scan, blockno, - bstrategy); + return context->scan->rs_rd->rd_tableam->scan_analyze_next_block(blockno, context); } /* @@ -1501,13 +1569,21 @@ table_scan_analyze_next_block(TableScanDesc scan, BlockNumber blockno, * tuples. */ static inline bool -table_scan_analyze_next_tuple(TableScanDesc scan, TransactionId OldestXmin, - double *liverows, double *deadrows, - TupleTableSlot *slot) +table_scan_analyze_next_tuple(TransactionId OldestXmin, AnalyzeSampleContext *context) +{ + return context->scan->rs_rd->rd_tableam->scan_analyze_next_tuple(OldestXmin, context); +} + +static inline void +table_scan_analyze_sample_tuple(Index sample, bool replace, AnalyzeSampleContext *context) +{ + context->scan->rs_rd->rd_tableam->scan_analyze_sample_tuple(sample, replace, context); +} + +static inline void +table_scan_analyze_endscan(AnalyzeSampleContext *context) { - return scan->rs_rd->rd_tableam->scan_analyze_next_tuple(scan, OldestXmin, - liverows, deadrows, - slot); + context->scan->rs_rd->rd_tableam->scan_analyze_endscan(context); } /* @@ -1783,6 +1859,32 @@ extern void table_block_relation_estimate_size(Relation rel, Size usable_bytes_per_page); /* ---------------------------------------------------------------------------- + * Helper functions to implement analyze scan. +j* ---------------------------------------------------------------------------- + */ +extern AnalyzeSampleContext * +CreateAnalyzeSampleContext(Relation onerel, List *cols, int targrows, + BufferAccessStrategy strategy); +extern void DestroyAnalyzeSampleContext(AnalyzeSampleContext *context); +extern TupleTableSlot * AnalyzeGetSampleSlot(AnalyzeSampleContext *context, + Relation onerel, AnalyzeSampleType type); +extern void AnalyzeRecordSampleRow(AnalyzeSampleContext *context, + TupleTableSlot *sample_slot, + HeapTuple sample_tuple, + AnalyzeSampleType type, int pos, + bool replace, bool withtid); +extern void InitAnalyzeSampleContextForChild(AnalyzeSampleContext *context, + Relation child, + int childtargrows); +extern void AnalyzeGetSampleStats(AnalyzeSampleContext *context, + int *totalsampledrows, + double *totalrows, + double *totaldeadrows); +extern HeapTuple * +AnalyzeGetSampleRows(AnalyzeSampleContext *context, AnalyzeSampleType type, int offset); +extern bool AnalyzeSampleIsValid(AnalyzeSampleContext *context, AnalyzeSampleType type); + +/* ---------------------------------------------------------------------------- * Functions in tableamapi.c * ---------------------------------------------------------------------------- */ diff --git a/src/include/foreign/fdwapi.h b/src/include/foreign/fdwapi.h index 8226860..e0da119 100644 --- a/src/include/foreign/fdwapi.h +++ b/src/include/foreign/fdwapi.h @@ -18,6 +18,7 @@ /* To avoid including explain.h here, reference ExplainState thus: */ struct ExplainState; +struct AnalyzeSampleContext; /* @@ -139,10 +140,8 @@ typedef void (*ExplainForeignModify_function) (ModifyTableState *mtstate, typedef void (*ExplainDirectModify_function) (ForeignScanState *node, struct ExplainState *es); -typedef int (*AcquireSampleRowsFunc) (Relation relation, int elevel, - HeapTuple *rows, int targrows, - double *totalrows, - double *totaldeadrows); +typedef void (*AcquireSampleRowsFunc) (Relation relation, int elevel, + struct AnalyzeSampleContext *context); typedef bool (*AnalyzeForeignTable_function) (Relation relation, AcquireSampleRowsFunc *func, -- 1.8.3.1