From 394ca0ba07e969fa83669afa5e2b11bba7992763 Mon Sep 17 00:00:00 2001 From: Hari Babu Date: Wed, 30 Aug 2017 12:41:15 +1000 Subject: [PATCH 2/8] Storage AM API hooks and related functions --- src/backend/access/heap/Makefile | 3 +- src/backend/access/heap/heapam_storage.c | 58 ++++++++ src/backend/access/heap/storageamapi.c | 103 ++++++++++++++ src/include/access/htup.h | 21 +++ src/include/access/storageamapi.h | 227 +++++++++++++++++++++++++++++++ src/include/catalog/pg_am.h | 3 + src/include/catalog/pg_proc.h | 5 + src/include/nodes/nodes.h | 1 + src/include/utils/tqual.h | 9 -- 9 files changed, 420 insertions(+), 10 deletions(-) create mode 100644 src/backend/access/heap/heapam_storage.c create mode 100644 src/backend/access/heap/storageamapi.c create mode 100644 src/include/access/storageamapi.h diff --git a/src/backend/access/heap/Makefile b/src/backend/access/heap/Makefile index b83d496bcd..02a3909e7c 100644 --- a/src/backend/access/heap/Makefile +++ b/src/backend/access/heap/Makefile @@ -12,6 +12,7 @@ subdir = src/backend/access/heap top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global -OBJS = heapam.o hio.o pruneheap.o rewriteheap.o syncscan.o tuptoaster.o visibilitymap.o +OBJS = heapam.o hio.o heapam_storage.o pruneheap.o rewriteheap.o storageamapi.o \ + syncscan.o tuptoaster.o visibilitymap.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/heap/heapam_storage.c b/src/backend/access/heap/heapam_storage.c new file mode 100644 index 0000000000..88827e7957 --- /dev/null +++ b/src/backend/access/heap/heapam_storage.c @@ -0,0 +1,58 @@ +/*------------------------------------------------------------------------- + * + * heapam_storage.c + * heap storage access method code + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/access/heap/heapam_storage.c + * + * + * NOTES + * This file contains the heap_ routines which implement + * the POSTGRES heap access method used for all POSTGRES + * relations. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/heapam.h" +#include "access/heapam_xlog.h" +#include "access/hio.h" +#include "access/htup_details.h" +#include "access/multixact.h" +#include "access/relscan.h" +#include "access/storageamapi.h" +#include "access/subtrans.h" +#include "access/tuptoaster.h" +#include "access/valid.h" +#include "access/visibilitymap.h" +#include "access/xloginsert.h" +#include "access/xact.h" +#include "catalog/catalog.h" +#include "miscadmin.h" +#include "pgstat.h" +#include "storage/bufmgr.h" +#include "storage/lmgr.h" +#include "storage/predicate.h" +#include "storage/procarray.h" +#include "storage/smgr.h" +#include "storage/spin.h" +#include "utils/builtins.h" +#include "utils/inval.h" +#include "utils/rel.h" +#include "utils/tqual.h" + + +Datum +heapam_storage_handler(PG_FUNCTION_ARGS) +{ + StorageAmRoutine *amroutine = makeNode(StorageAmRoutine); + + + PG_RETURN_POINTER(amroutine); +} diff --git a/src/backend/access/heap/storageamapi.c b/src/backend/access/heap/storageamapi.c new file mode 100644 index 0000000000..bcbe14588b --- /dev/null +++ b/src/backend/access/heap/storageamapi.c @@ -0,0 +1,103 @@ +/*---------------------------------------------------------------------- + * + * storageamapi.c + * Support routines for API for Postgres storage access methods + * + * FIXME: looks like this should be in amapi.c. + * + * Copyright (c) 2016, PostgreSQL Global Development Group + * + * src/backend/access/heap/storageamapi.c + *---------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/htup_details.h" +#include "access/storageamapi.h" +#include "catalog/pg_am.h" +#include "catalog/pg_proc.h" +#include "utils/syscache.h" +#include "utils/memutils.h" + + +/* + * GetStorageAmRoutine + * Call the specified access method handler routine to get its + * StorageAmRoutine struct, which will be palloc'd in the caller's + * memory context. + */ +StorageAmRoutine * +GetStorageAmRoutine(Oid amhandler) +{ + Datum datum; + StorageAmRoutine *routine; + + datum = OidFunctionCall0(amhandler); + routine = (StorageAmRoutine *) DatumGetPointer(datum); + + if (routine == NULL || !IsA(routine, StorageAmRoutine)) + elog(ERROR, "storage access method handler %u did not return a StorageAmRoutine struct", + amhandler); + + return routine; +} + +/* A crock */ +StorageAmRoutine * +GetHeapamStorageAmRoutine(void) +{ + Datum datum; + static StorageAmRoutine * HeapamStorageAmRoutine = NULL; + + if (HeapamStorageAmRoutine == NULL) + { + MemoryContext oldcxt; + + oldcxt = MemoryContextSwitchTo(TopMemoryContext); + datum = OidFunctionCall0(HEAPAM_STORAGE_AM_HANDLER_OID); + HeapamStorageAmRoutine = (StorageAmRoutine *) DatumGetPointer(datum); + MemoryContextSwitchTo(oldcxt); + } + + return HeapamStorageAmRoutine; +} + +/* + * GetStorageAmRoutineByAmId - look up the handler of the storage access + * method with the given OID, and get its StorageAmRoutine struct. + */ +StorageAmRoutine * +GetStorageAmRoutineByAmId(Oid amoid) +{ + regproc amhandler; + HeapTuple tuple; + Form_pg_am amform; + + /* Get handler function OID for the access method */ + tuple = SearchSysCache1(AMOID, ObjectIdGetDatum(amoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for access method %u", + amoid); + amform = (Form_pg_am) GETSTRUCT(tuple); + + /* Check that it is a storage access method */ + if (amform->amtype != AMTYPE_STORAGE) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("access method \"%s\" is not of type %s", + NameStr(amform->amname), "STORAGE"))); + + amhandler = amform->amhandler; + + /* Complain if handler OID is invalid */ + if (!RegProcedureIsValid(amhandler)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("storage access method \"%s\" does not have a handler", + NameStr(amform->amname)))); + + ReleaseSysCache(tuple); + + /* And finally, call the handler function to get the API struct. */ + return GetStorageAmRoutine(amhandler); +} diff --git a/src/include/access/htup.h b/src/include/access/htup.h index 61b3e68639..6459435c78 100644 --- a/src/include/access/htup.h +++ b/src/include/access/htup.h @@ -26,6 +26,27 @@ typedef struct MinimalTupleData MinimalTupleData; typedef MinimalTupleData *MinimalTuple; +typedef enum tuple_visibility_type +{ + MVCC_VISIBILITY = 0, /* HeapTupleSatisfiesMVCC */ + SELF_VISIBILITY, /* HeapTupleSatisfiesSelf */ + ANY_VISIBILITY, /* HeapTupleSatisfiesAny */ + TOAST_VISIBILITY, /* HeapTupleSatisfiesToast */ + DIRTY_VISIBILITY, /* HeapTupleSatisfiesDirty */ + HISTORIC_MVCC_VISIBILITY, /* HeapTupleSatisfiesHistoricMVCC */ + + END_OF_VISIBILITY +} tuple_visibility_type; + +/* Result codes for HeapTupleSatisfiesVacuum */ +typedef enum +{ + HEAPTUPLE_DEAD, /* tuple is dead and deletable */ + HEAPTUPLE_LIVE, /* tuple is live (committed, no deleter) */ + HEAPTUPLE_RECENTLY_DEAD, /* tuple is dead, but not deletable yet */ + HEAPTUPLE_INSERT_IN_PROGRESS, /* inserting xact is still in progress */ + HEAPTUPLE_DELETE_IN_PROGRESS /* deleting xact is still in progress */ +} HTSV_Result; /* * HeapTupleData is an in-memory data structure that points to a tuple. diff --git a/src/include/access/storageamapi.h b/src/include/access/storageamapi.h new file mode 100644 index 0000000000..2b08c6f4ff --- /dev/null +++ b/src/include/access/storageamapi.h @@ -0,0 +1,227 @@ +/*--------------------------------------------------------------------- + * + * storageamapi.h + * API for Postgres storage access methods + * + * Copyright (c) 2016, PostgreSQL Global Development Group + * + * src/include/access/storageamapi.h + *--------------------------------------------------------------------- + */ +#ifndef STORAGEAMAPI_H +#define STORAGEAMAPI_H + +#include "access/htup.h" +#include "access/heapam.h" +#include "access/sdir.h" +#include "access/skey.h" +#include "executor/tuptable.h" +#include "utils/relcache.h" +#include "utils/snapshot.h" + +/* A physical tuple coming from a storage AM scan */ +typedef void *StorageTuple; + +typedef union tuple_data +{ + TransactionId xid; + CommandId cid; + ItemPointerData tid; +} tuple_data; + +typedef enum tuple_data_flags +{ + XMIN = 0, + UPDATED_XID, + CMIN, + TID, + CTID +} tuple_data_flags; + + +typedef HeapScanDesc (*ScanBegin_function) (Relation relation, + Snapshot snapshot, + int nkeys, ScanKey key, + ParallelHeapScanDesc parallel_scan, + bool allow_strat, + bool allow_sync, + bool allow_pagemode, + bool is_bitmapscan, + bool is_samplescan, + bool temp_snap); +typedef void (*ScanSetlimits_function) (HeapScanDesc sscan, BlockNumber startBlk, BlockNumber numBlks); + +/* must return a TupleTableSlot? */ +typedef StorageTuple(*ScanGetnext_function) (HeapScanDesc scan, + ScanDirection direction); + +typedef TupleTableSlot *(*ScanGetnextSlot_function) (HeapScanDesc scan, + ScanDirection direction, TupleTableSlot *slot); + +typedef void (*ScanEnd_function) (HeapScanDesc scan); + + +typedef void (*ScanGetpage_function) (HeapScanDesc scan, BlockNumber page); +typedef void (*ScanRescan_function) (HeapScanDesc scan, ScanKey key, bool set_params, + bool allow_strat, bool allow_sync, bool allow_pagemode); +typedef void (*ScanUpdateSnapshot_function) (HeapScanDesc scan, Snapshot snapshot); + +typedef bool (*HotSearchBuffer_function) (ItemPointer tid, Relation relation, + Buffer buffer, Snapshot snapshot, HeapTuple heapTuple, + bool *all_dead, bool first_call); + +typedef Oid (*TupleInsert_function) (Relation relation, + TupleTableSlot *tupslot, + CommandId cid, + int options, + BulkInsertState bistate); + +typedef HTSU_Result (*TupleDelete_function) (Relation relation, + ItemPointer tid, + CommandId cid, + Snapshot crosscheck, + bool wait, + HeapUpdateFailureData *hufd); + +typedef HTSU_Result (*TupleUpdate_function) (Relation relation, + ItemPointer otid, + TupleTableSlot *slot, + CommandId cid, + Snapshot crosscheck, + bool wait, + HeapUpdateFailureData *hufd, + LockTupleMode *lockmode); + +typedef bool (*TupleFetch_function) (Relation relation, + ItemPointer tid, + Snapshot snapshot, + StorageTuple * tuple, + Buffer *userbuf, + bool keep_buf, + Relation stats_relation); + +typedef HTSU_Result (*TupleLock_function) (Relation relation, + ItemPointer tid, + StorageTuple * tuple, + CommandId cid, + LockTupleMode mode, + LockWaitPolicy wait_policy, + bool follow_update, + Buffer *buffer, + HeapUpdateFailureData *hufd); + +typedef void (*MultiInsert_function) (Relation relation, HeapTuple *tuples, int ntuples, + CommandId cid, int options, BulkInsertState bistate); + +typedef bool (*TupleFreeze_function) (HeapTupleHeader tuple, TransactionId cutoff_xid, + TransactionId cutoff_multi); + +typedef void (*TupleGetLatestTid_function) (Relation relation, + Snapshot snapshot, + ItemPointer tid); + +typedef tuple_data(*GetTupleData_function) (StorageTuple tuple, tuple_data_flags flags); + +typedef StorageTuple(*TupleFromDatum_function) (Datum data, Oid tableoid); + +typedef void (*SlotStoreTuple_function) (TupleTableSlot *slot, + StorageTuple tuple, + bool shouldFree, + bool minumumtuple); +typedef void (*SlotClearTuple_function) (TupleTableSlot *slot); +typedef Datum (*SlotGetattr_function) (TupleTableSlot *slot, + int attnum, bool *isnull); +typedef void (*SlotVirtualizeTuple_function) (TupleTableSlot *slot, int16 upto); + +typedef HeapTuple (*SlotGetTuple_function) (TupleTableSlot *slot, bool palloc_copy); +typedef MinimalTuple (*SlotGetMinTuple_function) (TupleTableSlot *slot, bool palloc_copy); + +typedef void (*SlotUpdateTableoid_function) (TupleTableSlot *slot, Oid tableoid); + +typedef void (*SpeculativeFinish_function) (Relation rel, + TupleTableSlot *slot); +typedef void (*SpeculativeAbort_function) (Relation rel, + TupleTableSlot *slot); + +typedef void (*RelationSync_function) (Relation relation); + +typedef bool (*SnapshotSatisfies_function) (StorageTuple htup, Snapshot snapshot, Buffer buffer); +typedef HTSU_Result (*SnapshotSatisfiesUpdate_function) (StorageTuple htup, CommandId curcid, Buffer buffer); +typedef HTSV_Result (*SnapshotSatisfiesVacuum_function) (StorageTuple htup, TransactionId OldestXmin, Buffer buffer); + +typedef struct StorageSlotAmRoutine +{ + /* Operations on TupleTableSlot */ + SlotStoreTuple_function slot_store_tuple; + SlotVirtualizeTuple_function slot_virtualize_tuple; + SlotClearTuple_function slot_clear_tuple; + SlotGetattr_function slot_getattr; + SlotGetTuple_function slot_tuple; + SlotGetMinTuple_function slot_min_tuple; + SlotUpdateTableoid_function slot_update_tableoid; +} StorageSlotAmRoutine; + +typedef StorageSlotAmRoutine * (*slot_storageam_hook) (void); + +/* + * API struct for a storage AM. Note this must be stored in a single palloc'd + * chunk of memory. + * + * XXX currently all functions are together in a single struct. Would it be + * worthwhile to split the slot-accessor functions to a different struct? + * That way, MinimalTuple could be handled without a complete StorageAmRoutine + * for them -- it'd only have a few functions in TupleTableSlotAmRoutine or so. + */ +typedef struct StorageAmRoutine +{ + NodeTag type; + + /* Operations on relation scans */ + ScanBegin_function scan_begin; + ScanSetlimits_function scansetlimits; + ScanGetnext_function scan_getnext; + ScanGetnextSlot_function scan_getnextslot; + ScanEnd_function scan_end; + ScanGetpage_function scan_getpage; + ScanRescan_function scan_rescan; + ScanUpdateSnapshot_function scan_update_snapshot; + HotSearchBuffer_function hot_search_buffer; /* heap_hot_search_buffer */ + + /* Operations on physical tuples */ + TupleInsert_function tuple_insert; /* heap_insert */ + TupleUpdate_function tuple_update; /* heap_update */ + TupleDelete_function tuple_delete; /* heap_delete */ + TupleFetch_function tuple_fetch; /* heap_fetch */ + TupleLock_function tuple_lock; /* heap_lock_tuple */ + MultiInsert_function multi_insert; /* heap_multi_insert */ + TupleFreeze_function tuple_freeze; /* heap_freeze_tuple */ + TupleGetLatestTid_function tuple_get_latest_tid; /* heap_get_latest_tid */ + + GetTupleData_function get_tuple_data; + TupleFromDatum_function tuple_from_datum; + + slot_storageam_hook slot_storageam; + + /* + * Speculative insertion support operations + * + * Setting a tuple's speculative token is a slot-only operation, so no + * need for a storage AM method, but after inserting a tuple containing a + * speculative token, the insertion must be completed by these routines: + */ + SpeculativeFinish_function speculative_finish; + SpeculativeAbort_function speculative_abort; + + + RelationSync_function relation_sync; /* heap_sync */ + + SnapshotSatisfies_function snapshot_satisfies; + SnapshotSatisfiesUpdate_function snapshot_satisfiesUpdate; /* HeapTupleSatisfiesUpdate */ + SnapshotSatisfiesVacuum_function snapshot_satisfiesVacuum; /* HeapTupleSatisfiesVacuum */ +} StorageAmRoutine; + +extern StorageAmRoutine * GetStorageAmRoutine(Oid amhandler); +extern StorageAmRoutine * GetStorageAmRoutineByAmId(Oid amoid); +extern StorageAmRoutine * GetHeapamStorageAmRoutine(void); + +#endif /* STORAGEAMAPI_H */ diff --git a/src/include/catalog/pg_am.h b/src/include/catalog/pg_am.h index dd9c263ade..2c3e33c104 100644 --- a/src/include/catalog/pg_am.h +++ b/src/include/catalog/pg_am.h @@ -84,5 +84,8 @@ DESCR("SP-GiST index access method"); DATA(insert OID = 3580 ( brin brinhandler i )); DESCR("block range index (BRIN) access method"); #define BRIN_AM_OID 3580 +DATA(insert OID = 4001 ( heapam heapam_storage_handler s )); +DESCR("heapam storage access method"); +#define HEAPAM_STORAGE_AM_OID 4001 #endif /* PG_AM_H */ diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index 4a8ab151d7..f1d0198848 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -558,6 +558,11 @@ DESCR("convert int4 to float4"); DATA(insert OID = 319 ( int4 PGNSP PGUID 12 1 0 0 0 f f f f t f i s 1 0 23 "700" _null_ _null_ _null_ _null_ _null_ ftoi4 _null_ _null_ _null_ )); DESCR("convert float4 to int4"); +/* Storage access method handlers */ +DATA(insert OID = 4002 ( heapam_storage_handler PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 3998 "2281" _null_ _null_ _null_ _null_ _null_ heapam_storage_handler _null_ _null_ _null_ )); +DESCR("row-oriented storage access method handler"); +#define HEAPAM_STORAGE_AM_HANDLER_OID 4002 + /* Index access method handlers */ DATA(insert OID = 330 ( bthandler PGNSP PGUID 12 1 0 0 0 f f f f t f v s 1 0 325 "2281" _null_ _null_ _null_ _null_ _null_ bthandler _null_ _null_ _null_ )); DESCR("btree index access method handler"); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index ffeeb4919b..805d623cf0 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -497,6 +497,7 @@ typedef enum NodeTag T_InlineCodeBlock, /* in nodes/parsenodes.h */ T_FdwRoutine, /* in foreign/fdwapi.h */ T_IndexAmRoutine, /* in access/amapi.h */ + T_StorageAmRoutine, /* in access/storageamapi.h */ T_TsmRoutine, /* in access/tsmapi.h */ T_ForeignKeyCacheInfo /* in utils/rel.h */ } NodeTag; diff --git a/src/include/utils/tqual.h b/src/include/utils/tqual.h index 96eaf01ca0..4fda7f8384 100644 --- a/src/include/utils/tqual.h +++ b/src/include/utils/tqual.h @@ -45,15 +45,6 @@ extern PGDLLIMPORT SnapshotData CatalogSnapshotData; #define HeapTupleSatisfiesVisibility(tuple, snapshot, buffer) \ ((*(snapshot)->satisfies) (tuple, snapshot, buffer)) -/* Result codes for HeapTupleSatisfiesVacuum */ -typedef enum -{ - HEAPTUPLE_DEAD, /* tuple is dead and deletable */ - HEAPTUPLE_LIVE, /* tuple is live (committed, no deleter) */ - HEAPTUPLE_RECENTLY_DEAD, /* tuple is dead, but not deletable yet */ - HEAPTUPLE_INSERT_IN_PROGRESS, /* inserting xact is still in progress */ - HEAPTUPLE_DELETE_IN_PROGRESS /* deleting xact is still in progress */ -} HTSV_Result; /* These are the "satisfies" test routines for the various snapshot types */ extern bool HeapTupleSatisfiesMVCC(HeapTuple htup, -- 2.14.2.windows.1