From 2f793936f4b1beb73dbcc13fabc8f4febc24a60d Mon Sep 17 00:00:00 2001
From: bucoo <bucoo@sohu.com>
Date: Mon, 25 Jan 2021 21:43:30 +0800
Subject: [PATCH] support parallel union/distinct/aggregate using batch hashagg
 and sort

---
 src/backend/commands/explain.c                    |  20 +
 src/backend/executor/Makefile                     |   1 +
 src/backend/executor/execAmi.c                    |   5 +
 src/backend/executor/execParallel.c               |  17 +
 src/backend/executor/execProcnode.c               |  10 +
 src/backend/executor/nodeAgg.c                    | 637 +++++++++++++++++++---
 src/backend/executor/nodeBatchSort.c              | 554 +++++++++++++++++++
 src/backend/nodes/copyfuncs.c                     |  22 +
 src/backend/nodes/outfuncs.c                      |  18 +
 src/backend/nodes/readfuncs.c                     |  19 +
 src/backend/optimizer/path/costsize.c             | 110 +++-
 src/backend/optimizer/plan/createplan.c           |  61 ++-
 src/backend/optimizer/plan/planner.c              | 305 +++++++++++
 src/backend/optimizer/plan/setrefs.c              |   1 +
 src/backend/optimizer/plan/subselect.c            |   6 +
 src/backend/optimizer/prep/prepunion.c            |  81 ++-
 src/backend/optimizer/util/pathnode.c             |  39 ++
 src/backend/optimizer/util/tlist.c                |  21 +
 src/backend/postmaster/pgstat.c                   |   6 +
 src/backend/storage/lmgr/lwlock.c                 |   2 -
 src/backend/utils/misc/guc.c                      |  26 +
 src/backend/utils/sort/Makefile                   |   3 +-
 src/backend/utils/sort/batchstore.c               | 466 ++++++++++++++++
 src/backend/utils/sort/sharedtuplestore.c         | 163 +++++-
 src/backend/utils/sort/tuplesort.c                |  20 +
 src/include/executor/nodeAgg.h                    |   2 +
 src/include/executor/nodeBatchSort.h              |  19 +
 src/include/nodes/execnodes.h                     |  19 +
 src/include/nodes/nodes.h                         |   6 +-
 src/include/nodes/pathnodes.h                     |  11 +
 src/include/nodes/plannodes.h                     |  15 +
 src/include/optimizer/cost.h                      |   7 +
 src/include/optimizer/pathnode.h                  |   7 +
 src/include/optimizer/tlist.h                     |   1 +
 src/include/pgstat.h                              |   4 +-
 src/include/storage/lwlock.h                      |   1 -
 src/include/utils/batchstore.h                    |  38 ++
 src/include/utils/sharedtuplestore.h              |  14 +
 src/include/utils/tuplesort.h                     |   1 +
 src/test/regress/expected/groupingsets.out        |  65 +++
 src/test/regress/expected/partition_aggregate.out | 138 +++++
 src/test/regress/expected/select_distinct.out     |  77 +++
 src/test/regress/expected/select_parallel.out     |  86 +++
 src/test/regress/expected/union.out               |  85 +++
 src/test/regress/sql/groupingsets.sql             |  10 +
 src/test/regress/sql/partition_aggregate.sql      |  45 ++
 src/test/regress/sql/select_distinct.sql          |  23 +
 src/test/regress/sql/select_parallel.sql          |  32 ++
 src/test/regress/sql/union.sql                    |  28 +
 49 files changed, 3223 insertions(+), 124 deletions(-)
 create mode 100644 src/backend/executor/nodeBatchSort.c
 create mode 100644 src/backend/utils/sort/batchstore.c
 create mode 100644 src/include/executor/nodeBatchSort.h
 create mode 100644 src/include/utils/batchstore.h

diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
index 5d7eb3574c..cd38289fbb 100644
--- a/src/backend/commands/explain.c
+++ b/src/backend/commands/explain.c
@@ -1283,6 +1283,9 @@ ExplainNode(PlanState *planstate, List *ancestors,
 		case T_Sort:
 			pname = sname = "Sort";
 			break;
+		case T_BatchSort:
+			pname = sname = "BatchSort";
+			break;
 		case T_IncrementalSort:
 			pname = sname = "Incremental Sort";
 			break;
@@ -1312,6 +1315,10 @@ ExplainNode(PlanState *planstate, List *ancestors,
 						pname = "MixedAggregate";
 						strategy = "Mixed";
 						break;
+					case AGG_BATCH_HASH:
+						pname = "BatchHashAggregate";
+						strategy = "BatchHashed";
+						break;
 					default:
 						pname = "Aggregate ???";
 						strategy = "???";
@@ -1946,6 +1953,18 @@ ExplainNode(PlanState *planstate, List *ancestors,
 			show_sort_keys(castNode(SortState, planstate), ancestors, es);
 			show_sort_info(castNode(SortState, planstate), es);
 			break;
+		case T_BatchSort:
+			{
+				BatchSort *bsort = (BatchSort*)plan;
+				show_sort_group_keys(planstate, "Sort Key",
+									 bsort->sort.numCols, 0, bsort->sort.sortColIdx,
+									 bsort->sort.sortOperators, bsort->sort.collations,
+									 bsort->sort.nullsFirst,
+									 ancestors, es);
+				if (es->verbose)
+					ExplainPropertyInteger("batches", NULL, bsort->numBatches, es);
+			}
+			break;
 		case T_IncrementalSort:
 			show_incremental_sort_keys(castNode(IncrementalSortState, planstate),
 									   ancestors, es);
@@ -3054,6 +3073,7 @@ show_hashagg_info(AggState *aggstate, ExplainState *es)
 	int64		memPeakKb = (aggstate->hash_mem_peak + 1023) / 1024;
 
 	if (agg->aggstrategy != AGG_HASHED &&
+		agg->aggstrategy != AGG_BATCH_HASH &&
 		agg->aggstrategy != AGG_MIXED)
 		return;
 
diff --git a/src/backend/executor/Makefile b/src/backend/executor/Makefile
index f990c6473a..a4855a8881 100644
--- a/src/backend/executor/Makefile
+++ b/src/backend/executor/Makefile
@@ -33,6 +33,7 @@ OBJS = \
 	instrument.o \
 	nodeAgg.o \
 	nodeAppend.o \
+	nodeBatchSort.o \
 	nodeBitmapAnd.o \
 	nodeBitmapHeapscan.o \
 	nodeBitmapIndexscan.o \
diff --git a/src/backend/executor/execAmi.c b/src/backend/executor/execAmi.c
index 23bdb53cd1..ecbcc09b51 100644
--- a/src/backend/executor/execAmi.c
+++ b/src/backend/executor/execAmi.c
@@ -17,6 +17,7 @@
 #include "executor/execdebug.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeAppend.h"
+#include "executor/nodeBatchSort.h"
 #include "executor/nodeBitmapAnd.h"
 #include "executor/nodeBitmapHeapscan.h"
 #include "executor/nodeBitmapIndexscan.h"
@@ -253,6 +254,10 @@ ExecReScan(PlanState *node)
 			ExecReScanSort((SortState *) node);
 			break;
 
+		case T_BatchSortState:
+			ExecReScanBatchSort((BatchSortState *)node);
+			break;
+
 		case T_IncrementalSortState:
 			ExecReScanIncrementalSort((IncrementalSortState *) node);
 			break;
diff --git a/src/backend/executor/execParallel.c b/src/backend/executor/execParallel.c
index c95d5170e4..98937302c2 100644
--- a/src/backend/executor/execParallel.c
+++ b/src/backend/executor/execParallel.c
@@ -27,6 +27,7 @@
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeAppend.h"
+#include "executor/nodeBatchSort.h"
 #include "executor/nodeBitmapHeapscan.h"
 #include "executor/nodeCustom.h"
 #include "executor/nodeForeignscan.h"
@@ -284,6 +285,10 @@ ExecParallelEstimate(PlanState *planstate, ExecParallelEstimateContext *e)
 			/* even when not parallel-aware, for EXPLAIN ANALYZE */
 			ExecSortEstimate((SortState *) planstate, e->pcxt);
 			break;
+		case T_BatchSortState:
+			if (planstate->plan->parallel_aware)
+				ExecBatchSortEstimate((BatchSortState*)planstate, e->pcxt);
+			break;
 		case T_IncrementalSortState:
 			/* even when not parallel-aware, for EXPLAIN ANALYZE */
 			ExecIncrementalSortEstimate((IncrementalSortState *) planstate, e->pcxt);
@@ -504,6 +509,10 @@ ExecParallelInitializeDSM(PlanState *planstate,
 			/* even when not parallel-aware, for EXPLAIN ANALYZE */
 			ExecSortInitializeDSM((SortState *) planstate, d->pcxt);
 			break;
+		case T_BatchSortState:
+			if (planstate->plan->parallel_aware)
+				ExecBatchSortInitializeDSM((BatchSortState*)planstate, d->pcxt);
+			break;
 		case T_IncrementalSortState:
 			/* even when not parallel-aware, for EXPLAIN ANALYZE */
 			ExecIncrementalSortInitializeDSM((IncrementalSortState *) planstate, d->pcxt);
@@ -990,6 +999,10 @@ ExecParallelReInitializeDSM(PlanState *planstate,
 		case T_IncrementalSortState:
 			/* these nodes have DSM state, but no reinitialization is required */
 			break;
+		case T_BatchSortState:
+			if (planstate->plan->parallel_aware)
+				ExecBatchSortReInitializeDSM((BatchSortState*)planstate, pcxt);
+			break;
 
 		default:
 			break;
@@ -1340,6 +1353,10 @@ ExecParallelInitializeWorker(PlanState *planstate, ParallelWorkerContext *pwcxt)
 			/* even when not parallel-aware, for EXPLAIN ANALYZE */
 			ExecSortInitializeWorker((SortState *) planstate, pwcxt);
 			break;
+		case T_BatchSortState:
+			if (planstate->plan->parallel_aware)
+				ExecBatchSortInitializeWorker((BatchSortState*)planstate, pwcxt);
+			break;
 		case T_IncrementalSortState:
 			/* even when not parallel-aware, for EXPLAIN ANALYZE */
 			ExecIncrementalSortInitializeWorker((IncrementalSortState *) planstate,
diff --git a/src/backend/executor/execProcnode.c b/src/backend/executor/execProcnode.c
index 414df50a05..d51750e4fb 100644
--- a/src/backend/executor/execProcnode.c
+++ b/src/backend/executor/execProcnode.c
@@ -75,6 +75,7 @@
 #include "executor/executor.h"
 #include "executor/nodeAgg.h"
 #include "executor/nodeAppend.h"
+#include "executor/nodeBatchSort.h"
 #include "executor/nodeBitmapAnd.h"
 #include "executor/nodeBitmapHeapscan.h"
 #include "executor/nodeBitmapIndexscan.h"
@@ -314,6 +315,11 @@ ExecInitNode(Plan *node, EState *estate, int eflags)
 												estate, eflags);
 			break;
 
+		case T_BatchSort:
+			result = (PlanState *) ExecInitBatchSort((BatchSort *) node,
+													 estate, eflags);
+			break;
+
 		case T_IncrementalSort:
 			result = (PlanState *) ExecInitIncrementalSort((IncrementalSort *) node,
 														   estate, eflags);
@@ -699,6 +705,10 @@ ExecEndNode(PlanState *node)
 			ExecEndSort((SortState *) node);
 			break;
 
+		case T_BatchSortState:
+			ExecEndBatchSort((BatchSortState *) node);
+			break;
+
 		case T_IncrementalSortState:
 			ExecEndIncrementalSort((IncrementalSortState *) node);
 			break;
diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c
index 601b6dab03..ad5e67eddf 100644
--- a/src/backend/executor/nodeAgg.c
+++ b/src/backend/executor/nodeAgg.c
@@ -256,7 +256,10 @@
 #include "optimizer/optimizer.h"
 #include "parser/parse_agg.h"
 #include "parser/parse_coerce.h"
+#include "pgstat.h"
+#include "storage/barrier.h"
 #include "utils/acl.h"
+#include "utils/batchstore.h"
 #include "utils/builtins.h"
 #include "utils/datum.h"
 #include "utils/dynahash.h"
@@ -311,6 +314,11 @@
  */
 #define CHUNKHDRSZ 16
 
+#define SHARED_AGG_MAGIC		UINT64CONST(0x4141bbcd61518e52)
+#define SHARED_AGG_KEY_INFO		UINT64CONST(0xD000000000000001)
+#define SHARED_AGG_KEY_BARRIER	UINT64CONST(0xD000000000000002)
+#define SHARED_AGG_KEY_FILE_SET	UINT64CONST(0xD000000000000003)
+
 /*
  * Track all tapes needed for a HashAgg that spills. We don't know the maximum
  * number of tapes needed at the start of the algorithm (because it can
@@ -446,7 +454,7 @@ static HashAggBatch *hashagg_batch_new(LogicalTapeSet *tapeset,
 									   int input_tapenum, int setno,
 									   int64 input_tuples, double input_card,
 									   int used_bits);
-static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
+static bool hashagg_batch_read(void *userdata, TupleTableSlot *slot, uint32 *hashp);
 static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
 							   int used_bits, double input_groups,
 							   double hashentrysize);
@@ -465,6 +473,8 @@ static void build_pertrans_for_aggref(AggStatePerTrans pertrans,
 									  Oid aggserialfn, Oid aggdeserialfn,
 									  Datum initValue, bool initValueIsNull,
 									  Oid *inputTypes, int numArguments);
+static TupleTableSlot *ExecBatchHashAggPrepare(PlanState *pstate);
+static TupleTableSlot *ExecBatchHashAggFirstReScan(PlanState *pstate);
 
 
 /*
@@ -1330,7 +1340,8 @@ finalize_aggregates(AggState *aggstate,
 		if (pertrans->numSortCols > 0)
 		{
 			Assert(aggstate->aggstrategy != AGG_HASHED &&
-				   aggstate->aggstrategy != AGG_MIXED);
+				   aggstate->aggstrategy != AGG_MIXED &&
+				   aggstate->aggstrategy != AGG_BATCH_HASH);
 
 			if (pertrans->numInputs == 1)
 				process_ordered_aggregate_single(aggstate,
@@ -1502,8 +1513,10 @@ build_hash_table(AggState *aggstate, int setno, long nbuckets)
 	MemoryContext hashcxt = aggstate->hashcontext->ecxt_per_tuple_memory;
 	MemoryContext tmpcxt = aggstate->tmpcontext->ecxt_per_tuple_memory;
 	Size		additionalsize;
+	bool		use_hash_iv;
 
 	Assert(aggstate->aggstrategy == AGG_HASHED ||
+		   aggstate->aggstrategy == AGG_BATCH_HASH ||
 		   aggstate->aggstrategy == AGG_MIXED);
 
 	/*
@@ -1514,6 +1527,15 @@ build_hash_table(AggState *aggstate, int setno, long nbuckets)
 	 */
 	additionalsize = aggstate->numtrans * sizeof(AggStatePerGroupData);
 
+	/*
+	 * In batch hash mode each worker must use the same hash algorithm,
+	 * because we need to ensure that the same records(group clauses) are in the same batch
+	 */
+	if (aggstate->aggstrategy == AGG_BATCH_HASH)
+		use_hash_iv = false;
+	else
+		use_hash_iv = DO_AGGSPLIT_SKIPFINAL(aggstate->aggsplit);
+
 	perhash->hashtable = BuildTupleHashTableExt(&aggstate->ss.ps,
 												perhash->hashslot->tts_tupleDescriptor,
 												perhash->numCols,
@@ -1526,7 +1548,7 @@ build_hash_table(AggState *aggstate, int setno, long nbuckets)
 												metacxt,
 												hashcxt,
 												tmpcxt,
-												DO_AGGSPLIT_SKIPFINAL(aggstate->aggsplit));
+												use_hash_iv);
 }
 
 /*
@@ -1625,10 +1647,15 @@ find_hash_columns(AggState *aggstate)
 			palloc(maxCols * sizeof(AttrNumber));
 		perhash->hashGrpColIdxHash =
 			palloc(perhash->numCols * sizeof(AttrNumber));
+		perhash->colnos_needed = bms_copy(aggregated_colnos);
 
 		/* Add all the grouping columns to colnos */
 		for (i = 0; i < perhash->numCols; i++)
+		{
 			colnos = bms_add_member(colnos, grpColIdx[i]);
+			perhash->colnos_needed = bms_add_member(perhash->colnos_needed,
+													grpColIdx[i]);
+		}
 
 		/*
 		 * First build mapping for columns directly hashed. These are the
@@ -1738,9 +1765,11 @@ hashagg_recompile_expressions(AggState *aggstate, bool minslot, bool nullcheck)
 	int			j = nullcheck ? 1 : 0;
 
 	Assert(aggstate->aggstrategy == AGG_HASHED ||
+		   aggstate->aggstrategy == AGG_BATCH_HASH ||
 		   aggstate->aggstrategy == AGG_MIXED);
 
-	if (aggstate->aggstrategy == AGG_HASHED)
+	if (aggstate->aggstrategy == AGG_HASHED ||
+		aggstate->aggstrategy == AGG_BATCH_HASH)
 		phase = &aggstate->phases[0];
 	else						/* AGG_MIXED */
 		phase = &aggstate->phases[1];
@@ -2168,6 +2197,9 @@ ExecAgg(PlanState *pstate)
 			case AGG_SORTED:
 				result = agg_retrieve_direct(node);
 				break;
+			case AGG_BATCH_HASH:
+				elog(ERROR, "batch hash should not run in function ExecAgg");
+				break;
 		}
 
 		if (!TupIsNull(result))
@@ -2568,35 +2600,20 @@ agg_fill_hash_table(AggState *aggstate)
 						   &aggstate->perhash[0].hashiter);
 }
 
-/*
- * If any data was spilled during hash aggregation, reset the hash table and
- * reprocess one batch of spilled data. After reprocessing a batch, the hash
- * table will again contain data, ready to be consumed by
- * agg_retrieve_hash_table_in_memory().
- *
- * Should only be called after all in memory hash table entries have been
- * finalized and emitted.
- *
- * Return false when input is exhausted and there's no more work to be done;
- * otherwise return true.
- */
-static bool
-agg_refill_hash_table(AggState *aggstate)
+static void
+agg_refill_hash_table_ex(AggState *aggstate,
+						 bool (*read_tup)(void *userdata, TupleTableSlot *slot, uint32 *hash),
+						 void *userdata,
+						 int used_bits,
+						 double input_groups,
+						 int setno)
 {
-	HashAggBatch *batch;
 	AggStatePerHash perhash;
 	HashAggSpill spill;
-	HashTapeInfo *tapeinfo = aggstate->hash_tapeinfo;
 	bool		spill_initialized = false;
 
-	if (aggstate->hash_batches == NIL)
-		return false;
-
-	batch = linitial(aggstate->hash_batches);
-	aggstate->hash_batches = list_delete_first(aggstate->hash_batches);
-
-	hash_agg_set_limits(aggstate->hashentrysize, batch->input_card,
-						batch->used_bits, &aggstate->hash_mem_limit,
+	hash_agg_set_limits(aggstate->hashentrysize, input_groups,
+						used_bits, &aggstate->hash_mem_limit,
 						&aggstate->hash_ngroups_limit, NULL);
 
 	/*
@@ -2628,7 +2645,7 @@ agg_refill_hash_table(AggState *aggstate)
 		aggstate->phase = &aggstate->phases[aggstate->current_phase];
 	}
 
-	select_current_set(aggstate, batch->setno, true);
+	select_current_set(aggstate, setno, true);
 
 	perhash = &aggstate->perhash[aggstate->current_set];
 
@@ -2646,31 +2663,27 @@ agg_refill_hash_table(AggState *aggstate)
 		TupleTableSlot *spillslot = aggstate->hash_spill_rslot;
 		TupleTableSlot *hashslot = perhash->hashslot;
 		TupleHashEntry entry;
-		MinimalTuple tuple;
 		uint32		hash;
 		bool		isnew = false;
 		bool	   *p_isnew = aggstate->hash_spill_mode ? NULL : &isnew;
 
 		CHECK_FOR_INTERRUPTS();
 
-		tuple = hashagg_batch_read(batch, &hash);
-		if (tuple == NULL)
+		if ((*read_tup)(userdata, spillslot, &hash) == false)
 			break;
 
-		ExecStoreMinimalTuple(tuple, spillslot, true);
 		aggstate->tmpcontext->ecxt_outertuple = spillslot;
 
 		prepare_hash_slot(perhash,
-						  aggstate->tmpcontext->ecxt_outertuple,
+						  spillslot,
 						  hashslot);
-		entry = LookupTupleHashEntryHash(
-										 perhash->hashtable, hashslot, p_isnew, hash);
+		entry = LookupTupleHashEntryHash(perhash->hashtable, hashslot, p_isnew, hash);
 
 		if (entry != NULL)
 		{
 			if (isnew)
 				initialize_hash_entry(aggstate, perhash->hashtable, entry);
-			aggstate->hash_pergroup[batch->setno] = entry->additional;
+			aggstate->hash_pergroup[setno] = entry->additional;
 			advance_aggregates(aggstate);
 		}
 		else
@@ -2682,13 +2695,13 @@ agg_refill_hash_table(AggState *aggstate)
 				 * that we don't assign tapes that will never be used.
 				 */
 				spill_initialized = true;
-				hashagg_spill_init(&spill, tapeinfo, batch->used_bits,
-								   batch->input_card, aggstate->hashentrysize);
+				hashagg_spill_init(&spill, aggstate->hash_tapeinfo, used_bits,
+								   input_groups, aggstate->hashentrysize);
 			}
 			/* no memory for a new group, spill */
 			hashagg_spill_tuple(aggstate, &spill, spillslot, hash);
 
-			aggstate->hash_pergroup[batch->setno] = NULL;
+			aggstate->hash_pergroup[setno] = NULL;
 		}
 
 		/*
@@ -2698,15 +2711,13 @@ agg_refill_hash_table(AggState *aggstate)
 		ResetExprContext(aggstate->tmpcontext);
 	}
 
-	hashagg_tapeinfo_release(tapeinfo, batch->input_tapenum);
-
 	/* change back to phase 0 */
 	aggstate->current_phase = 0;
 	aggstate->phase = &aggstate->phases[aggstate->current_phase];
 
 	if (spill_initialized)
 	{
-		hashagg_spill_finish(aggstate, &spill, batch->setno);
+		hashagg_spill_finish(aggstate, &spill, setno);
 		hash_agg_update_metrics(aggstate, true, spill.npartitions);
 	}
 	else
@@ -2715,9 +2726,43 @@ agg_refill_hash_table(AggState *aggstate)
 	aggstate->hash_spill_mode = false;
 
 	/* prepare to walk the first hash table */
-	select_current_set(aggstate, batch->setno, true);
-	ResetTupleHashIterator(aggstate->perhash[batch->setno].hashtable,
-						   &aggstate->perhash[batch->setno].hashiter);
+	select_current_set(aggstate, setno, true);
+	ResetTupleHashIterator(aggstate->perhash[setno].hashtable,
+						   &aggstate->perhash[setno].hashiter);
+}
+
+/*
+ * If any data was spilled during hash aggregation, reset the hash table and
+ * reprocess one batch of spilled data. After reprocessing a batch, the hash
+ * table will again contain data, ready to be consumed by
+ * agg_retrieve_hash_table_in_memory().
+ *
+ * Should only be called after all in memory hash table entries have been
+ * finalized and emitted.
+ *
+ * Return false when input is exhausted and there's no more work to be done;
+ * otherwise return true.
+ */
+static bool
+agg_refill_hash_table(AggState *aggstate)
+{
+	HashAggBatch *batch;
+
+	if (aggstate->hash_batches == NIL)
+		return false;
+
+	batch = linitial(aggstate->hash_batches);
+	aggstate->hash_batches = list_delete_first(aggstate->hash_batches);
+
+	agg_refill_hash_table_ex(aggstate,
+							 hashagg_batch_read,
+							 batch,
+							 batch->used_bits,
+							 batch->input_card,
+							 batch->setno);
+
+	hashagg_tapeinfo_release(aggstate->hash_tapeinfo,
+							 batch->input_tapenum);
 
 	pfree(batch);
 
@@ -3056,11 +3101,12 @@ hashagg_batch_new(LogicalTapeSet *tapeset, int tapenum, int setno,
 
 /*
  * read_spilled_tuple
- * 		read the next tuple from a batch's tape.  Return NULL if no more.
+ * 		read the next tuple from a batch's tape.  Return false if no more.
  */
-static MinimalTuple
-hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
+static bool
+hashagg_batch_read(void *userdata, TupleTableSlot *slot, uint32 *hashp)
 {
+	HashAggBatch *batch = userdata;
 	LogicalTapeSet *tapeset = batch->tapeset;
 	int			tapenum = batch->input_tapenum;
 	MinimalTuple tuple;
@@ -3070,7 +3116,7 @@ hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
 
 	nread = LogicalTapeRead(tapeset, tapenum, &hash, sizeof(uint32));
 	if (nread == 0)
-		return NULL;
+		return false;
 	if (nread != sizeof(uint32))
 		ereport(ERROR,
 				(errcode_for_file_access(),
@@ -3098,7 +3144,8 @@ hashagg_batch_read(HashAggBatch *batch, uint32 *hashp)
 				 errmsg("unexpected EOF for tape %d: requested %zu bytes, read %zu bytes",
 						tapenum, t_len - sizeof(uint32), nread)));
 
-	return tuple;
+	ExecStoreMinimalTuple(tuple, slot, true);
+	return true;
 }
 
 /*
@@ -3261,6 +3308,7 @@ ExecInitAgg(Agg *node, EState *estate, int eflags)
 	int			i = 0;
 	int			j = 0;
 	bool		use_hashing = (node->aggstrategy == AGG_HASHED ||
+							   node->aggstrategy == AGG_BATCH_HASH ||
 							   node->aggstrategy == AGG_MIXED);
 
 	/* check for unsupported flags */
@@ -3272,7 +3320,10 @@ ExecInitAgg(Agg *node, EState *estate, int eflags)
 	aggstate = makeNode(AggState);
 	aggstate->ss.ps.plan = (Plan *) node;
 	aggstate->ss.ps.state = estate;
-	aggstate->ss.ps.ExecProcNode = ExecAgg;
+	if (node->aggstrategy == AGG_BATCH_HASH)
+		aggstate->ss.ps.ExecProcNode = ExecBatchHashAggPrepare;
+	else
+		aggstate->ss.ps.ExecProcNode = ExecAgg;
 
 	aggstate->aggs = NIL;
 	aggstate->numaggs = 0;
@@ -3319,7 +3370,8 @@ ExecInitAgg(Agg *node, EState *estate, int eflags)
 			 * additional AGG_HASHED aggs become part of phase 0, but all
 			 * others add an extra phase.
 			 */
-			if (agg->aggstrategy != AGG_HASHED)
+			if (agg->aggstrategy != AGG_HASHED &&
+				agg->aggstrategy != AGG_BATCH_HASH)
 				++numPhases;
 			else
 				++numHashes;
@@ -3366,7 +3418,8 @@ ExecInitAgg(Agg *node, EState *estate, int eflags)
 	 * If we are doing a hashed aggregation then the child plan does not need
 	 * to handle REWIND efficiently; see ExecReScanAgg.
 	 */
-	if (node->aggstrategy == AGG_HASHED)
+	if (node->aggstrategy == AGG_HASHED ||
+		node->aggstrategy == AGG_BATCH_HASH)
 		eflags &= ~EXEC_FLAG_REWIND;
 	outerPlan = outerPlan(node);
 	outerPlanState(aggstate) = ExecInitNode(outerPlan, estate, eflags);
@@ -3374,9 +3427,20 @@ ExecInitAgg(Agg *node, EState *estate, int eflags)
 	/*
 	 * initialize source tuple type.
 	 */
-	aggstate->ss.ps.outerops =
-		ExecGetResultSlotOps(outerPlanState(&aggstate->ss),
-							 &aggstate->ss.ps.outeropsfixed);
+	if (node->aggstrategy == AGG_BATCH_HASH)
+	{
+		/*
+		 * When we prepared tuples from outer tree, we always read
+		 * tuples from saved files and it is MinimalTuple
+		 */
+		aggstate->ss.ps.outerops = &TTSOpsMinimalTuple;
+		aggstate->ss.ps.outeropsfixed = true;
+	}else
+	{
+		aggstate->ss.ps.outerops =
+			ExecGetResultSlotOps(outerPlanState(&aggstate->ss),
+								 &aggstate->ss.ps.outeropsfixed);
+	}
 	aggstate->ss.ps.outeropsset = true;
 
 	ExecCreateScanSlotFromOuterPlan(estate, &aggstate->ss,
@@ -3484,6 +3548,7 @@ ExecInitAgg(Agg *node, EState *estate, int eflags)
 		Assert(phase <= 1 || sortnode);
 
 		if (aggnode->aggstrategy == AGG_HASHED
+			|| aggnode->aggstrategy == AGG_BATCH_HASH
 			|| aggnode->aggstrategy == AGG_MIXED)
 		{
 			AggStatePerPhase phasedata = &aggstate->phases[0];
@@ -3696,7 +3761,8 @@ ExecInitAgg(Agg *node, EState *estate, int eflags)
 	 * hashing is being done too, then phase 0 is processed last); but if only
 	 * hashing is being done, then phase 0 is all there is.
 	 */
-	if (node->aggstrategy == AGG_HASHED)
+	if (node->aggstrategy == AGG_HASHED ||
+		node->aggstrategy == AGG_BATCH_HASH)
 	{
 		aggstate->current_phase = 0;
 		initialize_phase(aggstate, 0);
@@ -4011,7 +4077,8 @@ ExecInitAgg(Agg *node, EState *estate, int eflags)
 			dohash = false;
 			dosort = true;
 		}
-		else if (phase->aggstrategy == AGG_HASHED)
+		else if (phase->aggstrategy == AGG_HASHED ||
+				 phase->aggstrategy == AGG_BATCH_HASH)
 		{
 			dohash = true;
 			dosort = false;
@@ -4470,6 +4537,53 @@ ExecReScanAgg(AggState *node)
 			return;
 		}
 	}
+	else if (node->aggstrategy == AGG_BATCH_HASH)
+	{
+		/* If we haven't yet filled the batch store then we can just return */
+		if (!node->batch_filled)
+			return;
+
+		/*
+		 * If we filled the batch store and subplan does not have
+		 * any parameter changes(except only gather parameter),
+		 * we can rescan from batch store
+		 */
+		if (outerPlan->chgParam == NULL ||
+			(aggnode->gatherParam >= 0 &&
+			 bms_membership(outerPlan->chgParam) == BMS_SINGLETON &&
+			 bms_is_member(aggnode->gatherParam, outerPlan->chgParam)))
+		{
+			/* Rescan each set's batch store */
+			for (setno = 0; setno < numGroupingSets; ++setno)
+				bs_rescan(node->perhash[setno].batch_store);
+
+			/*
+			 * We don't want call ExecBatchHashAggNextBatch() for now,
+			 * this will immediately take up a batch, which is not what we want
+			 * in parallel mode. Because it is possible that other workers
+			 * will finish processing other batches very quickly,
+			 * and we did occupy a batch and we have been in no hurry to deal with it.
+			 *
+			 * So we use a new function ExecBatchHashAggFirstReScan call
+			 * ExecBatchHashAggNextBatch
+			 */
+			ExecSetExecProcNode(&node->ss.ps, ExecBatchHashAggFirstReScan);
+
+			return;
+		}
+
+		/* Clear all set's data */
+		for (setno = 0; setno < numGroupingSets; ++setno)
+			bs_clear(node->perhash[setno].batch_store);
+
+		/* Reset build barrier */
+		if (node->batch_barrier)
+			BarrierInit(node->batch_barrier, 0);
+
+		ExecSetExecProcNode(&node->ss.ps, ExecBatchHashAggPrepare);
+		node->batch_filled = false;
+		node->current_batch_set = 0;
+	}
 
 	/* Make sure we have closed any open tuplesorts */
 	for (transno = 0; transno < node->numtrans; transno++)
@@ -4512,11 +4626,13 @@ ExecReScanAgg(AggState *node)
 	MemSet(econtext->ecxt_aggnulls, 0, sizeof(bool) * node->numaggs);
 
 	/*
-	 * With AGG_HASHED/MIXED, the hash table is allocated in a sub-context of
+	 * With AGG_HASHED/AGG_BATCH_HASH/MIXED, the hash table is allocated in a sub-context of
 	 * the hashcontext. This used to be an issue, but now, resetting a context
 	 * automatically deletes sub-contexts too.
 	 */
-	if (node->aggstrategy == AGG_HASHED || node->aggstrategy == AGG_MIXED)
+	if (node->aggstrategy == AGG_HASHED ||
+		node->aggstrategy == AGG_BATCH_HASH ||
+		node->aggstrategy == AGG_MIXED)
 	{
 		hashagg_reset_spill_state(node);
 
@@ -4533,7 +4649,8 @@ ExecReScanAgg(AggState *node)
 		hashagg_recompile_expressions(node, false, false);
 	}
 
-	if (node->aggstrategy != AGG_HASHED)
+	if (node->aggstrategy != AGG_HASHED &&
+		node->aggstrategy != AGG_BATCH_HASH)
 	{
 		/*
 		 * Reset the per-group state (in particular, mark transvalues null)
@@ -4738,12 +4855,277 @@ AggRegisterCallback(FunctionCallInfo fcinfo,
 	elog(ERROR, "aggregate function cannot register a callback in this context");
 }
 
+/* Read tuple from batch store, Return false if no more */
+static bool
+batchstore_read(void *userdata, TupleTableSlot *slot, uint32 *hashp)
+{
+	MinimalTuple mtup = bs_read_hash(userdata, hashp);
+	if (unlikely(mtup == NULL))
+		return false;
+	ExecStoreMinimalTuple(mtup, slot, false);
+	return true;
+}
+
+/*
+ * Check have next batch. if not return false
+ * else open it and fill tuples into hash table
+ */
+static bool
+ExecBatchHashAggNextBatch(AggState *node)
+{
+	while (bs_next_batch(node->perhash[node->current_batch_set].batch_store, false) == false)
+	{
+		++node->current_batch_set;
+		if (node->current_batch_set >= node->num_hashes)
+		{
+			node->agg_done = true;
+			return false;
+		}
+	}
+
+	/* refill hash table from batch store */
+	agg_refill_hash_table_ex(node,
+							 batchstore_read,
+							 node->perhash[node->current_batch_set].batch_store,
+							 0,
+							 node->perhash[node->current_batch_set].aggnode->numGroups,
+							 node->current_batch_set);
+	return true;
+}
+
+static TupleTableSlot *
+ExecBatchHashAgg(PlanState *pstate)
+{
+	AggState	   *node = castNode(AggState, pstate);
+	TupleTableSlot *result;
+
+reloop:
+	result = agg_retrieve_hash_table_in_memory(node);
+	if (unlikely(result == NULL))
+	{
+		if (agg_refill_hash_table(node) == false &&
+			ExecBatchHashAggNextBatch(node) == false)
+		{
+			return NULL;
+		}else
+		{
+			goto reloop;
+		}
+	}
+
+	return result;
+}
+
+/*
+ * Test have next batch, set real execute function and call it if have next batch,
+ * else return an empty TupleTableSlot
+ */
+TupleTableSlot *
+ExecBatchHashAggFirstReScan(PlanState *pstate)
+{
+	AggState   *node = castNode(AggState, pstate);
+
+	node->current_batch_set = 0;
+	if (ExecBatchHashAggNextBatch(node) == false)
+		return ExecClearTuple(pstate->ps_ResultTupleSlot);
+
+	ExecSetExecProcNode(&node->ss.ps, ExecBatchHashAgg);
+	return ExecBatchHashAgg(pstate);
+}
+
+/*
+ * ExecBatchHashAggPrepare -
+ *  ExecBatchHashAggPrepare receives tuples from its outer subplan and save
+ *  into batchstore. after finish save tuples set execute function as
+ *  ExecBatchHashAgg and call it.
+ */
+static TupleTableSlot *
+ExecBatchHashAggPrepare(PlanState *pstate)
+{
+	int				i,x,max_colno_needed;
+	MinimalTuple	mtup;
+	TupleTableSlot *inputslot;
+	PlanState	   *outer = outerPlanState(pstate);
+	AggState	   *node = castNode(AggState, pstate);
+	ExprContext	   *tmpcontext = node->tmpcontext;
+	bool		   *isnull;
+	Bitmapset	   *colnos_needed;
+	Bitmapset	  **colnos_neededs;
+	Assert(node->aggstrategy == AGG_BATCH_HASH);
+	Assert(node->perhash[0].batch_store == NULL ||
+		   node->batch_barrier != NULL);
+
+	if (node->agg_done)
+		return NULL;
+
+	/* create batch store if not parallel */
+	if (node->perhash[0].batch_store == NULL)
+	{
+		MemoryContext	oldcontext = MemoryContextSwitchTo(GetMemoryChunkContext(pstate));
+		Agg			   *agg = castNode(Agg, pstate->plan);
+		ListCell	   *lc;
+
+		node->perhash[0].batch_store = bs_begin_hash(agg->numBatches);
+
+		i = 1;
+		foreach (lc, agg->chain)
+		{
+			Agg *subagg = lfirst_node(Agg, lc);
+			Assert(subagg->aggstrategy == AGG_BATCH_HASH);
+			Assert(i < node->num_hashes);
+			node->perhash[i].batch_store = bs_begin_hash(subagg->numBatches);
+			++i;
+		}
+
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	if (node->batch_barrier &&
+		BarrierAttach(node->batch_barrier) > 0)
+	{
+		BarrierDetach(node->batch_barrier);
+		goto batches_already_done_;
+	}
+
+	/* read for make minimal tuple */
+	isnull = palloc(sizeof(isnull[0]) * node->hash_spill_wslot->tts_tupleDescriptor->natts);
+	memset(isnull, true, sizeof(isnull[0]) * node->hash_spill_wslot->tts_tupleDescriptor->natts);
+	max_colno_needed = node->max_colno_needed;
+
+	/* convert Attribute numbers to index(start with 0) */
+	colnos_neededs = palloc(sizeof(colnos_neededs[0]) * node->num_hashes);
+	for (i=0;i<node->num_hashes;++i)
+	{
+		AggStatePerHash	perhash = &node->perhash[i];
+		colnos_needed = NULL;
+		x = -1;
+		while ((x=bms_next_member(perhash->colnos_needed, x)) >= 0)
+		{
+			Assert(x > 0);
+			colnos_needed = bms_add_member(colnos_needed, x-1);
+		}
+		colnos_neededs[i] = colnos_needed;
+	}
+
+	for (;;)
+	{
+		CHECK_FOR_INTERRUPTS();
+		inputslot = ExecProcNode(outer);
+		if (TupIsNull(inputslot))
+			break;
+
+		tmpcontext->ecxt_outertuple = inputslot;
+		slot_getsomeattrs(inputslot, max_colno_needed);
+
+		for (i=0;i<node->num_hashes;++i)
+		{
+			AggStatePerHash	perhash = &node->perhash[i];
+			TupleTableSlot *hashslot = perhash->hashslot;
+
+			CHECK_FOR_INTERRUPTS();
+
+			/* mark unneeded columns as null */
+			memset(isnull, true, sizeof(isnull[0]) * max_colno_needed);
+			colnos_needed = colnos_neededs[i];
+			x = -1;
+			while ((x = bms_next_member(colnos_needed, x)) >= 0)
+				isnull[x] = inputslot->tts_isnull[x];
+			/* make minimal tuple from we needed columns for this set */
+			mtup = heap_form_minimal_tuple(inputslot->tts_tupleDescriptor,
+										   inputslot->tts_values,
+										   isnull);
+
+			prepare_hash_slot(perhash, inputslot, hashslot);
+
+			bs_write_hash(perhash->batch_store,
+						  mtup,
+						  TupleHashTableHash(perhash->hashtable, hashslot));
+			pfree(mtup);
+			ResetExprContext(tmpcontext);
+		}
+	}
+
+	/* Before read, must call bs_end_write() function */
+	for (i=0;i<node->num_hashes;++i)
+		bs_end_write(node->perhash[i].batch_store);
+
+	if (node->batch_barrier)
+	{
+		/* wait other workers finish write */
+		BarrierArriveAndWait(node->batch_barrier, WAIT_EVENT_BATCH_HASH_BUILD);
+		BarrierDetach(node->batch_barrier);
+	}
+
+	/* clear temp memory */
+	for (i=0;i<node->num_hashes;++i)
+		bms_free(colnos_neededs[i]);
+	pfree(colnos_neededs);
+	pfree(isnull);
+
+batches_already_done_:
+	node->batch_filled = true;
+	node->current_batch_set = 0;
+	if (ExecBatchHashAggNextBatch(node) == false)
+		return NULL;
+
+	ExecSetExecProcNode(pstate, ExecBatchHashAgg);
+	return ExecBatchHashAgg(pstate);
+}
 
 /* ----------------------------------------------------------------
  *						Parallel Query Support
  * ----------------------------------------------------------------
  */
 
+static Size
+ExecAggEstimateToc(AggState *node, ParallelContext *pcxt)
+{
+	Size				size;
+	shm_toc_estimator	estimator;
+	ListCell		   *lc;
+
+	/* don't need this if no workers */
+	if (pcxt->nworkers == 0)
+		return 0;
+	/* don't need this if not instrumenting and not batch hash agg */
+	if (!node->ss.ps.instrument &&
+		node->aggstrategy != AGG_BATCH_HASH)
+		return 0;
+
+	shm_toc_initialize_estimator(&estimator);
+	if (node->ss.ps.instrument)
+	{
+		size = mul_size(pcxt->nworkers, sizeof(AggregateInstrumentation));
+		size = add_size(size, offsetof(SharedAggInfo, sinstrument));
+		shm_toc_estimate_chunk(&estimator, size);
+		shm_toc_estimate_keys(&estimator, 1);
+	}
+
+	if (node->aggstrategy == AGG_BATCH_HASH)
+	{
+		int nparticipants = pcxt->nworkers + 1;
+		shm_toc_estimate_chunk(&estimator, BUFFERALIGN(sizeof(Barrier)));
+		shm_toc_estimate_chunk(&estimator, BUFFERALIGN(sizeof(SharedFileSet)));
+		shm_toc_estimate_keys(&estimator, 2);
+
+		size = bs_parallel_hash_estimate(castNode(Agg, node->ss.ps.plan)->numBatches,
+										 nparticipants);
+		shm_toc_estimate_chunk(&estimator, BUFFERALIGN(size));
+		shm_toc_estimate_keys(&estimator, 1);
+
+		foreach (lc, castNode(Agg, node->ss.ps.plan)->chain)
+		{
+			Agg *agg = lfirst_node(Agg, lc);
+			Assert(agg->aggstrategy == AGG_BATCH_HASH);
+			size = bs_parallel_hash_estimate(agg->numBatches, nparticipants);
+			shm_toc_estimate_chunk(&estimator, BUFFERALIGN(size));
+			shm_toc_estimate_keys(&estimator, 1);
+		}
+	}
+
+	return shm_toc_estimate(&estimator);
+}
+
  /* ----------------------------------------------------------------
   *		ExecAggEstimate
   *
@@ -4753,14 +5135,7 @@ AggRegisterCallback(FunctionCallInfo fcinfo,
 void
 ExecAggEstimate(AggState *node, ParallelContext *pcxt)
 {
-	Size		size;
-
-	/* don't need this if not instrumenting or no workers */
-	if (!node->ss.ps.instrument || pcxt->nworkers == 0)
-		return;
-
-	size = mul_size(pcxt->nworkers, sizeof(AggregateInstrumentation));
-	size = add_size(size, offsetof(SharedAggInfo, sinstrument));
+	Size size = ExecAggEstimateToc(node, pcxt);
 	shm_toc_estimate_chunk(&pcxt->estimator, size);
 	shm_toc_estimate_keys(&pcxt->estimator, 1);
 }
@@ -4775,19 +5150,82 @@ void
 ExecAggInitializeDSM(AggState *node, ParallelContext *pcxt)
 {
 	Size		size;
+	shm_toc	   *toc;
+	void	   *addr;
 
-	/* don't need this if not instrumenting or no workers */
-	if (!node->ss.ps.instrument || pcxt->nworkers == 0)
+	size = ExecAggEstimateToc(node, pcxt);
+	if (size == 0)
 		return;
 
-	size = offsetof(SharedAggInfo, sinstrument)
-		+ pcxt->nworkers * sizeof(AggregateInstrumentation);
-	node->shared_info = shm_toc_allocate(pcxt->toc, size);
-	/* ensure any unfilled slots will contain zeroes */
-	memset(node->shared_info, 0, size);
-	node->shared_info->num_workers = pcxt->nworkers;
-	shm_toc_insert(pcxt->toc, node->ss.ps.plan->plan_node_id,
-				   node->shared_info);
+	addr = shm_toc_allocate(pcxt->toc, size);
+	toc = shm_toc_create(SHARED_AGG_MAGIC, addr, size);
+	shm_toc_insert(pcxt->toc, node->ss.ps.plan->plan_node_id, addr);
+
+	if (node->ss.ps.instrument)
+	{
+		size = offsetof(SharedAggInfo, sinstrument)
+			+ pcxt->nworkers * sizeof(AggregateInstrumentation);
+		node->shared_info = shm_toc_allocate(toc, size);
+		/* ensure any unfilled slots will contain zeroes */
+		memset(node->shared_info, 0, size);
+		node->shared_info->num_workers = pcxt->nworkers;
+		shm_toc_insert(toc, SHARED_AGG_KEY_INFO, node->shared_info);
+	}
+
+	if (node->aggstrategy == AGG_BATCH_HASH)
+	{
+		int				nparticipants = pcxt->nworkers + 1;
+		int				i = 0;
+		ListCell	   *lc;
+		Agg			   *agg;
+
+		/* Initialize shared file set */
+		SharedFileSet  *fset = shm_toc_allocate(toc, sizeof(SharedFileSet));
+		SharedFileSetInit(fset, pcxt->seg);
+		shm_toc_insert(toc, SHARED_AGG_KEY_FILE_SET, fset);
+
+		/* Initialize build barrier */
+		node->batch_barrier = shm_toc_allocate(toc, sizeof(Barrier));
+		BarrierInit(node->batch_barrier, 0);
+		shm_toc_insert(toc, SHARED_AGG_KEY_BARRIER, node->batch_barrier);
+
+		/* Initialize batch store */
+		agg = castNode(Agg, node->ss.ps.plan);
+		Assert(agg->numBatches > 0);
+		size = bs_parallel_hash_estimate(agg->numBatches, nparticipants);
+		addr = shm_toc_allocate(toc, size);
+		shm_toc_insert(toc, 0, addr);
+		node->perhash[0].batch_store = bs_init_parallel_hash(agg->numBatches,
+															 nparticipants,
+															 0,
+															 addr,
+															 pcxt->seg,
+															 fset,
+															 "BatchHashAgg");
+
+		/* Initialize batch store for other sets(if has) */
+		i = 1;
+		foreach (lc, agg->chain)
+		{
+			Agg	   *subagg = lfirst_node(Agg, lc);
+			char	name[30];
+			Assert(subagg->aggstrategy == AGG_BATCH_HASH &&
+				   subagg->numBatches > 0);
+			Assert(i < node->num_hashes);
+			size = bs_parallel_hash_estimate(subagg->numBatches, nparticipants);
+			addr = shm_toc_allocate(toc, size);
+			shm_toc_insert(toc, i, addr);
+			sprintf(name, "BatchHashAgg%d", i);
+			node->perhash[i].batch_store = bs_init_parallel_hash(subagg->numBatches,
+																 nparticipants,
+																 0,
+																 addr,
+																 pcxt->seg,
+																 fset,
+																 name);
+			++i;
+		}
+	}
 }
 
 /* ----------------------------------------------------------------
@@ -4799,8 +5237,43 @@ ExecAggInitializeDSM(AggState *node, ParallelContext *pcxt)
 void
 ExecAggInitializeWorker(AggState *node, ParallelWorkerContext *pwcxt)
 {
-	node->shared_info =
-		shm_toc_lookup(pwcxt->toc, node->ss.ps.plan->plan_node_id, true);
+	shm_toc	   *toc;
+	void	   *addr = shm_toc_lookup(pwcxt->toc, node->ss.ps.plan->plan_node_id, true);
+	if (addr == NULL)
+	{
+		Assert(node->aggstrategy != AGG_BATCH_HASH);
+		return;
+	}
+	toc = shm_toc_attach(SHARED_AGG_MAGIC, addr);
+	node->shared_info = shm_toc_lookup(toc, SHARED_AGG_KEY_INFO, true);
+
+	if (node->aggstrategy == AGG_BATCH_HASH)
+	{
+		int				i;
+		ListCell	   *lc;
+		Agg			   *agg = castNode(Agg, node->ss.ps.plan);
+		SharedFileSet  *fset = shm_toc_lookup(toc, SHARED_AGG_KEY_FILE_SET, false);
+
+		node->batch_barrier = shm_toc_lookup(toc, SHARED_AGG_KEY_BARRIER, false);
+		node->perhash[0].batch_store =
+			bs_attach_parallel_hash(shm_toc_lookup(toc, 0, false),
+									pwcxt->seg,
+									fset,
+									ParallelWorkerNumber+1);
+
+		i = 1;
+		foreach (lc, agg->chain)
+		{
+			Assert(lfirst_node(Agg, lc)->aggstrategy == AGG_BATCH_HASH);
+			Assert (i<node->num_hashes);
+			node->perhash[i].batch_store =
+				bs_attach_parallel_hash(shm_toc_lookup(toc, i, false),
+										pwcxt->seg,
+										fset,
+										ParallelWorkerNumber+1);
+			++i;
+		}
+	}
 }
 
 /* ----------------------------------------------------------------
diff --git a/src/backend/executor/nodeBatchSort.c b/src/backend/executor/nodeBatchSort.c
new file mode 100644
index 0000000000..a157909374
--- /dev/null
+++ b/src/backend/executor/nodeBatchSort.c
@@ -0,0 +1,554 @@
+#include "postgres.h"
+
+#include "common/hashfn.h"
+#include "executor/executor.h"
+#include "executor/nodeBatchSort.h"
+#include "fmgr.h"
+#include "miscadmin.h"
+#include "pgstat.h"
+#include "port/atomics.h"
+#include "storage/barrier.h"
+#include "utils/builtins.h"
+#include "utils/tuplesort.h"
+#include "utils/typcache.h"
+
+typedef struct ParallelBatchSort
+{
+	Barrier				barrier;		/* data build barrier */
+	pg_atomic_uint32	attached;		/* how many worker attached */
+	pg_atomic_uint32	cur_batch;		/* current not final sort batch number */
+	Size				tuplesort_size;	/* MAXIMUM_ALIGNOF*n */
+}ParallelBatchSort;
+
+#define PARALLEL_BATCH_SORT_SIZE		MAXALIGN(sizeof(ParallelBatchSort))
+#define PARALLEL_BATCH_SORT_SHARED(p,n)	\
+	(Sharedsort*)(((char*)(p)) + PARALLEL_BATCH_SORT_SIZE + (p)->tuplesort_size * (n))
+
+#define BUILD_BATCH_DONE	1
+
+/*
+ * if we can scan next batch, then return true and open it,
+ * else return false
+ */
+static bool
+ExecNextParallelBatchSort(BatchSortState *state)
+{
+	ParallelBatchSort  *parallel = state->parallel;
+	BatchSort		   *plan = castNode(BatchSort, state->ps.plan);
+	SortCoordinateData	coord;
+	uint32				cur_batch;
+	Assert(parallel != NULL);
+
+	/* do we opened other batch? */
+	if (state->curBatch >= 0 &&
+		state->curBatch < plan->numBatches &&
+		state->batches[state->curBatch] != NULL)
+	{
+		/* close it */
+		tuplesort_end(state->batches[state->curBatch]);
+		state->batches[state->curBatch] = NULL;
+	}
+
+	/* get next batch number and set next number in shared memory */
+	cur_batch = pg_atomic_fetch_add_u32(&parallel->cur_batch, 1);
+	if (cur_batch >= plan->numBatches)
+	{
+		/* out of batch range, no more batch */
+		state->curBatch = plan->numBatches;
+		return false;
+	}
+
+	/* open tuplesort as leader */
+	Assert(state->batches[cur_batch] == NULL);
+	state->curBatch = cur_batch;
+	coord.isWorker = false;		/* we are leader */
+	coord.nParticipants = pg_atomic_read_u32(&parallel->attached);
+	coord.sharedsort = PARALLEL_BATCH_SORT_SHARED(parallel, cur_batch);
+	state->batches[cur_batch] = tuplesort_begin_heap(ExecGetResultType(outerPlanState(state)),
+													 plan->sort.numCols,
+													 plan->sort.sortColIdx,
+													 plan->sort.sortOperators,
+													 plan->sort.collations,
+													 plan->sort.nullsFirst,
+													 work_mem,
+													 &coord,
+													 false);
+	tuplesort_performsort(state->batches[cur_batch]);
+	return true;
+}
+
+/*
+ * return a empty TupleTableSlot
+ */
+static TupleTableSlot *
+ExecEmptyBatchSort(PlanState *pstate)
+{
+	return ExecClearTuple(pstate->ps_ResultTupleSlot);
+}
+
+/*
+ * scan tuple slot from tuplesort,
+ * all batch must be sort completed(called function ExecBatchSortPrepare)
+ */
+static TupleTableSlot *
+ExecBatchSort(PlanState *pstate)
+{
+	TupleTableSlot *slot = pstate->ps_ResultTupleSlot;
+	BatchSortState *state = castNode(BatchSortState, pstate);
+	Assert(state->sort_Done);
+
+re_get_:
+	/*
+	 * Get tuple from current tuplesort
+	 */
+	if (likely(tuplesort_gettupleslot(state->batches[state->curBatch],
+									  true,
+									  false,
+									  slot,
+									  NULL)))
+		return slot;
+
+	/*
+	 * Is all batches scanned?
+	 */
+	if (state->curBatch < castNode(BatchSort, pstate->plan)->numBatches-1)
+	{
+		/* Try next batch */
+		if (state->parallel)
+		{
+			if (ExecNextParallelBatchSort(state) == false)
+			{
+				/* No nore batches */
+				ExecSetExecProcNode(pstate, ExecEmptyBatchSort);
+				return ExecClearTuple(slot);
+			}
+			/*
+			 * OK, we can scan next batch,
+			 * function ExecNextParallelBatchSort changed curBatch
+			 */
+		}else
+		{
+			state->curBatch++;
+		}
+		goto re_get_;
+	}
+
+	/*
+	 * no more batches, return empty TupleTableSlot
+	 * tuplesort_gettupleslot cleaned slot
+	 */
+	Assert(TupIsNull(slot));
+	return slot;
+}
+
+/*
+ * Sorts tuples from the outer subree of the node using tuplesort
+ */
+static TupleTableSlot *
+ExecBatchSortPrepare(PlanState *pstate)
+{
+	BatchSort		   *node = castNode(BatchSort, pstate->plan);
+	BatchSortState	   *state = castNode(BatchSortState, pstate);
+	PlanState		   *outerNode = outerPlanState(pstate);
+	TupleTableSlot	   *slot;
+	ListCell		   *lc;
+	ParallelBatchSort  *parallel = state->parallel;
+	SortCoordinateData	coord;
+	FunctionCallInfo	fcinfo;
+	uint32				hash;
+	int					i;
+	AttrNumber			maxAttr;
+	Assert(state->sort_Done == false ||
+		   (parallel && BarrierPhase(&parallel->barrier) >= BUILD_BATCH_DONE));
+	Assert(list_length(state->groupFuns) == node->numGroupCols);
+
+	/* Is parallel mode */
+	if (parallel)
+	{
+		/* Attach to barrier and test current state */
+		if (BarrierAttach(&parallel->barrier) >= BUILD_BATCH_DONE)
+		{
+			/*
+			 * all tuples sorted by other workers
+			 * we don't need scan tuple from outer subtree
+			 * and can not change batches state(too late)
+			 */
+			goto build_already_done_;
+		}
+		/* let other workers known we are attached */
+		pg_atomic_add_fetch_u32(&parallel->attached, 1);
+	}
+
+	/* Make all tuplesort */
+	for (i=node->numBatches;i>0;)
+	{
+		--i;
+		if (parallel)
+		{
+			coord.isWorker = true;		/* We are not leader */
+			coord.nParticipants = -1;
+			coord.sharedsort = PARALLEL_BATCH_SORT_SHARED(parallel, i);
+		}
+		state->batches[i] = tuplesort_begin_heap(ExecGetResultType(outerNode),
+												 node->sort.numCols,
+												 node->sort.sortColIdx,
+												 node->sort.sortOperators,
+												 node->sort.collations,
+												 node->sort.nullsFirst,
+												 work_mem / node->numBatches,
+												 parallel ? &coord : NULL,
+												 false);
+	}
+
+	/*
+	 * Get max using attribute number,
+	 * we don't want call slot_getsomeattrs() more than once
+	 */
+	maxAttr = 0;
+	for (i=node->numGroupCols;i>0;)
+	{
+		if (maxAttr < node->grpColIdx[--i])
+			maxAttr = node->grpColIdx[i];
+	}
+	for (i=node->sort.numCols;i>0;)
+	{
+		if (maxAttr < node->sort.sortColIdx[--i])
+			maxAttr = node->sort.sortColIdx[i];
+	}
+	Assert(maxAttr > 0);
+
+	for (;;)
+	{
+		CHECK_FOR_INTERRUPTS();
+		slot = ExecProcNode(outerNode);
+		if (TupIsNull(slot))
+			break;
+
+		/* Get all we using datums from tuple */
+		slot_getsomeattrs(slot, maxAttr);
+
+		/* Get hash value */
+		hash = 0;
+		i = 0;
+		foreach(lc, state->groupFuns)
+		{
+			AttrNumber att = node->grpColIdx[i++]-1;
+			if (slot->tts_isnull[att] == false)
+			{
+				fcinfo = lfirst(lc);
+				fcinfo->args[0].value = slot->tts_values[att];
+				hash = hash_combine(hash, DatumGetUInt32(FunctionCallInvoke(fcinfo)));
+				Assert(fcinfo->isnull == false);
+			}
+		}
+
+		/* Put tuple to differed batch using hash value */
+		tuplesort_puttupleslot(state->batches[hash%node->numBatches], slot);
+	}
+
+	/* Finish all tuplesort put */
+	for (i=node->numBatches;i>0;)
+	{
+		--i;
+		tuplesort_performsort(state->batches[i]);
+		if (parallel)
+		{
+			/*
+			 * In parallel mode we need reopen tuplesort as leader
+			 * and almost impossible get all batches,
+			 * so we close it for free resource
+			 */
+			tuplesort_end(state->batches[i]);
+			state->batches[i] = NULL;
+		}
+	}
+
+	if (parallel)
+	{
+		/* Wait other workers finish sort */
+		BarrierArriveAndWait(&parallel->barrier, WAIT_EVENT_BATCH_SORT_BUILD);
+		Assert(BarrierPhase(&parallel->barrier) >= BUILD_BATCH_DONE);
+	}
+
+build_already_done_:
+	/* All tuplesort already done */
+	state->sort_Done = true;
+
+	/*
+	 * Change execute function.
+	 * Separating the preparation and scanning data functions helps to improve performance,
+	 * because we don't have to judge whether the data is ready every time
+	 */
+	if (parallel)
+	{
+		/* we don't need barrier any more */
+		BarrierDetach(&parallel->barrier);
+
+		if (ExecNextParallelBatchSort(state))
+			ExecSetExecProcNode(pstate, ExecBatchSort);	/* got a batch, we can scan it */
+		else
+			ExecSetExecProcNode(pstate, ExecEmptyBatchSort);	/* no more batch, so no more tuple */
+	}else
+	{
+		/* scan tuplesort from first batch */
+		state->curBatch = 0;
+		ExecSetExecProcNode(pstate, ExecBatchSort);
+	}
+
+	/* return first tuple or empty */
+	return (*pstate->ExecProcNodeReal)(pstate);
+}
+
+/*
+ * Create the run-time state information for the batch sort node
+ */
+BatchSortState *
+ExecInitBatchSort(BatchSort *node, EState *estate, int eflags)
+{
+	BatchSortState *state;
+	TypeCacheEntry *typentry;
+	TupleDesc		desc;
+	int				i;
+
+	state = makeNode(BatchSortState);
+	state->ps.plan = (Plan*) node;
+	state->ps.state = estate;
+	state->ps.ExecProcNode = ExecBatchSortPrepare;
+
+	state->sort_Done = false;
+	state->batches = palloc0(node->numBatches * sizeof(Tuplesortstate*));
+
+	outerPlanState(state) = ExecInitNode(outerPlan(node), estate, eflags);
+
+	/*
+	 * Initialize return slot and type. No need to initialize projection info
+	 * because this node doesn't do projections.
+	 */
+	ExecInitResultTupleSlotTL(&state->ps, &TTSOpsMinimalTuple);
+	state->ps.ps_ProjInfo = NULL;
+
+	/*
+	 * create hash function call info data
+	 */
+	Assert(node->numGroupCols > 0);
+	desc = ExecGetResultType(outerPlanState(state));
+	for (i=0;i<node->numGroupCols;++i)
+	{
+		FmgrInfo			   *flinfo;
+		FunctionCallInfo		fcinfo;
+		Form_pg_attribute		attr = TupleDescAttr(desc, node->grpColIdx[i]-1);
+		typentry = lookup_type_cache(attr->atttypid, TYPECACHE_HASH_PROC);
+		if (!OidIsValid(typentry->hash_proc))
+			ereport(ERROR,
+					(errcode(ERRCODE_UNDEFINED_FUNCTION),
+					 errmsg("could not identify an extended hash function for type %s",
+							format_type_be(attr->atttypid))));
+
+		/* Alloc memory for hash function call info data */
+		flinfo = palloc0(sizeof(*flinfo));
+		fcinfo = palloc0(SizeForFunctionCallInfo(1));	/* hash function only have one argument */
+		fmgr_info(typentry->hash_proc, flinfo);
+		InitFunctionCallInfoData(*fcinfo, flinfo, 1, attr->attcollation, NULL, NULL);
+		fcinfo->args[0].isnull = false;	/* first argument always not null */
+
+		/* Save call info data to run-time state */
+		state->groupFuns = lappend(state->groupFuns, fcinfo);
+	}
+
+	return state;
+}
+
+/*
+ * Free tuplesort resources
+ * and set "all tuplesort already done" as false
+ */
+static void
+CleanBatchSort(BatchSortState *node)
+{
+	int i;
+
+	ExecClearTuple(node->ps.ps_ResultTupleSlot);
+	if (node->sort_Done)
+	{
+		for (i=castNode(BatchSort, node->ps.plan)->numBatches;i>0;)
+		{
+			if (node->batches[--i] != NULL)
+			{
+				tuplesort_end(node->batches[i]);
+				node->batches[i] = NULL;
+			}
+		}
+		node->sort_Done = false;
+	}
+}
+
+void
+ExecEndBatchSort(BatchSortState *node)
+{
+	ExecClearTuple(node->ps.ps_ResultTupleSlot);
+	CleanBatchSort(node);
+	ExecEndNode(outerPlanState(node));
+}
+
+void
+ExecReScanBatchSort(BatchSortState *node)
+{
+	int					i;
+	Bitmapset		   *outerParams;
+	ParallelBatchSort  *parallel;
+	BatchSort		   *plan;
+
+	/* If we haven't sorted yet, just return. */
+	if (node->sort_Done == false)
+		return;
+
+	parallel = node->parallel;
+	outerParams = outerPlanState(node)->chgParam;
+	plan = castNode(BatchSort, node->ps.plan);
+
+	/* must drop pointer to sort result tuple */
+	ExecClearTuple(node->ps.ps_ResultTupleSlot);
+
+	if (bms_is_empty(outerParams) ||
+		/*
+		 * In parallel mod in parallel mode don't need rescan outer
+		 * when it only have gather param change,
+		 * because we cached all tuple from it and we can read in any worker
+		 */
+		(parallel != NULL &&
+		 plan->gather_param >= 0 &&
+		 bms_membership(outerParams) == BMS_SINGLETON &&
+		 bms_is_member(plan->gather_param, outerParams)))
+	{
+		if (parallel)
+		{
+			/* restart batches */
+			pg_atomic_write_u32(&parallel->cur_batch, 0);
+
+			/*
+			 * We want call ExecNextParallelBatchSort() function when
+			 * parent call ExecProcNode(), not now. Because it is possible that
+			 * other workers will finish processing other batches very quickly,
+			 * and we did occupy a batch and we have been in no hurry to deal with it.
+			 * So we don't want occupy a batch for now.
+			 */
+			ExecSetExecProcNode(&node->ps, ExecBatchSortPrepare);
+		}else
+		{
+			/* restart batches */
+			node->curBatch = 0;
+
+			/* rescan tuplesort */
+			i = plan->numBatches;
+			while (i>0)
+				tuplesort_rescan(node->batches[--i]);
+			ExecSetExecProcNode(&node->ps, ExecBatchSort);
+		}
+	}else
+	{
+		/* clean opend tuplesort */
+		CleanBatchSort(node);
+
+		if (parallel)
+		{
+			/* reset tuplesort */
+			i = plan->numBatches;
+			while (i>0)
+			{
+				--i;
+				tuplesort_reset_shared(PARALLEL_BATCH_SORT_SHARED(parallel, i));
+			}
+
+			/* reset shared memory */
+			BarrierInit(&parallel->barrier, 0);
+			pg_atomic_write_u32(&parallel->attached, 0);
+			pg_atomic_write_u32(&parallel->cur_batch, 0);
+		}
+		ExecSetExecProcNode(&node->ps, ExecBatchSortPrepare);
+		node->sort_Done = false;
+	}
+}
+
+void
+ExecShutdownBatchSort(BatchSortState *node)
+{
+	CleanBatchSort(node);
+}
+
+/* ----------------------------------------------------------------
+ *						Parallel Query Support
+ * ----------------------------------------------------------------
+ */
+
+/* ----------------------------------------------------------------
+ *		ExecBatchSortEstimate
+ *
+ *		Estimate space required to propagate sort statistics.
+ * ----------------------------------------------------------------
+ */
+void
+ExecBatchSortEstimate(BatchSortState *node, ParallelContext *pcxt)
+{
+	Size size = mul_size(MAXALIGN(tuplesort_estimate_shared(pcxt->nworkers+1)),
+						 castNode(BatchSort, node->ps.plan)->numBatches);
+	size = add_size(size, PARALLEL_BATCH_SORT_SIZE);
+
+	shm_toc_estimate_chunk(&pcxt->estimator, size);
+	shm_toc_estimate_keys(&pcxt->estimator, 1);
+}
+
+/* Initialize shared memory for batch sort */
+static void
+InitializeBatchSortParallel(ParallelBatchSort *parallel,
+							int num_batches,
+							int num_workers,
+							dsm_segment *seg)
+{
+	int i;
+	BarrierInit(&parallel->barrier, 0);
+	pg_atomic_init_u32(&parallel->attached, 0);
+	pg_atomic_init_u32(&parallel->cur_batch, 0);
+	for (i=0;i<num_batches;++i)
+	{
+		tuplesort_initialize_shared(PARALLEL_BATCH_SORT_SHARED(parallel, i),
+									num_workers,
+									seg);
+	}
+}
+
+void
+ExecBatchSortInitializeDSM(BatchSortState *node, ParallelContext *pcxt)
+{
+	ParallelBatchSort  *parallel;
+	BatchSort		   *plan = castNode(BatchSort, node->ps.plan);
+	Size				tuplesort_size = MAXALIGN(tuplesort_estimate_shared(pcxt->nworkers+1));
+	Size				size = mul_size(tuplesort_size, plan->numBatches);
+	size = add_size(PARALLEL_BATCH_SORT_SIZE, size);
+
+	node->parallel = parallel = shm_toc_allocate(pcxt->toc, size);
+	parallel->tuplesort_size = tuplesort_size;
+	InitializeBatchSortParallel(parallel, plan->numBatches, pcxt->nworkers+1, pcxt->seg);
+	shm_toc_insert(pcxt->toc, plan->sort.plan.plan_node_id, parallel);
+}
+
+void
+ExecBatchSortReInitializeDSM(BatchSortState *node, ParallelContext *pcxt)
+{
+	pg_atomic_write_u32(&node->parallel->cur_batch, 0);
+}
+
+void
+ExecBatchSortInitializeWorker(BatchSortState *node, ParallelWorkerContext *pwcxt)
+{
+	uint32				i;
+	BatchSort		   *plan = castNode(BatchSort, node->ps.plan);
+	ParallelBatchSort  *parallel = shm_toc_lookup(pwcxt->toc,
+												  plan->sort.plan.plan_node_id,
+												  false);
+	node->parallel = parallel;
+	for (i=0;i<plan->numBatches;++i)
+	{
+		tuplesort_attach_shared(PARALLEL_BATCH_SORT_SHARED(parallel, i),
+								pwcxt->seg);
+	}
+}
diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c
index ba3ccc712c..7ee5d673f1 100644
--- a/src/backend/nodes/copyfuncs.c
+++ b/src/backend/nodes/copyfuncs.c
@@ -959,6 +959,23 @@ _copySort(const Sort *from)
 	return newnode;
 }
 
+/*
+ * _copyBatchSort
+ */
+static BatchSort *
+_copyBatchSort(const BatchSort *from)
+{
+	BatchSort	   *newnode = makeNode(BatchSort);
+
+	CopySortFields(&from->sort, &newnode->sort);
+
+	COPY_SCALAR_FIELD(numGroupCols);
+	COPY_SCALAR_FIELD(numBatches);
+	COPY_SCALAR_FIELD(gather_param);
+	COPY_POINTER_FIELD(grpColIdx, from->numGroupCols * sizeof(AttrNumber));
+
+	return newnode;
+}
 
 /*
  * _copyIncrementalSort
@@ -1024,6 +1041,8 @@ _copyAgg(const Agg *from)
 	COPY_BITMAPSET_FIELD(aggParams);
 	COPY_NODE_FIELD(groupingSets);
 	COPY_NODE_FIELD(chain);
+	COPY_SCALAR_FIELD(numBatches);
+	COPY_SCALAR_FIELD(gatherParam);
 
 	return newnode;
 }
@@ -4948,6 +4967,9 @@ copyObjectImpl(const void *from)
 		case T_Sort:
 			retval = _copySort(from);
 			break;
+		case T_BatchSort:
+			retval = _copyBatchSort(from);
+			break;
 		case T_IncrementalSort:
 			retval = _copyIncrementalSort(from);
 			break;
diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c
index 8392be6d44..61187685a6 100644
--- a/src/backend/nodes/outfuncs.c
+++ b/src/backend/nodes/outfuncs.c
@@ -785,6 +785,8 @@ _outAgg(StringInfo str, const Agg *node)
 	WRITE_BITMAPSET_FIELD(aggParams);
 	WRITE_NODE_FIELD(groupingSets);
 	WRITE_NODE_FIELD(chain);
+	WRITE_INT_FIELD(numBatches);
+	WRITE_INT_FIELD(gatherParam);
 }
 
 static void
@@ -854,6 +856,19 @@ _outSort(StringInfo str, const Sort *node)
 	_outSortInfo(str, node);
 }
 
+static void
+_outBatchSort(StringInfo str, const BatchSort *node)
+{
+	WRITE_NODE_TYPE("BATCHSORT");
+
+	_outSortInfo(str, &node->sort);
+
+	WRITE_INT_FIELD(numGroupCols);
+	WRITE_INT_FIELD(numBatches);
+	WRITE_INT_FIELD(gather_param);
+	WRITE_ATTRNUMBER_ARRAY(grpColIdx, node->numGroupCols);
+}
+
 static void
 _outIncrementalSort(StringInfo str, const IncrementalSort *node)
 {
@@ -3836,6 +3851,9 @@ outNode(StringInfo str, const void *obj)
 			case T_Sort:
 				_outSort(str, obj);
 				break;
+			case T_BatchSort:
+				_outBatchSort(str, obj);
+				break;
 			case T_IncrementalSort:
 				_outIncrementalSort(str, obj);
 				break;
diff --git a/src/backend/nodes/readfuncs.c b/src/backend/nodes/readfuncs.c
index d2c8d58070..e0d225aa85 100644
--- a/src/backend/nodes/readfuncs.c
+++ b/src/backend/nodes/readfuncs.c
@@ -2182,6 +2182,21 @@ _readSort(void)
 	READ_DONE();
 }
 
+static BatchSort *
+_readBatchSort(void)
+{
+	READ_LOCALS(BatchSort);
+
+	ReadCommonSort(&local_node->sort);
+
+	READ_INT_FIELD(numGroupCols);
+	READ_INT_FIELD(numBatches);
+	READ_INT_FIELD(gather_param);
+	READ_ATTRNUMBER_ARRAY(grpColIdx, local_node->numGroupCols);
+
+	READ_DONE();
+}
+
 /*
  * _readIncrementalSort
  */
@@ -2236,6 +2251,8 @@ _readAgg(void)
 	READ_BITMAPSET_FIELD(aggParams);
 	READ_NODE_FIELD(groupingSets);
 	READ_NODE_FIELD(chain);
+	READ_INT_FIELD(numBatches);
+	READ_INT_FIELD(gatherParam);
 
 	READ_DONE();
 }
@@ -2835,6 +2852,8 @@ parseNodeString(void)
 		return_value = _readMaterial();
 	else if (MATCH("SORT", 4))
 		return_value = _readSort();
+	else if (MATCH("BATCHSORT", 9))
+		return_value = _readBatchSort();
 	else if (MATCH("INCREMENTALSORT", 15))
 		return_value = _readIncrementalSort();
 	else if (MATCH("GROUP", 5))
diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c
index 380336518f..c0a75e31c1 100644
--- a/src/backend/optimizer/path/costsize.c
+++ b/src/backend/optimizer/path/costsize.c
@@ -147,6 +147,8 @@ bool		enable_partitionwise_aggregate = false;
 bool		enable_parallel_append = true;
 bool		enable_parallel_hash = true;
 bool		enable_partition_pruning = true;
+int			max_sort_batches = 0;
+int			max_hashagg_batches = 0;
 
 typedef struct
 {
@@ -1958,6 +1960,84 @@ cost_sort(Path *path, PlannerInfo *root,
 	path->total_cost = startup_cost + run_cost;
 }
 
+void cost_batchsort(Path *path, PlannerInfo *root,
+					List *batchkeys, Cost input_cost,
+					double tuples, int width,
+					Cost comparison_cost, int sort_mem,
+					uint32 numGroupCols, uint32 numBatches)
+{
+	Cost		startup_cost = input_cost;
+	Cost		run_cost = 0;
+	double		input_bytes = relation_byte_size(tuples, width);
+	double		batch_bytes = input_bytes / numBatches;
+	double		batch_tuples = tuples / numBatches;
+	long		sort_mem_bytes = sort_mem * 1024L;
+
+	if (sort_mem_bytes < (64*1024))
+		sort_mem_bytes = (64*1024);
+
+	/* hash cost */
+	startup_cost += cpu_operator_cost * numGroupCols * tuples;
+
+	path->rows = tuples;
+
+	/*
+	 * We want to be sure the cost of a sort is never estimated as zero, even
+	 * if passed-in tuple count is zero.  Besides, mustn't do log(0)...
+	 */
+	if (tuples < 2.0)
+		tuples = 2.0;
+
+	if (batch_bytes > sort_mem_bytes)
+	{
+		/*
+		 * We'll have to use a disk-based sort of all the tuples
+		 */
+		double		npages = ceil(batch_bytes / BLCKSZ);
+		double		nruns = batch_bytes / sort_mem_bytes;
+		double		mergeorder = tuplesort_merge_order(sort_mem_bytes);
+		double		log_runs;
+		double		npageaccesses;
+
+		/*
+		 * CPU costs
+		 *
+		 * Assume about N log2 N comparisons
+		 */
+		startup_cost += comparison_cost * batch_tuples * LOG2(batch_tuples) * numBatches;
+
+		/* Disk costs */
+
+		/* Compute logM(r) as log(r) / log(M) */
+		if (nruns > mergeorder)
+			log_runs = ceil(log(nruns) / log(mergeorder));
+		else
+			log_runs = 1.0;
+		npageaccesses = 2.0 * npages * log_runs;
+		/* Assume 3/4ths of accesses are sequential, 1/4th are not */
+		startup_cost += npageaccesses * numBatches *
+			(seq_page_cost * 0.75 + random_page_cost * 0.25);
+
+	}else
+	{
+		/* We'll use plain quicksort on all the input tuples */
+		startup_cost += comparison_cost * tuples * LOG2(tuples);
+	}
+
+	/*
+	 * Also charge a small amount (arbitrarily set equal to operator cost) per
+	 * extracted tuple.  We don't charge cpu_tuple_cost because a Sort node
+	 * doesn't do qual-checking or projection, so it has less overhead than
+	 * most plan nodes.  Note it's correct to use tuples not output_tuples
+	 * here --- the upper LIMIT will pro-rate the run cost so we'd be double
+	 * counting the LIMIT otherwise.
+	 */
+	run_cost += cpu_operator_cost * tuples;
+
+	path->startup_cost = startup_cost;
+	path->total_cost = startup_cost + run_cost;
+}
+
 /*
  * append_nonpartial_cost
  *	  Estimate the cost of the non-partial paths in a Parallel Append.
@@ -2332,7 +2412,7 @@ cost_agg(Path *path, PlannerInfo *root,
 	/* Use all-zero per-aggregate costs if NULL is passed */
 	if (aggcosts == NULL)
 	{
-		Assert(aggstrategy == AGG_HASHED);
+		Assert(aggstrategy == AGG_HASHED || aggstrategy == AGG_BATCH_HASH);
 		MemSet(&dummy_aggcosts, 0, sizeof(AggClauseCosts));
 		aggcosts = &dummy_aggcosts;
 	}
@@ -2391,10 +2471,13 @@ cost_agg(Path *path, PlannerInfo *root,
 	}
 	else
 	{
-		/* must be AGG_HASHED */
+		/* must be AGG_HASHED or AGG_BATCH_HASH */
+		Assert(aggstrategy == AGG_HASHED || max_hashagg_batches > 0);
 		startup_cost = input_total_cost;
-		if (!enable_hashagg)
+		if (aggstrategy == AGG_HASHED && !enable_hashagg)
+		{
 			startup_cost += disable_cost;
+		}
 		startup_cost += aggcosts->transCost.startup;
 		startup_cost += aggcosts->transCost.per_tuple * input_tuples;
 		/* cost of computing hash value */
@@ -2406,6 +2489,15 @@ cost_agg(Path *path, PlannerInfo *root,
 		/* cost of retrieving from hash table */
 		total_cost += cpu_tuple_cost * numGroups;
 		output_tuples = numGroups;
+
+		if (aggstrategy == AGG_BATCH_HASH)
+		{
+			double	nbytes = relation_byte_size(input_tuples, input_width);
+			double	npages = ceil(nbytes / BLCKSZ);
+			double	material_cost = (seq_page_cost * npages);
+			startup_cost += material_cost;
+			total_cost += material_cost;
+		}
 	}
 
 	/*
@@ -2421,7 +2513,9 @@ cost_agg(Path *path, PlannerInfo *root,
 	 * Accrue writes (spilled tuples) to startup_cost and to total_cost;
 	 * accrue reads only to total_cost.
 	 */
-	if (aggstrategy == AGG_HASHED || aggstrategy == AGG_MIXED)
+	if (aggstrategy == AGG_HASHED ||
+		aggstrategy == AGG_BATCH_HASH ||
+		aggstrategy == AGG_MIXED)
 	{
 		double		pages;
 		double		pages_written = 0.0;
@@ -2434,6 +2528,14 @@ cost_agg(Path *path, PlannerInfo *root,
 		int			num_partitions;
 		int			depth;
 
+		if (aggstrategy == AGG_BATCH_HASH &&
+			numGroups > max_hashagg_batches)
+		{
+			numGroups /= max_hashagg_batches;
+			if (numGroups < 1.0)
+				numGroups = 1.0;
+		}
+
 		/*
 		 * Estimate number of batches based on the computed limits. If less
 		 * than or equal to one, all groups are expected to fit in memory;
diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c
index 25d4750ca6..baa645529c 100644
--- a/src/backend/optimizer/plan/createplan.c
+++ b/src/backend/optimizer/plan/createplan.c
@@ -30,6 +30,7 @@
 #include "optimizer/cost.h"
 #include "optimizer/optimizer.h"
 #include "optimizer/paramassign.h"
+#include "optimizer/pathnode.h"
 #include "optimizer/paths.h"
 #include "optimizer/placeholder.h"
 #include "optimizer/plancat.h"
@@ -98,6 +99,7 @@ static Plan *create_projection_plan(PlannerInfo *root,
 									int flags);
 static Plan *inject_projection_plan(Plan *subplan, List *tlist, bool parallel_safe);
 static Sort *create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags);
+static BatchSort *create_batchsort_plan(PlannerInfo *root, BatchSortPath *best_path, int flags);
 static IncrementalSort *create_incrementalsort_plan(PlannerInfo *root,
 													IncrementalSortPath *best_path, int flags);
 static Group *create_group_plan(PlannerInfo *root, GroupPath *best_path);
@@ -468,6 +470,11 @@ create_plan_recurse(PlannerInfo *root, Path *best_path, int flags)
 											 (SortPath *) best_path,
 											 flags);
 			break;
+		case T_BatchSort:
+			plan = (Plan *) create_batchsort_plan(root,
+												  (BatchSortPath*) best_path,
+												  flags);
+			break;
 		case T_IncrementalSort:
 			plan = (Plan *) create_incrementalsort_plan(root,
 														(IncrementalSortPath *) best_path,
@@ -2009,6 +2016,40 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags)
 	return plan;
 }
 
+static BatchSort *create_batchsort_plan(PlannerInfo *root, BatchSortPath *best_path, int flags)
+{
+	BatchSort	   *plan;
+	Plan		   *subplan;
+
+	subplan = create_plan_recurse(root, best_path->subpath,
+								  flags | CP_SMALL_TLIST);
+
+	plan = makeNode(BatchSort);
+	subplan = prepare_sort_from_pathkeys(subplan,
+										 best_path->batchkeys,
+										 IS_OTHER_REL(best_path->subpath->parent) ?
+											 best_path->path.parent->relids : NULL,
+										 NULL,
+										 false,
+										 &plan->sort.numCols,
+										 &plan->sort.sortColIdx,
+										 &plan->sort.sortOperators,
+										 &plan->sort.collations,
+										 &plan->sort.nullsFirst);
+	plan->sort.plan.targetlist = subplan->targetlist;
+	plan->sort.plan.qual = NIL;
+	outerPlan(plan) = subplan;
+	innerPlan(plan) = NULL;
+	plan->numBatches = best_path->numBatches;
+	plan->numGroupCols = list_length(best_path->batchgroup);
+	plan->gather_param = -1;
+	plan->grpColIdx = extract_grouping_cols(best_path->batchgroup,
+											subplan->targetlist);
+
+	copy_generic_path_info(&plan->sort.plan, &best_path->path);
+	return plan;
+}
+
 /*
  * create_incrementalsort_plan
  *
@@ -2085,6 +2126,12 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag
 {
 	Unique	   *plan;
 	Plan	   *subplan;
+	List	   *pathkeys;
+
+	if (IsA(best_path->subpath, BatchSortPath))
+		pathkeys = ((BatchSortPath*)best_path->subpath)->batchkeys;
+	else
+		pathkeys = best_path->path.pathkeys;
 
 	/*
 	 * Unique doesn't project, so tlist requirements pass through; moreover we
@@ -2094,7 +2141,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag
 								  flags | CP_LABEL_TLIST);
 
 	plan = make_unique_from_pathkeys(subplan,
-									 best_path->path.pathkeys,
+									 pathkeys,
 									 best_path->numkeys);
 
 	copy_generic_path_info(&plan->plan, (Path *) best_path);
@@ -2282,7 +2329,9 @@ create_groupingsets_plan(PlannerInfo *root, GroupingSetsPath *best_path)
 			if (!rollup->is_hashed)
 				is_first_sort = false;
 
-			if (rollup->is_hashed)
+			if (best_path->aggstrategy == AGG_BATCH_HASH)
+				strat = AGG_BATCH_HASH;
+			else if (rollup->is_hashed)
 				strat = AGG_HASHED;
 			else if (list_length(linitial(rollup->gsets)) == 0)
 				strat = AGG_PLAIN;
@@ -6373,12 +6422,20 @@ make_agg(List *tlist, List *qual,
 	node->aggParams = NULL;		/* SS_finalize_plan() will fill this */
 	node->groupingSets = groupingSets;
 	node->chain = chain;
+	node->gatherParam = -1;
 
 	plan->qual = qual;
 	plan->targetlist = tlist;
 	plan->lefttree = lefttree;
 	plan->righttree = NULL;
 
+	if (aggstrategy == AGG_BATCH_HASH)
+	{
+		node->numBatches = (int32)numGroups;
+		if (node->numBatches > max_hashagg_batches)
+			node->numBatches = max_hashagg_batches;
+	}
+
 	return node;
 }
 
diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 4e6497ff32..0307ee094d 100644
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -175,6 +175,12 @@ static void consider_groupingsets_paths(PlannerInfo *root,
 										grouping_sets_data *gd,
 										const AggClauseCosts *agg_costs,
 										double dNumGroups);
+static void consider_parallel_hash_groupingsets_paths(PlannerInfo *root,
+													  RelOptInfo *grouped_rel,
+													  Path *path,
+													  grouping_sets_data *gd,
+													  const AggClauseCosts *agg_costs,
+													  double dNumGroups);
 static RelOptInfo *create_window_paths(PlannerInfo *root,
 									   RelOptInfo *input_rel,
 									   PathTarget *input_target,
@@ -3866,6 +3872,11 @@ create_grouping_paths(PlannerInfo *root,
 		extra.havingQual = parse->havingQual;
 		extra.targetList = parse->targetList;
 		extra.partial_costs_set = false;
+		if (parse->groupClause != NIL &&
+			(gd == NULL || gd->rollups == NIL))
+			extra.hashable_groups = grouping_get_hashable(parse->groupClause);
+		else
+			extra.hashable_groups = NIL;
 
 		/*
 		 * Determine whether partitionwise aggregation is in theory possible.
@@ -4522,6 +4533,78 @@ consider_groupingsets_paths(PlannerInfo *root,
 										  dNumGroups));
 }
 
+static void
+consider_parallel_hash_groupingsets_paths(PlannerInfo *root,
+										  RelOptInfo *grouped_rel,
+										  Path *path,
+										  grouping_sets_data *gd,
+										  const AggClauseCosts *agg_costs,
+										  double dNumGroups)
+{
+	int			hash_mem = get_hash_mem();
+	List	   *new_rollups = NIL;
+	List	   *sets_data;
+	ListCell   *lc;
+	RollupData *rollup;
+	GroupingSetData *gs;
+	double		hashsize;
+	double		numGroups;
+
+	sets_data = list_copy(gd->unsortable_sets);
+	foreach (lc, gd->rollups)
+	{
+		rollup = lfirst_node(RollupData, lc);
+		if (rollup->hashable == false)
+		{
+			list_free(sets_data);
+			return;
+		}
+		sets_data = list_concat(sets_data, rollup->gsets_data);
+	}
+	foreach (lc, sets_data)
+	{
+		gs = lfirst_node(GroupingSetData, lc);
+		numGroups = gs->numGroups / max_hashagg_batches;
+		if (numGroups < 1.0)
+			numGroups = 1.0;
+		hashsize = estimate_hashagg_tablesize(root,
+											  path,
+											  agg_costs,
+											  numGroups);
+		if (hashsize > hash_mem * 1024L)
+		{
+			list_free(sets_data);
+			list_free_deep(new_rollups);
+			return;
+		}
+
+		rollup = makeNode(RollupData);
+		rollup->groupClause = preprocess_groupclause(root, gs->set);
+		rollup->gsets_data = list_make1(gs);
+		rollup->gsets = remap_to_groupclause_idx(rollup->groupClause,
+												 rollup->gsets_data,
+												 gd->tleref_to_colnum_map);
+		rollup->numGroups = gs->numGroups;
+		rollup->hashable = true;
+		rollup->is_hashed = true;
+		new_rollups = lappend(new_rollups, rollup);
+	}
+
+	numGroups = dNumGroups / path->parallel_workers;
+	if (numGroups < list_length(new_rollups))
+		numGroups = list_length(new_rollups);
+	path = (Path*)create_groupingsets_path(root,
+										   grouped_rel,
+										   path,
+										   (List*) root->parse->havingQual,
+										   AGG_BATCH_HASH,
+										   new_rollups,
+										   agg_costs,
+										   numGroups);
+	path->parallel_aware = true;
+	add_partial_path(grouped_rel, path);
+}
+
 /*
  * create_window_paths
  *
@@ -4795,6 +4878,8 @@ create_distinct_paths(PlannerInfo *root,
 											  cheapest_input_path->rows,
 											  NULL);
 	}
+	distinct_rel->rows = numDistinctRows;
+	distinct_rel->reltarget = root->upper_targets[UPPERREL_DISTINCT];
 
 	/*
 	 * Consider sort-based implementations of DISTINCT, if possible.
@@ -4814,6 +4899,7 @@ create_distinct_paths(PlannerInfo *root,
 		 * the other.)
 		 */
 		List	   *needed_pathkeys;
+		List	   *hashable_clause;
 
 		if (parse->hasDistinctOn &&
 			list_length(root->distinct_pathkeys) <
@@ -4860,6 +4946,39 @@ create_distinct_paths(PlannerInfo *root,
 										  path,
 										  list_length(root->distinct_pathkeys),
 										  numDistinctRows));
+
+		/* add parallel unique */
+		if (max_sort_batches > 0 &&
+			distinct_rel->consider_parallel &&
+			input_rel->partial_pathlist != NIL &&
+			numDistinctRows >= 2.0 &&
+			(hashable_clause = grouping_get_hashable(parse->distinctClause)) != NIL)
+		{
+			double	numPartialDistinctRows;
+			uint32	num_batchs = (uint32)numDistinctRows;
+			if (num_batchs > max_sort_batches)
+				num_batchs = max_sort_batches;
+
+			foreach (lc, input_rel->partial_pathlist)
+			{
+				Path *path = (Path*)create_batchsort_path(root,
+														  distinct_rel,
+														  lfirst(lc),
+														  needed_pathkeys,
+														  hashable_clause,
+														  num_batchs,
+														  true);
+				numPartialDistinctRows = numDistinctRows / path->parallel_workers;
+				if (numPartialDistinctRows < 1.0)
+					numPartialDistinctRows = 1.0;
+				path = (Path*)create_upper_unique_path(root,
+													   distinct_rel,
+													   path,
+													   list_length(root->distinct_pathkeys),
+													   numPartialDistinctRows);
+				add_partial_path(distinct_rel, path);
+			}
+		}
 	}
 
 	/*
@@ -4895,8 +5014,34 @@ create_distinct_paths(PlannerInfo *root,
 								 NIL,
 								 NULL,
 								 numDistinctRows));
+
+		/* Generate parallel batch hashed aggregate path */
+		if (max_hashagg_batches > 0 &&
+			distinct_rel->consider_parallel &&
+			input_rel->partial_pathlist != NIL &&
+			numDistinctRows > 1.0)
+		{
+			Path *path = linitial(input_rel->partial_pathlist);
+			double numRows = numDistinctRows / path->parallel_workers;
+			if (numRows < 1.0)
+				numRows = 1.0;
+			path = (Path *)create_agg_path(root,
+										   distinct_rel,
+										   path,
+										   path->pathtarget,
+										   AGG_BATCH_HASH,
+										   AGGSPLIT_SIMPLE,
+										   parse->distinctClause,
+										   NIL,
+										   NULL,
+										   numRows);
+			path->parallel_aware = true;
+			add_partial_path(distinct_rel, path);
+		}
 	}
 
+	generate_useful_gather_paths(root, distinct_rel, false);
+
 	/* Give a helpful error if we failed to find any implementation */
 	if (distinct_rel->pathlist == NIL)
 		ereport(ERROR,
@@ -6654,6 +6799,58 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel,
 			}
 		}
 
+		/*
+		 * create simple agg using parallel
+		 */
+		if (max_sort_batches > 0 &&
+			grouped_rel->consider_parallel &&
+			extra->hashable_groups != NIL &&
+			input_rel->partial_pathlist != NIL &&
+			dNumGroups >= 2.0)
+		{
+			Path	   *path;
+			double		numGroups;
+			uint32		numBatches = (uint32)dNumGroups;
+
+			if (numBatches > max_sort_batches)
+				numBatches = max_sort_batches;
+			numGroups = dNumGroups / numBatches;
+			if (numGroups < 1.0)
+				numGroups = 1.0;
+
+			Assert(parse->groupingSets == NIL);
+			Assert(parse->groupClause != NIL);
+			foreach (lc, input_rel->partial_pathlist)
+			{
+				path = (Path*)create_batchsort_path(root,
+													grouped_rel,
+													lfirst(lc),
+													root->group_pathkeys,
+													extra->hashable_groups,
+													numBatches,
+													true);
+				if (parse->hasAggs)
+					path = (Path*)create_agg_path(root,
+												  grouped_rel,
+												  path,
+												  grouped_rel->reltarget,
+												  AGG_SORTED,
+												  AGGSPLIT_SIMPLE,
+												  parse->groupClause,
+												  havingQual,
+												  agg_costs,
+												  numGroups);
+				else
+					path = (Path*)create_group_path(root,
+													grouped_rel,
+													path,
+													parse->groupClause,
+													havingQual,
+													numGroups);
+				add_partial_path(grouped_rel, path);
+			}
+		}
+
 		/*
 		 * Instead of operating directly on the input relation, we can
 		 * consider finalizing a partially aggregated path.
@@ -6757,6 +6954,56 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel,
 											   havingQual,
 											   dNumGroups));
 			}
+
+			/* create parallel batch sort aggregate paths */
+			if (max_sort_batches > 0 &&
+				grouped_rel->consider_parallel &&
+				extra->hashable_groups != NIL &&
+				partially_grouped_rel->partial_pathlist != NIL &&
+				dNumGroups >= 2.0)
+			{
+				Path	   *path;
+				double		numGroups;
+				uint32		numBatches = (uint32)dNumGroups;
+
+				if (numBatches > max_sort_batches)
+					numBatches = max_sort_batches;
+				numGroups = dNumGroups / numBatches;
+				if (numGroups < 1.0)
+					numGroups = 1.0;
+
+				Assert(parse->groupingSets == NIL);
+				Assert(parse->groupClause != NIL);
+				foreach (lc, partially_grouped_rel->partial_pathlist)
+				{
+					path = (Path*)create_batchsort_path(root,
+														grouped_rel,
+														lfirst(lc),
+														root->group_pathkeys,
+														extra->hashable_groups,
+														numBatches,
+														true);
+					if (parse->hasAggs)
+						path = (Path*)create_agg_path(root,
+													  grouped_rel,
+													  path,
+													  grouped_rel->reltarget,
+													  AGG_SORTED,
+													  AGGSPLIT_FINAL_DESERIAL,
+													  parse->groupClause,
+													  havingQual,
+													  agg_costs,
+													  numGroups);
+					else
+						path = (Path*)create_group_path(root,
+														grouped_rel,
+														path,
+														parse->groupClause,
+														havingQual,
+														numGroups);
+					add_partial_path(grouped_rel, path);
+				}
+			}
 		}
 	}
 
@@ -6770,6 +7017,15 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel,
 			consider_groupingsets_paths(root, grouped_rel,
 										cheapest_path, false, true,
 										gd, agg_costs, dNumGroups);
+			if (max_hashagg_batches > 0 &&
+				grouped_rel->consider_parallel &&
+				input_rel->partial_pathlist != NIL)
+				consider_parallel_hash_groupingsets_paths(root,
+														  grouped_rel,
+														  linitial(input_rel->partial_pathlist),
+														  gd,
+														  agg_costs,
+														  dNumGroups);
 		}
 		else
 		{
@@ -6787,6 +7043,29 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel,
 									 havingQual,
 									 agg_costs,
 									 dNumGroups));
+
+			if (max_hashagg_batches > 0 &&
+				grouped_rel->consider_parallel &&
+				input_rel->partial_pathlist != NIL &&
+				dNumGroups >= 2.0)
+			{
+				Path   *path = linitial(input_rel->partial_pathlist);
+				double	numGroups = dNumGroups / path->parallel_workers;
+				if (numGroups < 1.0)
+					numGroups = 1.0;
+				path = (Path*)create_agg_path(root,
+											  grouped_rel,
+											  path,
+											  grouped_rel->reltarget,
+											  AGG_BATCH_HASH,
+											  AGGSPLIT_SIMPLE,
+											  parse->groupClause,
+											  havingQual,
+											  agg_costs,
+											  numGroups);
+				path->parallel_aware = true;
+				add_partial_path(grouped_rel, path);
+			}
 		}
 
 		/*
@@ -6809,6 +7088,32 @@ add_paths_to_grouping_rel(PlannerInfo *root, RelOptInfo *input_rel,
 									 agg_final_costs,
 									 dNumGroups));
 		}
+
+		/*
+		 * Generate a Finalize BatchHashAgg
+		 */
+		if (max_hashagg_batches > 0 &&
+			dNumGroups >= 2.0 &&
+			partially_grouped_rel &&
+			partially_grouped_rel->partial_pathlist)
+		{
+			Path   *path = linitial(partially_grouped_rel->partial_pathlist);
+			double	numGroups = dNumGroups / path->parallel_workers;
+			if (numGroups < 1.0)
+				numGroups = 1.0;
+			path = (Path*)create_agg_path(root,
+										  grouped_rel,
+										  path,
+										  grouped_rel->reltarget,
+										  AGG_BATCH_HASH,
+										  AGGSPLIT_FINAL_DESERIAL,
+										  parse->groupClause,
+										  havingQual,
+										  agg_final_costs,
+										  numGroups);
+			path->parallel_aware = true;
+			add_partial_path(grouped_rel, path);
+		}
 	}
 
 	/*
diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c
index c3c36be13e..d2060dbeff 100644
--- a/src/backend/optimizer/plan/setrefs.c
+++ b/src/backend/optimizer/plan/setrefs.c
@@ -737,6 +737,7 @@ set_plan_refs(PlannerInfo *root, Plan *plan, int rtoffset)
 
 		case T_Material:
 		case T_Sort:
+		case T_BatchSort:
 		case T_IncrementalSort:
 		case T_Unique:
 		case T_SetOp:
diff --git a/src/backend/optimizer/plan/subselect.c b/src/backend/optimizer/plan/subselect.c
index 6d4cc1bcce..4556ec9d2a 100644
--- a/src/backend/optimizer/plan/subselect.c
+++ b/src/backend/optimizer/plan/subselect.c
@@ -2698,6 +2698,8 @@ finalize_plan(PlannerInfo *root, Plan *plan,
 										  &aggcontext);
 					agg->aggParams = aggcontext.paramids;
 				}
+
+				agg->gatherParam = gather_param;
 			}
 			break;
 
@@ -2759,6 +2761,10 @@ finalize_plan(PlannerInfo *root, Plan *plan,
 			/* no node-type-specific fields need fixing */
 			break;
 
+		case T_BatchSort:
+			((BatchSort*)plan)->gather_param = gather_param;
+			break;
+
 		default:
 			elog(ERROR, "unrecognized node type: %d",
 				 (int) nodeTag(plan));
diff --git a/src/backend/optimizer/prep/prepunion.c b/src/backend/optimizer/prep/prepunion.c
index 86f794c193..1faab8a6ef 100644
--- a/src/backend/optimizer/prep/prepunion.c
+++ b/src/backend/optimizer/prep/prepunion.c
@@ -67,7 +67,7 @@ static List *plan_union_children(PlannerInfo *root,
 								 List *refnames_tlist,
 								 List **tlist_list);
 static Path *make_union_unique(SetOperationStmt *op, Path *path, List *tlist,
-							   PlannerInfo *root);
+							   PlannerInfo *root, List *groupList, List *sortKeys);
 static void postprocess_setop_rel(PlannerInfo *root, RelOptInfo *rel);
 static bool choose_hashed_setop(PlannerInfo *root, List *groupClauses,
 								Path *input_path,
@@ -354,6 +354,7 @@ recurse_set_operations(Node *setOp, PlannerInfo *root,
 			rel = generate_nonunion_paths(op, root,
 										  refnames_tlist,
 										  pTargetList);
+		generate_useful_gather_paths(root, rel, false);
 		if (pNumGroups)
 			*pNumGroups = rel->rows;
 
@@ -552,6 +553,8 @@ generate_union_paths(SetOperationStmt *op, PlannerInfo *root,
 	List	   *tlist_list;
 	List	   *tlist;
 	Path	   *path;
+	List	   *groupList = NIL;
+	List	   *sortKeys = NIL;
 
 	/*
 	 * If plain UNION, tell children to fetch all tuples.
@@ -587,6 +590,14 @@ generate_union_paths(SetOperationStmt *op, PlannerInfo *root,
 
 	*pTargetList = tlist;
 
+	if (!op->all)
+	{
+		/* Identify the grouping semantics */
+		groupList = generate_setop_grouplist(op, tlist);
+		if (grouping_is_sortable(groupList))
+			sortKeys = make_pathkeys_for_sortclauses(root, groupList, tlist);
+	}
+
 	/* Build path lists and relid set. */
 	foreach(lc, rellist)
 	{
@@ -627,7 +638,7 @@ generate_union_paths(SetOperationStmt *op, PlannerInfo *root,
 	 * node(s) to remove duplicates.
 	 */
 	if (!op->all)
-		path = make_union_unique(op, path, tlist, root);
+		path = make_union_unique(op, path, tlist, root, groupList, sortKeys);
 
 	add_path(result_rel, path);
 
@@ -646,6 +657,7 @@ generate_union_paths(SetOperationStmt *op, PlannerInfo *root,
 	{
 		Path	   *ppath;
 		ListCell   *lc;
+		List	   *hashable_list;
 		int			parallel_workers = 0;
 
 		/* Find the highest number of workers requested for any subpath. */
@@ -678,11 +690,64 @@ generate_union_paths(SetOperationStmt *op, PlannerInfo *root,
 							   NIL, NULL,
 							   parallel_workers, enable_parallel_append,
 							   NIL, -1);
+
+		/* create parallel batch sort union */
+		if (!op->all &&
+			max_sort_batches > 0 &&
+			sortKeys != NIL &&
+			ppath->rows >= 2.0 &&
+			(hashable_list = grouping_get_hashable(groupList)) != NIL)
+		{
+			Path   *partial_path;
+			uint32	numBatches = ppath->rows;
+			if (numBatches > max_sort_batches)
+				numBatches = max_sort_batches;
+			Assert(list_length(sortKeys) >= list_length(hashable_list));
+			partial_path = (Path*)create_batchsort_path(root,
+														result_rel,
+														ppath,
+														sortKeys,
+														hashable_list,
+														numBatches,
+														true);
+			partial_path = (Path*) create_upper_unique_path(root,
+															result_rel,
+															partial_path,
+															list_length(sortKeys),
+															partial_path->rows);
+			add_partial_path(result_rel, partial_path);
+		}
+
+		/* create parallel batch hashed union */
+		if (!op->all &&
+			max_hashagg_batches > 0 &&
+			ppath->rows > 1.0 &&
+			grouping_is_hashable(groupList) &&
+			ppath->pathtarget->width * ppath->rows / max_hashagg_batches <= get_hash_mem() * 1024L)
+		{
+			Path   *partial_path;
+			double	dNumGroups = ppath->rows / ppath->parallel_workers;
+			if (dNumGroups < 1.0)
+				dNumGroups = 1.0;
+			partial_path = (Path*)create_agg_path(root,
+												  result_rel,
+												  ppath,
+												  create_pathtarget(root, tlist),
+												  AGG_BATCH_HASH,
+												  AGGSPLIT_SIMPLE,
+												  groupList,
+												  NIL,
+												  NULL,
+												  dNumGroups);
+			partial_path->parallel_aware = true;
+			add_partial_path(result_rel, partial_path);
+		}
+
 		ppath = (Path *)
 			create_gather_path(root, result_rel, ppath,
 							   result_rel->reltarget, NULL, NULL);
 		if (!op->all)
-			ppath = make_union_unique(op, ppath, tlist, root);
+			ppath = make_union_unique(op, ppath, tlist, root, groupList, sortKeys);
 		add_path(result_rel, ppath);
 	}
 
@@ -933,15 +998,11 @@ plan_union_children(PlannerInfo *root,
  */
 static Path *
 make_union_unique(SetOperationStmt *op, Path *path, List *tlist,
-				  PlannerInfo *root)
+				  PlannerInfo *root, List *groupList, List *sortKeys)
 {
 	RelOptInfo *result_rel = fetch_upper_rel(root, UPPERREL_SETOP, NULL);
-	List	   *groupList;
 	double		dNumGroups;
 
-	/* Identify the grouping semantics */
-	groupList = generate_setop_grouplist(op, tlist);
-
 	/*
 	 * XXX for the moment, take the number of distinct groups as equal to the
 	 * total input size, ie, the worst case.  This is too conservative, but
@@ -976,9 +1037,7 @@ make_union_unique(SetOperationStmt *op, Path *path, List *tlist,
 				create_sort_path(root,
 								 result_rel,
 								 path,
-								 make_pathkeys_for_sortclauses(root,
-															   groupList,
-															   tlist),
+								 sortKeys,
 								 -1.0);
 		path = (Path *) create_upper_unique_path(root,
 												 result_rel,
diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c
index d465b9e213..f88762fada 100644
--- a/src/backend/optimizer/util/pathnode.c
+++ b/src/backend/optimizer/util/pathnode.c
@@ -2880,6 +2880,45 @@ create_sort_path(PlannerInfo *root,
 	return pathnode;
 }
 
+BatchSortPath *
+create_batchsort_path(PlannerInfo *root,
+					  RelOptInfo *rel,
+					  Path *subpath,
+					  List *pathkeys,
+					  List *groupClause,
+					  uint32 numBatches,
+					  bool parallel_sort)
+{
+	BatchSortPath   *pathnode = makeNode(BatchSortPath);
+	Assert(numBatches > 0);
+
+	pathnode->path.pathtype = T_BatchSort;
+	pathnode->path.parent = rel;
+	/* Sort doesn't project, so use source path's pathtarget */
+	pathnode->path.pathtarget = subpath->pathtarget;
+	/* For now, assume we are above any joins, so no parameterization */
+	pathnode->path.param_info = NULL;
+	pathnode->path.parallel_aware = parallel_sort;
+	pathnode->path.parallel_safe = rel->consider_parallel &&
+		subpath->parallel_safe;
+	pathnode->path.parallel_workers = subpath->parallel_workers;
+	pathnode->batchkeys = pathkeys;
+	pathnode->batchgroup = groupClause;
+	pathnode->numBatches = numBatches;
+
+	pathnode->subpath = subpath;
+
+	cost_batchsort(&pathnode->path, root, pathkeys,
+				   subpath->total_cost, subpath->rows,
+				   subpath->pathtarget->width,
+				   0.0,				/* XXX comparison_cost shouldn't be 0? */
+				   work_mem/numBatches,
+				   list_length(groupClause),
+				   numBatches);
+
+	return pathnode;
+}
+
 /*
  * create_group_path
  *	  Creates a pathnode that represents performing grouping of presorted input
diff --git a/src/backend/optimizer/util/tlist.c b/src/backend/optimizer/util/tlist.c
index 89853a0630..7725139573 100644
--- a/src/backend/optimizer/util/tlist.c
+++ b/src/backend/optimizer/util/tlist.c
@@ -588,6 +588,27 @@ grouping_is_hashable(List *groupClause)
 	return true;
 }

+/*
+ * grouping_get_hashable - extract all hashable clause
+ *
+ * Some time we don't all clauses can be hash
+ */
+List *
+grouping_get_hashable(List *groupClause)
+{
+	ListCell   *lc;
+	List	   *result = NIL;
+
+	foreach (lc, groupClause)
+	{
+		SortGroupClause *groupcl = lfirst_node(SortGroupClause, lc);
+
+		if (groupcl->hashable)
+			result = lappend(result, groupcl);
+	}
+
+	return result;
+}
 
 /*****************************************************************************
  *		PathTarget manipulation functions
diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c
index 3f24a33ef1..5025724fd4 100644
--- a/src/backend/postmaster/pgstat.c
+++ b/src/backend/postmaster/pgstat.c
@@ -4045,6 +4045,12 @@ pgstat_get_wait_ipc(WaitEventIPC w)
 		case WAIT_EVENT_XACT_GROUP_UPDATE:
 			event_name = "XactGroupUpdate";
 			break;
+		case WAIT_EVENT_BATCH_SORT_BUILD:
+			event_name = "Batch/Sort/Building";
+			break;
+		case WAIT_EVENT_BATCH_HASH_BUILD:
+			event_name = "Batch/Hash/Building";
+			break;
 			/* no default case, so that compiler will warn */
 	}
 
diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c
index db7e59f8b7..1fd839904c 100644
--- a/src/backend/storage/lmgr/lwlock.c
+++ b/src/backend/storage/lmgr/lwlock.c
@@ -170,8 +170,6 @@ static const char *const BuiltinTrancheNames[] = {
 	"PerSessionRecordType",
 	/* LWTRANCHE_PER_SESSION_RECORD_TYPMOD: */
 	"PerSessionRecordTypmod",
-	/* LWTRANCHE_SHARED_TUPLESTORE: */
-	"SharedTupleStore",
 	/* LWTRANCHE_SHARED_TIDBITMAP: */
 	"SharedTidBitmap",
 	/* LWTRANCHE_PARALLEL_APPEND: */
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 17579eeaca..d9d1ea728e 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -3422,6 +3422,32 @@ static struct config_int ConfigureNamesInt[] =
 		check_huge_page_size, NULL, NULL
 	},
 
+	{
+		{"max_sort_batches", PGC_USERSET, QUERY_TUNING_METHOD,
+			gettext_noop("Sets the maximum number of batches for sort"),
+			NULL
+		},
+		&max_sort_batches,
+		/* boot and min value 0 mean is disable */
+		0, 0,
+		/* I think too many batches will make each batch not enough working memory when sorting */
+		MAX_PARALLEL_WORKER_LIMIT * 3,
+		NULL, NULL, NULL
+	},
+
+	{
+		{"max_hashagg_batches", PGC_USERSET, QUERY_TUNING_METHOD,
+			gettext_noop("Sets the maximum number of batches for hash aggregate"),
+			NULL
+		},
+		&max_hashagg_batches,
+		/* boot and min value 0 mean is disable, I think max_files_per_process/2 is a good value */
+		0, 0,
+		/* I think too many batches will make FD will open and close files frequently, this is guessed-at value */
+		10000,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"debug_invalidate_system_caches_always", PGC_SUSET, DEVELOPER_OPTIONS,
 			gettext_noop("Aggressively invalidate system caches for debugging purposes."),
diff --git a/src/backend/utils/sort/Makefile b/src/backend/utils/sort/Makefile
index 7ac3659261..f82f5aa8cd 100644
--- a/src/backend/utils/sort/Makefile
+++ b/src/backend/utils/sort/Makefile
@@ -19,7 +19,8 @@ OBJS = \
 	sharedtuplestore.o \
 	sortsupport.o \
 	tuplesort.o \
-	tuplestore.o
+	tuplestore.o \
+	batchstore.o
 
 tuplesort.o: qsort_tuple.c
 
diff --git a/src/backend/utils/sort/batchstore.c b/src/backend/utils/sort/batchstore.c
new file mode 100644
index 0000000000..4578c718d3
--- /dev/null
+++ b/src/backend/utils/sort/batchstore.c
@@ -0,0 +1,466 @@
+#include "postgres.h"
+
+#include "access/htup_details.h"
+#include "access/parallel.h"
+#include "commands/tablespace.h"
+#include "executor/nodeHash.h"
+#include "port/atomics.h"
+#include "storage/buffile.h"
+#include "utils/batchstore.h"
+#include "utils/memutils.h"
+#include "utils/sharedtuplestore.h"
+
+#define InvalidBatch UINT32_MAX
+
+typedef enum BatchMethod
+{
+	BSM_HASH = 1,
+	BSM_PARALLEL_HASH
+}BatchMethod;
+
+typedef struct BatchStoreParallelHashData
+{
+	pg_atomic_uint32	cur_batches;
+	uint32				num_batches;
+	uint32				num_participants;
+}BatchStoreParallelHashData;
+
+typedef struct BatchStoreData
+{
+	BatchStoreFuncs	func;
+	BatchMethod	method;
+	uint32		num_batches;
+	void	   *cur_batch_ptr;
+	uint32		cur_batch_num;
+	union
+	{
+		/* for hash */
+		struct
+		{
+			StringInfoData	hash_read_buf;
+		};
+		/* for parallel hash */
+		struct
+		{
+			dsm_segment		   *dsm_seg;
+			MemoryContext		accessor_mcontext;
+			/* in shared memory, next idle parallel hash batch number */
+			pg_atomic_uint32   *shm_ph_batch_num;
+		};
+	};
+	void	 *all_batches[FLEXIBLE_ARRAY_MEMBER];
+}BatchStoreData;
+
+static void bs_write_normal_hash(BatchStore bs, MinimalTuple mtup, uint32 hash);
+static MinimalTuple bs_read_normal_hash(BatchStore bs, uint32 *hash);
+
+static void bs_write_parallel_hash(BatchStore bs, MinimalTuple mtup, uint32 hash);
+static void bs_write_parallel_one_batch_hash(BatchStore bs, MinimalTuple mtup, uint32 hash);
+static MinimalTuple bs_read_parallel_hash(BatchStore bs, uint32 *hash);
+
+/*
+ * make an empty batch store
+ */
+static inline BatchStore
+make_empty_batch_store(uint32 num_batches, BatchMethod method)
+{
+	BatchStore bs = palloc0(offsetof(BatchStoreData, all_batches) +
+								sizeof(void*) * num_batches);
+
+	bs->method = method;
+	bs->num_batches = num_batches;
+	bs->cur_batch_num = InvalidBatch;
+
+	return bs;
+}
+
+/*
+ * Create a normal batch store
+ */
+BatchStore
+bs_begin_hash(uint32 num_batches)
+{
+	BatchStore bs = make_empty_batch_store(num_batches, BSM_HASH);
+
+	/* Initialize hash read buffer for MinimalTuple */
+	initStringInfo(&bs->hash_read_buf);
+	enlargeStringInfo(&bs->hash_read_buf, MINIMAL_TUPLE_DATA_OFFSET);
+	MemSet(bs->hash_read_buf.data, 0, MINIMAL_TUPLE_DATA_OFFSET);
+
+	PrepareTempTablespaces();
+
+	bs->func.hash_write = bs_write_normal_hash;
+	bs->func.hash_read = bs_read_normal_hash;
+	return bs;
+}
+
+size_t
+bs_parallel_hash_estimate(uint32 num_batches, uint32 nparticipants)
+{
+	return MAXALIGN(sizeof(struct BatchStoreParallelHashData)) +
+				MAXALIGN(sts_estimate(nparticipants)) * num_batches;
+}
+
+/*
+ * Create or attach shared batch store
+ */
+static BatchStore
+bs_begin_parallel_hash(BatchStoreParallelHash bsph,
+					   uint32 my_participant_num, bool init,
+					   SharedFileSet *fileset, const char *name,
+					   dsm_segment *dsm_seg)
+{
+	uint32			i;
+	MemoryContext	oldcontext;
+	char		   *addr;
+	char			buffer[24];
+	Size			sts_size = MAXALIGN(sts_estimate(bsph->num_participants));
+	BatchStore		bs = make_empty_batch_store(bsph->num_batches, BSM_PARALLEL_HASH);
+
+	bs->shm_ph_batch_num = &bsph->cur_batches;
+
+	bs->accessor_mcontext = AllocSetContextCreate(CurrentMemoryContext,
+												  "batch parallel hash",
+												  ALLOCSET_DEFAULT_SIZES);
+	oldcontext = MemoryContextSwitchTo(bs->accessor_mcontext);
+	addr = ((char*)bsph) + MAXALIGN(sizeof(*bsph));
+	for (i=bsph->num_batches;i>0;)
+	{
+		--i;
+		if (init)
+		{
+			sprintf(buffer, "%s_%u", name, i);
+			bs->all_batches[i] = sts_initialize((SharedTuplestore*)addr,
+												bsph->num_participants,
+												my_participant_num,
+												sizeof(uint32),
+												0,
+												fileset,
+												buffer);
+		}else
+		{
+			bs->all_batches[i] = sts_attach((SharedTuplestore*)addr,
+											my_participant_num,
+											fileset);
+		}
+		addr += sts_size;
+	}
+	MemoryContextSwitchTo(oldcontext);
+
+	bs->dsm_seg = dsm_seg;
+	bs->func.hash_read = bs_read_parallel_hash;
+	if (bs->num_batches == 1)
+		bs->func.hash_write = bs_write_parallel_one_batch_hash;
+	else
+		bs->func.hash_write = bs_write_parallel_hash;
+
+	return bs;
+}
+
+/*
+ * Create a parallel batch store
+ */
+BatchStore
+bs_init_parallel_hash(uint32 num_batches,
+					  uint32 nparticipants, uint32 my_participant_num,
+					  BatchStoreParallelHash bsph, dsm_segment *dsm_seg,
+					  SharedFileSet *fileset, const char *name)
+{
+	Assert(name != NULL && fileset != NULL);
+	bsph->num_batches = num_batches;
+	bsph->num_participants = nparticipants;
+	pg_atomic_init_u32(&bsph->cur_batches, InvalidBatch);
+
+	return bs_begin_parallel_hash(bsph, my_participant_num, true, fileset, name, dsm_seg);
+}
+
+/*
+ * Attach from parallel batch store
+ */
+BatchStore
+bs_attach_parallel_hash(BatchStoreParallelHash bsph, dsm_segment *dsm_seg,
+						SharedFileSet *fileset, uint32 my_participant_num)
+{
+	return bs_begin_parallel_hash(bsph, my_participant_num, false, fileset, NULL, dsm_seg);
+}
+
+/* Destory batch store */
+void
+bs_destory(BatchStore bs)
+{
+	uint32	i;
+	if (bs == NULL)
+		return;
+
+	switch(bs->method)
+	{
+	case BSM_HASH:
+		for(i=0;i<bs->num_batches;++i)
+		{
+			if (bs->all_batches[i])
+				BufFileClose(bs->all_batches[i]);
+		}
+		pfree(bs->hash_read_buf.data);
+		break;
+	case BSM_PARALLEL_HASH:
+		{
+			BatchStoreParallelHash bsph = (BatchStoreParallelHash)(((char*)bs->shm_ph_batch_num) -
+											offsetof(BatchStoreParallelHashData, cur_batches));
+			uint32 count = bsph->num_batches;
+			while (count > 0)
+				sts_detach(bs->all_batches[--count]);
+			MemoryContextDelete(bs->accessor_mcontext);
+		}
+		break;
+	default:
+		ereport(ERROR,
+				(errmsg("unknown batch store method %u", bs->method)));
+		break;
+	}
+
+	pfree(bs);
+}
+
+/*
+ * Write MinimalTuple and hash value to normal batch store
+ */
+static void
+bs_write_normal_hash(BatchStore bs, MinimalTuple mtup, uint32 hash)
+{
+	uint32 batch = hash % bs->num_batches;
+	uint32 data_len = mtup->t_len - MINIMAL_TUPLE_DATA_OFFSET;
+	BufFile *buffile = bs->all_batches[batch];
+
+	if (unlikely(buffile == NULL))
+	{
+		MemoryContext oldcontext = MemoryContextSwitchTo(GetMemoryChunkContext(bs));
+		buffile = BufFileCreateTemp(false);
+		bs->all_batches[batch] = buffile;
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	BufFileWrite(buffile, &hash, sizeof(hash));
+	BufFileWrite(buffile, &mtup->t_len, sizeof(mtup->t_len));
+	BufFileWrite(buffile, ((char*)mtup) + MINIMAL_TUPLE_DATA_OFFSET, data_len);
+}
+
+/*
+ * Read MinimalTuple and hash value from normal batch store
+ */
+static MinimalTuple
+bs_read_normal_hash(BatchStore bs, uint32 *hash)
+{
+	MinimalTuple	mtup;
+	size_t			nread;
+	uint32			head[2];
+	uint32			data_len;
+
+	/* Read hash value and tuple length */
+	nread = BufFileRead(bs->cur_batch_ptr, head, sizeof(head));
+	if (nread == 0)
+		return NULL;
+
+	if (nread != sizeof(head))
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not read from batch store temporary file: %m")));
+	*hash = head[0];
+
+	/* Enlarge buffer and read tuple data */
+	enlargeStringInfo(&bs->hash_read_buf, head[1]);
+	mtup = (MinimalTuple)bs->hash_read_buf.data;
+	mtup->t_len = head[1];
+	data_len = head[1] - MINIMAL_TUPLE_DATA_OFFSET;
+	if (BufFileRead(bs->cur_batch_ptr, ((char*)mtup) + MINIMAL_TUPLE_DATA_OFFSET, data_len) != data_len)
+		ereport(ERROR,
+				(errcode_for_file_access(),
+				 errmsg("could not read from batch store temporary file: %m")));
+
+	return mtup;
+}
+
+/* end write batch store, ready for read */
+void
+bs_end_write(BatchStore bs)
+{
+	uint32 i;
+	switch(bs->method)
+	{
+	case BSM_HASH:
+		/* nothing to do */
+		break;
+	case BSM_PARALLEL_HASH:
+		for (i=bs->num_batches;i>0;)
+			sts_end_write(bs->all_batches[--i]);
+		bs->cur_batch_ptr = NULL;
+		break;
+	default:
+		ereport(ERROR,
+				(errmsg("unknown batch store method %u", bs->method)));
+		break;
+	}
+}
+
+/*
+ * Write MinimalTuple and hash value to parallel batch store
+ */
+static void
+bs_write_parallel_hash(BatchStore bs, MinimalTuple mtup, uint32 hash)
+{
+	sts_puttuple(bs->all_batches[hash%bs->num_batches],
+				 &hash,
+				 mtup);
+}
+
+/*
+ * Write MinimalTuple and hash value to parallel batch store,
+ * only for only have one batch
+ */
+static void
+bs_write_parallel_one_batch_hash(BatchStore bs, MinimalTuple mtup, uint32 hash)
+{
+	Assert(bs->num_batches == 1);
+	sts_puttuple(bs->all_batches[0],
+				 &hash,
+				 mtup);
+}
+
+/*
+ * Read MinimalTuple and hash value from parallel batch store
+ */
+static MinimalTuple
+bs_read_parallel_hash(BatchStore bs, uint32 *hash)
+{
+	return sts_scan_next(bs->cur_batch_ptr, hash);
+}
+
+/*
+ * Get next batch from batch store, return false if no more.
+ */
+bool
+bs_next_batch(BatchStore bs, bool no_parallel)
+{
+	uint32 batch;
+	switch(bs->method)
+	{
+	case BSM_HASH:
+
+		batch = bs->cur_batch_num;
+		++batch;
+
+		for (;batch < bs->num_batches;++batch)
+		{
+			if (bs->all_batches[batch])
+			{
+				bs->cur_batch_ptr = bs->all_batches[batch];
+				bs->cur_batch_num = batch;
+				if (BufFileSeek(bs->cur_batch_ptr, 0, 0, SEEK_SET) != 0)
+				{
+					ereport(ERROR,
+							(errcode_for_file_access(),
+							 errmsg("can not seek batch store file to head")));
+				}
+				return true;
+			}
+		}
+		break;
+	case BSM_PARALLEL_HASH:
+		if (no_parallel)
+		{
+			batch = bs->cur_batch_num;
+			++batch;
+		}else
+		{
+			batch = pg_atomic_add_fetch_u32(bs->shm_ph_batch_num, 1);
+		}
+
+		if (batch < bs->num_batches)
+		{
+			bs->cur_batch_num = batch;
+			bs->cur_batch_ptr = bs->all_batches[batch];
+			sts_begin_scan(bs->cur_batch_ptr);
+			return true;
+		}
+		break;
+	default:
+		ereport(ERROR,
+				(errmsg("unknown batch store method %u", bs->method)));
+		break;
+	}
+
+	return false;
+}
+
+void
+bs_rescan(BatchStore bs)
+{
+	switch(bs->method)
+	{
+	case BSM_HASH:
+		break;
+	case BSM_PARALLEL_HASH:
+		for (uint32 i=bs->num_batches;i>0;)
+			sts_reinitialize(bs->all_batches[--i]);
+		pg_atomic_write_u32(bs->shm_ph_batch_num, InvalidBatch);
+		break;
+	default:
+		ereport(ERROR,
+				(errmsg("unknown batch store method %u", bs->method)));
+		break;
+	}
+	bs->cur_batch_ptr = NULL;
+	bs->cur_batch_num = InvalidBatch;
+}
+
+/*
+ * Delete all the contents of a batch store
+ */
+void
+bs_clear(BatchStore bs)
+{
+	uint32	i;
+	switch (bs->method)
+	{
+	case BSM_HASH:
+		for (i=bs->num_batches;i>0;)
+		{
+			--i;
+			if (bs->all_batches[i])
+			{
+				BufFileClose(bs->all_batches[i]);
+				bs->all_batches[i] = 0;
+			}
+		}
+		break;
+	case BSM_PARALLEL_HASH:
+		pg_atomic_write_u32(bs->shm_ph_batch_num, InvalidBatch);
+		for (i=bs->num_batches;i>0;)
+			sts_clear(bs->all_batches[--i]);
+		break;
+	default:
+		ereport(ERROR,
+				(errmsg("unknown batch store method %u", bs->method)));
+		break;
+	}
+	bs->cur_batch_ptr = NULL;
+	bs->cur_batch_num = InvalidBatch;
+}
+
+void
+bs_end_cur_batch(BatchStore bs)
+{
+	switch(bs->method)
+	{
+	case BSM_HASH:
+		bs->cur_batch_ptr = NULL;
+		break;
+	case BSM_PARALLEL_HASH:
+		sts_end_scan(bs->cur_batch_ptr);
+		bs->cur_batch_ptr = NULL;
+		break;
+	default:
+		ereport(ERROR,
+				(errmsg("unknown batch store method %u", bs->method)));
+		break;
+	}
+}
diff --git a/src/backend/utils/sort/sharedtuplestore.c b/src/backend/utils/sort/sharedtuplestore.c
index 57e35db4f8..03472c080e 100644
--- a/src/backend/utils/sort/sharedtuplestore.c
+++ b/src/backend/utils/sort/sharedtuplestore.c
@@ -24,9 +24,10 @@
 #include "access/htup.h"
 #include "access/htup_details.h"
 #include "miscadmin.h"
+#include "port/atomics.h"
 #include "storage/buffile.h"
-#include "storage/lwlock.h"
 #include "storage/sharedfileset.h"
+#include "utils/memutils.h"
 #include "utils/sharedtuplestore.h"
 
 /*
@@ -50,8 +51,7 @@ typedef struct SharedTuplestoreChunk
 /* Per-participant shared state. */
 typedef struct SharedTuplestoreParticipant
 {
-	LWLock		lock;
-	BlockNumber read_page;		/* Page number for next read. */
+	pg_atomic_uint32 read_page;	/* Page number for next read. */
 	BlockNumber npages;			/* Number of pages written. */
 	bool		writing;		/* Used only for assertions. */
 } SharedTuplestoreParticipant;
@@ -72,6 +72,8 @@ struct SharedTuplestore
 struct SharedTuplestoreAccessor
 {
 	int			participant;	/* My participant number. */
+	bool		is_read_only;	/* is read only attach? */
+	bool		is_normal_scan;	/* is not parallel scan? */
 	SharedTuplestore *sts;		/* The shared state. */
 	SharedFileSet *fileset;		/* The SharedFileSet holding files. */
 	MemoryContext context;		/* Memory context for buffers. */
@@ -155,9 +157,8 @@ sts_initialize(SharedTuplestore *sts, int participants,
 
 	for (i = 0; i < participants; ++i)
 	{
-		LWLockInitialize(&sts->participants[i].lock,
-						 LWTRANCHE_SHARED_TUPLESTORE);
-		sts->participants[i].read_page = 0;
+		pg_atomic_init_u32(&sts->participants[i].read_page, 0);
+		sts->participants[i].npages = 0;
 		sts->participants[i].writing = false;
 	}
 
@@ -192,6 +193,27 @@ sts_attach(SharedTuplestore *sts,
 	return accessor;
 }
 
+/*
+ * Like function sts_attach, but only for read, can't write any tuple
+ */
+SharedTuplestoreAccessor *
+sts_attach_read_only(SharedTuplestore *sts,
+					 SharedFileSet *fileset)
+{
+	SharedTuplestoreAccessor *accessor;
+
+	Assert(sts->nparticipants > 0);
+
+	accessor = palloc0(sizeof(SharedTuplestoreAccessor));
+	accessor->is_read_only = true;
+	accessor->participant = 0;
+	accessor->sts = sts;
+	accessor->fileset = fileset;
+	accessor->context = CurrentMemoryContext;
+
+	return accessor;
+}
+
 static void
 sts_flush_chunk(SharedTuplestoreAccessor *accessor)
 {
@@ -242,10 +264,42 @@ sts_reinitialize(SharedTuplestoreAccessor *accessor)
 	 */
 	for (i = 0; i < accessor->sts->nparticipants; ++i)
 	{
-		accessor->sts->participants[i].read_page = 0;
+		pg_atomic_init_u32(&accessor->sts->participants[i].read_page, 0);
 	}
 }
 
+/*
+ * Delete all the contents of shared tuplestore, and reset read page to the start
+ */
+void
+sts_clear(SharedTuplestoreAccessor *accessor)
+{
+	int					i;
+	char				name[MAXPGPATH];
+	SharedTuplestore   *sts = accessor->sts;
+
+	/* Must not in read and write */
+	Assert(accessor->read_file == NULL &&
+		   accessor->write_file == NULL);
+
+	/* Delete all created files */
+	for (i=0;i<sts->nparticipants;++i)
+	{
+		Assert(sts->participants[i].writing == false);
+		if (sts->participants[i].npages > 0)
+		{
+			sts_filename(name, accessor, i);
+			BufFileDeleteShared(accessor->fileset, name);
+			pg_atomic_write_u32(&sts->participants[i].read_page, 0);
+			sts->participants[i].npages = 0;
+		}
+	}
+
+	accessor->write_page = 0;
+	accessor->write_pointer = NULL;
+	accessor->write_end = NULL;
+}
+
 /*
  * Begin scanning the contents in parallel.
  */
@@ -272,6 +326,8 @@ sts_begin_parallel_scan(SharedTuplestoreAccessor *accessor)
 	accessor->read_participant = accessor->participant;
 	accessor->read_file = NULL;
 	accessor->read_next_page = 0;
+	accessor->read_ntuples = 0;
+	accessor->read_ntuples_available = 0;
 }
 
 /*
@@ -302,15 +358,23 @@ sts_puttuple(SharedTuplestoreAccessor *accessor, void *meta_data,
 {
 	size_t		size;
 
+	if (unlikely(accessor->is_read_only))
+	{
+		ereport(ERROR,
+				(errmsg("shard tuplestore is attached read only")));
+	}
+
 	/* Do we have our own file yet? */
 	if (accessor->write_file == NULL)
 	{
 		SharedTuplestoreParticipant *participant;
-		char		name[MAXPGPATH];
+		char			name[MAXPGPATH];
+		MemoryContext	oldcontext = MemoryContextSwitchTo(accessor->context);
 
 		/* Create one.  Only this backend will write into it. */
 		sts_filename(name, accessor, accessor->participant);
 		accessor->write_file = BufFileCreateShared(accessor->fileset, name);
+		MemoryContextSwitchTo(oldcontext);
 
 		/* Set up the shared state for this backend's file. */
 		participant = &accessor->sts->participants[accessor->participant];
@@ -532,20 +596,36 @@ sts_parallel_scan_next(SharedTuplestoreAccessor *accessor, void *meta_data)
 		/* Find the location of a new chunk to read. */
 		p = &accessor->sts->participants[accessor->read_participant];
 
-		LWLockAcquire(&p->lock, LW_EXCLUSIVE);
-		/* We can skip directly past overflow pages we know about. */
-		if (p->read_page < accessor->read_next_page)
-			p->read_page = accessor->read_next_page;
-		eof = p->read_page >= p->npages;
-		if (!eof)
+		if (accessor->is_normal_scan)
+		{
+			eof = accessor->read_next_page >= p->npages;
+			if (!eof)
+			{
+				read_page = accessor->read_next_page;
+				accessor->read_next_page += STS_CHUNK_PAGES;
+			}
+		}else
 		{
 			/* Claim the next chunk. */
-			read_page = p->read_page;
-			/* Advance the read head for the next reader. */
-			p->read_page += STS_CHUNK_PAGES;
-			accessor->read_next_page = p->read_page;
+			read_page = pg_atomic_read_u32(&p->read_page);
+			/* We can skip directly past overflow pages we know about. */
+			while (read_page < accessor->read_next_page)
+			{
+				if (pg_atomic_compare_exchange_u32(&p->read_page,
+												   &read_page,
+												   accessor->read_next_page))
+					break;
+			}
+			while ((eof = read_page >= p->npages) == false)
+			{
+				/* Advance the read head for the next reader. */
+				accessor->read_next_page = read_page + STS_CHUNK_PAGES;
+				if (pg_atomic_compare_exchange_u32(&p->read_page,
+												   &read_page,
+												   accessor->read_next_page))
+					break;
+			}
 		}
-		LWLockRelease(&p->lock);
 
 		if (!eof)
 		{
@@ -556,10 +636,12 @@ sts_parallel_scan_next(SharedTuplestoreAccessor *accessor, void *meta_data)
 			if (accessor->read_file == NULL)
 			{
 				char		name[MAXPGPATH];
+				MemoryContext oldcontext = MemoryContextSwitchTo(accessor->context);
 
 				sts_filename(name, accessor, accessor->read_participant);
 				accessor->read_file =
 					BufFileOpenShared(accessor->fileset, name, O_RDONLY);
+				MemoryContextSwitchTo(oldcontext);
 			}
 
 			/* Seek and load the chunk header. */
@@ -626,3 +708,46 @@ sts_filename(char *name, SharedTuplestoreAccessor *accessor, int participant)
 {
 	snprintf(name, MAXPGPATH, "%s.p%d", accessor->sts->name, participant);
 }
+
+/*
+ * Begin scanning the contents.
+ */
+void
+sts_begin_scan(SharedTuplestoreAccessor *accessor)
+{
+	sts_begin_parallel_scan(accessor);
+	accessor->is_normal_scan = true;
+}
+
+/*
+ * Finish a normal scan, freeing associated backend-local resources.
+ */
+void
+sts_end_scan(SharedTuplestoreAccessor *accessor)
+{
+	Assert(accessor->is_normal_scan);
+	sts_end_parallel_scan(accessor);
+	accessor->is_normal_scan = false;
+}
+
+/*
+ * Get the next tuple in the current normal scan.
+ */
+MinimalTuple
+sts_scan_next(SharedTuplestoreAccessor *accessor,
+					   void *meta_data)
+{
+	Assert(accessor->is_normal_scan);
+	return sts_parallel_scan_next(accessor, meta_data);
+}
+
+/*
+ * Detach SharedTuplestore and freeing associated backend-local resources.
+ */
+void
+sts_detach(SharedTuplestoreAccessor *accessor)
+{
+	sts_end_write(accessor);
+	sts_end_parallel_scan(accessor);
+	pfree(accessor);
+}
diff --git a/src/backend/utils/sort/tuplesort.c b/src/backend/utils/sort/tuplesort.c
index 7d0f96afb7..fc38d26a68 100644
--- a/src/backend/utils/sort/tuplesort.c
+++ b/src/backend/utils/sort/tuplesort.c
@@ -4573,6 +4573,26 @@ tuplesort_initialize_shared(Sharedsort *shared, int nWorkers, dsm_segment *seg)
 	}
 }
 
+/*
+ * tuplesort_reset_shared - reset shared tuplesort state
+ *
+ * Must be called from leader process before workers are launched,
+ * can be reuse after this call
+ */
+void
+tuplesort_reset_shared(Sharedsort *shared)
+{
+	int			i;
+
+	shared->currentWorker = 0;
+	shared->workersFinished = 0;
+	SharedFileSetDeleteAll(&shared->fileset);
+	for (i = 0; i < shared->nTapes; i++)
+	{
+		shared->tapes[i].firstblocknumber = 0L;
+	}
+}
+
 /*
  * tuplesort_attach_shared - attach to shared tuplesort state
  *
diff --git a/src/include/executor/nodeAgg.h b/src/include/executor/nodeAgg.h
index 398446d11f..68b277b379 100644
--- a/src/include/executor/nodeAgg.h
+++ b/src/include/executor/nodeAgg.h
@@ -310,6 +310,8 @@ typedef struct AggStatePerHashData
 	int			largestGrpColIdx;	/* largest col required for hashing */
 	AttrNumber *hashGrpColIdxInput; /* hash col indices in input slot */
 	AttrNumber *hashGrpColIdxHash;	/* indices in hash table tuples */
+	Bitmapset  *colnos_needed;	/* all columns needed from the outer plan */
+	struct BatchStoreData *batch_store;	/* grouping set batch store hash */
 	Agg		   *aggnode;		/* original Agg node, for numGroups etc. */
 }			AggStatePerHashData;
 
diff --git a/src/include/executor/nodeBatchSort.h b/src/include/executor/nodeBatchSort.h
new file mode 100644
index 0000000000..66c68e0125
--- /dev/null
+++ b/src/include/executor/nodeBatchSort.h
@@ -0,0 +1,19 @@
+
+#ifndef NODE_BATCH_SORT_H
+#define NODE_BATCH_SORT_H
+
+#include "access/parallel.h"
+#include "nodes/execnodes.h"
+
+extern BatchSortState *ExecInitBatchSort(BatchSort *node, EState *estate, int eflags);
+extern void ExecEndBatchSort(BatchSortState *node);
+extern void ExecReScanBatchSort(BatchSortState *node);
+
+/* parallel scan support */
+extern void ExecBatchSortEstimate(BatchSortState *node, ParallelContext *pcxt);
+extern void ExecBatchSortInitializeDSM(BatchSortState *node, ParallelContext *pcxt);
+extern void ExecBatchSortReInitializeDSM(BatchSortState *node, ParallelContext *pcxt);
+extern void ExecBatchSortInitializeWorker(BatchSortState *node, ParallelWorkerContext *pwcxt);
+extern void ExecShutdownBatchSort(BatchSortState *node);
+
+#endif							/* NODE_BATCH_SORT_H */
\ No newline at end of file
diff --git a/src/include/nodes/execnodes.h b/src/include/nodes/execnodes.h
index 48c3f570fa..8ad2a24ab7 100644
--- a/src/include/nodes/execnodes.h
+++ b/src/include/nodes/execnodes.h
@@ -2011,6 +2011,22 @@ typedef struct SortState
 	SharedSortInfo *shared_info;	/* one entry per worker */
 } SortState;
 
+/* ----------------
+ *	 BatchSortState information
+ * ----------------
+ */
+typedef struct BatchSortState
+{
+	PlanState	ps;				/* its first field is NodeTag */
+	struct Tuplesortstate
+			  **batches;		/* private state of tuplesort.c */
+	List	   *groupFuns;		/* hash function call info for each group-key */
+	struct ParallelBatchSort
+			   *parallel;		/* parallel info, private in nodeBatchSort.c */
+	int			curBatch;		/* current batch index */
+	bool		sort_Done;		/* sort completed yet? */
+}BatchSortState;
+
 /* ----------------
  *	 Instrumentation information for IncrementalSort
  * ----------------
@@ -2202,6 +2218,9 @@ typedef struct AggState
 										 * ->hash_pergroup */
 	ProjectionInfo *combinedproj;	/* projection machinery */
 	SharedAggInfo *shared_info; /* one entry per worker */
+	struct Barrier *batch_barrier;		/* for parallel batch */
+	int			current_batch_set;		/* current batch grouping set */
+	bool		batch_filled;			/* is all batches filled? */
 } AggState;
 
 /* ----------------
diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index caed683ba9..8da83c7269 100644
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -74,6 +74,7 @@ typedef enum NodeTag
 	T_HashJoin,
 	T_Material,
 	T_Sort,
+	T_BatchSort,
 	T_IncrementalSort,
 	T_Group,
 	T_Agg,
@@ -131,6 +132,7 @@ typedef enum NodeTag
 	T_HashJoinState,
 	T_MaterialState,
 	T_SortState,
+	T_BatchSortState,
 	T_IncrementalSortState,
 	T_GroupState,
 	T_AggState,
@@ -245,6 +247,7 @@ typedef enum NodeTag
 	T_ProjectionPath,
 	T_ProjectSetPath,
 	T_SortPath,
+	T_BatchSortPath,
 	T_IncrementalSortPath,
 	T_GroupPath,
 	T_UpperUniquePath,
@@ -759,7 +762,8 @@ typedef enum AggStrategy
 	AGG_PLAIN,					/* simple agg across all input rows */
 	AGG_SORTED,					/* grouped agg, input must be sorted */
 	AGG_HASHED,					/* grouped agg, use internal hashtable */
-	AGG_MIXED					/* grouped agg, hash and sort both used */
+	AGG_MIXED,					/* grouped agg, hash and sort both used */
+	AGG_BATCH_HASH				/* grouped agg, use batch hash */
 } AggStrategy;
 
 /*
diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h
index cde2637798..2db484a7da 100644
--- a/src/include/nodes/pathnodes.h
+++ b/src/include/nodes/pathnodes.h
@@ -1654,6 +1654,16 @@ typedef struct SortPath
 	Path	   *subpath;		/* path representing input source */
 } SortPath;
 
+typedef struct BatchSortPath
+{
+	Path		path;
+	Path	   *subpath;		/* path representing input source */
+	List	   *batchkeys;		/* our result is not all ordered, only for each batch,
+								 * so we can not use Path::pathkeys */
+	List	   *batchgroup;		/* a list of SortGroupClause for hash */
+	uint32		numBatches;
+}BatchSortPath;
+
 /*
  * IncrementalSortPath represents an incremental sort step
  *
@@ -2500,6 +2510,7 @@ typedef struct
 	bool		target_parallel_safe;
 	Node	   *havingQual;
 	List	   *targetList;
+	List	   *hashable_groups;
 	PartitionwiseAggregateType patype;
 } GroupPathExtraData;
 
diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h
index 43160439f0..675850de13 100644
--- a/src/include/nodes/plannodes.h
+++ b/src/include/nodes/plannodes.h
@@ -774,6 +774,19 @@ typedef struct Sort
 	bool	   *nullsFirst;		/* NULLS FIRST/LAST directions */
 } Sort;
 
+/* ----------------
+ *		batch sort node
+ * ----------------
+ */
+typedef struct BatchSort
+{
+	Sort		sort;
+	int			numGroupCols;	/* number of group-key columns */
+	int			numBatches;		/* number of group */
+	int			gather_param;	/* ID of Param of plan Gather or GatherMerge */
+	AttrNumber *grpColIdx;		/* their indexes in the target list */
+}BatchSort;
+
 /* ----------------
  *		incremental sort node
  * ----------------
@@ -828,6 +841,8 @@ typedef struct Agg
 	/* Note: planner provides numGroups & aggParams only in HASHED/MIXED case */
 	List	   *groupingSets;	/* grouping sets to use */
 	List	   *chain;			/* chained Agg/Sort nodes */
+	int			numBatches;		/* valid in AGG_BATCH_HASH */
+	int			gatherParam;	/* param of gather */
 } Agg;
 
 /* ----------------
diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h
index ed2e4af4be..f44796ad68 100644
--- a/src/include/optimizer/cost.h
+++ b/src/include/optimizer/cost.h
@@ -66,6 +66,8 @@ extern PGDLLIMPORT bool enable_parallel_append;
 extern PGDLLIMPORT bool enable_parallel_hash;
 extern PGDLLIMPORT bool enable_partition_pruning;
 extern PGDLLIMPORT int constraint_exclusion;
+extern PGDLLIMPORT int max_sort_batches;
+extern PGDLLIMPORT int max_hashagg_batches;
 
 extern double index_pages_fetched(double tuples_fetched, BlockNumber pages,
 								  double index_pages, PlannerInfo *root);
@@ -102,6 +104,11 @@ extern void cost_sort(Path *path, PlannerInfo *root,
 					  List *pathkeys, Cost input_cost, double tuples, int width,
 					  Cost comparison_cost, int sort_mem,
 					  double limit_tuples);
+extern void cost_batchsort(Path *path, PlannerInfo *root,
+						   List *batchkeys, Cost input_cost,
+						   double tuples, int width,
+						   Cost comparison_cost, int sort_mem,
+						   uint32 numGroupCols, uint32 numBatchs);
 extern void cost_incremental_sort(Path *path,
 								  PlannerInfo *root, List *pathkeys, int presorted_keys,
 								  Cost input_startup_cost, Cost input_total_cost,
diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h
index 23dec14cbd..9f87d71b95 100644
--- a/src/include/optimizer/pathnode.h
+++ b/src/include/optimizer/pathnode.h
@@ -195,6 +195,13 @@ extern IncrementalSortPath *create_incremental_sort_path(PlannerInfo *root,
 														 List *pathkeys,
 														 int presorted_keys,
 														 double limit_tuples);
+extern BatchSortPath *create_batchsort_path(PlannerInfo *root,
+											RelOptInfo *rel,
+											Path *subpath,
+											List *pathkeys,
+											List *groupClause,
+											uint32 numBatches,
+											bool parallel_sort);
 extern GroupPath *create_group_path(PlannerInfo *root,
 									RelOptInfo *rel,
 									Path *subpath,
diff --git a/src/include/optimizer/tlist.h b/src/include/optimizer/tlist.h
index e081ef2d5e..188a91e0c2 100644
--- a/src/include/optimizer/tlist.h
+++ b/src/include/optimizer/tlist.h
@@ -36,6 +36,7 @@ extern Oid *extract_grouping_collations(List *groupClause, List *tlist);
 extern AttrNumber *extract_grouping_cols(List *groupClause, List *tlist);
 extern bool grouping_is_sortable(List *groupClause);
 extern bool grouping_is_hashable(List *groupClause);
+extern List *grouping_get_hashable(List *groupClause);
 
 extern PathTarget *make_pathtarget_from_tlist(List *tlist);
 extern List *make_tlist_from_pathtarget(PathTarget *target);
diff --git a/src/include/pgstat.h b/src/include/pgstat.h
index c38b689710..67138fa566 100644
--- a/src/include/pgstat.h
+++ b/src/include/pgstat.h
@@ -964,7 +964,9 @@ typedef enum
 	WAIT_EVENT_REPLICATION_SLOT_DROP,
 	WAIT_EVENT_SAFE_SNAPSHOT,
 	WAIT_EVENT_SYNC_REP,
-	WAIT_EVENT_XACT_GROUP_UPDATE
+	WAIT_EVENT_XACT_GROUP_UPDATE,
+	WAIT_EVENT_BATCH_SORT_BUILD,
+	WAIT_EVENT_BATCH_HASH_BUILD
 } WaitEventIPC;
 
 /* ----------
diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h
index cbf2510fbf..cee0a540c9 100644
--- a/src/include/storage/lwlock.h
+++ b/src/include/storage/lwlock.h
@@ -214,7 +214,6 @@ typedef enum BuiltinTrancheIds
 	LWTRANCHE_PER_SESSION_DSA,
 	LWTRANCHE_PER_SESSION_RECORD_TYPE,
 	LWTRANCHE_PER_SESSION_RECORD_TYPMOD,
-	LWTRANCHE_SHARED_TUPLESTORE,
 	LWTRANCHE_SHARED_TIDBITMAP,
 	LWTRANCHE_PARALLEL_APPEND,
 	LWTRANCHE_PER_XACT_PREDICATE_LIST,
diff --git a/src/include/utils/batchstore.h b/src/include/utils/batchstore.h
new file mode 100644
index 0000000000..f8d276a0d7
--- /dev/null
+++ b/src/include/utils/batchstore.h
@@ -0,0 +1,38 @@
+#ifndef BATCH_STORE_H
+#define BATCH_STORE_H
+
+#include "access/htup.h"
+#include "storage/dsm.h"
+#include "storage/sharedfileset.h"
+
+typedef struct BatchStoreData* BatchStore;
+typedef struct BatchStoreParallelHashData* BatchStoreParallelHash;
+
+typedef struct BatchStoreFuncs
+{
+	void (*hash_write)(BatchStore bs, MinimalTuple mtup, uint32 hash);
+	MinimalTuple (*hash_read)(BatchStore bs, uint32 *hash);
+}BatchStoreFuncs;
+
+#define bs_write_hash(bs, mtup, hash) (*((BatchStoreFuncs*)bs)->hash_write)(bs, mtup, hash)
+#define bs_read_hash(bs, phash) (*((BatchStoreFuncs*)bs)->hash_read)(bs, phash)
+
+extern BatchStore bs_begin_hash(uint32 num_batches);
+
+extern size_t bs_parallel_hash_estimate(uint32 num_batches, uint32 nparticipants);
+extern BatchStore bs_init_parallel_hash(uint32 num_batches,
+										uint32 nparticipants, uint32 my_participant_num,
+										BatchStoreParallelHash bsph, dsm_segment *dsm_seg,
+										SharedFileSet *fileset, const char *name);
+extern BatchStore bs_attach_parallel_hash(BatchStoreParallelHash bsph, dsm_segment *dsm_seg,
+										  SharedFileSet *fileset, uint32 my_participant_num);
+
+extern void bs_destory(BatchStore bs);
+extern void bs_clear(BatchStore bs);
+
+extern void bs_end_write(BatchStore bs);
+
+extern bool bs_next_batch(BatchStore bs, bool no_parallel);
+extern void bs_rescan(BatchStore bs);
+extern void bs_end_cur_batch(BatchStore bs);
+#endif /* BATCH_STORE_H */
\ No newline at end of file
diff --git a/src/include/utils/sharedtuplestore.h b/src/include/utils/sharedtuplestore.h
index 01ad6efe51..1848dc8d78 100644
--- a/src/include/utils/sharedtuplestore.h
+++ b/src/include/utils/sharedtuplestore.h
@@ -43,10 +43,15 @@ extern SharedTuplestoreAccessor *sts_attach(SharedTuplestore *sts,
 											int my_participant_number,
 											SharedFileSet *fileset);
 
+extern SharedTuplestoreAccessor *sts_attach_read_only(SharedTuplestore *sts,
+													  SharedFileSet *fileset);
+
 extern void sts_end_write(SharedTuplestoreAccessor *accessor);
 
 extern void sts_reinitialize(SharedTuplestoreAccessor *accessor);
 
+extern void sts_clear(SharedTuplestoreAccessor *accessor);
+
 extern void sts_begin_parallel_scan(SharedTuplestoreAccessor *accessor);
 
 extern void sts_end_parallel_scan(SharedTuplestoreAccessor *accessor);
@@ -58,4 +63,13 @@ extern void sts_puttuple(SharedTuplestoreAccessor *accessor,
 extern MinimalTuple sts_parallel_scan_next(SharedTuplestoreAccessor *accessor,
 										   void *meta_data);
 
+extern void sts_begin_scan(SharedTuplestoreAccessor *accessor);
+
+extern void sts_end_scan(SharedTuplestoreAccessor *accessor);
+
+extern MinimalTuple sts_scan_next(SharedTuplestoreAccessor *accessor,
+								  void *meta_data);
+
+extern void sts_detach(SharedTuplestoreAccessor *accessor);
+
 #endif							/* SHAREDTUPLESTORE_H */
diff --git a/src/include/utils/tuplesort.h b/src/include/utils/tuplesort.h
index f94949370b..19f92ee946 100644
--- a/src/include/utils/tuplesort.h
+++ b/src/include/utils/tuplesort.h
@@ -266,6 +266,7 @@ extern Size tuplesort_estimate_shared(int nworkers);
 extern void tuplesort_initialize_shared(Sharedsort *shared, int nWorkers,
 										dsm_segment *seg);
 extern void tuplesort_attach_shared(Sharedsort *shared, dsm_segment *seg);
+extern void tuplesort_reset_shared(Sharedsort *shared);
 
 /*
  * These routines may only be called if randomAccess was specified 'true'.
diff --git a/src/test/regress/expected/groupingsets.out b/src/test/regress/expected/groupingsets.out
index 7c844c6e09..bc1310ad23 100644
--- a/src/test/regress/expected/groupingsets.out
+++ b/src/test/regress/expected/groupingsets.out
@@ -1929,4 +1929,69 @@ set work_mem to default;
 
 drop table gs_group_1;
 drop table gs_hash_1;
+-- parallel grouping sets
+BEGIN;
+set max_hashagg_batches = 512;
+set min_parallel_table_scan_size = 0;
+set parallel_setup_cost = 10;
+explain (costs off)
+select sum(unique1),count(unique1),two,four,ten,twenty from tenk1 group by grouping sets(two,four,ten,(two,twenty),()) order by 3,4,5,6;
+                  QUERY PLAN                  
+----------------------------------------------
+ Gather Merge
+   Workers Planned: 2
+   ->  Sort
+         Sort Key: two, four, ten, twenty
+         ->  Parallel BatchHashAggregate
+               Group Key: two, twenty
+               Group Key: two
+               Group Key: ()
+               Group Key: four
+               Group Key: ten
+               ->  Parallel Seq Scan on tenk1
+(11 rows)
+
+select sum(unique1),count(unique1),two,four,ten,twenty from tenk1 group by grouping sets(two,four,ten,(two,twenty),()) order by 3,4,5,6;
+   sum    | count | two | four | ten | twenty 
+----------+-------+-----+------+-----+--------
+  2495000 |   500 |   0 |      |     |      0
+  2496000 |   500 |   0 |      |     |      2
+  2497000 |   500 |   0 |      |     |      4
+  2498000 |   500 |   0 |      |     |      6
+  2499000 |   500 |   0 |      |     |      8
+  2500000 |   500 |   0 |      |     |     10
+  2501000 |   500 |   0 |      |     |     12
+  2502000 |   500 |   0 |      |     |     14
+  2503000 |   500 |   0 |      |     |     16
+  2504000 |   500 |   0 |      |     |     18
+ 24995000 |  5000 |   0 |      |     |       
+  2495500 |   500 |   1 |      |     |      1
+  2496500 |   500 |   1 |      |     |      3
+  2497500 |   500 |   1 |      |     |      5
+  2498500 |   500 |   1 |      |     |      7
+  2499500 |   500 |   1 |      |     |      9
+  2500500 |   500 |   1 |      |     |     11
+  2501500 |   500 |   1 |      |     |     13
+  2502500 |   500 |   1 |      |     |     15
+  2503500 |   500 |   1 |      |     |     17
+  2504500 |   500 |   1 |      |     |     19
+ 25000000 |  5000 |   1 |      |     |       
+ 12495000 |  2500 |     |    0 |     |       
+ 12497500 |  2500 |     |    1 |     |       
+ 12500000 |  2500 |     |    2 |     |       
+ 12502500 |  2500 |     |    3 |     |       
+  4995000 |  1000 |     |      |   0 |       
+  4996000 |  1000 |     |      |   1 |       
+  4997000 |  1000 |     |      |   2 |       
+  4998000 |  1000 |     |      |   3 |       
+  4999000 |  1000 |     |      |   4 |       
+  5000000 |  1000 |     |      |   5 |       
+  5001000 |  1000 |     |      |   6 |       
+  5002000 |  1000 |     |      |   7 |       
+  5003000 |  1000 |     |      |   8 |       
+  5004000 |  1000 |     |      |   9 |       
+ 49995000 | 10000 |     |      |     |       
+(37 rows)
+
+ABORT;
 -- end
diff --git a/src/test/regress/expected/partition_aggregate.out b/src/test/regress/expected/partition_aggregate.out
index dfa4b036b5..899291f9e9 100644
--- a/src/test/regress/expected/partition_aggregate.out
+++ b/src/test/regress/expected/partition_aggregate.out
@@ -1517,3 +1517,141 @@ SELECT x, sum(y), avg(y), count(*) FROM pagg_tab_para GROUP BY x HAVING avg(y) <
  21 | 6000 | 6.0000000000000000 |  1000
 (6 rows)
 
+-- simple agg in parallel
+BEGIN;
+SET max_sort_batches = 6;
+SET min_parallel_table_scan_size = 0;
+SET parallel_tuple_cost = 0;
+SET parallel_setup_cost = 0;
+SET enable_indexonlyscan = OFF;
+-- using batch sort
+EXPLAIN (COSTS OFF)
+SELECT unique2,count(*) FROM tenk1 GROUP BY 1;
+                  QUERY PLAN                  
+----------------------------------------------
+ Gather
+   Workers Planned: 2
+   ->  GroupAggregate
+         Group Key: unique2
+         ->  Parallel BatchSort
+               Sort Key: unique2
+               ->  Parallel Seq Scan on tenk1
+(7 rows)
+
+EXPLAIN (COSTS OFF)
+SELECT count(*) FROM (SELECT unique2,count(*) FROM tenk1 GROUP BY 1) foo;
+                     QUERY PLAN                     
+----------------------------------------------------
+ Aggregate
+   ->  Gather
+         Workers Planned: 2
+         ->  GroupAggregate
+               Group Key: tenk1.unique2
+               ->  Parallel BatchSort
+                     Sort Key: tenk1.unique2
+                     ->  Parallel Seq Scan on tenk1
+(8 rows)
+
+SELECT count(*) FROM (SELECT unique2,count(*) FROM tenk1 GROUP BY 1) foo;
+ count 
+-------
+ 10000
+(1 row)
+
+EXPLAIN (COSTS OFF)
+SELECT count(*) FROM
+  (SELECT unique2,count(*) id FROM tenk1 GROUP BY 1) t1
+    INNER JOIN
+  (SELECT unique2  id FROM tenk1) t2
+  USING(id);
+                               QUERY PLAN                               
+------------------------------------------------------------------------
+ Finalize Aggregate
+   ->  Gather
+         Workers Planned: 2
+         ->  Partial Aggregate
+               ->  Parallel Hash Join
+                     Hash Cond: ((count(*)) = tenk1.unique2)
+                     ->  GroupAggregate
+                           Group Key: tenk1_1.unique2
+                           ->  Parallel BatchSort
+                                 Sort Key: tenk1_1.unique2
+                                 ->  Parallel Seq Scan on tenk1 tenk1_1
+                     ->  Parallel Hash
+                           ->  Parallel Seq Scan on tenk1
+(13 rows)
+
+SELECT count(*) FROM
+  (SELECT unique2,count(*) id FROM tenk1 GROUP BY 1) t1
+    INNER JOIN
+  (SELECT unique2  id FROM tenk1) t2
+  USING(id);
+ count 
+-------
+ 10000
+(1 row)
+
+-- using batch hash
+SET max_sort_batches = 0;
+SET max_hashagg_batches = 512;
+EXPLAIN (COSTS OFF)
+SELECT unique2,count(*) FROM tenk1 GROUP BY 1;
+               QUERY PLAN               
+----------------------------------------
+ Gather
+   Workers Planned: 2
+   ->  Parallel BatchHashAggregate
+         Group Key: unique2
+         ->  Parallel Seq Scan on tenk1
+(5 rows)
+
+EXPLAIN (COSTS OFF)
+SELECT count(*) FROM (SELECT unique2,count(*) FROM tenk1 GROUP BY 1) foo;
+                  QUERY PLAN                  
+----------------------------------------------
+ Aggregate
+   ->  Gather
+         Workers Planned: 2
+         ->  Parallel BatchHashAggregate
+               Group Key: tenk1.unique2
+               ->  Parallel Seq Scan on tenk1
+(6 rows)
+
+SELECT count(*) FROM (SELECT unique2,count(*) FROM tenk1 GROUP BY 1) foo;
+ count 
+-------
+ 10000
+(1 row)
+
+EXPLAIN (COSTS OFF)
+SELECT count(*) FROM
+  (SELECT unique2,count(*) id FROM tenk1 GROUP BY 1) t1
+    INNER JOIN
+  (SELECT unique2  id FROM tenk1) t2
+  USING(id);
+                            QUERY PLAN                            
+------------------------------------------------------------------
+ Finalize Aggregate
+   ->  Gather
+         Workers Planned: 2
+         ->  Partial Aggregate
+               ->  Parallel Hash Join
+                     Hash Cond: ((count(*)) = tenk1.unique2)
+                     ->  Parallel BatchHashAggregate
+                           Group Key: tenk1_1.unique2
+                           ->  Parallel Seq Scan on tenk1 tenk1_1
+                     ->  Parallel Hash
+                           ->  Parallel Seq Scan on tenk1
+(11 rows)
+
+SELECT count(*) FROM
+  (SELECT unique2,count(*) id FROM tenk1 GROUP BY 1) t1
+    INNER JOIN
+  (SELECT unique2  id FROM tenk1) t2
+  USING(id);
+ count 
+-------
+ 10000
+(1 row)
+
+ABORT;
diff --git a/src/test/regress/expected/select_distinct.out b/src/test/regress/expected/select_distinct.out
index 11c6f50fbf..27d77a33ba 100644
--- a/src/test/regress/expected/select_distinct.out
+++ b/src/test/regress/expected/select_distinct.out
@@ -306,3 +306,80 @@ SELECT null IS NOT DISTINCT FROM null as "yes";
  t
 (1 row)
 
+-- parallel distinct
+BEGIN;
+SET max_sort_batches = 6;
+SET min_parallel_table_scan_size =0;
+SET parallel_tuple_cost = 0;
+SET parallel_setup_cost = 0;
+SET enable_indexonlyscan = OFF;
+-- using batch sort
+EXPLAIN (costs off)
+SELECT COUNT(*) FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+                     QUERY PLAN                     
+----------------------------------------------------
+ Aggregate
+   ->  Gather
+         Workers Planned: 2
+         ->  Unique
+               ->  Parallel BatchSort
+                     Sort Key: tenk1.unique2
+                     ->  Parallel Seq Scan on tenk1
+(7 rows)
+
+SELECT COUNT(*) FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+ count 
+-------
+ 10000
+(1 row)
+
+explain (costs off)
+SELECT DISTINCT * FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+                        QUERY PLAN                        
+----------------------------------------------------------
+ Gather
+   Workers Planned: 2
+   ->  Unique
+         ->  Parallel BatchSort
+               Sort Key: tenk1.unique2
+               ->  Unique
+                     ->  Parallel BatchSort
+                           Sort Key: tenk1.unique2
+                           ->  Parallel Seq Scan on tenk1
+(9 rows)
+
+-- using batch hash
+SET max_sort_batches = 0;
+SET max_hashagg_batches = 512;
+EXPLAIN (costs off)
+SELECT COUNT(*) FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+                  QUERY PLAN                  
+----------------------------------------------
+ Aggregate
+   ->  Gather
+         Workers Planned: 2
+         ->  Parallel BatchHashAggregate
+               Group Key: tenk1.unique2
+               ->  Parallel Seq Scan on tenk1
+(6 rows)
+
+SELECT COUNT(*) FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+ count 
+-------
+ 10000
+(1 row)
+
+explain (costs off)
+SELECT DISTINCT * FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+                  QUERY PLAN                  
+----------------------------------------------
+ Gather
+   Workers Planned: 2
+   ->  Parallel BatchHashAggregate
+         Group Key: tenk1.unique2
+         ->  Parallel BatchHashAggregate
+               Group Key: tenk1.unique2
+               ->  Parallel Seq Scan on tenk1
+(7 rows)
+
+ABORT;
diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out
index 9b0c418db7..0d399f3d18 100644
--- a/src/test/regress/expected/select_parallel.out
+++ b/src/test/regress/expected/select_parallel.out
@@ -845,6 +845,92 @@ select * from
  VVVVxx  |  2500 | 3
 (12 rows)
 
+--test rescan for batch sort
+SET max_sort_batches = 12;
+explain (costs off)
+select * from
+  (select string4, count(unique2)
+   from tenk1 group by string4) ss
+  right join (values (1),(2),(3)) v(x) on true order by 3,1;
+                        QUERY PLAN                        
+----------------------------------------------------------
+ Sort
+   Sort Key: "*VALUES*".column1, tenk1.string4
+   ->  Nested Loop Left Join
+         ->  Values Scan on "*VALUES*"
+         ->  Gather
+               Workers Planned: 4
+               ->  GroupAggregate
+                     Group Key: tenk1.string4
+                     ->  Parallel BatchSort
+                           Sort Key: tenk1.string4
+                           ->  Parallel Seq Scan on tenk1
+(11 rows)
+
+select * from
+  (select string4, count(unique2)
+   from tenk1 group by string4) ss
+  right join (values (1),(2),(3)) v(x) on true order by 3,1;
+ string4 | count | x 
+---------+-------+---
+ AAAAxx  |  2500 | 1
+ HHHHxx  |  2500 | 1
+ OOOOxx  |  2500 | 1
+ VVVVxx  |  2500 | 1
+ AAAAxx  |  2500 | 2
+ HHHHxx  |  2500 | 2
+ OOOOxx  |  2500 | 2
+ VVVVxx  |  2500 | 2
+ AAAAxx  |  2500 | 3
+ HHHHxx  |  2500 | 3
+ OOOOxx  |  2500 | 3
+ VVVVxx  |  2500 | 3
+(12 rows)
+
+reset max_sort_batches;
+--test rescan for batch hash agg
+SET max_hashagg_batches = 512;
+explain (costs off)
+select * from
+  (select string4, count(unique2)
+   from tenk1 group by string4) ss
+  right join (values (1),(2),(3)) v(x) on true order by 3,1;
+                        QUERY PLAN                        
+----------------------------------------------------------
+ Sort
+   Sort Key: "*VALUES*".column1, tenk1.string4
+   ->  Nested Loop Left Join
+         ->  Values Scan on "*VALUES*"
+         ->  Gather Merge
+               Workers Planned: 4
+               ->  Sort
+                     Sort Key: tenk1.string4
+                     ->  Parallel BatchHashAggregate
+                           Group Key: tenk1.string4
+                           ->  Parallel Seq Scan on tenk1
+(11 rows)
+
+select * from
+  (select string4, count(unique2)
+   from tenk1 group by string4) ss
+  right join (values (1),(2),(3)) v(x) on true order by 3,1;
+ string4 | count | x 
+---------+-------+---
+ AAAAxx  |  2500 | 1
+ HHHHxx  |  2500 | 1
+ OOOOxx  |  2500 | 1
+ VVVVxx  |  2500 | 1
+ AAAAxx  |  2500 | 2
+ HHHHxx  |  2500 | 2
+ OOOOxx  |  2500 | 2
+ VVVVxx  |  2500 | 2
+ AAAAxx  |  2500 | 3
+ HHHHxx  |  2500 | 3
+ OOOOxx  |  2500 | 3
+ VVVVxx  |  2500 | 3
+(12 rows)
+
+reset max_hashagg_batches;
 reset enable_material;
 reset enable_hashagg;
 -- check parallelized int8 aggregate (bug #14897)
diff --git a/src/test/regress/expected/union.out b/src/test/regress/expected/union.out
index 75f78db8f5..8083a4802a 100644
--- a/src/test/regress/expected/union.out
+++ b/src/test/regress/expected/union.out
@@ -1420,3 +1420,88 @@ where (x = 0) or (q1 >= q2 and q1 <= q2);
  4567890123456789 |  4567890123456789 | 1
 (6 rows)
 
+-- parallel union
+BEGIN;
+SET max_sort_batches = 6;
+SET min_parallel_table_scan_size =0;
+SET parallel_tuple_cost = 0;
+SET parallel_setup_cost = 0;
+SET enable_indexonlyscan = OFF;
+-- using batch sort
+EXPLAIN (costs off)
+SELECT count(*) FROM (SELECT unique2 FROM tenk1 UNION SELECT unique2 FROM tenk1) foo;
+                            QUERY PLAN                            
+------------------------------------------------------------------
+ Aggregate
+   ->  Gather
+         Workers Planned: 2
+         ->  Unique
+               ->  Parallel BatchSort
+                     Sort Key: tenk1.unique2
+                     ->  Parallel Append
+                           ->  Parallel Seq Scan on tenk1
+                           ->  Parallel Seq Scan on tenk1 tenk1_1
+(9 rows)
+
+SELECT count(*) FROM (SELECT unique2 FROM tenk1 UNION SELECT unique2 FROM tenk1) foo;
+ count 
+-------
+ 10000
+(1 row)
+
+EXPLAIN (costs off)
+SELECT count(*) FROM (SELECT unique2 FROM tenk1 UNION SELECT unique2 FROM tenk1) t1 INNER JOIN tenk1 USING(unique2);
+                                  QUERY PLAN                                  
+------------------------------------------------------------------------------
+ Finalize Aggregate
+   ->  Gather
+         Workers Planned: 2
+         ->  Partial Aggregate
+               ->  Parallel Hash Join
+                     Hash Cond: (tenk1_1.unique2 = tenk1.unique2)
+                     ->  Unique
+                           ->  Parallel BatchSort
+                                 Sort Key: tenk1_1.unique2
+                                 ->  Parallel Append
+                                       ->  Parallel Seq Scan on tenk1 tenk1_1
+                                       ->  Parallel Seq Scan on tenk1 tenk1_2
+                     ->  Parallel Hash
+                           ->  Parallel Seq Scan on tenk1
+(14 rows)
+
+SELECT count(*) FROM (SELECT unique2 FROM tenk1 UNION SELECT unique2 FROM tenk1) t1 INNER JOIN tenk1 USING(unique2);
+ count 
+-------
+ 10000
+(1 row)
+
+-- using batch hash
+SET max_sort_batches = 0;
+SET max_hashagg_batches = 512;
+EXPLAIN (costs off)
+SELECT count(*) from (
+  SELECT hundred from tenk1
+    union
+  SELECT hundred from tenk1) foo;
+                         QUERY PLAN                         
+------------------------------------------------------------
+ Aggregate
+   ->  Gather
+         Workers Planned: 2
+         ->  Parallel BatchHashAggregate
+               Group Key: tenk1.hundred
+               ->  Parallel Append
+                     ->  Parallel Seq Scan on tenk1
+                     ->  Parallel Seq Scan on tenk1 tenk1_1
+(8 rows)
+
+SELECT count(*) from (
+  SELECT hundred from tenk1
+    union
+  SELECT hundred from tenk1) foo;
+ count 
+-------
+   100
+(1 row)
+
+ABORT;
diff --git a/src/test/regress/sql/groupingsets.sql b/src/test/regress/sql/groupingsets.sql
index 18ae803e9d..1913ee78ba 100644
--- a/src/test/regress/sql/groupingsets.sql
+++ b/src/test/regress/sql/groupingsets.sql
@@ -529,4 +529,14 @@ set work_mem to default;
 drop table gs_group_1;
 drop table gs_hash_1;
 
+-- parallel grouping sets
+BEGIN;
+set max_hashagg_batches = 512;
+set min_parallel_table_scan_size = 0;
+set parallel_setup_cost = 10;
+explain (costs off)
+select sum(unique1),count(unique1),two,four,ten,twenty from tenk1 group by grouping sets(two,four,ten,(two,twenty),()) order by 3,4,5,6;
+select sum(unique1),count(unique1),two,four,ten,twenty from tenk1 group by grouping sets(two,four,ten,(two,twenty),()) order by 3,4,5,6;
+ABORT;
+
 -- end
diff --git a/src/test/regress/sql/partition_aggregate.sql b/src/test/regress/sql/partition_aggregate.sql
index c17294b15b..2948d8c2b4 100644
--- a/src/test/regress/sql/partition_aggregate.sql
+++ b/src/test/regress/sql/partition_aggregate.sql
@@ -331,3 +331,48 @@ RESET parallel_setup_cost;
 EXPLAIN (COSTS OFF)
 SELECT x, sum(y), avg(y), count(*) FROM pagg_tab_para GROUP BY x HAVING avg(y) < 7 ORDER BY 1, 2, 3;
 SELECT x, sum(y), avg(y), count(*) FROM pagg_tab_para GROUP BY x HAVING avg(y) < 7 ORDER BY 1, 2, 3;
+
+-- simple agg in parallel
+BEGIN;
+SET max_sort_batches = 6;
+SET min_parallel_table_scan_size = 0;
+SET parallel_tuple_cost = 0;
+SET parallel_setup_cost = 0;
+SET enable_indexonlyscan = OFF;
+-- using batch sort
+EXPLAIN (COSTS OFF)
+SELECT unique2,count(*) FROM tenk1 GROUP BY 1;
+EXPLAIN (COSTS OFF)
+SELECT count(*) FROM (SELECT unique2,count(*) FROM tenk1 GROUP BY 1) foo;
+SELECT count(*) FROM (SELECT unique2,count(*) FROM tenk1 GROUP BY 1) foo;
+EXPLAIN (COSTS OFF)
+SELECT count(*) FROM
+  (SELECT unique2,count(*) id FROM tenk1 GROUP BY 1) t1
+    INNER JOIN
+  (SELECT unique2  id FROM tenk1) t2
+  USING(id);
+SELECT count(*) FROM
+  (SELECT unique2,count(*) id FROM tenk1 GROUP BY 1) t1
+    INNER JOIN
+  (SELECT unique2  id FROM tenk1) t2
+  USING(id);
+-- using batch hash
+SET max_sort_batches = 0;
+SET max_hashagg_batches = 512;
+EXPLAIN (COSTS OFF)
+SELECT unique2,count(*) FROM tenk1 GROUP BY 1;
+EXPLAIN (COSTS OFF)
+SELECT count(*) FROM (SELECT unique2,count(*) FROM tenk1 GROUP BY 1) foo;
+SELECT count(*) FROM (SELECT unique2,count(*) FROM tenk1 GROUP BY 1) foo;
+EXPLAIN (COSTS OFF)
+SELECT count(*) FROM
+  (SELECT unique2,count(*) id FROM tenk1 GROUP BY 1) t1
+    INNER JOIN
+  (SELECT unique2  id FROM tenk1) t2
+  USING(id);
+SELECT count(*) FROM
+  (SELECT unique2,count(*) id FROM tenk1 GROUP BY 1) t1
+    INNER JOIN
+  (SELECT unique2  id FROM tenk1) t2
+  USING(id);
+ABORT;
diff --git a/src/test/regress/sql/select_distinct.sql b/src/test/regress/sql/select_distinct.sql
index 33102744eb..c98926309a 100644
--- a/src/test/regress/sql/select_distinct.sql
+++ b/src/test/regress/sql/select_distinct.sql
@@ -135,3 +135,26 @@ SELECT 1 IS NOT DISTINCT FROM 2 as "no";
 SELECT 2 IS NOT DISTINCT FROM 2 as "yes";
 SELECT 2 IS NOT DISTINCT FROM null as "no";
 SELECT null IS NOT DISTINCT FROM null as "yes";
+
+-- parallel distinct
+BEGIN;
+SET max_sort_batches = 6;
+SET min_parallel_table_scan_size =0;
+SET parallel_tuple_cost = 0;
+SET parallel_setup_cost = 0;
+SET enable_indexonlyscan = OFF;
+-- using batch sort
+EXPLAIN (costs off)
+SELECT COUNT(*) FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+SELECT COUNT(*) FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+explain (costs off)
+SELECT DISTINCT * FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+-- using batch hash
+SET max_sort_batches = 0;
+SET max_hashagg_batches = 512;
+EXPLAIN (costs off)
+SELECT COUNT(*) FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+SELECT COUNT(*) FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+explain (costs off)
+SELECT DISTINCT * FROM (SELECT DISTINCT unique2 FROM tenk1) foo;
+ABORT;
\ No newline at end of file
diff --git a/src/test/regress/sql/select_parallel.sql b/src/test/regress/sql/select_parallel.sql
index 5a01a98b26..7ee69e1b97 100644
--- a/src/test/regress/sql/select_parallel.sql
+++ b/src/test/regress/sql/select_parallel.sql
@@ -312,6 +312,38 @@ select * from
    from tenk1 group by string4 order by string4) ss
   right join (values (1),(2),(3)) v(x) on true;
 
+--test rescan for batch sort
+SET max_sort_batches = 12;
+
+explain (costs off)
+select * from
+  (select string4, count(unique2)
+   from tenk1 group by string4) ss
+  right join (values (1),(2),(3)) v(x) on true order by 3,1;
+
+select * from
+  (select string4, count(unique2)
+   from tenk1 group by string4) ss
+  right join (values (1),(2),(3)) v(x) on true order by 3,1;
+
+reset max_sort_batches;
+
+--test rescan for batch hash agg
+SET max_hashagg_batches = 512;
+
+explain (costs off)
+select * from
+  (select string4, count(unique2)
+   from tenk1 group by string4) ss
+  right join (values (1),(2),(3)) v(x) on true order by 3,1;
+
+select * from
+  (select string4, count(unique2)
+   from tenk1 group by string4) ss
+  right join (values (1),(2),(3)) v(x) on true order by 3,1;
+
+reset max_hashagg_batches;
+
 reset enable_material;
 
 reset enable_hashagg;
diff --git a/src/test/regress/sql/union.sql b/src/test/regress/sql/union.sql
index ce22f34c71..171f6e41ec 100644
--- a/src/test/regress/sql/union.sql
+++ b/src/test/regress/sql/union.sql
@@ -540,3 +540,31 @@ select * from
    union all
    select *, 1 as x from int8_tbl b) ss
 where (x = 0) or (q1 >= q2 and q1 <= q2);
+
+-- parallel union
+BEGIN;
+SET max_sort_batches = 6;
+SET min_parallel_table_scan_size =0;
+SET parallel_tuple_cost = 0;
+SET parallel_setup_cost = 0;
+SET enable_indexonlyscan = OFF;
+-- using batch sort
+EXPLAIN (costs off)
+SELECT count(*) FROM (SELECT unique2 FROM tenk1 UNION SELECT unique2 FROM tenk1) foo;
+SELECT count(*) FROM (SELECT unique2 FROM tenk1 UNION SELECT unique2 FROM tenk1) foo;
+EXPLAIN (costs off)
+SELECT count(*) FROM (SELECT unique2 FROM tenk1 UNION SELECT unique2 FROM tenk1) t1 INNER JOIN tenk1 USING(unique2);
+SELECT count(*) FROM (SELECT unique2 FROM tenk1 UNION SELECT unique2 FROM tenk1) t1 INNER JOIN tenk1 USING(unique2);
+-- using batch hash
+SET max_sort_batches = 0;
+SET max_hashagg_batches = 512;
+EXPLAIN (costs off)
+SELECT count(*) from (
+  SELECT hundred from tenk1
+    union
+  SELECT hundred from tenk1) foo;
+SELECT count(*) from (
+  SELECT hundred from tenk1
+    union
+  SELECT hundred from tenk1) foo;
+ABORT;
\ No newline at end of file
-- 
2.16.3

