From 705980a0b7d38ee0f33a76de8cdc8cd0e7579436 Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki.linnakangas@iki.fi>
Date: Wed, 13 Mar 2019 21:08:35 +0200
Subject: [PATCH 2/2] Andrey Borodin's test_blockset tool, adapted for
 SparseBitset

WIP: I don't know if we want to commit something like this in the end, but
it's pretty useful for manual testing.

from https://www.postgresql.org/message-id/FD68CBB1-716F-4344-9E3F-C27C14B8678B%40yandex-team.ru
---
 src/test/modules/test_blockset/.gitignore     |   4 +
 src/test/modules/test_blockset/Makefile       |  21 ++
 src/test/modules/test_blockset/README         |   8 +
 .../test_blockset/expected/test_blockset.out  |   7 +
 .../test_blockset/sql/test_blockset.sql       |   3 +
 .../test_blockset/test_blockset--1.0.sql      |   8 +
 .../modules/test_blockset/test_blockset.c     | 236 ++++++++++++++++++
 .../test_blockset/test_blockset.control       |   4 +
 src/test/regress/expected/gist.out            |   4 +-
 src/test/regress/sql/gist.sql                 |   4 +-
 10 files changed, 293 insertions(+), 6 deletions(-)
 create mode 100644 src/test/modules/test_blockset/.gitignore
 create mode 100644 src/test/modules/test_blockset/Makefile
 create mode 100644 src/test/modules/test_blockset/README
 create mode 100644 src/test/modules/test_blockset/expected/test_blockset.out
 create mode 100644 src/test/modules/test_blockset/sql/test_blockset.sql
 create mode 100644 src/test/modules/test_blockset/test_blockset--1.0.sql
 create mode 100644 src/test/modules/test_blockset/test_blockset.c
 create mode 100644 src/test/modules/test_blockset/test_blockset.control

diff --git a/src/test/modules/test_blockset/.gitignore b/src/test/modules/test_blockset/.gitignore
new file mode 100644
index 00000000000..5dcb3ff9723
--- /dev/null
+++ b/src/test/modules/test_blockset/.gitignore
@@ -0,0 +1,4 @@
+# Generated subdirectories
+/log/
+/results/
+/tmp_check/
diff --git a/src/test/modules/test_blockset/Makefile b/src/test/modules/test_blockset/Makefile
new file mode 100644
index 00000000000..091cf8c0958
--- /dev/null
+++ b/src/test/modules/test_blockset/Makefile
@@ -0,0 +1,21 @@
+# src/test/modules/test_blockset/Makefile
+
+MODULE_big = test_blockset
+OBJS = test_blockset.o $(WIN32RES)
+PGFILEDESC = "test_blockset - test code for block set library"
+
+EXTENSION = test_blockset
+DATA = test_blockset--1.0.sql
+
+REGRESS = test_blockset
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = src/test/modules/test_blockset
+top_builddir = ../../../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
diff --git a/src/test/modules/test_blockset/README b/src/test/modules/test_blockset/README
new file mode 100644
index 00000000000..730951ff03c
--- /dev/null
+++ b/src/test/modules/test_blockset/README
@@ -0,0 +1,8 @@
+test_blockset overview
+=========================
+
+test_blockset is a test harness module for testing block set data structure.
+There are two main tests:
+1. Test of comliance with Bitmapset on numbers avaiable to Bitmapset
+2. Test of numbers that can exceed INT32_MAX but are just shifted on one bit
+from numbers kept in Bitmapset
\ No newline at end of file
diff --git a/src/test/modules/test_blockset/expected/test_blockset.out b/src/test/modules/test_blockset/expected/test_blockset.out
new file mode 100644
index 00000000000..02c29d5fc0c
--- /dev/null
+++ b/src/test/modules/test_blockset/expected/test_blockset.out
@@ -0,0 +1,7 @@
+CREATE EXTENSION test_blockset;
+SELECT test_blockset();
+ test_blockset 
+---------------
+ 
+(1 row)
+
diff --git a/src/test/modules/test_blockset/sql/test_blockset.sql b/src/test/modules/test_blockset/sql/test_blockset.sql
new file mode 100644
index 00000000000..85d2886676e
--- /dev/null
+++ b/src/test/modules/test_blockset/sql/test_blockset.sql
@@ -0,0 +1,3 @@
+CREATE EXTENSION test_blockset;
+
+SELECT test_blockset();
\ No newline at end of file
diff --git a/src/test/modules/test_blockset/test_blockset--1.0.sql b/src/test/modules/test_blockset/test_blockset--1.0.sql
new file mode 100644
index 00000000000..04eea8a6146
--- /dev/null
+++ b/src/test/modules/test_blockset/test_blockset--1.0.sql
@@ -0,0 +1,8 @@
+/* src/test/modules/test_blockset/test_blockset--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION test_blockset" to load this file. \quit
+
+CREATE FUNCTION test_blockset()
+RETURNS pg_catalog.void STRICT
+AS 'MODULE_PATHNAME' LANGUAGE C;
diff --git a/src/test/modules/test_blockset/test_blockset.c b/src/test/modules/test_blockset/test_blockset.c
new file mode 100644
index 00000000000..218942a587b
--- /dev/null
+++ b/src/test/modules/test_blockset/test_blockset.c
@@ -0,0 +1,236 @@
+/*--------------------------------------------------------------------------
+ *
+ * test_blockset.c
+ *		Test block set data structure.
+ *
+ * Copyright (c) 2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *		src/test/modules/test_blockset/test_blockset.c
+ *
+ * -------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "fmgr.h"
+#include "lib/sparsebitset.h"
+#include "nodes/bitmapset.h"
+#include "utils/memutils.h"
+#include "storage/block.h"
+#include "miscadmin.h"
+
+PG_MODULE_MAGIC;
+
+PG_FUNCTION_INFO_V1(test_blockset);
+
+static void test_blockset_bms_compliance(int32_t);
+static void test_blockset_big_block_numbers(int32_t);
+
+/*
+ * SQL-callable entry point to perform all tests.
+ */
+Datum
+test_blockset(PG_FUNCTION_ARGS)
+{
+	test_blockset_bms_compliance(0);
+	test_blockset_bms_compliance(1);
+	test_blockset_bms_compliance(2);
+	test_blockset_bms_compliance(1337);
+	test_blockset_bms_compliance(100000);
+	test_blockset_big_block_numbers(1337);
+	test_blockset_big_block_numbers(31337);
+	test_blockset_big_block_numbers(5000000);
+	PG_RETURN_VOID();
+}
+
+/*
+ * This test creates random bitmap with test_limit members
+ * and checks that block set behavior is similar to Bitmapset
+ */
+static void test_blockset_bms_compliance(int32_t test_limit)
+{
+	SparseBitset *sbs = NULL;
+	Bitmapset *bms = NULL;
+	BlockNumber blockno;
+	int index;
+	int32_t point_index = 0;
+
+	sbs = sbs_create();
+
+	for (int32_t i = 0; i < test_limit; i++)
+	{
+		blockno = random() & INT32_MAX;
+		/* bms does not support block numbers above INT32_MAX */
+		bms = bms_add_member(bms, (int)blockno);
+	}
+
+	/*
+	 * SparseBitset requires that the items are added in-order, so we populate
+	 * the SparseBitset after the Bitmapset, by iterating through all the values
+	 * from the Bitmapset in the right order.
+	 */
+	{
+		int x = -1;
+
+		while ((x = bms_next_member(bms, x)) >= 0)
+			sbs_add_member(sbs, (BlockNumber) (x));
+	}
+
+	index = -1;
+	blockno = InvalidBlockNumber;
+
+	sbs_begin_iterate(sbs);
+	while (true)
+	{
+		bool found;
+		BlockNumber next_bn = sbs_iterate_next(sbs, &found);
+		int next_index = bms_next_member(bms, index);
+
+		if (!found)
+			next_bn = InvalidBlockNumber;
+
+		point_index++;
+
+		if (next_bn == InvalidBlockNumber && next_index == -2)
+			return; /* We have found everything */
+
+		if (((BlockNumber)next_index) != next_bn)
+		{
+			elog(ERROR,
+				 "Bitmapset returned value %X different from block set %X,"
+				 " test_limit %d, point index %d",
+				 next_index, next_bn, test_limit, point_index);
+		}
+
+		if (!sbs_is_member(sbs, next_bn))
+		{
+			elog(ERROR,
+				 "Block set did not found present item %X"
+				 " test_limit %d, point index %d",
+				 next_bn, test_limit, point_index);
+		}
+
+		index = next_index;
+		blockno = next_bn;
+	}
+
+	for (int32_t i = 0; i < test_limit; i++)
+	{
+		blockno = random() & INT32_MAX;
+		if (sbs_is_member(sbs, blockno) != bms_is_member((int)blockno, bms))
+		{
+			elog(ERROR,
+				 "Block set did agree with bitmapset item %X"
+				 " test_limit %d, point index %d",
+				 blockno, test_limit, point_index);
+		}
+	}
+
+	sbs_free(sbs);
+	bms_free(bms);
+}
+
+/* 
+ * This test is similar to test_blockset_bms_compliance()
+ * except that it shifts BlockNumbers by one bit to ensure that blockset
+ * operates correctly on values higher that INT32_MAX
+ * This function is copy-pasted from previous with the exception of barrel
+ * shifts for BlockNumbers. I've tried various refactorings, but they all
+ * looked ugly.
+ */
+static void test_blockset_big_block_numbers(int32_t test_limit)
+{
+	SparseBitset *sbs;
+	Bitmapset *bms = NULL;
+	BlockNumber blockno;
+	int index;
+	int32_t point_index = 0;
+	MemoryContext old_cxt;
+
+	MemoryContext test_cxt;
+	
+	fprintf(stderr, "TEST %d:\n", test_limit);
+
+	test_cxt = AllocSetContextCreate(CurrentMemoryContext,
+									 "blockset test",
+									 ALLOCSET_SMALL_SIZES);
+	old_cxt = MemoryContextSwitchTo(test_cxt);
+	sbs = sbs_create();
+	MemoryContextSwitchTo(old_cxt);
+
+	for (int32_t i = 0; i < test_limit; i++)
+	{
+		blockno = random() & INT32_MAX;
+		/* bms does not support block numbers above INT32_MAX */
+		bms = bms_add_member(bms, (int)blockno);
+
+		CHECK_FOR_INTERRUPTS();
+	}
+
+	/*
+	 * SparseBitset requires that the items are added in-order, so we populate
+	 * the SparseBitset after the Bitmapset, by iterating through all the values
+	 * from the Bitmapset in the right order.
+	 */
+	{
+		int x = -1;
+
+		while ((x = bms_next_member(bms, x)) >= 0)
+			sbs_add_member(sbs, ((BlockNumber) (x)) << 1);
+	}
+
+	MemoryContextStats(test_cxt);
+
+	index = -1;
+	blockno = InvalidBlockNumber;
+
+	sbs_begin_iterate(sbs);
+	while (true)
+	{
+		bool		found;
+		BlockNumber next_bn = sbs_iterate_next(sbs, &found);
+		int next_index = bms_next_member(bms, index);
+
+		if (!found)
+			next_bn = InvalidBlockNumber;
+		
+		point_index++;
+
+		if (next_bn == InvalidBlockNumber && next_index == -2)
+			break; /* We have found everything */
+
+		if (((BlockNumber)next_index) != (next_bn >> 1))
+		{
+			elog(ERROR,
+				 "Bitmapset returned value %X different from block set %X,"
+				 " test_limit %d, point index %d",
+				 next_index, next_bn, test_limit, point_index);
+		}
+
+		if (!sbs_is_member(sbs, next_bn))
+		{
+			elog(ERROR,
+				 "Block set did not found present item %X"
+				 " test_limit %d, point index %d",
+				 next_bn, test_limit, point_index);
+		}
+
+		index = next_index;
+		blockno = next_bn;
+	}
+
+	for (int32_t i = 0; i < test_limit; i++)
+	{
+		blockno = random() & INT32_MAX;
+		if (sbs_is_member(sbs, blockno << 1) != bms_is_member((int)blockno, bms))
+		{
+			elog(ERROR,
+				 "Block set did agree with bitmapset item %X"
+				 " test_limit %d, point index %d",
+				 blockno, test_limit, point_index);
+		}
+	}
+
+	sbs_free(sbs);
+	bms_free(bms);
+}
diff --git a/src/test/modules/test_blockset/test_blockset.control b/src/test/modules/test_blockset/test_blockset.control
new file mode 100644
index 00000000000..fdb7598c5a7
--- /dev/null
+++ b/src/test/modules/test_blockset/test_blockset.control
@@ -0,0 +1,4 @@
+comment = 'Test code for block set library'
+default_version = '1.0'
+module_pathname = '$libdir/test_blockset'
+relocatable = true
diff --git a/src/test/regress/expected/gist.out b/src/test/regress/expected/gist.out
index f5a2993aaf2..5b92f08c747 100644
--- a/src/test/regress/expected/gist.out
+++ b/src/test/regress/expected/gist.out
@@ -27,9 +27,7 @@ insert into gist_point_tbl (id, p)
 select g+100000, point(g*10+1, g*10+1) from generate_series(1, 10000) g;
 -- To test vacuum, delete some entries from all over the index.
 delete from gist_point_tbl where id % 2 = 1;
--- And also delete some concentration of values. (GiST doesn't currently
--- attempt to delete pages even when they become empty, but if it did, this
--- would exercise it)
+-- And also delete some concentration of values.
 delete from gist_point_tbl where id < 10000;
 vacuum analyze gist_point_tbl;
 -- rebuild the index with a different fillfactor
diff --git a/src/test/regress/sql/gist.sql b/src/test/regress/sql/gist.sql
index bae722fe13c..e66396e851b 100644
--- a/src/test/regress/sql/gist.sql
+++ b/src/test/regress/sql/gist.sql
@@ -28,9 +28,7 @@ select g+100000, point(g*10+1, g*10+1) from generate_series(1, 10000) g;
 -- To test vacuum, delete some entries from all over the index.
 delete from gist_point_tbl where id % 2 = 1;
 
--- And also delete some concentration of values. (GiST doesn't currently
--- attempt to delete pages even when they become empty, but if it did, this
--- would exercise it)
+-- And also delete some concentration of values.
 delete from gist_point_tbl where id < 10000;
 
 vacuum analyze gist_point_tbl;
-- 
2.20.1

