From 705980a0b7d38ee0f33a76de8cdc8cd0e7579436 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 13 Mar 2019 21:08:35 +0200 Subject: [PATCH 2/2] Andrey Borodin's test_blockset tool, adapted for SparseBitset WIP: I don't know if we want to commit something like this in the end, but it's pretty useful for manual testing. from https://www.postgresql.org/message-id/FD68CBB1-716F-4344-9E3F-C27C14B8678B%40yandex-team.ru --- src/test/modules/test_blockset/.gitignore | 4 + src/test/modules/test_blockset/Makefile | 21 ++ src/test/modules/test_blockset/README | 8 + .../test_blockset/expected/test_blockset.out | 7 + .../test_blockset/sql/test_blockset.sql | 3 + .../test_blockset/test_blockset--1.0.sql | 8 + .../modules/test_blockset/test_blockset.c | 236 ++++++++++++++++++ .../test_blockset/test_blockset.control | 4 + src/test/regress/expected/gist.out | 4 +- src/test/regress/sql/gist.sql | 4 +- 10 files changed, 293 insertions(+), 6 deletions(-) create mode 100644 src/test/modules/test_blockset/.gitignore create mode 100644 src/test/modules/test_blockset/Makefile create mode 100644 src/test/modules/test_blockset/README create mode 100644 src/test/modules/test_blockset/expected/test_blockset.out create mode 100644 src/test/modules/test_blockset/sql/test_blockset.sql create mode 100644 src/test/modules/test_blockset/test_blockset--1.0.sql create mode 100644 src/test/modules/test_blockset/test_blockset.c create mode 100644 src/test/modules/test_blockset/test_blockset.control diff --git a/src/test/modules/test_blockset/.gitignore b/src/test/modules/test_blockset/.gitignore new file mode 100644 index 00000000000..5dcb3ff9723 --- /dev/null +++ b/src/test/modules/test_blockset/.gitignore @@ -0,0 +1,4 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ diff --git a/src/test/modules/test_blockset/Makefile b/src/test/modules/test_blockset/Makefile new file mode 100644 index 00000000000..091cf8c0958 --- /dev/null +++ b/src/test/modules/test_blockset/Makefile @@ -0,0 +1,21 @@ +# src/test/modules/test_blockset/Makefile + +MODULE_big = test_blockset +OBJS = test_blockset.o $(WIN32RES) +PGFILEDESC = "test_blockset - test code for block set library" + +EXTENSION = test_blockset +DATA = test_blockset--1.0.sql + +REGRESS = test_blockset + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = src/test/modules/test_blockset +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif diff --git a/src/test/modules/test_blockset/README b/src/test/modules/test_blockset/README new file mode 100644 index 00000000000..730951ff03c --- /dev/null +++ b/src/test/modules/test_blockset/README @@ -0,0 +1,8 @@ +test_blockset overview +========================= + +test_blockset is a test harness module for testing block set data structure. +There are two main tests: +1. Test of comliance with Bitmapset on numbers avaiable to Bitmapset +2. Test of numbers that can exceed INT32_MAX but are just shifted on one bit +from numbers kept in Bitmapset \ No newline at end of file diff --git a/src/test/modules/test_blockset/expected/test_blockset.out b/src/test/modules/test_blockset/expected/test_blockset.out new file mode 100644 index 00000000000..02c29d5fc0c --- /dev/null +++ b/src/test/modules/test_blockset/expected/test_blockset.out @@ -0,0 +1,7 @@ +CREATE EXTENSION test_blockset; +SELECT test_blockset(); + test_blockset +--------------- + +(1 row) + diff --git a/src/test/modules/test_blockset/sql/test_blockset.sql b/src/test/modules/test_blockset/sql/test_blockset.sql new file mode 100644 index 00000000000..85d2886676e --- /dev/null +++ b/src/test/modules/test_blockset/sql/test_blockset.sql @@ -0,0 +1,3 @@ +CREATE EXTENSION test_blockset; + +SELECT test_blockset(); \ No newline at end of file diff --git a/src/test/modules/test_blockset/test_blockset--1.0.sql b/src/test/modules/test_blockset/test_blockset--1.0.sql new file mode 100644 index 00000000000..04eea8a6146 --- /dev/null +++ b/src/test/modules/test_blockset/test_blockset--1.0.sql @@ -0,0 +1,8 @@ +/* src/test/modules/test_blockset/test_blockset--1.0.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION test_blockset" to load this file. \quit + +CREATE FUNCTION test_blockset() +RETURNS pg_catalog.void STRICT +AS 'MODULE_PATHNAME' LANGUAGE C; diff --git a/src/test/modules/test_blockset/test_blockset.c b/src/test/modules/test_blockset/test_blockset.c new file mode 100644 index 00000000000..218942a587b --- /dev/null +++ b/src/test/modules/test_blockset/test_blockset.c @@ -0,0 +1,236 @@ +/*-------------------------------------------------------------------------- + * + * test_blockset.c + * Test block set data structure. + * + * Copyright (c) 2019, PostgreSQL Global Development Group + * + * IDENTIFICATION + * src/test/modules/test_blockset/test_blockset.c + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "fmgr.h" +#include "lib/sparsebitset.h" +#include "nodes/bitmapset.h" +#include "utils/memutils.h" +#include "storage/block.h" +#include "miscadmin.h" + +PG_MODULE_MAGIC; + +PG_FUNCTION_INFO_V1(test_blockset); + +static void test_blockset_bms_compliance(int32_t); +static void test_blockset_big_block_numbers(int32_t); + +/* + * SQL-callable entry point to perform all tests. + */ +Datum +test_blockset(PG_FUNCTION_ARGS) +{ + test_blockset_bms_compliance(0); + test_blockset_bms_compliance(1); + test_blockset_bms_compliance(2); + test_blockset_bms_compliance(1337); + test_blockset_bms_compliance(100000); + test_blockset_big_block_numbers(1337); + test_blockset_big_block_numbers(31337); + test_blockset_big_block_numbers(5000000); + PG_RETURN_VOID(); +} + +/* + * This test creates random bitmap with test_limit members + * and checks that block set behavior is similar to Bitmapset + */ +static void test_blockset_bms_compliance(int32_t test_limit) +{ + SparseBitset *sbs = NULL; + Bitmapset *bms = NULL; + BlockNumber blockno; + int index; + int32_t point_index = 0; + + sbs = sbs_create(); + + for (int32_t i = 0; i < test_limit; i++) + { + blockno = random() & INT32_MAX; + /* bms does not support block numbers above INT32_MAX */ + bms = bms_add_member(bms, (int)blockno); + } + + /* + * SparseBitset requires that the items are added in-order, so we populate + * the SparseBitset after the Bitmapset, by iterating through all the values + * from the Bitmapset in the right order. + */ + { + int x = -1; + + while ((x = bms_next_member(bms, x)) >= 0) + sbs_add_member(sbs, (BlockNumber) (x)); + } + + index = -1; + blockno = InvalidBlockNumber; + + sbs_begin_iterate(sbs); + while (true) + { + bool found; + BlockNumber next_bn = sbs_iterate_next(sbs, &found); + int next_index = bms_next_member(bms, index); + + if (!found) + next_bn = InvalidBlockNumber; + + point_index++; + + if (next_bn == InvalidBlockNumber && next_index == -2) + return; /* We have found everything */ + + if (((BlockNumber)next_index) != next_bn) + { + elog(ERROR, + "Bitmapset returned value %X different from block set %X," + " test_limit %d, point index %d", + next_index, next_bn, test_limit, point_index); + } + + if (!sbs_is_member(sbs, next_bn)) + { + elog(ERROR, + "Block set did not found present item %X" + " test_limit %d, point index %d", + next_bn, test_limit, point_index); + } + + index = next_index; + blockno = next_bn; + } + + for (int32_t i = 0; i < test_limit; i++) + { + blockno = random() & INT32_MAX; + if (sbs_is_member(sbs, blockno) != bms_is_member((int)blockno, bms)) + { + elog(ERROR, + "Block set did agree with bitmapset item %X" + " test_limit %d, point index %d", + blockno, test_limit, point_index); + } + } + + sbs_free(sbs); + bms_free(bms); +} + +/* + * This test is similar to test_blockset_bms_compliance() + * except that it shifts BlockNumbers by one bit to ensure that blockset + * operates correctly on values higher that INT32_MAX + * This function is copy-pasted from previous with the exception of barrel + * shifts for BlockNumbers. I've tried various refactorings, but they all + * looked ugly. + */ +static void test_blockset_big_block_numbers(int32_t test_limit) +{ + SparseBitset *sbs; + Bitmapset *bms = NULL; + BlockNumber blockno; + int index; + int32_t point_index = 0; + MemoryContext old_cxt; + + MemoryContext test_cxt; + + fprintf(stderr, "TEST %d:\n", test_limit); + + test_cxt = AllocSetContextCreate(CurrentMemoryContext, + "blockset test", + ALLOCSET_SMALL_SIZES); + old_cxt = MemoryContextSwitchTo(test_cxt); + sbs = sbs_create(); + MemoryContextSwitchTo(old_cxt); + + for (int32_t i = 0; i < test_limit; i++) + { + blockno = random() & INT32_MAX; + /* bms does not support block numbers above INT32_MAX */ + bms = bms_add_member(bms, (int)blockno); + + CHECK_FOR_INTERRUPTS(); + } + + /* + * SparseBitset requires that the items are added in-order, so we populate + * the SparseBitset after the Bitmapset, by iterating through all the values + * from the Bitmapset in the right order. + */ + { + int x = -1; + + while ((x = bms_next_member(bms, x)) >= 0) + sbs_add_member(sbs, ((BlockNumber) (x)) << 1); + } + + MemoryContextStats(test_cxt); + + index = -1; + blockno = InvalidBlockNumber; + + sbs_begin_iterate(sbs); + while (true) + { + bool found; + BlockNumber next_bn = sbs_iterate_next(sbs, &found); + int next_index = bms_next_member(bms, index); + + if (!found) + next_bn = InvalidBlockNumber; + + point_index++; + + if (next_bn == InvalidBlockNumber && next_index == -2) + break; /* We have found everything */ + + if (((BlockNumber)next_index) != (next_bn >> 1)) + { + elog(ERROR, + "Bitmapset returned value %X different from block set %X," + " test_limit %d, point index %d", + next_index, next_bn, test_limit, point_index); + } + + if (!sbs_is_member(sbs, next_bn)) + { + elog(ERROR, + "Block set did not found present item %X" + " test_limit %d, point index %d", + next_bn, test_limit, point_index); + } + + index = next_index; + blockno = next_bn; + } + + for (int32_t i = 0; i < test_limit; i++) + { + blockno = random() & INT32_MAX; + if (sbs_is_member(sbs, blockno << 1) != bms_is_member((int)blockno, bms)) + { + elog(ERROR, + "Block set did agree with bitmapset item %X" + " test_limit %d, point index %d", + blockno, test_limit, point_index); + } + } + + sbs_free(sbs); + bms_free(bms); +} diff --git a/src/test/modules/test_blockset/test_blockset.control b/src/test/modules/test_blockset/test_blockset.control new file mode 100644 index 00000000000..fdb7598c5a7 --- /dev/null +++ b/src/test/modules/test_blockset/test_blockset.control @@ -0,0 +1,4 @@ +comment = 'Test code for block set library' +default_version = '1.0' +module_pathname = '$libdir/test_blockset' +relocatable = true diff --git a/src/test/regress/expected/gist.out b/src/test/regress/expected/gist.out index f5a2993aaf2..5b92f08c747 100644 --- a/src/test/regress/expected/gist.out +++ b/src/test/regress/expected/gist.out @@ -27,9 +27,7 @@ insert into gist_point_tbl (id, p) select g+100000, point(g*10+1, g*10+1) from generate_series(1, 10000) g; -- To test vacuum, delete some entries from all over the index. delete from gist_point_tbl where id % 2 = 1; --- And also delete some concentration of values. (GiST doesn't currently --- attempt to delete pages even when they become empty, but if it did, this --- would exercise it) +-- And also delete some concentration of values. delete from gist_point_tbl where id < 10000; vacuum analyze gist_point_tbl; -- rebuild the index with a different fillfactor diff --git a/src/test/regress/sql/gist.sql b/src/test/regress/sql/gist.sql index bae722fe13c..e66396e851b 100644 --- a/src/test/regress/sql/gist.sql +++ b/src/test/regress/sql/gist.sql @@ -28,9 +28,7 @@ select g+100000, point(g*10+1, g*10+1) from generate_series(1, 10000) g; -- To test vacuum, delete some entries from all over the index. delete from gist_point_tbl where id % 2 = 1; --- And also delete some concentration of values. (GiST doesn't currently --- attempt to delete pages even when they become empty, but if it did, this --- would exercise it) +-- And also delete some concentration of values. delete from gist_point_tbl where id < 10000; vacuum analyze gist_point_tbl; -- 2.20.1