From 5f546b4d05747eff04d81dc15437de27bbaad7fb Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Sun, 25 Sep 2022 16:57:01 -0700
Subject: [PATCH v2 3/3] wip: meson: Add installcheck equivalent

run all tests that support running against existing server:
$ meson test --setup running

run just the main pg_regress tests against existing server:
$ meson test --setup running main-running/regress
---
 contrib/basic_archive/meson.build             |  1 +
 contrib/pg_freespacemap/meson.build           |  3 +
 contrib/pg_stat_statements/meson.build        |  4 +
 contrib/pg_walinspect/meson.build             |  3 +
 contrib/test_decoding/meson.build             |  5 ++
 src/interfaces/ecpg/test/meson.build          |  1 +
 src/test/isolation/meson.build                |  1 +
 src/test/modules/commit_ts/meson.build        |  3 +
 src/test/modules/snapshot_too_old/meson.build |  3 +
 src/test/modules/test_oat_hooks/meson.build   |  1 +
 src/test/modules/test_pg_dump/meson.build     |  2 +
 src/test/modules/worker_spi/meson.build       |  4 +-
 src/test/regress/meson.build                  |  1 +
 .cirrus.yml                                   | 15 ++++
 meson.build                                   | 87 ++++++++++++++++---
 15 files changed, 120 insertions(+), 14 deletions(-)

diff --git a/contrib/basic_archive/meson.build b/contrib/basic_archive/meson.build
index b67cbef60bd..0600c04940b 100644
--- a/contrib/basic_archive/meson.build
+++ b/contrib/basic_archive/meson.build
@@ -19,5 +19,6 @@ tests += {
     'regress_args': [
       '--temp-config', files('basic_archive.conf'),
     ],
+    'runningcheck': false,
   },
 }
diff --git a/contrib/pg_freespacemap/meson.build b/contrib/pg_freespacemap/meson.build
index f795014d7ca..a8703660c73 100644
--- a/contrib/pg_freespacemap/meson.build
+++ b/contrib/pg_freespacemap/meson.build
@@ -25,5 +25,8 @@ tests += {
     'regress_args': [
       '--temp-config', files('pg_freespacemap.conf')
     ],
+    # Disabled because these tests require "autovacuum=off", which
+    # typical runningcheck users do not have (e.g. buildfarm clients).
+    'runningcheck': false,
   },
 }
diff --git a/contrib/pg_stat_statements/meson.build b/contrib/pg_stat_statements/meson.build
index ac117d2fc1d..07c5d5d4442 100644
--- a/contrib/pg_stat_statements/meson.build
+++ b/contrib/pg_stat_statements/meson.build
@@ -31,5 +31,9 @@ tests += {
       'pg_stat_statements',
     ],
     'regress_args': ['--temp-config', files('pg_stat_statements.conf')],
+    # Disabled because these tests require
+    # "shared_preload_libraries=pg_stat_statements", which typical
+    # runningcheck users do not have (e.g. buildfarm clients).
+    'runningcheck': false,
   },
 }
diff --git a/contrib/pg_walinspect/meson.build b/contrib/pg_walinspect/meson.build
index d6b27877dd0..15a052ec879 100644
--- a/contrib/pg_walinspect/meson.build
+++ b/contrib/pg_walinspect/meson.build
@@ -22,6 +22,9 @@ tests += {
     'sql': [
       'pg_walinspect',
     ],
+    # Disabled because these tests require "wal_level=replica", which
+    # some runningcheck users do not have (e.g. buildfarm clients).
     'regress_args': ['--temp-config', files('walinspect.conf')],
+    'runningcheck': false,
   },
 }
diff --git a/contrib/test_decoding/meson.build b/contrib/test_decoding/meson.build
index dd7cb0101ad..48b9c866a43 100644
--- a/contrib/test_decoding/meson.build
+++ b/contrib/test_decoding/meson.build
@@ -37,6 +37,9 @@ tests += {
     'regress_args': [
       '--temp-config', files('logical.conf'),
     ],
+    # Disabled because these tests require "wal_level=logical", which
+    # typical runningcheck users do not have (e.g. buildfarm clients).
+    'runningcheck': false,
   },
   'isolation': {
     'specs': [
@@ -54,6 +57,8 @@ tests += {
     'regress_args': [
       '--temp-config', files('logical.conf'),
     ],
+    # see above
+    'runningcheck': false,
   },
   'tap': {
     'tests': [
diff --git a/src/interfaces/ecpg/test/meson.build b/src/interfaces/ecpg/test/meson.build
index f0ace641f0c..21331501a5a 100644
--- a/src/interfaces/ecpg/test/meson.build
+++ b/src/interfaces/ecpg/test/meson.build
@@ -79,6 +79,7 @@ tests += {
     'test_kwargs': {
       'depends': ecpg_test_dependencies,
     },
+    'dbname': 'ecpg1_regression,ecpg2_regression',
     'regress_args': ecpg_regress_args,
   },
 }
diff --git a/src/test/isolation/meson.build b/src/test/isolation/meson.build
index c7656fd4609..6c748b08209 100644
--- a/src/test/isolation/meson.build
+++ b/src/test/isolation/meson.build
@@ -54,5 +54,6 @@ tests += {
       'priority': 40,
       'timeout': 1000,
     },
+    'dbname': 'isolation_regression',
   },
 }
diff --git a/src/test/modules/commit_ts/meson.build b/src/test/modules/commit_ts/meson.build
index 60cb12164d2..fa86e70e880 100644
--- a/src/test/modules/commit_ts/meson.build
+++ b/src/test/modules/commit_ts/meson.build
@@ -6,6 +6,9 @@ tests += {
     'sql': [
       'commit_timestamp',
     ],
+    # Disabled because these tests require "track_commit_timestamp = on",
+    # which typical runningcheck users do not have (e.g. buildfarm clients).
+    'runningcheck': false,
   },
   'tap': {
     'tests': [
diff --git a/src/test/modules/snapshot_too_old/meson.build b/src/test/modules/snapshot_too_old/meson.build
index efd3f1f113b..5094b52949b 100644
--- a/src/test/modules/snapshot_too_old/meson.build
+++ b/src/test/modules/snapshot_too_old/meson.build
@@ -10,5 +10,8 @@ tests += {
       'sto_using_hash_index',
     ],
     'regress_args': ['--temp-config', files('sto.conf')],
+    # Disabled because these tests require "old_snapshot_threshold" >= 0, which
+    # typical runningcheck users do not have (e.g. buildfarm clients).
+    'runningcheck': false,
   },
 }
diff --git a/src/test/modules/test_oat_hooks/meson.build b/src/test/modules/test_oat_hooks/meson.build
index 5faf0459777..b2faab98a5f 100644
--- a/src/test/modules/test_oat_hooks/meson.build
+++ b/src/test/modules/test_oat_hooks/meson.build
@@ -14,5 +14,6 @@ tests += {
       'test_oat_hooks',
     ],
     'regress_args': ['--no-locale', '--encoding=UTF8'],
+    'runningcheck': false,
   },
 }
diff --git a/src/test/modules/test_pg_dump/meson.build b/src/test/modules/test_pg_dump/meson.build
index 41021829f3a..73f1fcf9428 100644
--- a/src/test/modules/test_pg_dump/meson.build
+++ b/src/test/modules/test_pg_dump/meson.build
@@ -13,6 +13,8 @@ tests += {
     'sql': [
       'test_pg_dump',
     ],
+    # doesn't delete its user
+    'runningcheck': false,
   },
   'tap': {
     'tests': [
diff --git a/src/test/modules/worker_spi/meson.build b/src/test/modules/worker_spi/meson.build
index 32acad883b2..5491248f16f 100644
--- a/src/test/modules/worker_spi/meson.build
+++ b/src/test/modules/worker_spi/meson.build
@@ -21,6 +21,8 @@ tests += {
     'sql': [
       'worker_spi',
     ],
-    'regress_args': ['--temp-config', files('dynamic.conf'), '--dbname=contrib_regression'],
+    'dbname': 'contrib_regression',
+    'regress_args': ['--temp-config', files('dynamic.conf')],
+    'runningcheck': false,
   },
 }
diff --git a/src/test/regress/meson.build b/src/test/regress/meson.build
index 03de591b0c7..18a5e1acf7c 100644
--- a/src/test/regress/meson.build
+++ b/src/test/regress/meson.build
@@ -67,5 +67,6 @@ tests += {
       'priority': 50,
       'timeout': 1000,
     },
+    'dbname': 'regression',
   },
 }
diff --git a/.cirrus.yml b/.cirrus.yml
index 531cfe96f65..e5416371049 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -122,6 +122,21 @@ task:
       meson test $MTEST_ARGS --num-processes ${TEST_JOBS}
     EOF
 
+  # test runningcheck, freebsd chosen because it's currently fast enough
+  test_running_script: |
+    su postgres <<-EOF
+      set -e
+      ulimit -c unlimited
+      meson test $MTEST_ARGS --quiet --suite setup
+      export LD_LIBRARY_PATH="$(pwd)/build/tmp_install/usr/local/pgsql/lib/:$LD_LIBRARY_PATH"
+      mkdir -p build/testrun
+      build/tmp_install/usr/local/pgsql/bin/initdb -N build/runningcheck --no-instructions -A trust
+      echo "include '$(pwd)/src/tools/ci/pg_ci_base.conf'" >> build/runningcheck/postgresql.conf
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -c -o '-c fsync=off' -D build/runningcheck -l build/testrun/runningcheck.log start
+      meson test $MTEST_ARGS --num-processes ${TEST_JOBS} --setup running
+      build/tmp_install/usr/local/pgsql/bin/pg_ctl -D build/runningcheck stop
+    EOF
+
   on_failure:
     <<: *on_failure_meson
     cores_script: src/tools/ci/cores_backtrace.sh freebsd /tmp/cores
diff --git a/meson.build b/meson.build
index 253994931e8..48e6a108bfe 100644
--- a/meson.build
+++ b/meson.build
@@ -2820,6 +2820,18 @@ endif
 # Test Generation
 ###############################################################
 
+# when using a meson version understanding exclude_suites, define a 'default'
+# test setup that excludes tests running against a pre-existing install and a
+# 'running' setup that conflicts with creation of the temporary installation
+# and tap tests (which don't support running against a running server).
+running_suites = []
+install_suites = []
+if meson.version().version_compare('>=0.57')
+  runningcheck = true
+else
+  runningcheck = false
+endif
+
 testwrap = files('src/tools/testwrap')
 
 foreach test_dir : tests
@@ -2827,7 +2839,6 @@ foreach test_dir : tests
     testwrap,
     '--basedir', meson.build_root(),
     '--srcdir', test_dir['sd'],
-    '--testgroup', test_dir['name'],
   ]
 
   foreach kind, v : test_dir
@@ -2840,41 +2851,51 @@ foreach test_dir : tests
     if kind in ['regress', 'isolation', 'ecpg']
       if kind == 'regress'
         runner = pg_regress
+        fallback_dbname = 'regression_@0@'
       elif kind == 'isolation'
         runner = pg_isolation_regress
+        fallback_dbname = 'isolation_regression_@0@'
       elif kind == 'ecpg'
         runner = pg_regress_ecpg
+        fallback_dbname = 'ecpg_regression_@0@'
       endif
 
-      test_output = test_result_dir / test_dir['name'] / kind
+      test_group = test_dir['name']
+      test_group_running = test_dir['name'] + '-running'
 
-      test_command = [
+      test_output = test_result_dir / test_group / kind
+      test_output_running = test_result_dir / test_group_running/ kind
+
+      # Unless specified by the test, choose a non-conflicting database name,
+      # to avoid conflicts when running against existing server.
+      dbname = t.get('dbname',
+        fallback_dbname.format(test_dir['name']))
+
+      test_command_base = [
         runner.full_path(),
         '--inputdir', t.get('inputdir', test_dir['sd']),
         '--expecteddir', t.get('expecteddir', test_dir['sd']),
-        '--outputdir', test_output,
-        '--temp-instance', test_output / 'tmp_check',
         '--bindir', '',
         '--dlpath', test_dir['bd'],
         '--max-concurrent-tests=20',
-        '--port', testport.to_string(),
+        '--dbname', dbname,
       ] + t.get('regress_args', [])
 
+      test_selection = []
       if t.has_key('schedule')
-        test_command += ['--schedule', t['schedule'],]
+        test_selection += ['--schedule', t['schedule'],]
       endif
 
       if kind == 'isolation'
-        test_command += t.get('specs', [])
+        test_selection += t.get('specs', [])
       else
-        test_command += t.get('sql', [])
+        test_selection += t.get('sql', [])
       endif
 
       env = test_env
       env.prepend('PATH', temp_install_bindir, test_dir['bd'])
 
       test_kwargs = {
-        'suite': [test_dir['name']],
         'priority': 10,
         'timeout': 1000,
         'depends': test_deps + t.get('deps', []),
@@ -2883,12 +2904,41 @@ foreach test_dir : tests
 
       test(test_dir['name'] / kind,
         python,
-        args: testwrap_base + [
+        args: [
+          testwrap_base,
+          '--testgroup', test_group,
           '--testname', kind,
-          '--', test_command,
+          '--',
+          test_command_base,
+          '--outputdir', test_output,
+          '--temp-instance', test_output / 'tmp_check',
+          '--port', testport.to_string(),
+          test_selection,
         ],
+        suite: test_group,
         kwargs: test_kwargs,
       )
+      install_suites += test_dir['name']
+
+      # some tests can't support running against running DB
+      if runningcheck and t.get('runningcheck', true)
+        test(test_group_running / kind,
+          python,
+          args: [
+            testwrap_base,
+            '--testgroup', test_group_running,
+            '--testname', kind,
+            '--',
+            test_command_base,
+            '--outputdir', test_output_running,
+            test_selection,
+          ],
+          is_parallel: t.get('runningcheck-parallel', true),
+          suite: test_group_running,
+          kwargs: test_kwargs,
+        )
+        running_suites += test_group_running
+      endif
 
       testport += 1
     elif kind == 'tap'
@@ -2911,9 +2961,10 @@ foreach test_dir : tests
         env.set(name, value)
       endforeach
 
+      suite = test_dir['name']
       test_kwargs = {
         'protocol': 'tap',
-        'suite': [test_dir['name']],
+        'suite': suite,
         'timeout': 1000,
         'depends': test_deps + t.get('deps', []),
         'env': env,
@@ -2933,12 +2984,14 @@ foreach test_dir : tests
           python,
           kwargs: test_kwargs,
           args: testwrap_base + [
+            '--testgroup', test_dir['name'],
             '--testname', onetap_p,
             '--', test_command,
             test_dir['sd'] / onetap,
           ],
         )
       endforeach
+      install_suites += suite
     else
       error('unknown kind @0@ of test in @1@'.format(kind, test_dir['sd']))
     endif
@@ -2947,6 +3000,14 @@ foreach test_dir : tests
 
 endforeach # directories with tests
 
+# repeat condition so meson realizes version dependency
+if meson.version().version_compare('>=0.57')
+  add_test_setup('default',
+    is_default: true,
+    exclude_suites: running_suites)
+  add_test_setup('running',
+    exclude_suites: ['setup'] + install_suites)
+endif
 
 
 ###############################################################
-- 
2.37.3.542.gdd3f6c4cae

