From 610ee9be2e5c497f186a6ac5f570558216fd9e36 Mon Sep 17 00:00:00 2001
From: Andres Freund <andres@anarazel.de>
Date: Mon, 15 Mar 2021 09:25:15 -0700
Subject: [PATCH v1 2/6] ci: Add CI for FreeBSD, Linux, MacOS and Windows,
 utilizing cirrus-ci.

---
 .cirrus.yml                     | 306 ++++++++++++++++++++++++++++++++
 .dockerignore                   |   3 +
 ci/docker/linux_debian_bullseye |   9 +
 ci/docker/windows_vs_2019       |  82 +++++++++
 ci/freebsd_gcp_repartition.sh   |  28 +++
 ci/pg_ci_base.conf              |  12 ++
 ci/windows_build_config.pl      |   3 +
 7 files changed, 443 insertions(+)
 create mode 100644 .cirrus.yml
 create mode 100644 .dockerignore
 create mode 100644 ci/docker/linux_debian_bullseye
 create mode 100644 ci/docker/windows_vs_2019
 create mode 100755 ci/freebsd_gcp_repartition.sh
 create mode 100644 ci/pg_ci_base.conf
 create mode 100644 ci/windows_build_config.pl

diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644
index 00000000000..9f513f7d10a
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,306 @@
+env:
+  # accelerate initial clone, but a bit of depth so that concurrent tasks work
+  CIRRUS_CLONE_DEPTH: 100
+  # Useful to be able to analyse what in a script takes long
+  CIRRUS_LOG_TIMESTAMP: true
+  # target to test, for all but windows
+  CHECK: check-world
+  CHECKFLAGS: -Otarget
+  PGCTLTIMEOUT: 120
+  CCACHE_MAXSIZE: "500M"
+  TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+
+
+task:
+  name: FreeBSD
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*freebsd.*'
+  compute_engine_instance:
+    image_project: pg-vm-images-aio
+    image: family/pg-aio-freebsd-13-0
+    platform: freebsd
+    cpu: 2
+    memory: 2G
+    disk: 50
+  env:
+    CCACHE_DIR: "/tmp/ccache_dir"
+
+  ccache_cache:
+    folder: "/tmp/ccache_dir"
+  sysinfo_script:
+    - export || true
+  sysconfig_script:
+    - sudo sysctl kern.corefile='/tmp/%N.%P.core'
+  repartition_script:
+    - ci/freebsd_gcp_repartition.sh
+  create_user_script:
+    - pw useradd postgres
+    - chown -R postgres:postgres .
+    - mkdir -p /tmp/ccache_dir
+    - chown -R postgres:postgres /tmp/ccache_dir
+
+  configure_script:
+    - su postgres -c './configure --enable-cassert --enable-debug --enable-tap-tests --with-includes=/usr/local/include --with-libs=/usr/local/lib CC="ccache cc"'
+  build_script:
+    - su postgres -c 'gmake -s -j3 && gmake -s -j3 -C contrib'
+  upload_caches:
+    - ccache
+
+  check_world_script:
+    - su postgres -c 'time gmake -s -j2 ${CHECK} ${CHECKFLAGS}'
+
+  on_failure:
+    core_script:
+      for corefile in $(find /tmp -name '*.core' 2>/dev/null) ; do binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECPATH | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ; echo dumping $corefile for $binary ; gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile ; done
+    log_artifacts:
+      path: "**/**.log"
+      type: text/plain
+    regress_diffs_artifacts:
+      path: "**/**.diffs"
+      type: text/plain
+    tap_artifacts:
+      path: "**/regress_log_*"
+      type: text/plain
+
+
+task:
+  name: Linux
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+  compute_engine_instance:
+    image_project: pg-vm-images-aio
+    image: family/pg-aio-bullseye
+    platform: linux
+    cpu: 4
+    memory: 8G
+    nested_virtualization: false
+  env:
+    CCACHE_DIR: "/tmp/ccache_dir"
+    DEBUGINFOD_URLS: "https://debuginfod.debian.net"
+    TIMEOUT_CMD: timeout -s KILL -v 25m
+  ccache_cache:
+    folder: "/tmp/ccache_dir"
+
+  sysinfo_script:
+    - id
+    - uname -a
+    - cat /proc/cmdline
+    - lsblk
+    - ulimit -a -H
+    - ulimit -a -S
+    - export
+  sysconfig_script:
+    - useradd -m postgres
+    - chown -R postgres:postgres .
+    - mkdir -p /tmp/ccache_dir
+    - chown -R postgres:postgres /tmp/ccache_dir
+    - echo '* - memlock 134217728' > /etc/security/limits.d/postgres.conf
+    - su postgres -c 'ulimit -l -H'
+    - su postgres -c 'ulimit -l -S'
+    - echo '/tmp/%e-%s-%p.core' > /proc/sys/kernel/core_pattern
+
+  configure_script:
+    - echo "COPT=-O0 -ggdb" > src/Makefile.custom
+    - su postgres -c './configure --enable-cassert --enable-debug --enable-tap-tests --with-tcl --with-python --with-perl --with-ldap --with-openssl --with-icu --with-llvm CC="ccache gcc" CXX="ccache g++" CLANG="ccache clang"'
+  build_script:
+    - su postgres -c 'make -s -j4 && make -s -j4 -C contrib'
+  upload_caches:
+    - ccache
+
+  tests_script:
+    - su postgres -c 'ulimit -c unlimited ; ${TIMEOUT_CMD} make -s ${CHECK} ${CHECKFLAGS} -j8'
+
+  on_failure:
+    cores_script:
+      - for corefile in $(find /tmp/ -name '*.core' 2>/dev/null) ; do binary=$(gdb -quiet -core $corefile -batch -ex 'info auxv' | grep AT_EXECFN | perl -pe "s/^.*\"(.*)\"\$/\$1/g") ; echo dumping $corefile for $binary ; gdb --batch --quiet -ex "thread apply all bt full" -ex "quit" $binary $corefile ; done
+    log_artifacts:
+      path: "**/**.log"
+      type: text/plain
+    regress_diffs_artifacts:
+      path: "**/**.diffs"
+      type: text/plain
+    tap_artifacts:
+      path: "**/regress_log_*"
+      type: text/plain
+
+
+task:
+  name: macOS
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(macos|darwin|osx).*'
+  osx_instance:
+    image: big-sur-base
+  env:
+    CIRRUS_WORKING_DIR: ${HOME}/pgsql/
+    TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+    CCACHE_DIR: ${HOME}/ccache
+    HOMEBREW_CACHE: ${HOME}/homebrew-cache
+    PERL5LIB: ${HOME}/perl5/lib/perl5
+
+  sysinfo_script:
+    - id
+    - export
+  ccache_cache:
+    folder: ${CCACHE_DIR}
+  homebrew_cache:
+    folder: ${HOMEBREW_CACHE}
+  perl_cache:
+    folder: ~/perl5
+
+  cpan_install_script:
+    - perl -mIPC::Run -e 1 || cpan -T IPC::Run
+    - perl -mIO::Pty -e 1 || cpan -T IO::Pty
+  upload_caches:
+    - perl
+  core_install_script:
+    - sudo chmod 777 /cores
+  homebrew_install_script:
+    - brew install make coreutils ccache
+  upload_caches:
+    - homebrew
+
+  configure_script:
+    - ./configure --prefix=$HOME/install --enable-cassert --enable-debug --enable-tap-tests CC="ccache gcc" CFLAGS="-O0"
+  build_script:
+    - gmake -s -j12 && gmake -s -j12 -C contrib
+  upload_caches:
+    - ccache
+
+  tests_script:
+    - ulimit -c unlimited
+    - ulimit -n 1024
+    - gtimeout -s KILL -v 20m gmake -s -j12 ${CHECK} ${CHECKFLAGS}
+
+  on_failure:
+    cores_script:
+      - for corefile in $(find /cores/ -name 'core.*' 2>/dev/null) ; do lldb -c $corefile --batch -o 'thread backtrace all' -o 'quit' ; done
+    log_artifacts:
+      path: "**/**.log"
+      type: text/plain
+    regress_diffs_artifacts:
+      path: "**/**.diffs"
+      type: text/plain
+    tap_artifacts:
+      path: "**/regress_log_*"
+      type: text/plain
+
+
+task:
+  name: Windows
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*windows.*'
+  windows_container:
+    dockerfile: ci/docker/windows_vs_2019
+    cpu: 4
+    memory: 4G
+  env:
+    PROVE_FLAGS: -j10
+    # The default working dir is in a directory msbuild complains about
+    CIRRUS_WORKING_DIR: "c:/cirrus"
+    TEMP_CONFIG: ${CIRRUS_WORKING_DIR}/ci/pg_ci_base.conf
+    # Avoid re-installing over and over
+    NO_TEMP_INSTALL: 1
+
+  sysinfo_script:
+    - chcp
+    - systeminfo
+    - powershell -Command get-psdrive -psprovider filesystem
+    - ps: Get-Item -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug'
+    - set
+
+  configure_script:
+    - copy ci\windows_build_config.pl src\tools\msvc\config.pl
+    - vcvarsall x64
+    - perl src/tools/msvc/mkvcbuild.pl
+  build_script:
+    - vcvarsall x64
+    # Disable file tracker, we're never going to rebuild...
+    - msbuild -m /p:TrackFileAccess=false pgsql.sln
+  tempinstall_script:
+    - perl src\tools\msvc\install.pl tmp_install
+
+  check_script:
+    - perl src/tools/msvc/vcregress.pl check parallel
+  startcreate_script:
+    - tmp_install\bin\pg_ctl.exe initdb -D tmp_check\db -l tmp_check\initdb.log
+    - echo include '%TEMP_CONFIG%' >> tmp_check\db\postgresql.conf
+    - tmp_install\bin\pg_ctl.exe start -D tmp_check\db -l tmp_check\postmaster.log
+  isolationcheck_script:
+    - perl src/tools/msvc/vcregress.pl isolationcheck
+  modulescheck_script:
+    - perl src/tools/msvc/vcregress.pl modulescheck
+  contribcheck_script:
+    - perl src/tools/msvc/vcregress.pl contribcheck
+  plcheck_script:
+    - perl src/tools/msvc/vcregress.pl plcheck
+  stop_script:
+    - tmp_install\bin\pg_ctl.exe stop -D tmp_check\db -l tmp_check\postmaster.log
+  subscriptioncheck_script:
+    - perl src/tools/msvc/vcregress.pl taptest .\src\test\subscription\
+  authentication_script:
+    - perl src/tools/msvc/vcregress.pl taptest .\src\test\authentication\
+  recoverycheck_script:
+    - perl src/tools/msvc/vcregress.pl recoverycheck
+  bincheck_script:
+    - perl src/tools/msvc/vcregress.pl bincheck
+  upgradecheck_script:
+    - perl src/tools/msvc/vcregress.pl upgradecheck
+  ecpgcheck_script:
+    # tries to build additional stuff
+    - vcvarsall x64
+    # References ecpg_regression.proj in the current dir
+    - cd src\tools\msvc
+    - perl vcregress.pl ecpgcheck
+
+  always:
+    cat_dumps_script:
+      - cat crashlog.txt || true
+    dump_artifacts:
+      path: "crashlog.txt"
+      type: text/plain
+
+  on_failure:
+    log_artifacts:
+      path: "**/**.log"
+      type: text/plain
+    regress_diffs_artifacts:
+      path: "**/**.diffs"
+      type: text/plain
+    tap_artifacts:
+      path: "**/regress_log_*"
+      type: text/plain
+
+
+task:
+  name: CompilerWarnings
+  depends_on:
+  - Linux
+  # not run task count as a success, so we need to recheck Linux' condition here :/
+  only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
+  container:
+    dockerfile: ci/docker/linux_debian_bullseye
+  env:
+    CCACHE_SIZE: "4GB"
+    CCACHE_DIR: "/tmp/ccache_dir"
+  ccache_cache:
+    folder: "/tmp/ccache_dir"
+  setup_script:
+    - echo "COPT=-Werror" > src/Makefile.custom
+    - gcc -v
+    - clang -v
+  # gcc with asserts disabled
+  always:
+    gcc_warning_script:
+      - ./configure --cache gcc.cache CC="ccache gcc"
+      - time make -s -j4 clean && time make -s -j4
+  # gcc with asserts enabled
+  always:
+    gcc_a_warning_script:
+      - ./configure --cache gcc.cache --enable-cassert CC="ccache gcc"
+      - time make -s -j4 clean && time make -s -j4
+  # clang with asserts disabled
+  always:
+    clang_warning_script:
+      - ./configure --cache clang.cache CC="ccache clang"
+      - time make -s -j4 clean && time make -s -j4
+  # clang with asserts enabled
+  always:
+    clang_a_warning_script:
+      - ./configure --cache clang.cache --enable-cassert CC="ccache clang"
+      - time make -s -j4 clean && time make -s -j4
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000000..3fceab2e97b
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,3 @@
+# Ignore everything, except ci/
+*
+!ci/*
diff --git a/ci/docker/linux_debian_bullseye b/ci/docker/linux_debian_bullseye
new file mode 100644
index 00000000000..3c1d1a4583a
--- /dev/null
+++ b/ci/docker/linux_debian_bullseye
@@ -0,0 +1,9 @@
+FROM debian:bullseye
+RUN apt-get -y update && apt-get -y upgrade && \
+  DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install \
+      gcc g++ libreadline-dev flex bison make perl libipc-run-perl clang \
+      llvm-dev libperl-dev libpython3-dev tcl-dev libldap2-dev libicu-dev \
+      docbook-xml docbook-xsl fop libxml2-utils xsltproc krb5-admin-server \
+      krb5-kdc krb5-user slapd ldap-utils libssl-dev pkg-config locales-all \
+      liburing-dev python3-distutils ccache gdb && \
+  apt-get clean
diff --git a/ci/docker/windows_vs_2019 b/ci/docker/windows_vs_2019
new file mode 100644
index 00000000000..e5b4810ac9e
--- /dev/null
+++ b/ci/docker/windows_vs_2019
@@ -0,0 +1,82 @@
+# escape=`
+
+# We used to use the visual studio container, but it's too outdated now
+FROM cirrusci/windowsservercore:2019
+
+SHELL ["powershell", "-NoLogo", "-NoProfile", "-Command"]
+
+
+RUN `
+    New-Item -Path 'HKLM:\SOFTWARE\Policies\Microsoft\VisualStudio' ; `
+    New-Item -Path 'HKLM:\SOFTWARE\Policies\Microsoft\VisualStudio\Setup' ; `
+    New-ItemProperty -Path 'HKLM:\SOFTWARE\Policies\Microsoft\VisualStudio\Setup' -Name KeepDownloadedPayloads -Value 0 -PropertyType DWord
+
+
+# Install commandline debugger and log all crashes to c:\cirrus\crashlog.txt
+#
+# Done manually as doing this via chocolatey / the installer directly, ends up
+# with a lot of unnecessary chaff, making the layer unnecessarily large.
+RUN `
+    mkdir c:\t ; `
+    cd c:\t ; `
+    curl.exe -sSL -o 'windsdksetup.exe' https://download.microsoft.com/download/9/7/9/97982c1d-d687-41be-9dd3-6d01e52ceb68/windowssdk/winsdksetup.exe ; `
+    Start-Process -FilePath ".\windsdksetup.exe" `
+      -ArgumentList '/Features OptionId.WindowsDesktopDebuggers /layout c:\t\sdk /quiet /norestart /log c:\t\sdk.log' `
+    -Wait  ; `
+    `
+    Start-Process -FilePath msiexec.exe `
+      -ArgumentList '/a \"C:\t\sdk\Installers\X64 Debuggers And Tools-x64_en-us.msi\" /qb /log install2.log' `
+    -Wait ; `
+    C:\Windows` Kits\10\Debuggers\x64\cdb.exe -version ; `
+    `
+    cd c:\ ; `
+    Remove-Item C:\t\* -Force -Recurse ; `
+    Remove-Item C:\t -Force -Recurse ; `
+    Set-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Debugger' -Value '\"C:\Windows Kits\10\Debuggers\x64\cdb.exe\" -p %ld -e %ld -g -kqm -c \".lines -e; .symfix+ ;.logappend c:\cirrus\crashlog.txt ; !peb; ~*kP ; .logclose ; q \"' ; `
+    New-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name 'Auto' -Value 1 -PropertyType DWord ; `
+    Get-ItemProperty -Path 'HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug' -Name Debugger; `
+    setx PATH \"C:\Windows Kits\10\Debuggers\x64;$Env:PATH\" /m
+
+
+# Install perl and bison.
+#
+# Done manually as choco takes a lot longer. I think it's download issues with
+# powershell's download stuff? That's wy curl.exe is directly used here at least...
+RUN `
+    mkdir c:\t ; `
+    cd c:\t ; `
+    `
+    curl.exe -sSL -o perl.zip `
+        https://strawberryperl.com/download/5.30.3.1/strawberry-perl-5.30.3.1-64bit-portable.zip ; `
+    7z.exe x .\perl.zip -xr!c -oc:\strawberry ; `
+    `
+    curl.exe -sSL -o winflexbison.zip `
+        https://github.com/lexxmark/winflexbison/releases/download/v2.5.24/win_flex_bison-2.5.24.zip ; `
+    7z.exe x .\winflexbison.zip -oc:\winflexbison ; `
+    Rename-Item -Path c:\winflexbison\win_flex.exe c:\winflexbison\flex.exe ; `
+    Rename-Item -Path c:\winflexbison\win_bison.exe c:\winflexbison\bison.exe ; `
+    `
+    cd c:\ ; `
+    Remove-Item C:\t -Force -Recurse ; `
+    setx PATH \"C:\strawberry\perl\bin;C:\winflexbison;C:\Program Files\Git\usr\bin;$Env:PATH\" /m
+
+
+# Install visual studio
+#
+# Adding VS path to vcvarsall.bat so user of container doesn't need to know the full path
+RUN `
+    mkdir c:\t ; `
+    cd c:\t ; `
+    curl.exe -sSL -o c:\t\vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe ; `
+    Start-Process -Wait `
+        -FilePath c:\t\vs_buildtools.exe `
+        -ArgumentList `
+          '--quiet', '--wait', '--norestart', '--nocache', `
+          '--installPath', 'c:\BuildTools', `
+          '--add', 'Microsoft.VisualStudio.Component.VC.Tools.x86.x64', `
+          '--add', 'Microsoft.VisualStudio.Component.Windows10SDK.20348'  ; `
+    cd c:\ ; `
+    Remove-Item C:\t -Force -Recurse ; `
+    Remove-Item -Force -Recurse ${Env:TEMP}\*; `
+    Remove-Item -Force -Recurse \"${Env:ProgramData}\Package Cache\"  ; `
+    setx PATH \"c:\BuildTools\VC\Auxiliary\Build;$Env:PATH\" /m
diff --git a/ci/freebsd_gcp_repartition.sh b/ci/freebsd_gcp_repartition.sh
new file mode 100755
index 00000000000..2d5e1738998
--- /dev/null
+++ b/ci/freebsd_gcp_repartition.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# The default filesystem on freebsd gcp images is very slow to run tests on,
+# due to its 32KB block size
+#
+# XXX: It'd probably better to fix this in the image, using something like
+# https://people.freebsd.org/~lidl/blog/re-root.html
+
+# fix backup partition table after resize
+gpart recover da0
+gpart show da0
+# kill swap, so we can delete a partition
+swapoff -a || true
+# (apparently we can only have 4!?)
+gpart delete -i 3 da0
+gpart add -t freebsd-ufs -l data8k -a 4096 da0
+gpart show da0
+newfs -U -b 8192 /dev/da0p3
+
+# Migrate working directory
+du -hs $CIRRUS_WORKING_DIR
+mv $CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR.orig
+mkdir $CIRRUS_WORKING_DIR
+mount -o noatime /dev/da0p3 $CIRRUS_WORKING_DIR
+cp -r $CIRRUS_WORKING_DIR.orig/* $CIRRUS_WORKING_DIR/
diff --git a/ci/pg_ci_base.conf b/ci/pg_ci_base.conf
new file mode 100644
index 00000000000..637e3cfb343
--- /dev/null
+++ b/ci/pg_ci_base.conf
@@ -0,0 +1,12 @@
+# Tends to produce too many core files, taking a long time
+restart_after_crash = false
+
+# So that tests using the "manually" started postgres on windows can use
+# prepared statements
+max_prepared_transactions=10
+
+# Settings that make logs more useful
+log_line_prefix='%m [%p][%b][%v:%x] '
+log_checkpoints = true
+log_connections = true
+log_disconnections = true
diff --git a/ci/windows_build_config.pl b/ci/windows_build_config.pl
new file mode 100644
index 00000000000..a8abfa2a02f
--- /dev/null
+++ b/ci/windows_build_config.pl
@@ -0,0 +1,3 @@
+$config->{"tap_tests"} = 1;
+$config->{"asserts"} = 1;
+1;
-- 
2.32.0.rc2

