From 115173017d45eec3e8d07e6c21bc6aee8e7f8c4e Mon Sep 17 00:00:00 2001 From: usernamedt Date: Mon, 17 Jan 2022 17:46:03 +0500 Subject: [PATCH 3/3] Turn on zlib compression for CI test runs --- .cirrus.yml | 2 + src/backend/utils/misc/guc.c | 2 +- src/common/zpq_stream.c | 27 +--- src/interfaces/libpq/fe-connect.c | 2 +- src/tools/ci/install_zlib.bat | 201 +++++++++++++++++++++++++++ src/tools/ci/windows_build_config.pl | 1 + src/tools/msvc/Solution.pm | 2 +- 7 files changed, 211 insertions(+), 26 deletions(-) create mode 100644 src/tools/ci/install_zlib.bat diff --git a/.cirrus.yml b/.cirrus.yml index 677bdf0e65..297ac51651 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -381,6 +381,8 @@ task: configure_script: # copy errors out when using forward slashes + - choco install -y cmake + - ./src/tools/ci/install_zlib.bat - copy src\tools\ci\windows_build_config.pl src\tools\msvc\config.pl - vcvarsall x64 - perl src/tools/msvc/mkvcbuild.pl diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 9d8fd9b817..11a43ba668 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -4601,7 +4601,7 @@ static struct config_string ConfigureNamesString[] = NULL }, &libpq_compress_algorithms, - "off", + "zlib", check_libpq_compression, NULL, NULL }, diff --git a/src/common/zpq_stream.c b/src/common/zpq_stream.c index a712200e45..de00a82896 100644 --- a/src/common/zpq_stream.c +++ b/src/common/zpq_stream.c @@ -156,24 +156,8 @@ zpq_build_msg_compression_map(ZpqStream * zpq) for (i = 0; i < 256; i++) { - zpq->compressor_by_msg_type[i] = -1; - } - - for (i = 0; i < zpq->n_compressors; i++) - { - /* compress CopyData, DataRow and Query messages */ - if (zpq->compressor_by_msg_type['d'] == -1) - { - zpq->compressor_by_msg_type['d'] = i; - } - if (zpq->compressor_by_msg_type['D'] == -1) - { - zpq->compressor_by_msg_type['D'] = i; - } - if (zpq->compressor_by_msg_type['Q'] == -1) - { - zpq->compressor_by_msg_type['Q'] = i; - } + /* Compress all messages during the CI test runs */ + zpq->compressor_by_msg_type[i] = 0; } } @@ -186,11 +170,8 @@ zpq_build_msg_compression_map(ZpqStream * zpq) static inline int zpq_choose_compressor(ZpqStream * zpq, char msg_type, uint32 msg_len) { - if (msg_len >= ZPQ_COMPRESS_THRESHOLD) - { - return zpq->compressor_by_msg_type[(unsigned char) msg_type]; - } - return -1; + /* Compress messages with any length during the CI test runs */ + return zpq->compressor_by_msg_type[(unsigned char) msg_type]; } /* diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index 42131abbb9..2cb0b2f4f0 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -340,7 +340,7 @@ static const internalPQconninfoOption PQconninfoOptions[] = { "Replication", "D", 5, offsetof(struct pg_conn, replication)}, - {"compression", "PGCOMPRESSION", "off", NULL, + {"compression", "PGCOMPRESSION", "zlib", NULL, "Libpq-compression", "", 16, offsetof(struct pg_conn, compression)}, diff --git a/src/tools/ci/install_zlib.bat b/src/tools/ci/install_zlib.bat new file mode 100644 index 0000000000..e8419377c8 --- /dev/null +++ b/src/tools/ci/install_zlib.bat @@ -0,0 +1,201 @@ +@echo off +setlocal EnableDelayedExpansion + +set ORIGIN=%cd% + +:: Configuration +set VERSION=1.2.11 +set VER=%VERSION:.=% +set URL=https://zlib.net/zlib%VER%.zip +set CMAKE_VS_PLATFORM_NAME=x64 +set TMPDIR=%HOMEDRIVE%%HOMEPATH%\zlib_tmp +set SRC_DIR=%TMPDIR%\zlib-%VERSION% +set ZIPFILE=zlib%VER%.zip +set ABS_ZIPFILE=%TMPDIR%\%ZIPFILE% +set BUILD_DIR=%SRC_DIR%\build +set LOGFILE=%TMPDIR%\zlib_install.log + +echo [0/6] Library(zlib==%VERSION%) + +if not exist %TMPDIR% (mkdir %TMPDIR% && cd /d %TMPDIR% || exit /B 1) +call :cleanup_src +call :cleanup_log +copy /y nul %LOGFILE% >nul 2>&1 +call :log_sysinfo >>%LOGFILE% 2>&1 + +if not defined ARCH ( + set ARCH=x64 +) +set GENERATOR=-DCMAKE_GENERATOR_PLATFORM=%ARCH% +if "%ARCH%" == "x64" ( + set PREFIX=-DCMAKE_INSTALL_PREFIX="C:\zlib" +) else ( + set PREFIX=-DCMAKE_INSTALL_PREFIX="C:\zlib" +) + +echo|set /p="[1/6] Checking cmake... " +call :setup_cmake_path >>%LOGFILE% 2>&1 +if not defined CMAKE ( + call :failed + echo[ + echo Please, install CMAKE: https://cmake.org/download/ + exit /B 1 +) else (echo done.) + +echo|set /p="[2/6] Downloading... " +echo Fetching %URL% >>%LOGFILE% 2>&1 +cd /d %TMPDIR% && call :winget "%URL%" >>%LOGFILE% 2>&1 +if %ERRORLEVEL% NEQ 0 (call :failed && exit /B 1) else (echo done.) + +echo|set /p="[3/6] Extracting... " +powershell.exe -nologo -noprofile -command "& { Add-Type -A 'System.IO.Compression.FileSystem'; [IO.Compression.ZipFile]::ExtractToDirectory('%ABS_ZIPFILE%', '%TMPDIR%'); }" >>%LOGFILE% 2>&1 +if %ERRORLEVEL% NEQ 0 (call :failed && exit /B 1) else (echo done.) + +echo|set /p="[4/6] Fixing CMakeLists.txt... " +set OLDSTR=RUNTIME DESTINATION ""\${INSTALL_BIN_DIR}\"" +set NEWSTR=RUNTIME DESTINATION ""bin\"" +call :search_replace "%OLDSTR%" "%NEWSTR%" +if %ERRORLEVEL% NEQ 0 (call :failed && exit /B 1) + +set OLDSTR=ARCHIVE DESTINATION ""\${INSTALL_LIB_DIR}\"" +set NEWSTR=ARCHIVE DESTINATION ""lib\"" +call :search_replace "%OLDSTR%" "%NEWSTR%" +if %ERRORLEVEL% NEQ 0 (call :failed && exit /B 1) + +set OLDSTR=LIBRARY DESTINATION ""\${INSTALL_LIB_DIR}\"" +set NEWSTR=LIBRARY DESTINATION ""lib\"" +call :search_replace "%OLDSTR%" "%NEWSTR%" +if %ERRORLEVEL% NEQ 0 (call :failed && exit /B 1) + +set OLDSTR=DESTINATION ""\${INSTALL_INC_DIR}\"" +set NEWSTR=DESTINATION ""include\"" +call :search_replace "%OLDSTR%" "%NEWSTR%" +if %ERRORLEVEL% NEQ 0 (call :failed && exit /B 1) else (echo done.) + +rd /S /Q %BUILD_DIR% >nul 2>&1 +mkdir %BUILD_DIR% && cd /d %BUILD_DIR% + +echo|set /p="[5/6] Configuring... " +"%CMAKE%" .. -DCMAKE_BUILD_TYPE=Release %PREFIX% %GENERATOR% >>%LOGFILE% 2>&1 +if %ERRORLEVEL% NEQ 0 (call :failed && exit /B 1) else (echo done.) + +echo [6/6] Compiling and installing... +"%CMAKE%" --build . --config Release --target install +if %ERRORLEVEL% NEQ 0 (call :failed && exit /B 1) else (echo done.) + +call :cleanup_src +call :goto_origin + +echo Details can be found at %LOGFILE%. + +@echo on +@goto :eof + +:setup_cmake_path +where cmake.exe +cmake.exe /? 2> NUL 1> NUL +if not %ERRORLEVEL%==9009 (echo cmake.exe is accessible by default. && set CMAKE=cmake.exe) +if not defined CMAKE (echo cmake.exe is not accessible by default. Looking into common installation dirs.) +if not defined CMAKE (call :setup_cmake_path_for "%PROGRAMFILES%\CMake\bin") +if not defined CMAKE (call :setup_cmake_path_for "C:\Program Files\CMake\bin") +if not defined CMAKE (call :setup_cmake_path_for "C:\Program Files (x86)\CMake\bin") +@goto :eof + +:setup_cmake_path_for +set DIR_PATH=%~1 +echo|set /p="Checking !DIR_PATH! ... " +if exist !DIR_PATH!\cmake.exe ( + echo found. + set CMAKE=!DIR_PATH!\cmake.exe +) else ( + echo not found. +) +@goto :eof + +:cleanup_src +del /Q %ABS_ZIPFILE% >nul 2>&1 +rd /S /Q %SRC_DIR% >nul 2>&1 +@goto :eof + +:cleanup_log +del /Q %LOGFILE% >nul 2>&1 +@goto :eof + +:log_sysinfo +echo -- Environment variables +set +echo[ + +echo -- System info +systeminfo | findstr /B /C:"OS Name" /C:"OS Version" +echo[ +@goto :eof + +:cmakefiles_debug +set FILE=%BUILD_DIR%\CMakeFiles\CMakeError.log +if exist %FILE% ( + echo -- %FILE% + type %FILE% + echo[ +) +set FILE=%BUILD_DIR%\CMakeFiles\CMakeOutput.log +if exist %FILE% ( + echo -- %FILE% + type %FILE% + echo[ +) +@goto :eof + +:failed +echo FAILED. +call :cmakefiles_debug >>%LOGFILE% 2>&1 +echo[ +echo ---------------------------------------- log begin ---------------------------------------- +type %LOGFILE% +echo ---------------------------------------- log end ---------------------------------------- +echo LOG: %LOGFILE% +call :cleanup_src +call :goto_origin +@goto :eof + +:goto_origin +cd /d %ORIGIN% >nul 2>&1 +@goto :eof + +:search_replace +set OLDSTR=%~1 +set NEWSTR=%~2 +cd /d %SRC_DIR% +set CMD="(gc CMakeLists.txt) -replace '%OLDSTR%', '%NEWSTR%' | Out-File -encoding ASCII CMakeLists.txt" +powershell -Command %CMD% >>%LOGFILE% 2>&1 +goto :eof + +:winget +set URL=%~1 +for %%F in (%URL%) do set FILE=%%~nxF + +setlocal EnableDelayedExpansion +set multiLine=$security_protcols = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::SystemDefault^ + +if ([Net.SecurityProtocolType].GetMember('Tls11').Count -gt 0) {^ + + $security_protcols = $security_protcols -bor [Net.SecurityProtocolType]::Tls11^ + +}^ + +if ([Net.SecurityProtocolType].GetMember('Tls12').Count -gt 0) {^ + +$security_protcols = $security_protcols -bor [Net.SecurityProtocolType]::Tls12^ + +}^ + +[Net.ServicePointManager]::SecurityProtocol = $security_protcols^ + +Write-Host 'Downloading %URL%... ' -NoNewLine^ + +(New-Object Net.WebClient).DownloadFile('%URL%', '%FILE%')^ + +Write-Host 'done.' + +powershell -Command !multiLine! +goto:eof \ No newline at end of file diff --git a/src/tools/ci/windows_build_config.pl b/src/tools/ci/windows_build_config.pl index b0d4360c74..6629f322a5 100644 --- a/src/tools/ci/windows_build_config.pl +++ b/src/tools/ci/windows_build_config.pl @@ -7,6 +7,7 @@ $config->{"tap_tests"} = 1; $config->{"asserts"} = 1; $config->{"openssl"} = "c:/openssl/1.1/"; +$config->{"zlib"} = "c:/zlib"; $config->{"perl"} = "c:/strawberry/$ENV{DEFAULT_PERL_VERSION}/perl/"; $config->{"python"} = "c:/python/"; diff --git a/src/tools/msvc/Solution.pm b/src/tools/msvc/Solution.pm index e402e3af15..387b2114d8 100644 --- a/src/tools/msvc/Solution.pm +++ b/src/tools/msvc/Solution.pm @@ -945,7 +945,7 @@ sub AddProject if ($self->{options}->{zlib}) { $proj->AddIncludeDir($self->{options}->{zlib} . '\include'); - $proj->AddLibrary($self->{options}->{zlib} . '\lib\zdll.lib'); + $proj->AddLibrary($self->{options}->{zlib} . '\lib\zlibstatic.lib'); } if ($self->{options}->{openssl}) { -- 2.30.2