diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml new file mode 100644 index 00000000..d3f35c1b --- /dev/null +++ b/.github/workflows/build-linux.yml @@ -0,0 +1,42 @@ +name: Linux +on: + push: + branches: + - "**" + pull_request: + branches: + - main + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + inputs: + build_versions: + description: Build versions + default: 11, 12, 13, 14, 15 + test_mode: + description: Test mode + default: basic +jobs: + build: + name: ${{ format('pg_probackup ({0}, v{1})', 'ubuntu-22.04', matrix.pg_version) }} + strategy: + matrix: + pg_version: ${{ fromJSON(github.event.inputs.build_versions && format('[{0}]', github.event.inputs.build_versions) || '[11, 12, 13, 14, 15]') }} + fail-fast: false + runs-on: + - ubuntu-22.04 + env: + PG_VERSION: ${{ matrix.pg_version }} + PG_BRANCH: ${{ matrix.pg_version == 16 && 'master' || format('REL_{0}_STABLE', matrix.pg_version) }} + PTRACK_PATCH_PG_BRANCH: ${{ matrix.pg_version == 16 && 'master' || format('REL_{0}_STABLE', matrix.pg_version) }} + PG_BASE: ${{ github.workspace }}/pgsql + PG_CONFIG: ${{ github.workspace }}/pgsql/bin/pg_config + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: sudo ci/install-deps.sh + - name: Build and install + run: ci/build-install.sh + - name: Test + run: ci/test.sh + continue-on-error: ${{ matrix.pg_version == 16 }} + diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml new file mode 100644 index 00000000..c3c6ed40 --- /dev/null +++ b/.github/workflows/build-windows.yml @@ -0,0 +1,24 @@ +name: Windows +on: + push: + branches: + - "**" + pull_request: + branches: + - main + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: +jobs: + build: + name: pg_probackup (windows-2022, v15) + runs-on: + - windows-2022 + steps: + - uses: actions/checkout@v2 + - name: Install dependencies + run: ci/install-deps.ps1 + - name: Build and install + run: ci/build-install.ps1 + - name: Test + run: ci/test.ps1 + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 6f99d0f2..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Build Probackup - -on: - push: - branches: - - "**" - # Runs triggered by pull requests are disabled to prevent executing potentially unsafe code from public pull requests - # pull_request: - # branches: - # - main - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - - build-win2019: - - runs-on: - - windows-2019 - - env: - zlib_dir: C:\dep\zlib - - steps: - - - uses: actions/checkout@v2 - - - name: Install pacman packages - run: | - $env:PATH += ";C:\msys64\usr\bin" - pacman -S --noconfirm --needed bison flex - - - name: Make zlib - run: | - git clone -b v1.2.11 --depth 1 https://github.com/madler/zlib.git - cd zlib - cmake -DCMAKE_INSTALL_PREFIX:PATH=C:\dep\zlib -G "Visual Studio 16 2019" . - cmake --build . --config Release --target ALL_BUILD - cmake --build . --config Release --target INSTALL - copy C:\dep\zlib\lib\zlibstatic.lib C:\dep\zlib\lib\zdll.lib - copy C:\dep\zlib\bin\zlib.dll C:\dep\zlib\lib - - - name: Get Postgres sources - run: git clone -b REL_14_STABLE https://github.com/postgres/postgres.git - - # Copy ptrack to contrib to build the ptrack extension - # Convert line breaks in the patch file to LF otherwise the patch doesn't apply - - name: Get Ptrack sources - run: | - git clone -b master --depth 1 https://github.com/postgrespro/ptrack.git - Copy-Item -Path ptrack -Destination postgres\contrib -Recurse - (Get-Content ptrack\patches\REL_14_STABLE-ptrack-core.diff -Raw).Replace("`r`n","`n") | Set-Content ptrack\patches\REL_14_STABLE-ptrack-core.diff -Force -NoNewline - cd postgres - git apply -3 ../ptrack/patches/REL_14_STABLE-ptrack-core.diff - - - name: Build Postgres - run: | - $env:PATH += ";C:\msys64\usr\bin" - cd postgres\src\tools\msvc - (Get-Content config_default.pl) -Replace "zlib *=>(.*?)(?=,? *#)", "zlib => '${{ env.zlib_dir }}'" | Set-Content config.pl - cmd.exe /s /c "`"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat`" amd64 && .\build.bat" - - - name: Build Probackup - run: cmd.exe /s /c "`"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat`" amd64 && perl .\gen_probackup_project.pl `"${{ github.workspace }}`"\postgres" - - - name: Install Postgres - run: | - cd postgres - src\tools\msvc\install.bat postgres_install - - - name: Install Testgres - run: | - git clone -b no-port-for --single-branch --depth 1 https://github.com/postgrespro/testgres.git - cd testgres - python setup.py install - - # Grant the Github runner user full control of the workspace for initdb to successfully process the data folder - - name: Test Probackup - run: | - icacls.exe "${{ github.workspace }}" /grant "${env:USERNAME}:(OI)(CI)F" - $env:PATH += ";${{ github.workspace }}\postgres\postgres_install\lib;${{ env.zlib_dir }}\lib" - $Env:LC_MESSAGES = "English" - $Env:PG_CONFIG = "${{ github.workspace }}\postgres\postgres_install\bin\pg_config.exe" - $Env:PGPROBACKUPBIN = "${{ github.workspace }}\postgres\Release\pg_probackup\pg_probackup.exe" - $Env:PG_PROBACKUP_PTRACK = "ON" - If (!$Env:MODE -Or $Env:MODE -Eq "basic") { - $Env:PG_PROBACKUP_TEST_BASIC = "ON" - python -m unittest -v tests - python -m unittest -v tests.init_test - } else { - python -m unittest -v tests.$Env:MODE - } - diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 074ae3d0..00000000 --- a/.travis.yml +++ /dev/null @@ -1,102 +0,0 @@ -os: linux - -dist: jammy - -language: c - -cache: ccache - -addons: - apt: - packages: - - sudo - - libc-dev - - bison - - flex - - libreadline-dev - - zlib1g-dev - - libzstd-dev - - libssl-dev - - perl - - libperl-dev - - libdbi-perl - - cpanminus - - locales - - python3 - - python3-dev - - python3-pip - - libicu-dev - - libgss-dev - - libkrb5-dev - - libxml2-dev - - libxslt1-dev - - libldap2-dev - - tcl-dev - - diffutils - - gdb - - gettext - - lcov - - openssh-client - - openssh-server - - libipc-run-perl - - libtime-hires-perl - - libtimedate-perl - - libdbd-pg-perl - -before_install: - - sudo travis/before-install.sh - -install: - - travis/install.sh - -before_script: - - sudo travis/before-script.sh - - travis/before-script-user.sh - -script: - - travis/script.sh - -notifications: - email: - on_success: change - on_failure: always - -# Default MODE is basic, i.e. all tests with PG_PROBACKUP_TEST_BASIC=ON -env: - - PG_VERSION=16 PG_BRANCH=master PTRACK_PATCH_PG_BRANCH=master - - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=REL_15_STABLE - - PG_VERSION=14 PG_BRANCH=REL_14_STABLE PTRACK_PATCH_PG_BRANCH=REL_14_STABLE - - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE - - PG_VERSION=12 PG_BRANCH=REL_12_STABLE PTRACK_PATCH_PG_BRANCH=REL_12_STABLE - - PG_VERSION=11 PG_BRANCH=REL_11_STABLE PTRACK_PATCH_PG_BRANCH=REL_11_STABLE - - PG_VERSION=10 PG_BRANCH=REL_10_STABLE - - PG_VERSION=9.6 PG_BRANCH=REL9_6_STABLE - - PG_VERSION=9.5 PG_BRANCH=REL9_5_STABLE - - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup_test.BackupTest.test_full_backup - - PG_VERSION=15 PG_BRANCH=REL_15_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=backup_test.BackupTest.test_full_backup_stream -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=backup -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=catchup -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=checkdb -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=compression -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=delta -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=locking -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=merge -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=option -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=page -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=ptrack -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=replica -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=OFF MODE=retention -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=restore -# - PG_VERSION=13 PG_BRANCH=REL_13_STABLE PTRACK_PATCH_PG_BRANCH=REL_13_STABLE MODE=time_consuming - -jobs: - allow_failures: - - if: env(PG_BRANCH) = master - - if: env(PG_BRANCH) = REL9_5_STABLE -# - if: env(MODE) IN (archive, backup, delta, locking, merge, replica, retention, restore) - -# Only run CI for master branch commits to limit our travis usage -#branches: -# only: -# - master - diff --git a/README.md b/README.md index 7486a6ca..ec64d70f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ +[![Linux](https://github.com/postgrespro/pg_probackup/actions/workflows/build-linux.yml/badge.svg)](https://github.com/postgrespro/pg_probackup/actions/workflows/build-linux.yml) +[![Windows](https://github.com/postgrespro/pg_probackup/actions/workflows/build-windows.yml/badge.svg)](https://github.com/postgrespro/pg_probackup/actions/workflows/build-windows.yml) [![GitHub release](https://img.shields.io/github/v/release/postgrespro/pg_probackup?include_prereleases)](https://github.com/postgrespro/pg_probackup/releases/latest) -[![Build Status](https://travis-ci.com/postgrespro/pg_probackup.svg?branch=master)](https://travis-ci.com/postgrespro/pg_probackup) # pg_probackup diff --git a/ci/build-install.ps1 b/ci/build-install.ps1 new file mode 100644 index 00000000..c4d4ff23 --- /dev/null +++ b/ci/build-install.ps1 @@ -0,0 +1,25 @@ +git clone -b REL_15_STABLE https://github.com/postgres/postgres.git +# Copy ptrack to contrib to build the ptrack extension +# Convert line breaks in the patch file to LF otherwise the patch doesn't apply +git clone -b master --depth 1 https://github.com/postgrespro/ptrack.git +Copy-Item -Path ptrack -Destination postgres\contrib -Recurse +(Get-Content ptrack\patches\REL_15_STABLE-ptrack-core.diff -Raw).Replace("`r`n","`n") | Set-Content ptrack\patches\REL_15_STABLE-ptrack-core.diff -Force -NoNewline +cd postgres +git apply -3 ../ptrack/patches/REL_15_STABLE-ptrack-core.diff + +# Pacman packages are installed into MSYS +$env:PATH += ";C:\msys64\usr\bin" + +# Build Postgres +cd "$env:GITHUB_WORKSPACE\postgres\src\tools\msvc" +(Get-Content config_default.pl) -Replace "zlib *=>(.*?)(?=,? *#)", "zlib => 'C:\dep\zlib'" | Set-Content config.pl +cmd.exe /s /c "`"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat`" amd64 && .\build.bat" +If (-Not $?) {exit 1} +# Install Postgres +.\install.bat postgres_install +If (-Not $?) {exit 1} + +# Build Pg_probackup +cd "$env:GITHUB_WORKSPACE" +cmd.exe /s /c "`"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat`" amd64 && perl .\gen_probackup_project.pl `"$env:GITHUB_WORKSPACE\postgres`"" + diff --git a/travis/install.sh b/ci/build-install.sh similarity index 62% rename from travis/install.sh rename to ci/build-install.sh index 43ada47b..69e2b250 100755 --- a/travis/install.sh +++ b/ci/build-install.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh -set -xe +set -x if [ -z ${PG_VERSION+x} ]; then echo PG_VERSION is not set! @@ -16,13 +16,10 @@ if [ -z ${PTRACK_PATCH_PG_BRANCH+x} ]; then PTRACK_PATCH_PG_BRANCH=OFF fi -# fix -sudo chown -R travis /home/travis/.ccache - -export PGHOME=/pg +export PG_SRC=$PWD/postgres # Clone Postgres -echo "############### Getting Postgres sources:" +echo "############### Getting Postgres sources..." git clone https://github.com/postgres/postgres.git -b $PG_BRANCH --depth=1 # Clone ptrack @@ -33,34 +30,36 @@ else export PG_PROBACKUP_PTRACK=OFF fi +cd postgres # Compile and install Postgres -echo "############### Compiling Postgres:" -cd postgres # Go to postgres dir +echo "############### Compiling Postgres..." if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then git apply -3 contrib/ptrack/patches/${PTRACK_PATCH_PG_BRANCH}-ptrack-core.diff fi -CC='ccache gcc' CFLAGS="-Og" ./configure --prefix=$PGHOME \ - --cache-file=~/.ccache/configure-cache \ +CFLAGS="-Og" ./configure --prefix=$PG_BASE \ --enable-debug --enable-cassert --enable-depend \ --enable-tap-tests --enable-nls make -s -j$(nproc) install make -s -j$(nproc) -C contrib/ install -# Override default Postgres instance -export PATH=$PGHOME/bin:$PATH -export LD_LIBRARY_PATH=$PGHOME/lib -export PG_CONFIG=$(which pg_config) +export PATH=$PATH:$PG_BASE/bin +export LD_LIBRARY_PATH=$PG_BASE/lib if [ "$PG_PROBACKUP_PTRACK" = "ON" ]; then - echo "############### Compiling Ptrack:" + echo "############### Compiling Ptrack..." make -C contrib/ptrack install fi # Get amcheck if missing if [ ! -d "contrib/amcheck" ]; then - echo "############### Getting missing amcheck:" + echo "############### Getting missing amcheck..." git clone https://github.com/petergeoghegan/amcheck.git --depth=1 contrib/amcheck make -C contrib/amcheck install fi -pip3 install testgres \ No newline at end of file +cd .. +# Build and install pg_probackup (using PG_CPPFLAGS and SHLIB_LINK for gcov) +echo "############### Compiling and installing pg_probackup..." +# make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" top_srcdir=$CUSTOM_PG_SRC install +make install USE_PGXS=1 top_srcdir=$PG_SRC PG_CONFIG=$PG_BASE/bin/pg_config + diff --git a/ci/install-deps.ps1 b/ci/install-deps.ps1 new file mode 100644 index 00000000..aef1d394 --- /dev/null +++ b/ci/install-deps.ps1 @@ -0,0 +1,18 @@ +$env:PATH += ";C:\msys64\usr\bin" +pacman -S --noconfirm --needed bison flex + +git clone -b v1.2.11 --depth 1 https://github.com/madler/zlib.git +cd zlib +cmake -DCMAKE_INSTALL_PREFIX:PATH=C:\dep\zlib -G "Visual Studio 17 2022" . +cmake --build . --config Release --target ALL_BUILD +cmake --build . --config Release --target INSTALL +Copy-Item "C:\dep\zlib\lib\zlibstatic.lib" -Destination "C:\dep\zlib\lib\zdll.lib" +Copy-Item "C:\dep\zlib\bin\zlib.dll" -Destination "C:\dep\zlib\lib" +If (-Not $?) {exit 1} + +#Install Testgres +cd "$env:GITHUB_WORKSPACE" +git clone -b no-port-for --single-branch --depth 1 https://github.com/postgrespro/testgres.git +cd testgres +python setup.py install + diff --git a/ci/install-deps.sh b/ci/install-deps.sh new file mode 100755 index 00000000..e25df9b5 --- /dev/null +++ b/ci/install-deps.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env sh + +apt -qq update && \ + DEBIAN_FRONTEND=noninteractive apt install -y \ + libc-dev \ + bison \ + flex \ + ccache \ + libreadline-dev \ + zlib1g-dev \ + libzstd-dev \ + libssl-dev \ + perl \ + libperl-dev \ + libdbi-perl \ + cpanminus \ + locales \ + python3 \ + python3-dev \ + python3-pip \ + libicu-dev \ + libgss-dev \ + libkrb5-dev \ + libxml2-dev \ + libxslt1-dev \ + libldap2-dev \ + tcl-dev \ + diffutils \ + gdb \ + gettext \ + lcov \ + openssh-client \ + openssh-server \ + libipc-run-perl \ + libtime-hires-perl \ + libtimedate-perl \ + libdbd-pg-perl + +pip3 install testgres + diff --git a/ci/test.ps1 b/ci/test.ps1 new file mode 100644 index 00000000..de8ac19b --- /dev/null +++ b/ci/test.ps1 @@ -0,0 +1,14 @@ +# Grant the Github runner user full control of the workspace for initdb to successfully process the data folder +icacls.exe "$env:GITHUB_WORKSPACE" /grant "${env:USERNAME}:(OI)(CI)F" +$env:PATH += ";$env:GITHUB_WORKSPACE\postgres\postgres_install\lib;C:\dep\zlib\lib" +$Env:LC_MESSAGES = "English" +$Env:PG_CONFIG = "$env:GITHUB_WORKSPACE\postgres\postgres_install\bin\pg_config.exe" +$Env:PGPROBACKUPBIN = "$env:GITHUB_WORKSPACE\postgres\Release\pg_probackup\pg_probackup.exe" +$Env:PG_PROBACKUP_PTRACK = "ON" +If (!$Env:MODE -Or $Env:MODE -Eq "basic") { + $Env:PG_PROBACKUP_TEST_BASIC = "ON" + python -m unittest -v tests +} else { + python -m unittest -v tests.$Env:MODE +} + diff --git a/travis/script.sh b/ci/test.sh similarity index 56% rename from travis/script.sh rename to ci/test.sh index 31ef0972..f45d3b05 100755 --- a/travis/script.sh +++ b/ci/test.sh @@ -1,17 +1,4 @@ -#!/usr/bin/env bash - -set -xe - -export PGHOME=/pg -export PG_SRC=$PWD/postgres -export PATH=$PGHOME/bin:$PATH -export LD_LIBRARY_PATH=$PGHOME/lib -export PG_CONFIG=$(which pg_config) - -# Build and install pg_probackup (using PG_CPPFLAGS and SHLIB_LINK for gcov) -echo "############### Compiling and installing pg_probackup:" -# make USE_PGXS=1 PG_CPPFLAGS="-coverage" SHLIB_LINK="-coverage" top_srcdir=$CUSTOM_PG_SRC install -make USE_PGXS=1 top_srcdir=$PG_SRC install +#!/usr/bin/env sh if [ -z ${MODE+x} ]; then MODE=basic @@ -21,7 +8,19 @@ if [ -z ${PGPROBACKUP_GDB+x} ]; then PGPROBACKUP_GDB=ON fi -echo "############### Testing:" +export PATH=$PATH:$PG_BASE/bin +export LD_LIBRARY_PATH=$PG_BASE/lib + +echo "############### pg_config path:" +which pg_config + +echo "############### pg_config:" +pg_config + +echo "############### Kernel parameters:" +sudo sysctl kernel.yama.ptrace_scope=0 + +echo "############### Testing..." echo PG_PROBACKUP_PARANOIA=${PG_PROBACKUP_PARANOIA} echo ARCHIVE_COMPRESSION=${ARCHIVE_COMPRESSION} echo PGPROBACKUPBIN_OLD=${PGPROBACKUPBIN_OLD} @@ -34,8 +33,8 @@ if [ "$MODE" = "basic" ]; then export PG_PROBACKUP_TEST_BASIC=ON echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} python3 -m unittest -v tests - python3 -m unittest -v tests.init_test else echo PG_PROBACKUP_TEST_BASIC=${PG_PROBACKUP_TEST_BASIC} python3 -m unittest -v tests.$MODE fi + diff --git a/travis/before-install.sh b/travis/before-install.sh deleted file mode 100755 index 376de5e6..00000000 --- a/travis/before-install.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -xe - -mkdir /pg -chown travis /pg \ No newline at end of file diff --git a/travis/before-script-user.sh b/travis/before-script-user.sh deleted file mode 100755 index d9c07f1e..00000000 --- a/travis/before-script-user.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -xe - -ssh-keygen -t rsa -f ~/.ssh/id_rsa -q -N "" -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -ssh-keyscan -H localhost >> ~/.ssh/known_hosts diff --git a/travis/before-script.sh b/travis/before-script.sh deleted file mode 100755 index ca59bcf2..00000000 --- a/travis/before-script.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -xe - -/etc/init.d/ssh start - -# Show pg_config path (just in case) -echo "############### pg_config path:" -which pg_config - -# Show pg_config just in case -echo "############### pg_config:" -pg_config - -# Show kernel parameters -echo "############### kernel params:" -cat /proc/sys/kernel/yama/ptrace_scope -sudo sysctl kernel.yama.ptrace_scope=0 -cat /proc/sys/kernel/yama/ptrace_scope