diff options
Diffstat (limited to '.github')
-rw-r--r-- | .github/workflows/build-kernel-module.yml | 113 | ||||
-rw-r--r-- | .github/workflows/build-programs.yml | 67 | ||||
-rw-r--r-- | .github/workflows/lint.yml | 35 | ||||
-rw-r--r-- | .github/workflows/release.yml | 65 | ||||
-rw-r--r-- | .github/workflows/test-cow-fuse.yml | 151 |
5 files changed, 431 insertions, 0 deletions
diff --git a/.github/workflows/build-kernel-module.yml b/.github/workflows/build-kernel-module.yml new file mode 100644 index 0000000..526bf8e --- /dev/null +++ b/.github/workflows/build-kernel-module.yml @@ -0,0 +1,113 @@ +name: Build dnbd3 kernel module + +on: + push: + branches: + - master + pull_request: + +jobs: + build: + strategy: + fail-fast: false + matrix: + config: + - name: CentOS 8.4 [4.18.0] + build-os: ubuntu-22.04 + kernel-type: centos-rpm + kernel-source: https://vault.centos.org/8.4.2105/BaseOS/Source/SPackages/kernel-4.18.0-305.25.1.el8_4.src.rpm + - name: CentOS 8.5 [4.18.0] + build-os: ubuntu-22.04 + kernel-type: centos-rpm + kernel-source: https://vault.centos.org/8.5.2111/BaseOS/Source/SPackages/kernel-4.18.0-348.2.1.el8_5.src.rpm + - name: CentOS 9.3 [5.14.0] + build-os: ubuntu-22.04 + kernel-type: centos-tar + kernel-source: https://files.bwlp.ks.uni-freiburg.de/stuff/centos/linux-5.14.0-362.18.1.el9_3.tar.xz + - name: CentOS 9.4 [5.14.0] + build-os: ubuntu-22.04 + kernel-type: centos-tar + kernel-source: https://files.bwlp.ks.uni-freiburg.de/stuff/centos/linux-5.14.0-427.el9.tar.xz + - name: Vanilla [4.19.x] + build-os: ubuntu-22.04 + kernel-type: vanilla + kernel-version: 4.19.y + - name: Vanilla [5.4.x] + build-os: ubuntu-22.04 + kernel-type: vanilla + kernel-version: 5.4.y + - name: Vanilla [5.10.x] + build-os: ubuntu-22.04 + kernel-type: vanilla + kernel-version: 5.10.y + - name: Vanilla [5.15.x] + build-os: ubuntu-22.04 + kernel-type: vanilla + kernel-version: 5.15.y + - name: Vanilla [6.1.x] + build-os: ubuntu-22.04 + kernel-type: vanilla + kernel-version: 6.1.y + - name: Vanilla [6.6.x] + build-os: ubuntu-22.04 + kernel-type: vanilla + kernel-version: 6.6.y + name: Build dnbd3 ${{ matrix.config.name }} + runs-on: ${{ matrix.config.build-os }} + steps: + - name: Install dnbd3 dependencies + run: | + sudo apt-get update -y -qq + sudo apt-get install -y -qq make \ + clang-format \ + libelf-dev \ + rpm2cpio \ + rpm + - name: Checkout dnbd3 repository + uses: actions/checkout@v4 + - name: Fetch dnbd3 repository tags + run: git fetch --prune --unshallow + - name: Checkout Vanilla kernel version [git] + if: matrix.config.kernel-type == 'vanilla' + run: git clone --depth 1 --branch "linux-${{ matrix.config.kernel-version }}" "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git" "../kernel" + - name: Checkout CentOS kernel version [rpm] + if: matrix.config.kernel-type == 'centos-rpm' + run: | + mkdir "${{ github.workspace }}/../kernel" + mkdir "${{ github.workspace }}/../kernel-download" + wget -q -O "${{ github.workspace }}/../kernel-download/kernel.rpm" "${{ matrix.config.kernel-source }}" + cd "${{ github.workspace }}/../kernel-download" + rpm2cpio "${{ github.workspace }}/../kernel-download/kernel.rpm" | cpio --extract --make-directories + tar --strip 1 -a -x -f linux-*.el*.tar.xz -C "${{ github.workspace }}/../kernel" + cd + rm -rf -- "${{ github.workspace }}/../kernel-download" + - name: Checkout CentOS kernel version [tar] + if: matrix.config.kernel-type == 'centos-tar' + run: | + mkdir "${{ github.workspace }}/../kernel" + mkdir "${{ github.workspace }}/../kernel-download" + wget -q -O "${{ github.workspace }}/../kernel-download/kernel.tar.xz" "${{ matrix.config.kernel-source }}" + tar --strip 1 -a -x -f "${{ github.workspace }}/../kernel-download/kernel.tar.xz" -C "${{ github.workspace }}/../kernel" + rm -rf -- "${{ github.workspace }}/../kernel-download" + - name: Configure kernel version + working-directory: ${{ github.workspace }}/../kernel + run: | + make defconfig + make modules_prepare + - name: Configure dnbd3 build + run: | + cmake -B ${{ github.workspace }}/build \ + -S ${{ github.workspace }} \ + -D CMAKE_BUILD_TYPE=Release \ + -D DNBD3_KERNEL_MODULE=ON \ + -D KERNEL_BUILD_DIR=${{ github.workspace }}/../kernel \ + -D KERNEL_INSTALL_DIR=${{ github.workspace }}/../kernel/extra \ + -D KERNEL_SCRIPTS_DIR=${{ github.workspace }}/../kernel/scripts \ + -D DNBD3_BENCHMARK=OFF \ + -D DNBD3_CLIENT_FUSE=OFF \ + -D DNBD3_SERVER=OFF \ + -D DNBD3_SERVER_FUSE=OFF \ + -D DNBD3_RELEASE_HARDEN=OFF + - name: Build dnbd3 kernel module + working-directory: ${{ github.workspace }}/build + run: make diff --git a/.github/workflows/build-programs.yml b/.github/workflows/build-programs.yml new file mode 100644 index 0000000..b361864 --- /dev/null +++ b/.github/workflows/build-programs.yml @@ -0,0 +1,67 @@ +name: Build dnbd3 programs + +on: + push: + branches: + - master + pull_request: + +jobs: + build: + strategy: + fail-fast: false + matrix: + config: + - name: debug [default] on Ubuntu 22.04 + build-os: ubuntu-22.04 + build-type: "Debug" + build-cfg-server-fuse: "OFF" + build-cfg-harden: "OFF" + - name: debug [server with fuse support] on Ubuntu 22.04 + build-os: ubuntu-22.04 + build-type: "Debug" + build-cfg-server-fuse: "ON" + build-cfg-harden: "OFF" + - name: release [default] on Ubuntu 22.04 + build-os: ubuntu-22.04 + build-type: "Release" + build-cfg-server-fuse: "OFF" + build-cfg-harden: "OFF" + - name: release [server with fuse support] on Ubuntu 22.04 + build-os: ubuntu-22.04 + build-type: "Release" + build-cfg-server-fuse: "ON" + build-cfg-harden: "OFF" + - name: release [default hardening] on Ubuntu 22.04 + build-os: ubuntu-22.04 + build-type: "Release" + build-cfg-server-fuse: "OFF" + build-cfg-harden: "ON" + name: Build dnbd3 ${{ matrix.config.name }} + runs-on: ${{ matrix.config.build-os }} + steps: + - name: Install dnbd3 dependencies + run: | + sudo apt-get update -y -qq + sudo apt-get install -y -qq make \ + clang-format \ + linux-headers-generic \ + libfuse-dev \ + libcurl4-openssl-dev \ + libjansson-dev + - name: Checkout dnbd3 repository + uses: actions/checkout@v4 + - name: Fetch dnbd3 repository tags + run: git fetch --prune --unshallow + - name: Configure dnbd3 build + run: | + cmake -B ${{ github.workspace }}/build \ + -S ${{ github.workspace }} \ + -D CMAKE_BUILD_TYPE=${{ matrix.config.build-type }} \ + -D DNBD3_KERNEL_MODULE=OFF \ + -D DNBD3_BENCHMARK=ON \ + -D DNBD3_SERVER_FUSE=${{ matrix.config.build-cfg-server-fuse }} \ + -D DNBD3_RELEASE_HARDEN=${{ matrix.config.build-cfg-harden }} + - name: Build dnbd3 artifacts + working-directory: ${{ github.workspace }}/build + run: make diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..14fc471 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,35 @@ +name: Lint dnbd3 + +on: + push: + branches: + - master + pull_request: + +jobs: + lint: + name: Lint dnbd3 + runs-on: ubuntu-22.04 + steps: + - name: Install dnbd3 dependencies + run: | + sudo apt-get update -y -qq + sudo apt-get install -y -qq make \ + clang-format \ + linux-headers-generic \ + libfuse-dev \ + libcurl4-openssl-dev \ + libjansson-dev \ + rpm + - name: Checkout dnbd3 repository + uses: actions/checkout@v4 + - name: Fetch dnbd3 repository tags + run: git fetch --prune --unshallow + - name: Configure dnbd3 build + run: | + cmake -B ${{ github.workspace }}/build \ + -S ${{ github.workspace }} \ + -D DNBD3_BENCHMARK=ON + - name: Lint dnbd3 artifacts + working-directory: ${{ github.workspace }}/build + run: make lint diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..28c260f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,65 @@ +name: Release dnbd3 + +on: + release: + types: [published] + +jobs: + release: + name: Release dnbd3 + # Use very old system (= libc) for building for best compatibility + runs-on: ubuntu-18.04 + steps: + - name: Install dnbd3 dependencies + run: | + sudo apt-get update -y -qq + sudo apt-get install -y -qq make \ + clang-format \ + linux-headers-generic \ + libfuse-dev \ + libjansson-dev \ + libcurl4-openssl-dev \ + rpm + - name: Checkout dnbd3 repository + uses: actions/checkout@v4 + - name: Fetch dnbd3 repository tags + run: git fetch --prune --unshallow + - name: Configure dnbd3 release + run: | + cmake -B ${{ github.workspace }}/build \ + -S ${{ github.workspace }} \ + -D CMAKE_BUILD_TYPE=Release \ + -D DNBD3_KERNEL_MODULE=OFF \ + -D DNBD3_BENCHMARK=OFF \ + -D DNBD3_SERVER_FUSE=OFF \ + -D DNBD3_RELEASE_HARDEN=OFF + - name: Build dnbd3 artifacts + working-directory: ${{ github.workspace }}/build + run: make package + - name: Create links to artifacts + working-directory: ${{ github.workspace }}/build + run: | + ln -s dnbd3_*.deb dnbd3_linux_x86_64.deb + ln -s dnbd3_*.rpm dnbd3_linux_x86_64.rpm + ln -s dnbd3_*.tar.gz dnbd3_linux_x86_64.tar.gz + - name: Attach Debian artifacts to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: ${{ github.workspace }}/build/dnbd3_linux_x86_64.deb + asset_name: dnbd3_linux_x86_64.deb + tag: ${{ github.ref }} + - name: Attach RedHat artifacts to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: ${{ github.workspace }}/build/dnbd3_linux_x86_64.rpm + asset_name: dnbd3_linux_x86_64.rpm + tag: ${{ github.ref }} + - name: Attach generic artifacts to release + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: ${{ github.workspace }}/build/dnbd3_linux_x86_64.tar.gz + asset_name: dnbd3_linux_x86_64.tar.gz + tag: ${{ github.ref }} diff --git a/.github/workflows/test-cow-fuse.yml b/.github/workflows/test-cow-fuse.yml new file mode 100644 index 0000000..46b6933 --- /dev/null +++ b/.github/workflows/test-cow-fuse.yml @@ -0,0 +1,151 @@ +name: test fuse cow +on: push +env: + ASAN_OPTIONS: leak_check_at_exit=false +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup dotnet + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.x.x' + - name: Install dnbd3 dependencies + working-directory: ${{ github.workspace }} + run: | + for i in 1 2 3 4; do sudo apt-get update -y -qq && break; sleep 5; done + for i in 1 2 3 4; do sudo apt-get install -y -qq make \ + clang-format \ + linux-headers-generic \ + libfuse-dev \ + libjansson-dev \ + libcurl4-openssl-dev && break; sleep 5; done + - name: Configure dnbd3 build + run: | + cmake -B ${{ github.workspace }}/build \ + -S ${{ github.workspace }} \ + -D CMAKE_BUILD_TYPE=${{ matrix.config.build-type }} \ + -D CMAKE_C_FLAGS=-fsanitize=address \ + -D DNBD3_KERNEL_MODULE=OFF \ + -D DNBD3_BENCHMARK=OFF \ + -D DNBD3_CLIENT_FUSE=ON \ + -D DNBD3_CLIENT_FUSE_COW_TEST=ON \ + -D DNBD3_SERVER=ON \ + -D DNBD3_SERVER_FUSE=OFF \ + -D DNBD3_SERVER_AFL=OFF \ + -D DNBD3_SERVER_DEBUG_LOCKS=OFF \ + -D DNBD3_SERVER_DEBUG_THREADS=OFF \ + -D DNBD3_RELEASE_HARDEN=OFF \ + -D DNBD3_PACKAGE_DOCKER=OFF + + - name: Build dnbd3 artifacts + working-directory: ${{ github.workspace }}/build + run: make + - name: Get cow_merger_service + working-directory: ${{ github.workspace }} + run: git clone --depth 1 --branch "master" "https://github.com/z0Kng/cow_merger_service.git" "../cow_server" + - name: Build cow_merger_service + working-directory: ${{ github.workspace }}/../cow_server/cow_merger_service + run: dotnet publish -c Release -o publish -p:PublishReadyToRun=true -p:PublishSingleFile=true -p:PublishReadyToRunShowWarnings=true --self-contained true --runtime linux-x64 + - name: Setup cow_merger_service + working-directory: ${{ github.workspace }}/../cow_server/ + run: | + sudo mkdir -m 777 /mnt/work + mkdir /mnt/work/WorkingDirectory + mkdir /mnt/work/OriginalImageDirectory + mkdir /mnt/work/Output + sed -i 's/^ "WorkingDirectory":.*/ "WorkingDirectory": "\/mnt\/work\/WorkingDirectory",/g' cow_merger_service/publish/appsettings.json + sed -i 's/^ "OriginalImageDirectory":.*/ "OriginalImageDirectory": "\/mnt\/work\/OriginalImageDirectory",/g' cow_merger_service/publish/appsettings.json + sed -i 's/^ "DestinationDirectory":.*/ "DestinationDirectory": "\/mnt\/work\/Output",/g' cow_merger_service/publish/appsettings.json + while read line; do echo $line; done < cow_merger_service/publish/appsettings.json + cd cow_merger_service/publish/ + FILENAME=cow_merger_service + FILESIZE=$(stat -c%s "$FILENAME") + echo "Size of $FILENAME = $FILESIZE bytes." + ./cow_merger_service &> log.out & + - name: Generate test file + working-directory: ${{ github.workspace }}/build/src/cowtest + run: ./dnbd3-fuse-cow-test -c /mnt/work/OriginalImageDirectory/test.r1 + - name: Setup dnbd3 server + working-directory: ${{ github.workspace }} + run: | + mkdir /mnt/work/dnbd3-server-config/ + cp pkg/config/server.conf /mnt/work/dnbd3-server-config/ + sed -i "s/^basePath=.*/basePath=\/mnt\/work\/OriginalImageDirectory/g" /mnt/work/dnbd3-server-config/server.conf + cd build/src/server + ./dnbd3-server --config /mnt/work/dnbd3-server-config/ + sleep 5 + - name: Run standard test + working-directory: ${{ github.workspace }}/build/src + run: | + mkdir /mnt/work/mount + mkdir /mnt/work/tmp + mkdir /mnt/work/logs + ./fuse/dnbd3-fuse "/mnt/work/mount" -f -h localhost -i test -c "/mnt/work/tmp" -C localhost:5000 --cow-stats-stdout --cow-stats-file &> /mnt/work/logs/standardLog1.out & + PID=$! + sleep 5 + echo PID: $PID + ./cowtest/dnbd3-fuse-cow-test -t "/mnt/work/mount/img" + echo unmounting: $PID + sudo umount /mnt/work/mount + wait $PID + echo unmounted + ./fuse/dnbd3-fuse "/mnt/work/mount" -f -h localhost -i test -L "/mnt/work/tmp" -C localhost:5000 --merge --cow-stats-stdout --cow-stats-file &> /mnt/work/logs/standardLog2.out & + PID=$! + sleep 5 + echo PID: $PID + ./cowtest/dnbd3-fuse-cow-test -v "/mnt/work/mount/img" + echo unmounting: $PID + sudo umount /mnt/work/mount + wait $PID + echo unmounted + sleep 30 + echo Output: $(ls /mnt/work/Output) + ./cowtest/dnbd3-fuse-cow-test -v /mnt/work/Output/test.r2 + rm /mnt/work/Output/test.r2 + rm -rf /mnt/work/tmp + - name: Run random test + working-directory: ${{ github.workspace }}/build/src + run: | + cp /mnt/work/OriginalImageDirectory/test.r1 /mnt/work/ + mkdir /mnt/work/tmp + ./fuse/dnbd3-fuse "/mnt/work/mount" -f -h localhost -i test -c "/mnt/work/tmp" -C localhost:5000 --cow-stats-stdout --cow-stats-file &> /mnt/work/logs/randomLog1.out & + PIDFUSE=$! + sleep 5 + echo PIDFUSE : $PIDFUSE + ./cowtest/dnbd3-fuse-cow-test --randomTest "/mnt/work/mount/img" "/mnt/work/test.r1" &> /mnt/work/logs/randomTestLog1.out & + PIDTEST=$! + sleep 60 + echo "stopping test" + kill -INT $PIDTEST + wait $PIDTEST + echo "testStopped" + echo unmounting: $PIDFUSE + sudo umount /mnt/work/mount + echo unmounted + wait $PIDFUSE + ./fuse/dnbd3-fuse "/mnt/work/mount" -f -h localhost -i test -L "/mnt/work/tmp" -C localhost:5000 --merge --cow-stats-stdout --cow-stats-file &> /mnt/work/logs/randomLog2.out & + PIDFUSE=$! + sleep 5 + echo PIDFUSE : $PIDFUSE + ./cowtest/dnbd3-fuse-cow-test --compare "/mnt/work/mount/img" "/mnt/work/test.r1" + echo unmounting: $PIDFUSE + sudo umount /mnt/work/mount + echo unmounted + wait $PIDFUSE + sleep 30 + echo Output: $(ls /mnt/work/Output) + ./cowtest/dnbd3-fuse-cow-test --compare "/mnt/work/Output/test.r2" "/mnt/work/test.r1" + - name: Print logs + if: always() + run: | + tail -n+0 \ + /mnt/work/logs/standardLog1.out \ + /mnt/work/logs/standardLog2.out \ + /mnt/work/logs/randomLog1.out \ + /mnt/work/logs/randomTestLog1.out \ + /mnt/work/logs/randomLog2.out \ + /mnt/work/tmp/status \ + ${{ github.workspace }}/build/src/server/dnbd3.log \ + ${{ github.workspace }}/../cow_server/cow_merger_service/publish/log.out |