diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8a439d2..ddee7ff8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,104 +5,195 @@ on: branches: - master schedule: - - cron: '00 01 * * *' + - cron: '00 01 * * *' jobs: test: name: test + env: + # For some builds, we use cross to test on 32-bit and big-endian + # systems. + CARGO: cargo + # When CARGO is set to CROSS, this is set to `--target matrix.target`. + TARGET_FLAGS: + # When CARGO is set to CROSS, TARGET_DIR includes matrix.target. + TARGET_DIR: ./target + # Emit backtraces on panics. + RUST_BACKTRACE: 1 + runs-on: ${{ matrix.os }} + strategy: + matrix: + build: + # We test ripgrep on a pinned version of Rust, along with the moving + # targets of 'stable' and 'beta' for good measure. + - pinned + - stable + - beta + # Our release builds are generated by a nightly compiler to take + # advantage of the latest optimizations/compile time improvements. So + # we test all of them here. (We don't do mips releases, but test on + # mips for big-endian coverage.) + - nightly + - nightly-musl + - nightly-32 + - nightly-mips + - nightly-arm + - macos + - win-msvc + - win-gnu + include: + - build: pinned + os: ubuntu-18.04 + rust: 1.41.0 + - build: stable + os: ubuntu-18.04 + rust: stable + - build: beta + os: ubuntu-18.04 + rust: beta + - build: nightly + os: ubuntu-18.04 + rust: nightly + - build: nightly-musl + os: ubuntu-18.04 + rust: nightly + target: x86_64-unknown-linux-musl + - build: nightly-32 + os: ubuntu-18.04 + rust: nightly + target: i686-unknown-linux-gnu + - build: nightly-mips + os: ubuntu-18.04 + rust: nightly + target: mips64-unknown-linux-gnuabi64 + - build: nightly-arm + os: ubuntu-18.04 + rust: nightly + # For stripping release binaries: + # docker run --rm -v $PWD/target:/target:Z \ + # rustembedded/cross:arm-unknown-linux-gnueabihf \ + # arm-linux-gnueabihf-strip \ + # /target/arm-unknown-linux-gnueabihf/debug/rg + target: arm-unknown-linux-gnueabihf + - build: macos + os: macos-latest + rust: nightly + - build: win-msvc + os: windows-2019 + rust: nightly + - build: win-gnu + os: windows-2019 + rust: nightly-x86_64-gnu + steps: + - name: Checkout repository + uses: actions/checkout@v1 + with: + fetch-depth: 1 + + - name: Install packages (Ubuntu) + if: matrix.os == 'ubuntu-18.04' + run: | + ci/ubuntu-install-packages + + - name: Install packages (macOS) + if: matrix.os == 'macos-latest' + run: | + ci/macos-install-packages + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + profile: minimal + override: true + + - name: Use Cross + if: matrix.target != '' + run: | + # FIXME: to work around bugs in latest cross release, install master. + # See: https://github.com/rust-embedded/cross/issues/357 + cargo install --git https://github.com/rust-embedded/cross + echo "::set-env name=CARGO::cross" + echo "::set-env name=TARGET_FLAGS::--target ${{ matrix.target }}" + echo "::set-env name=TARGET_DIR::./target/${{ matrix.target }}" + + - name: Show command used for Cargo + run: | + echo "cargo command is: ${{ env.CARGO }}" + echo "target flag is: ${{ env.TARGET_FLAGS }}" + + - name: Build ripgrep and all crates + run: ${{ env.CARGO }} build --verbose --all ${{ env.TARGET_FLAGS }} + + - name: Build ripgrep with PCRE2 + run: ${{ env.CARGO }} build --verbose --all --features pcre2 ${{ env.TARGET_FLAGS }} + + # This is useful for debugging problems when the expected build artifacts + # (like shell completions and man pages) aren't generated. + - name: Show build.rs stderr + shell: bash + run: | + set +x + stderr="$(find "${{ env.TARGET_DIR }}/debug" -name stderr -print0 | xargs -0 ls -t | head -n1)" + if [ -s "$stderr" ]; then + echo "===== $stderr ===== " + cat "$stderr" + echo "=====" + fi + set -x + + - name: Run tests with PCRE2 (sans cross) + if: matrix.target == '' + run: ${{ env.CARGO }} test --verbose --all --features pcre2 ${{ env.TARGET_FLAGS }} + + - name: Run tests without PCRE2 (with cross) + # These tests should actually work, but they almost double the runtime. + # Every integration test spins up qemu to run 'rg', and when PCRE2 is + # enabled, every integration test is run twice: one with the default + # regex engine and once with PCRE2. + if: matrix.target != '' + run: ${{ env.CARGO }} test --verbose --all ${{ env.TARGET_FLAGS }} + + - name: Test for existence of build artifacts (Windows) + if: matrix.os == 'windows-2019' + shell: bash + run: | + outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")" + ls "$outdir/_rg.ps1" && file "$outdir/_rg.ps1" + + - name: Test for existence of build artifacts (Unix) + if: matrix.os != 'windows-2019' + shell: bash + run: | + outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")" + for f in rg.bash rg.fish rg.1; do + # We could use file -E here, but it isn't supported on macOS. + ls "$outdir/$f" && file "$outdir/$f" + done + + - name: Test zsh shell completions (Unix, sans cross) + # We could test this when using Cross, but we'd have to execute the + # 'rg' binary (done in test_complete.sh) with qemu, which is a pain and + # doesn't really gain us much. If shell completion works in one place, + # it probably works everywhere. + if: matrix.target == '' && matrix.os != 'windows-2019' + shell: bash + run: ci/test_complete.sh + + rustfmt: + name: rustfmt runs-on: ubuntu-18.04 steps: - - name: no-op - run: echo hello - -# test: -# name: test -# runs-on: ${{ matrix.os }} -# strategy: -# matrix: -# # The docs seem to suggest that we can have a matrix with just an -# # include directive, but it result in a "matrix must define at least -# # one vector" error in the CI system. -# build: -# # - pinned-glibc -# - pinned-musl -# - stable -# # - beta -# # We test musl with nightly because every once in a while, this will -# # catch an upstream regression. -# # - nightly-glibc -# # - nightly-musl -# # - macos -# # - win-msvc-32 -# # - win-msvc-64 -# # - win-gnu-32 -# # - win-gnu-64 -# include: -# # - build: pinned-glibc -# # os: ubuntu-18.04 -# # rust: 1.34.0 -# # target: x86_64-unknown-linux-gnu -# - build: pinned-musl -# os: ubuntu-18.04 -# rust: 1.34.0 -# target: x86_64-unknown-linux-musl -# - build: stable -# os: ubuntu-18.04 -# rust: stable -# target: x86_64-unknown-linux-gnu -# # - build: beta -# # os: ubuntu-18.04 -# # rust: beta -# # target: x86_64-unknown-linux-gnu -# # - build: nightly-glibc -# # os: ubuntu-18.04 -# # rust: nightly -# # target: x86_64-unknown-linux-gnu -# # - build: nightly-musl -# # os: ubuntu-18.04 -# # rust: nightly -# # target: x86_64-unknown-linux-musl -# # - build: macos -# # os: macOS-10.14 -# # rust: stable -# # target: x86_64-apple-darwin -# # - build: win-msvc-32 -# # os: windows-2019 -# # rust: stable -# # target: i686-pc-windows-msvc -# # - build: win-msvc-64 -# # os: windows-2019 -# # rust: stable -# # target: x86_64-pc-windows-msvc -# # - build: win-gnu-32 -# # os: windows-2019 -# # rust: stable-i686-gnu -# # target: i686-pc-windows-gnu -# # - build: win-gnu-64 -# # os: windows-2019 -# # rust: stable-x86_64-gnu -# # target: x86_64-pc-windows-gnu -# steps: -# - name: Checkout repository -# uses: actions/checkout@v1 -# with: -# fetch-depth: 1 -# - name: Install Rust -# uses: hecrj/setup-rust-action@v1 -# with: -# rust-version: ${{ matrix.rust }} -# - name: Install Rust Target -# run: rustup target add ${{ matrix.target }} -# - name: Install musl-gcc -# if: contains(matrix.target, 'musl') -# run: | -# sudo apt-get install musl-tools -# - name: Build everything -# run: cargo build --verbose --target ${{ matrix.target }} --all --features pcre2 -# - name: Install zsh -# if: matrix.build == 'stable' -# run: sudo apt-get install zsh -# - name: Test zsh auto-completions -# if: matrix.build == 'stable' -# run: ./ci/test_complete.sh -# - name: Run tests -# run: cargo test --verbose --target ${{ matrix.target }} --all --features pcre2 + - name: Checkout repository + uses: actions/checkout@v1 + with: + fetch-depth: 1 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + profile: minimal + components: rustfmt + - name: Check formatting + run: | + cargo fmt --all -- --check diff --git a/Cross.toml b/Cross.toml index d16d10f0..3b0e4caa 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,2 +1,11 @@ [target.x86_64-unknown-linux-musl] image = "burntsushi/cross:x86_64-unknown-linux-musl" + +[target.i686-unknown-linux-gnu] +image = "burntsushi/cross:i686-unknown-linux-gnu" + +[target.mips64-unknown-linux-gnuabi64] +image = "burntsushi/cross:mips64-unknown-linux-gnuabi64" + +[target.arm-unknown-linux-gnueabihf] +image = "burntsushi/cross:arm-unknown-linux-gnueabihf" diff --git a/ci/cargo-out-dir b/ci/cargo-out-dir new file mode 100755 index 00000000..2b08d616 --- /dev/null +++ b/ci/cargo-out-dir @@ -0,0 +1,19 @@ +#!/bin/bash + +# Finds Cargo's `OUT_DIR` directory from the most recent build. +# +# This requires one parameter corresponding to the target directory +# to search for the build output. + +if [ $# != 1 ]; then + echo "Usage: $(basename "$0") " >&2 + exit 2 +fi + +# This works by finding the most recent stamp file, which is produced by +# every ripgrep build. +target_dir="$1" +find "$target_dir" -name ripgrep-stamp -print0 \ + | xargs -0 ls -t \ + | head -n1 \ + | xargs dirname diff --git a/ci/docker/README.md b/ci/docker/README.md new file mode 100644 index 00000000..58f367ef --- /dev/null +++ b/ci/docker/README.md @@ -0,0 +1,24 @@ +These are Docker images used for cross compilation in CI builds (or locally) +via the [Cross](https://github.com/rust-embedded/cross) tool. + +The Cross tool actually provides its own Docker images, and all Docker images +in this directory are derived from one of them. We provide our own in order +to customize the environment. For example, we need to install some things like +`asciidoc` in order to generate man pages. We also install compression tools +like `xz` so that tests for the `-z/--search-zip` flag are run. + +If you make a change to a Docker image, then you can re-build it. `cd` into the +directory containing the `Dockerfile` and run: + + $ cd x86_64-unknown-linux-musl + $ ./build + +At this point, subsequent uses of `cross` will now use your built image since +Docker prefers local images over remote images. In order to make these changes +stick, they need to be pushed to Docker Hub: + + $ docker push burntsushi/cross:x86_64-unknown-linux-musl + +Of course, only I (BurntSushi) can push to that location. To make `cross` use +a different location, then edit `Cross.toml` in the root of this repo to use +a different image name for the desired target. diff --git a/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile b/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile new file mode 100644 index 00000000..59901cb5 --- /dev/null +++ b/ci/docker/arm-unknown-linux-gnueabihf/Dockerfile @@ -0,0 +1,4 @@ +FROM rustembedded/cross:arm-unknown-linux-gnueabihf + +COPY stage/ubuntu-install-packages / +RUN /ubuntu-install-packages diff --git a/ci/docker/arm-unknown-linux-gnueabihf/build b/ci/docker/arm-unknown-linux-gnueabihf/build new file mode 100755 index 00000000..107b8369 --- /dev/null +++ b/ci/docker/arm-unknown-linux-gnueabihf/build @@ -0,0 +1,5 @@ +#!/bin/sh + +mkdir -p stage +cp ../../ubuntu-install-packages ./stage/ +docker build -t burntsushi/cross:arm-unknown-linux-gnueabihf . diff --git a/ci/docker/i686-unknown-linux-gnu/Dockerfile b/ci/docker/i686-unknown-linux-gnu/Dockerfile new file mode 100644 index 00000000..9c9107ea --- /dev/null +++ b/ci/docker/i686-unknown-linux-gnu/Dockerfile @@ -0,0 +1,4 @@ +FROM rustembedded/cross:i686-unknown-linux-gnu + +COPY stage/ubuntu-install-packages / +RUN /ubuntu-install-packages diff --git a/ci/docker/i686-unknown-linux-gnu/build b/ci/docker/i686-unknown-linux-gnu/build new file mode 100755 index 00000000..5837c828 --- /dev/null +++ b/ci/docker/i686-unknown-linux-gnu/build @@ -0,0 +1,5 @@ +#!/bin/sh + +mkdir -p stage +cp ../../ubuntu-install-packages ./stage/ +docker build -t burntsushi/cross:i686-unknown-linux-gnu . diff --git a/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile b/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile new file mode 100644 index 00000000..4b14616b --- /dev/null +++ b/ci/docker/mips64-unknown-linux-gnuabi64/Dockerfile @@ -0,0 +1,4 @@ +FROM rustembedded/cross:mips64-unknown-linux-gnuabi64 + +COPY stage/ubuntu-install-packages / +RUN /ubuntu-install-packages diff --git a/ci/docker/mips64-unknown-linux-gnuabi64/build b/ci/docker/mips64-unknown-linux-gnuabi64/build new file mode 100755 index 00000000..a15ca352 --- /dev/null +++ b/ci/docker/mips64-unknown-linux-gnuabi64/build @@ -0,0 +1,5 @@ +#!/bin/sh + +mkdir -p stage +cp ../../ubuntu-install-packages ./stage/ +docker build -t burntsushi/cross:mips64-unknown-linux-gnuabi64 . diff --git a/ci/docker/x86_64-unknown-linux-musl/Dockerfile b/ci/docker/x86_64-unknown-linux-musl/Dockerfile index 0405c0c4..8818b605 100644 --- a/ci/docker/x86_64-unknown-linux-musl/Dockerfile +++ b/ci/docker/x86_64-unknown-linux-musl/Dockerfile @@ -1,5 +1,4 @@ FROM rustembedded/cross:x86_64-unknown-linux-musl -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - libxslt1-dev asciidoc docbook-xsl xsltproc libxml2-utils +COPY stage/ubuntu-install-packages / +RUN /ubuntu-install-packages diff --git a/ci/docker/x86_64-unknown-linux-musl/build b/ci/docker/x86_64-unknown-linux-musl/build new file mode 100755 index 00000000..7fce7c2a --- /dev/null +++ b/ci/docker/x86_64-unknown-linux-musl/build @@ -0,0 +1,5 @@ +#!/bin/sh + +mkdir -p stage +cp ../../ubuntu-install-packages ./stage/ +docker build -t burntsushi/cross:x86_64-unknown-linux-musl . diff --git a/ci/macos-install-packages b/ci/macos-install-packages new file mode 100755 index 00000000..309562f7 --- /dev/null +++ b/ci/macos-install-packages @@ -0,0 +1,3 @@ +#!/bin/sh + +brew install asciidoc docbook-xsl diff --git a/ci/test_complete.sh b/ci/test_complete.sh index 0268de89..ff11c802 100755 --- a/ci/test_complete.sh +++ b/ci/test_complete.sh @@ -18,7 +18,7 @@ get_comp_args() { main() { local diff - local rg="${0:a:h}/../target/${TARGET:-}/release/rg" + local rg="${0:a:h}/../${TARGET_DIR:-target}/release/rg" local _rg="${0:a:h}/../complete/_rg" local -a help_args comp_args diff --git a/ci/ubuntu-install-packages b/ci/ubuntu-install-packages new file mode 100755 index 00000000..edaac6f4 --- /dev/null +++ b/ci/ubuntu-install-packages @@ -0,0 +1,6 @@ +#!/bin/sh + +sudo apt-get update +sudo apt-get install -y --no-install-recommends \ + libxslt1-dev asciidoc docbook-xsl xsltproc libxml2-utils \ + zsh xz-utils liblz4-tool musl-tools