From feefe690943e42d87b76acd8e78bdbd48bf826d5 Mon Sep 17 00:00:00 2001 From: BlackDex Date: Wed, 16 Dec 2020 19:31:39 +0100 Subject: [PATCH] Updated Github Actions, Fixed Dockerfile - Updated the Github Actions to build just one binary with all DB Backends. - Created a hadolint workflow to check and verify Dockerfiles. - Fixed current hadolint errors. - Fixed a bug in the Dockerfile.j2 which prevented the correct libraries and tools to be installed on the Alpine images. - Deleted travis.yml since that is not used anymore --- .github/workflows/build.yml | 125 ++++++++++++++++++++++++++ .github/workflows/hadolint.yml | 34 +++++++ .github/workflows/workspace.yml | 148 ------------------------------- .travis.yml | 21 ----- docker/Dockerfile.j2 | 25 +++--- docker/amd64/Dockerfile.alpine | 2 + docker/arm32v6/Dockerfile | 8 +- docker/arm32v7/Dockerfile | 8 +- docker/arm32v7/Dockerfile.alpine | 1 + docker/arm64v8/Dockerfile | 8 +- 10 files changed, 185 insertions(+), 195 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/hadolint.yml delete mode 100644 .github/workflows/workspace.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..d7ed5618 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,125 @@ +name: Build + +on: + push: + # Ignore when there are only changes done too one of these paths + paths-ignore: + - "**.md" + - "**.txt" + - "azure-pipelines.yml" + - "docker/**" + - "hooks/**" + - "tools/**" + +jobs: + build: + strategy: + fail-fast: false + matrix: + channel: + - nightly + # - stable + target-triple: + - x86_64-unknown-linux-gnu + # - x86_64-unknown-linux-musl + include: + - target-triple: x86_64-unknown-linux-gnu + host-triple: x86_64-unknown-linux-gnu + features: "sqlite,mysql,postgresql" + channel: nightly + os: ubuntu-18.04 + ext: + # - target-triple: x86_64-unknown-linux-gnu + # host-triple: x86_64-unknown-linux-gnu + # features: "sqlite,mysql,postgresql" + # channel: stable + # os: ubuntu-18.04 + # ext: + # - target-triple: x86_64-unknown-linux-musl + # host-triple: x86_64-unknown-linux-gnu + # features: "sqlite,postgresql" + # channel: nightly + # os: ubuntu-18.04 + # ext: + # - target-triple: x86_64-unknown-linux-musl + # host-triple: x86_64-unknown-linux-gnu + # features: "sqlite,postgresql" + # channel: stable + # os: ubuntu-18.04 + # ext: + + name: Building ${{ matrix.channel }}-${{ matrix.target-triple }} + runs-on: ${{ matrix.os }} + steps: + # Checkout the repo + - name: Checkout + uses: actions/checkout@v2 + # End Checkout the repo + + + # Install musl-tools when needed + - name: Install musl tools + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends musl-dev musl-tools cmake + if: matrix.target-triple == 'x86_64-unknown-linux-musl' + # End Install musl-tools when needed + + + # Install dependencies + - name: Install dependencies Ubuntu + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends openssl sqlite build-essential libmariadb-dev-compat libpq-dev libssl-dev pkgconf + if: startsWith( matrix.os, 'ubuntu' ) + # End Install dependencies + + + # Enable Rust Caching + - uses: Swatinem/rust-cache@v1 + # End Enable Rust Caching + + + # Uses the rust-toolchain file to determine version + - name: 'Install ${{ matrix.channel }}-${{ matrix.host-triple }} for target: ${{ matrix.target-triple }}' + uses: actions-rs/toolchain@v1 + with: + profile: minimal + target: ${{ matrix.target-triple }} + # End Uses the rust-toolchain file to determine version + + + # Run cargo tests (In release mode to speed up cargo build afterwards) + - name: '`cargo test --release --features ${{ matrix.features }} --target ${{ matrix.target-triple }}`' + uses: actions-rs/cargo@v1 + with: + command: test + args: --release --features ${{ matrix.features }} --target ${{ matrix.target-triple }} + # End Run cargo tests + + + # Build the binary + - name: '`cargo build --release --features ${{ matrix.features }} --target ${{ matrix.target-triple }}`' + uses: actions-rs/cargo@v1 + with: + command: build + args: --release --features ${{ matrix.features }} --target ${{ matrix.target-triple }} + # End Build the binary + + + # Upload artifact to Github Actions + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: bitwarden_rs-${{ matrix.target-triple }}${{ matrix.ext }} + path: target/${{ matrix.target-triple }}/release/bitwarden_rs${{ matrix.ext }} + # End Upload artifact to Github Actions + + + ## This is not used at the moment + ## We could start using this when we can build static binaries + # Upload to github actions release + # - name: Release + # uses: Shopify/upload-to-release@1 + # if: startsWith(github.ref, 'refs/tags/') + # with: + # name: bitwarden_rs-${{ matrix.target-triple }}${{ matrix.ext }} + # path: target/${{ matrix.target-triple }}/release/bitwarden_rs${{ matrix.ext }} + # repo-token: ${{ secrets.GITHUB_TOKEN }} + # End Upload to github actions release diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml new file mode 100644 index 00000000..7b799554 --- /dev/null +++ b/.github/workflows/hadolint.yml @@ -0,0 +1,34 @@ +name: Hadolint + +on: + pull_request: + # Ignore when there are only changes done too one of these paths + paths: + - "docker/**" + +jobs: + hadolint: + name: Validate Dockerfile syntax + runs-on: ubuntu-20.04 + steps: + # Checkout the repo + - name: Checkout + uses: actions/checkout@v2 + # End Checkout the repo + + + # Download hadolint + - name: Download hadolint + shell: bash + run: | + sudo curl -L https://github.com/hadolint/hadolint/releases/download/v$HADOLINT_VERSION/hadolint-$(uname -s)-$(uname -m) -o /usr/local/bin/hadolint && \ + sudo chmod +x /usr/local/bin/hadolint + env: + HADOLINT_VERSION: 1.19.0 + # End Download hadolint + + # Test Dockerfiles + - name: Run hadolint + shell: bash + run: git ls-files --exclude='docker/*/Dockerfile*' --ignored | xargs hadolint + # End Test Dockerfiles diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml deleted file mode 100644 index b7c73285..00000000 --- a/.github/workflows/workspace.yml +++ /dev/null @@ -1,148 +0,0 @@ -name: Workflow - -on: - push: - paths-ignore: - - "**.md" - #pull_request: - # paths-ignore: - # - "**.md" - -jobs: - build: - name: Build - strategy: - fail-fast: false - matrix: - db-backend: [sqlite, mysql, postgresql] - target: - - x86_64-unknown-linux-gnu - # - x86_64-unknown-linux-musl - # - x86_64-apple-darwin - # - x86_64-pc-windows-msvc - include: - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - ext: - # - target: x86_64-unknown-linux-musl - # os: ubuntu-latest - # ext: - # - target: x86_64-apple-darwin - # os: macOS-latest - # ext: - # - target: x86_64-pc-windows-msvc - # os: windows-latest - # ext: .exe - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v1 - - # - name: Cache choco cache - # uses: actions/cache@v1.0.3 - # if: matrix.os == 'windows-latest' - # with: - # path: ~\AppData\Local\Temp\chocolatey - # key: ${{ runner.os }}-choco-cache-${{ matrix.db-backend }} - - - name: Cache vcpkg installed - uses: actions/cache@v1.0.3 - if: matrix.os == 'windows-latest' - with: - path: $VCPKG_ROOT/installed - key: ${{ runner.os }}-vcpkg-cache-${{ matrix.db-backend }} - env: - VCPKG_ROOT: 'C:\vcpkg' - - - name: Cache vcpkg downloads - uses: actions/cache@v1.0.3 - if: matrix.os == 'windows-latest' - with: - path: $VCPKG_ROOT/downloads - key: ${{ runner.os }}-vcpkg-cache-${{ matrix.db-backend }} - env: - VCPKG_ROOT: 'C:\vcpkg' - - # - name: Cache homebrew - # uses: actions/cache@v1.0.3 - # if: matrix.os == 'macOS-latest' - # with: - # path: ~/Library/Caches/Homebrew - # key: ${{ runner.os }}-brew-cache - - # - name: Cache apt - # uses: actions/cache@v1.0.3 - # if: matrix.os == 'ubuntu-latest' - # with: - # path: /var/cache/apt/archives - # key: ${{ runner.os }}-apt-cache - - # Install dependencies - - name: Install dependencies macOS - run: brew update; brew install openssl sqlite libpq mysql - if: matrix.os == 'macOS-latest' - - - name: Install dependencies Ubuntu - run: sudo apt-get update && sudo apt-get install --no-install-recommends openssl sqlite libpq-dev libmysql++-dev - if: matrix.os == 'ubuntu-latest' - - - name: Install dependencies Windows - run: vcpkg integrate install; vcpkg install sqlite3:x64-windows openssl:x64-windows libpq:x64-windows libmysql:x64-windows - if: matrix.os == 'windows-latest' - env: - VCPKG_ROOT: 'C:\vcpkg' - # End Install dependencies - - # Install rust nightly toolchain - - name: Cache cargo registry - uses: actions/cache@v1.0.3 - with: - path: ~/.cargo/registry - key: ${{ runner.os }}-${{matrix.db-backend}}-cargo-registry-${{ hashFiles('**/Cargo.lock') }} - - name: Cache cargo index - uses: actions/cache@v1.0.3 - with: - path: ~/.cargo/git - key: ${{ runner.os }}-${{matrix.db-backend}}-cargo-index-${{ hashFiles('**/Cargo.lock') }} - - name: Cache cargo build - uses: actions/cache@v1.0.3 - with: - path: target - key: ${{ runner.os }}-${{matrix.db-backend}}-cargo-build-target-${{ hashFiles('**/Cargo.lock') }} - - - name: Install latest nightly - uses: actions-rs/toolchain@v1.0.5 - with: - # Uses rust-toolchain to determine version - profile: minimal - target: ${{ matrix.target }} - - # Build - - name: Build Win - if: matrix.os == 'windows-latest' - run: cargo.exe build --features ${{ matrix.db-backend }} --release --target ${{ matrix.target }} - env: - RUSTFLAGS: -Ctarget-feature=+crt-static - VCPKG_ROOT: 'C:\vcpkg' - - - name: Build macOS / Ubuntu - if: matrix.os == 'macOS-latest' || matrix.os == 'ubuntu-latest' - run: cargo build --verbose --features ${{ matrix.db-backend }} --release --target ${{ matrix.target }} - - # Test - - name: Run tests - run: cargo test --features ${{ matrix.db-backend }} - - # Upload & Release - - name: Upload artifact - uses: actions/upload-artifact@v1.0.0 - with: - name: bitwarden_rs-${{ matrix.db-backend }}-${{ matrix.target }}${{ matrix.ext }} - path: target/${{ matrix.target }}/release/bitwarden_rs${{ matrix.ext }} - - - name: Release - uses: Shopify/upload-to-release@1.0.0 - if: startsWith(github.ref, 'refs/tags/') - with: - name: bitwarden_rs-${{ matrix.db-backend }}-${{ matrix.target }}${{ matrix.ext }} - path: target/${{ matrix.target }}/release/bitwarden_rs${{ matrix.ext }} - repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ae8d8137..00000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -dist: xenial - -env: - global: - - HADOLINT_VERSION=1.17.1 - -language: rust -rust: nightly -cache: cargo - -before_install: - - sudo curl -L https://github.com/hadolint/hadolint/releases/download/v$HADOLINT_VERSION/hadolint-$(uname -s)-$(uname -m) -o /usr/local/bin/hadolint - - sudo chmod +rx /usr/local/bin/hadolint - - rustup set profile minimal - -# Nothing to install -install: true -script: -- git ls-files --exclude='Dockerfile*' --ignored | xargs --max-lines=1 hadolint -- cargo test --features "sqlite" -- cargo test --features "mysql" diff --git a/docker/Dockerfile.j2 b/docker/Dockerfile.j2 index 413c7144..38c62900 100644 --- a/docker/Dockerfile.j2 +++ b/docker/Dockerfile.j2 @@ -62,16 +62,19 @@ FROM bitwardenrs/web-vault@{{ vault_image_hash }} as vault FROM {{ build_stage_base_image }} as build {% if "alpine" in target_file %} -{% if "amd64" in target_file -%} +{% if "amd64" in target_file %} # Alpine-based AMD64 (musl) does not support mysql/mariadb during compile time. ARG DB=sqlite,postgresql -{% else -%} +{% set features = "sqlite,postgresql" %} +{% else %} # Alpine-based ARM (musl) only supports sqlite during compile time. ARG DB=sqlite -{% endif %} +{% set features = "sqlite" %} +{% endif %} {% else %} # Debian-based builds support multidb ARG DB=sqlite,mysql,postgresql +{% set features = "sqlite,mysql,postgresql" %} {% endif %} # Build time options to avoid dpkg warnings and help with reproducible builds. @@ -137,12 +140,10 @@ COPY ./build.rs ./build.rs # We at least need libmariadb3:amd64 installed for the x86_64 version of libmariadb.so (client) # We also need the libmariadb-dev-compat:amd64 but it can not be installed together with the {{ package_arch_prefix }} version. # What we can do is a force install, because nothing important is overlapping each other. -RUN apt-get install -y libmariadb3:amd64 && \ - mkdir -pv /tmp/dpkg && \ - cd /tmp/dpkg && \ +RUN apt-get install -y --no-install-recommends libmariadb3:amd64 && \ apt-get download libmariadb-dev-compat:amd64 && \ - dpkg --force-all -i *.deb && \ - rm -rf /tmp/dpkg + dpkg --force-all -i ./libmariadb-dev-compat*.deb && \ + rm -rvf ./libmariadb-dev-compat*.deb # For Diesel-RS migrations_macros to compile with PostgreSQL we need to do some magic. # The libpq5{{ package_arch_prefix }} package seems to not provide a symlink to libpq.so.5 with the name libpq.so. @@ -203,11 +204,13 @@ RUN [ "cross-build-start" ] RUN apk add --no-cache \ openssl \ curl \ -{% if "sqlite" in target_file %} +{% if "sqlite" in features %} sqlite \ -{% elif "mysql" in target_file %} +{% endif %} +{% if "mysql" in features %} mariadb-connector-c \ -{% elif "postgresql" in target_file %} +{% endif %} +{% if "postgresql" in features %} postgresql-libs \ {% endif %} ca-certificates diff --git a/docker/amd64/Dockerfile.alpine b/docker/amd64/Dockerfile.alpine index ac558f1c..3ede25b5 100644 --- a/docker/amd64/Dockerfile.alpine +++ b/docker/amd64/Dockerfile.alpine @@ -74,6 +74,8 @@ ENV SSL_CERT_DIR=/etc/ssl/certs RUN apk add --no-cache \ openssl \ curl \ + sqlite \ + postgresql-libs \ ca-certificates RUN mkdir /data diff --git a/docker/arm32v6/Dockerfile b/docker/arm32v6/Dockerfile index f0f0b3c8..33c74f57 100644 --- a/docker/arm32v6/Dockerfile +++ b/docker/arm32v6/Dockerfile @@ -70,12 +70,10 @@ COPY ./build.rs ./build.rs # We at least need libmariadb3:amd64 installed for the x86_64 version of libmariadb.so (client) # We also need the libmariadb-dev-compat:amd64 but it can not be installed together with the :armel version. # What we can do is a force install, because nothing important is overlapping each other. -RUN apt-get install -y libmariadb3:amd64 && \ - mkdir -pv /tmp/dpkg && \ - cd /tmp/dpkg && \ +RUN apt-get install -y --no-install-recommends libmariadb3:amd64 && \ apt-get download libmariadb-dev-compat:amd64 && \ - dpkg --force-all -i *.deb && \ - rm -rf /tmp/dpkg + dpkg --force-all -i ./libmariadb-dev-compat*.deb && \ + rm -rvf ./libmariadb-dev-compat*.deb # For Diesel-RS migrations_macros to compile with PostgreSQL we need to do some magic. # The libpq5:armel package seems to not provide a symlink to libpq.so.5 with the name libpq.so. diff --git a/docker/arm32v7/Dockerfile b/docker/arm32v7/Dockerfile index f69baa57..65827f29 100644 --- a/docker/arm32v7/Dockerfile +++ b/docker/arm32v7/Dockerfile @@ -70,12 +70,10 @@ COPY ./build.rs ./build.rs # We at least need libmariadb3:amd64 installed for the x86_64 version of libmariadb.so (client) # We also need the libmariadb-dev-compat:amd64 but it can not be installed together with the :armhf version. # What we can do is a force install, because nothing important is overlapping each other. -RUN apt-get install -y libmariadb3:amd64 && \ - mkdir -pv /tmp/dpkg && \ - cd /tmp/dpkg && \ +RUN apt-get install -y --no-install-recommends libmariadb3:amd64 && \ apt-get download libmariadb-dev-compat:amd64 && \ - dpkg --force-all -i *.deb && \ - rm -rf /tmp/dpkg + dpkg --force-all -i ./libmariadb-dev-compat*.deb && \ + rm -rvf ./libmariadb-dev-compat*.deb # For Diesel-RS migrations_macros to compile with PostgreSQL we need to do some magic. # The libpq5:armhf package seems to not provide a symlink to libpq.so.5 with the name libpq.so. diff --git a/docker/arm32v7/Dockerfile.alpine b/docker/arm32v7/Dockerfile.alpine index d7cff76e..047ff966 100644 --- a/docker/arm32v7/Dockerfile.alpine +++ b/docker/arm32v7/Dockerfile.alpine @@ -77,6 +77,7 @@ RUN [ "cross-build-start" ] RUN apk add --no-cache \ openssl \ curl \ + sqlite \ ca-certificates RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/community catatonit diff --git a/docker/arm64v8/Dockerfile b/docker/arm64v8/Dockerfile index 42db845e..6a7c0bb1 100644 --- a/docker/arm64v8/Dockerfile +++ b/docker/arm64v8/Dockerfile @@ -70,12 +70,10 @@ COPY ./build.rs ./build.rs # We at least need libmariadb3:amd64 installed for the x86_64 version of libmariadb.so (client) # We also need the libmariadb-dev-compat:amd64 but it can not be installed together with the :arm64 version. # What we can do is a force install, because nothing important is overlapping each other. -RUN apt-get install -y libmariadb3:amd64 && \ - mkdir -pv /tmp/dpkg && \ - cd /tmp/dpkg && \ +RUN apt-get install -y --no-install-recommends libmariadb3:amd64 && \ apt-get download libmariadb-dev-compat:amd64 && \ - dpkg --force-all -i *.deb && \ - rm -rf /tmp/dpkg + dpkg --force-all -i ./libmariadb-dev-compat*.deb && \ + rm -rvf ./libmariadb-dev-compat*.deb # For Diesel-RS migrations_macros to compile with PostgreSQL we need to do some magic. # The libpq5:arm64 package seems to not provide a symlink to libpq.so.5 with the name libpq.so.