From 6eddd52a14a2c327951d4683e16e40772f054e4d Mon Sep 17 00:00:00 2001 From: Aleksander <170264518+t-aleksander@users.noreply.github.com> Date: Tue, 25 Jun 2024 09:49:15 +0200 Subject: [PATCH] ci: Build deb/rpm packages (#649) * embed ui into the binary * make sure all required files are embedded * make the packaging actually work * add newlines * prepare workflow file for testing * add packaging step for .deb * defguard-core -> defguard * place the binary in /usr/bin * try without path interpolation * try without cross compilation * try with env passthrough * env -> conf * add frontend building step * set node version * add cache dependency path * remove node version from matrix * fix path * frozen lockfile * build rpm * cleanup and fix docker * fix .env file * remove possibly unnecessary step * cleanup * cleanup 2 * sort cargo toml * add new lines * change dockerfile * sort whole file * remove unused file --- .env | 2 +- .fpm | 6 ++++ .github/workflows/release.yml | 56 +++++++++++++++++++++++++++++++++++ Cargo.lock | 37 +++++++++++++++++++++++ Cargo.toml | 8 +++-- Cross.toml | 8 ++--- Dockerfile | 28 ++++++++---------- Dockerfile.ci | 18 ----------- defguard.service | 23 ++++++++++++++ src/assets.rs | 48 ++++++++++++++++++++++++++++++ src/headers.rs | 3 +- src/lib.rs | 23 +++++++------- 12 files changed, 207 insertions(+), 53 deletions(-) create mode 100644 .fpm delete mode 100644 Dockerfile.ci create mode 100644 defguard.service create mode 100644 src/assets.rs diff --git a/.env b/.env index 344ede6fc..60c27c65a 100644 --- a/.env +++ b/.env @@ -13,7 +13,7 @@ DEFGUARD_DEFAULT_ADMIN_PASSWORD=pass123 ### Proxy configuration ### # Optional. URL of proxy gRPC server -# DEFGUARD_PROXY_URL: http://localhost:50051 +# DEFGUARD_PROXY_URL=http://localhost:50051 ### LDAP configuration ### DEFGUARD_LDAP_URL=ldap://localhost:389 diff --git a/.fpm b/.fpm new file mode 100644 index 000000000..b982fd832 --- /dev/null +++ b/.fpm @@ -0,0 +1,6 @@ +-s dir +--name defguard +--architecture x86_64 +--description "defguard core service" +--url "https://defguard.net/" +--maintainer "teonite" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d32fa6625..26ce3bb53 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -112,6 +112,26 @@ jobs: [registry."docker.io"] mirrors = ["dockerhub-proxy.teonite.net"] + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + version: 9 + + - name: Use Node.js 20 + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'pnpm' + cache-dependency-path: ./web/pnpm-lock.yaml + + - name: Install frontend dependencies + run: pnpm install --ignore-scripts --frozen-lockfile + working-directory: web + + - name: Build frontend + run: pnpm build + working-directory: web + - name: Build release binary uses: actions-rs/cargo@v1 with: @@ -139,3 +159,39 @@ jobs: asset_path: defguard-${{ github.ref_name }}-${{ matrix.target }}.tar.gz asset_name: defguard-${{ github.ref_name }}-${{ matrix.target }}.tar.gz asset_content_type: application/octet-stream + + - name: Build DEB package + if: matrix.build == 'linux' + uses: bpicode/github-action-fpm@master + with: + fpm_args: "defguard-${{ github.ref_name }}-${{ matrix.target }}=/usr/bin/defguard defguard.service=/usr/lib/systemd/system/defguard.service .env=/etc/defguard/core.conf" + fpm_opts: "--debug --output-type deb --version ${{ env.VERSION }} --package defguard-${{ env.VERSION }}-${{ matrix.target }}.deb" + + - name: Upload DEB + if: matrix.build == 'linux' + uses: actions/upload-release-asset@v1.0.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: defguard-${{ env.VERSION }}-${{ matrix.target }}.deb + asset_name: defguard-${{ env.VERSION }}-${{ matrix.target }}.deb + asset_content_type: application/octet-stream + + - name: Build RPM package + if: matrix.build == 'linux' + uses: bpicode/github-action-fpm@master + with: + fpm_args: "defguard-${{ github.ref_name }}-${{ matrix.target }}=/usr/bin/defguard defguard.service=/usr/lib/systemd/system/defguard.service .env=/etc/defguard/core.conf" + fpm_opts: "--debug --output-type rpm --version ${{ env.VERSION }} --package defguard-${{ env.VERSION }}-${{ matrix.target }}.rpm" + + - name: Upload RPM + if: matrix.build == 'linux' + uses: actions/upload-release-asset@v1.0.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create-release.outputs.upload_url }} + asset_path: defguard-${{ env.VERSION }}-${{ matrix.target }}.rpm + asset_name: defguard-${{ env.VERSION }}-${{ matrix.target }}.rpm + asset_content_type: application/octet-stream diff --git a/Cargo.lock b/Cargo.lock index 2224fd62a..60085dcaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -986,6 +986,7 @@ dependencies = [ "lettre", "matches", "md4", + "mime_guess", "model_derive", "openidconnect", "otpauth", @@ -997,6 +998,7 @@ dependencies = [ "regex", "reqwest", "rsa", + "rust-embed", "rust-ini", "secp256k1", "secrecy", @@ -3469,6 +3471,41 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rust-embed" +version = "8.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19549741604902eb99a7ed0ee177a0663ee1eda51a29f71401f166e47e77806a" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb9f96e283ec64401f30d3df8ee2aaeb2561f34c824381efa24a35f79bf40ee4" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.60", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c74a686185620830701348de757fd36bef4aa9680fd23c49fc539ddcc1af32" +dependencies = [ + "globset", + "sha2", + "walkdir", +] + [[package]] name = "rust-ini" version = "0.20.0" diff --git a/Cargo.toml b/Cargo.toml index 810ad3d1b..03dffaa5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,6 @@ repository = "https://github.com/DefGuard/defguard" [workspace] [dependencies] -model_derive = { path = "model-derive" } anyhow = "1.0" argon2 = { version = "0.5", features = ["std"] } axum = { version = "0.7" } @@ -35,16 +34,19 @@ jsonwebtoken = "9.2" ldap3 = { version = "0.11", default-features = false, features = ["tls"] } lettre = { version = "0.11", features = ["tokio1", "tokio1-native-tls"] } md4 = "0.10" -otpauth = "0.4" +mime_guess = "2.0" +model_derive = { path = "model-derive" } openidconnect = { version = "3.4", default-features = false, optional = true } -pulldown-cmark = "0.9" +otpauth = "0.4" prost = "0.12" +pulldown-cmark = "0.9" rand = "0.8" rand_core = { version = "0.6", default-features = false, features = [ "getrandom", ] } reqwest = { version = "0.11", features = ["json"] } rsa = { version = "0.9", features = ["pem"] } +rust-embed = { version = "8.4", features = ["include-exclude"] } rust-ini = "0.20" secp256k1 = { version = "0.28", features = [ "recovery", diff --git a/Cross.toml b/Cross.toml index 2f0e1f514..499cd583a 100644 --- a/Cross.toml +++ b/Cross.toml @@ -7,7 +7,7 @@ pre-build = [ "apt-get update && apt-get install --assume-yes libssl-dev unzip", "PB_REL='https://github.com/protocolbuffers/protobuf/releases'", "PB_VERSION='3.20.0' && curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip", - "unzip -o protoc-$PB_VERSION-linux-x86_64.zip bin/protoc include/google/* -d /usr" + "unzip -o protoc-$PB_VERSION-linux-x86_64.zip bin/protoc include/google/* -d /usr", ] [target.armv7-unknown-linux-gnueabihf] @@ -17,7 +17,7 @@ pre-build = [ "apt-get update && apt-get install --assume-yes libssl-dev libssl-dev:$CROSS_DEB_ARCH unzip", "PB_REL='https://github.com/protocolbuffers/protobuf/releases'", "PB_VERSION='3.20.0' && curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip", - "unzip -o protoc-$PB_VERSION-linux-x86_64.zip bin/protoc include/google/* -d /usr" + "unzip -o protoc-$PB_VERSION-linux-x86_64.zip bin/protoc include/google/* -d /usr", ] [target.aarch64-unknown-linux-gnu] @@ -27,7 +27,7 @@ pre-build = [ "apt-get update && apt-get install --assume-yes libssl-dev libssl-dev:$CROSS_DEB_ARCH unzip", "PB_REL='https://github.com/protocolbuffers/protobuf/releases'", "PB_VERSION='3.20.0' && curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip", - "unzip -o protoc-$PB_VERSION-linux-x86_64.zip bin/protoc include/google/* -d /usr" + "unzip -o protoc-$PB_VERSION-linux-x86_64.zip bin/protoc include/google/* -d /usr", ] @@ -37,5 +37,5 @@ pre-build = [ "apt-get update && apt-get install --assume-yes libssl-dev unzip", "PB_REL='https://github.com/protocolbuffers/protobuf/releases'", "PB_VERSION='3.20.0' && curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip", - "unzip -o protoc-$PB_VERSION-linux-x86_64.zip bin/protoc include/google/* -d /usr" + "unzip -o protoc-$PB_VERSION-linux-x86_64.zip bin/protoc include/google/* -d /usr", ] diff --git a/Dockerfile b/Dockerfile index e25ee8b1d..8762ea004 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,13 @@ +FROM node:20-alpine as web + +WORKDIR /app +COPY web/package.json web/pnpm-lock.yaml web/.npmrc . +RUN npm i -g pnpm +RUN pnpm install --ignore-scripts --frozen-lockfile +COPY web/ . +RUN pnpm run generate-translation-types +RUN pnpm build + FROM rust:1.77 as chef WORKDIR /build @@ -20,6 +30,9 @@ COPY --from=planner /build/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json # build project +COPY --from=web /app/dist ./web/dist +COPY web/src/shared/images/svg ./web/src/shared/images/svg +COPY user_agent_header_regexes.yaml /build/user_agent_header_regexes.yaml RUN apt-get update && apt-get -y install protobuf-compiler libprotobuf-dev COPY Cargo.toml Cargo.lock build.rs ./ COPY .sqlx .sqlx @@ -30,26 +43,11 @@ COPY proto proto COPY migrations migrations RUN cargo install --locked --path . --root /build -FROM node:20.5-alpine3.17 as web - -WORKDIR /app -COPY web/package.json . -COPY web/pnpm-lock.yaml . -COPY web/.npmrc . -RUN npm i -g pnpm -RUN pnpm install --ignore-scripts --frozen-lockfile -COPY web/ . -RUN pnpm run generate-translation-types -RUN pnpm build - # run FROM debian:bookworm-slim as runtime RUN apt-get update -y && \ apt-get install --no-install-recommends -y ca-certificates libssl-dev && \ rm -rf /var/lib/apt/lists/* -COPY user_agent_header_regexes.yaml /app/user_agent_header_regexes.yaml WORKDIR /app COPY --from=builder /build/bin/defguard . -COPY --from=web /app/dist ./web/dist -COPY web/src/shared/images/svg ./web/src/shared/images/svg ENTRYPOINT ["./defguard"] diff --git a/Dockerfile.ci b/Dockerfile.ci deleted file mode 100644 index 0f77ae45b..000000000 --- a/Dockerfile.ci +++ /dev/null @@ -1,18 +0,0 @@ -FROM node:20.5-alpine3.17 as web -WORKDIR /app -COPY web/package.json . -COPY web/pnpm-lock.yaml . -COPY web/.npmrc . -RUN npm i -g pnpm -RUN pnpm i --frozen-lockfile --ignore-scripts -COPY web/ . -RUN pnpm build - -FROM debian:bullseye-slim -RUN apt-get update -y && \ - apt-get install --no-install-recommends -y ca-certificates && \ - rm -rf /var/lib/apt/lists/* -COPY build/bin/defguard . -COPY --from=web /app/dist ./web -USER 1000 -ENTRYPOINT ["./defguard"] diff --git a/defguard.service b/defguard.service new file mode 100644 index 000000000..49e572123 --- /dev/null +++ b/defguard.service @@ -0,0 +1,23 @@ +[Unit] +Description=defguard core service +Documentation=https://defguard.gitbook.io/defguard/ +Wants=network-online.target +After=network-online.target + +[Service] +DynamicUser=yes +User=defguard +ExecReload=/bin/kill -HUP $MAINPID +EnvironmentFile=/etc/defguard/core.conf +ExecStart=/usr/bin/defguard +KillMode=process +KillSignal=SIGINT +LimitNOFILE=65536 +LimitNPROC=infinity +Restart=on-failure +RestartSec=2 +TasksMax=infinity +OOMScoreAdjust=-1000 + +[Install] +WantedBy=multi-user.target diff --git a/src/assets.rs b/src/assets.rs new file mode 100644 index 000000000..0ae3ddc74 --- /dev/null +++ b/src/assets.rs @@ -0,0 +1,48 @@ +use axum::{ + http::{header, StatusCode, Uri}, + response::{IntoResponse, Response}, +}; +use rust_embed::Embed; + +pub async fn web_asset(uri: Uri) -> impl IntoResponse { + let mut path = uri.path().trim_start_matches('/').to_string(); + // Rewrite the path to match the structure of the embedded files + path.insert_str(0, "dist/"); + StaticFile(path) +} + +pub async fn index() -> impl IntoResponse { + web_asset(Uri::from_static("/index.html")).await +} + +pub async fn svg(uri: Uri) -> impl IntoResponse { + let mut path = uri.path().trim_start_matches('/').to_string(); + // Rewrite the path to match the structure of the embedded files + path.insert_str(0, "src/shared/images/"); + StaticFile(path) +} + +#[derive(Embed)] +#[folder = "web/"] +#[include = "dist/*"] +#[include = "src/shared/images/*"] +struct WebAsset; + +pub struct StaticFile(pub T); + +impl IntoResponse for StaticFile +where + T: Into, +{ + fn into_response(self) -> Response { + let path = self.0.into(); + + match WebAsset::get(path.as_str()) { + Some(content) => { + let mime = mime_guess::from_path(path).first_or_octet_stream(); + ([(header::CONTENT_TYPE, mime.as_ref())], content.data).into_response() + } + None => (StatusCode::NOT_FOUND, "404 Not Found").into_response(), + } + } +} diff --git a/src/headers.rs b/src/headers.rs index 8dc6ace92..e20a816bc 100644 --- a/src/headers.rs +++ b/src/headers.rs @@ -12,9 +12,10 @@ use crate::{ #[must_use] pub fn create_user_agent_parser() -> Arc { + let regexes = include_bytes!("../user_agent_header_regexes.yaml"); Arc::new( UserAgentParser::builder() - .build_from_yaml("user_agent_header_regexes.yaml") + .build_from_bytes(regexes) .expect("Parser creation failed"), ) } diff --git a/src/lib.rs b/src/lib.rs index 85155852d..275d08d49 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,12 +6,12 @@ use std::{ use anyhow::anyhow; use axum::{ - handler::HandlerWithoutStateExt, http::{Request, StatusCode}, routing::{delete, get, patch, post, put}, serve, Extension, Router, }; +use assets::{index, svg, web_asset}; use handlers::ssh_authorized_keys::{ add_authentication_key, delete_authentication_key, fetch_authentication_keys, }; @@ -30,10 +30,7 @@ use tokio::{ OnceCell, }, }; -use tower_http::{ - services::{ServeDir, ServeFile}, - trace::{DefaultOnResponse, TraceLayer}, -}; +use tower_http::trace::{DefaultOnResponse, TraceLayer}; use tracing::Level; use uaparser::UserAgentParser; @@ -108,6 +105,7 @@ use self::{ }; pub mod appstate; +pub mod assets; pub mod auth; pub mod config; pub mod db; @@ -164,10 +162,15 @@ pub fn build_webapp( user_agent_parser: Arc, failed_logins: Arc>, ) -> Router { - let serve_web_dir = ServeDir::new("web/dist").fallback(ServeFile::new("web/dist/index.html")); - let serve_images = - ServeDir::new("web/src/shared/images/svg").not_found_service(handle_404.into_service()); - let webapp = Router::new().nest( + let webapp: Router = Router::new() + .route("/", get(index)) + .route("/*path", get(index)) + .route("/fonts/*path", get(web_asset)) + .route("/assets/*path", get(web_asset)) + .route("/svg/*path", get(svg)) + .fallback_service(get(handle_404)); + + let webapp = webapp.nest( "/api/v1", Router::new() .route("/health", get(health_check)) @@ -343,8 +346,6 @@ pub fn build_webapp( ); webapp - .nest_service("/svg", serve_images) - .nest_service("/", serve_web_dir) .with_state(AppState::new( pool, webhook_tx,