diff --git a/container/Containerfile b/container/Containerfile
index b1969ce..5847ac1 100644
--- a/container/Containerfile
+++ b/container/Containerfile
@@ -12,11 +12,19 @@ RUN npm run build
 # We use musl to get a truly static binary
 # that runs cleanly without depending on glibc.
 FROM docker.io/rust:latest as builder
-WORKDIR /build
-ADD server /build
 RUN apt update
 RUN apt install musl musl-dev musl-tools -y
 RUN rustup target add x86_64-unknown-linux-musl
+
+# Some hacks to make podman cache build deps, see: 
+# https://whitfin.io/blog/speeding-up-rust-docker-builds/
+RUN cargo new build-container
+WORKDIR /build-container
+COPY server/Cargo.toml server/Cargo.lock ./
+RUN cargo build --target x86_64-unknown-linux-musl --release
+RUN rm src/*.rs
+ADD server /build-container
+
 RUN cargo build --target x86_64-unknown-linux-musl --release
 
 # Final stage, copy the server binary and the frontend build
@@ -29,7 +37,7 @@ RUN addgroup -S user && adduser -S user -G user
 
 WORKDIR /runner
 # Copy the server binary and the public directory
-COPY --from=builder /build/target/x86_64-unknown-linux-musl/release/server /runner/server
+COPY --from=builder /build-container/target/x86_64-unknown-linux-musl/release/server /runner/server
 # Copy the frontend SPA build into public
 COPY --from=client /build/dist /runner/public
 
diff --git a/container/ContainerfileDebug b/container/ContainerfileDebug
index 8b9166b..5f745c2 100644
--- a/container/ContainerfileDebug
+++ b/container/ContainerfileDebug
@@ -9,11 +9,19 @@
 # We use musl to get a truly static binary
 # that runs cleanly without depending on glibc.
 FROM docker.io/rust:latest as builder
-WORKDIR /build
-ADD server /build
 RUN apt update
 RUN apt install musl musl-dev musl-tools -y
 RUN rustup target add x86_64-unknown-linux-musl
+
+# Some hacks to make podman cache build deps, see: 
+# https://whitfin.io/blog/speeding-up-rust-docker-builds/
+RUN cargo new build-container
+WORKDIR /build-container
+COPY server/Cargo.toml server/Cargo.lock ./
+RUN cargo build --target x86_64-unknown-linux-musl
+RUN rm src/*.rs
+ADD server /build-container
+
 # Note that '--release' is missing here, so we build in debug mode
 RUN cargo build --target x86_64-unknown-linux-musl
 
@@ -27,7 +35,7 @@ RUN addgroup -S user && adduser -S user -G user
 
 WORKDIR /runner
 # Copy the server binary and the public directory, note the debug binary
-COPY --from=builder /build/target/x86_64-unknown-linux-musl/debug/server /runner/server
+COPY --from=builder /build-container/target/x86_64-unknown-linux-musl/debug/server /runner/server
 RUN mkdir /runner/public
 RUN echo "Debug build!" > /runner/public/index.html