Implement prestart fetch with S3 cache for dependencies
- Prestart task fetches from S3 binary cache into shared volume - Server task serves from shared volume (read-only) - Build uses S3 cache as substituter (ultra-fast builds for shared deps) - Push entire closure to cache (derivation + dependencies) - No host involvement, pure container isolation Architecture: - Site A builds nodejs_20 + vite → pushed to cache - Site B builds → pulls nodejs_20 + vite from cache (instant) - Only builds site-specific code Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
01d6a3e779
commit
297001e5fd
1 changed files with 70 additions and 24 deletions
|
|
@ -42,7 +42,15 @@ runs:
|
||||||
- name: Build site with Nix
|
- name: Build site with Nix
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
nix build ${{ inputs.flake-output }} --print-build-logs
|
# Configure S3 as substituter to pull cached dependencies
|
||||||
|
export AWS_ACCESS_KEY_ID="${{ env.S3_ACCESS_KEY }}"
|
||||||
|
export AWS_SECRET_ACCESS_KEY="${{ env.S3_SECRET_KEY }}"
|
||||||
|
|
||||||
|
# Build with S3 cache as substituter (fetches cached deps)
|
||||||
|
nix build ${{ inputs.flake-output }} \
|
||||||
|
--print-build-logs \
|
||||||
|
--option substituters "https://cache.nixos.org s3://nix-cache?endpoint=${{ inputs.s3-endpoint }}&scheme=https" \
|
||||||
|
--option trusted-public-keys "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= $(cat /tmp/cache-pub-key.pem 2>/dev/null || echo '')"
|
||||||
|
|
||||||
# Get the store path
|
# Get the store path
|
||||||
STORE_PATH=$(readlink -f result)
|
STORE_PATH=$(readlink -f result)
|
||||||
|
|
@ -59,20 +67,21 @@ runs:
|
||||||
export AWS_ACCESS_KEY_ID="${{ env.S3_ACCESS_KEY }}"
|
export AWS_ACCESS_KEY_ID="${{ env.S3_ACCESS_KEY }}"
|
||||||
export AWS_SECRET_ACCESS_KEY="${{ env.S3_SECRET_KEY }}"
|
export AWS_SECRET_ACCESS_KEY="${{ env.S3_SECRET_KEY }}"
|
||||||
|
|
||||||
# Push to S3 binary cache
|
# Push entire closure (derivation + all dependencies) to cache
|
||||||
nix copy \
|
nix copy \
|
||||||
--to "s3://nix-cache?endpoint=${{ inputs.s3-endpoint }}&scheme=https&secret-key=${{ env.NIX_SIGNING_KEY }}" \
|
--to "s3://nix-cache?endpoint=${{ inputs.s3-endpoint }}&scheme=https&secret-key=${{ env.NIX_SIGNING_KEY }}" \
|
||||||
|
--derivation \
|
||||||
"$STORE_PATH"
|
"$STORE_PATH"
|
||||||
|
|
||||||
echo "✅ Pushed to binary cache: $STORE_HASH"
|
echo "✅ Pushed to binary cache: $STORE_HASH (with all dependencies)"
|
||||||
|
|
||||||
- name: Pull to host and deploy via Nomad
|
- name: Deploy via Nomad
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
# First, pull the store path to the host's /nix/store
|
# Get S3 credentials for the fetch task
|
||||||
ssh alvin "nix copy --from 's3://nix-cache?endpoint=${{ inputs.s3-endpoint }}&scheme=https' '$STORE_PATH'"
|
S3_ACCESS_KEY="${{ env.S3_ACCESS_KEY }}"
|
||||||
|
S3_SECRET_KEY="${{ env.S3_SECRET_KEY }}"
|
||||||
|
|
||||||
# Now deploy Nomad job that mounts this specific store path
|
|
||||||
cat > /tmp/deploy-${{ inputs.site-name }}.nomad.json <<NOMAD_EOF
|
cat > /tmp/deploy-${{ inputs.site-name }}.nomad.json <<NOMAD_EOF
|
||||||
{
|
{
|
||||||
"Job": {
|
"Job": {
|
||||||
|
|
@ -106,25 +115,62 @@ runs:
|
||||||
"traefik.http.routers.${{ inputs.site-name }}.tls.certresolver=letsencrypt"
|
"traefik.http.routers.${{ inputs.site-name }}.tls.certresolver=letsencrypt"
|
||||||
]
|
]
|
||||||
}],
|
}],
|
||||||
"Tasks": [{
|
"Volumes": [{
|
||||||
"Name": "server",
|
"Name": "site-data",
|
||||||
"Driver": "docker",
|
"Type": "host",
|
||||||
"Config": {
|
"Source": "site-data"
|
||||||
"image": "joseluisq/static-web-server:2",
|
}],
|
||||||
"ports": ["http"],
|
"Tasks": [
|
||||||
"volumes": [
|
{
|
||||||
"$STORE_PATH:/var/www:ro"
|
"Name": "fetch",
|
||||||
]
|
"Driver": "docker",
|
||||||
|
"Lifecycle": {
|
||||||
|
"Hook": "prestart",
|
||||||
|
"Sidecar": false
|
||||||
|
},
|
||||||
|
"Config": {
|
||||||
|
"image": "nixos/nix:latest",
|
||||||
|
"command": "/bin/sh",
|
||||||
|
"args": [
|
||||||
|
"-c",
|
||||||
|
"nix copy --from 's3://nix-cache?endpoint=${{ inputs.s3-endpoint }}&scheme=https' '$STORE_PATH' && cp -r $STORE_PATH/* /alloc/data/"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"Env": {
|
||||||
|
"AWS_ACCESS_KEY_ID": "$S3_ACCESS_KEY",
|
||||||
|
"AWS_SECRET_ACCESS_KEY": "$S3_SECRET_KEY"
|
||||||
|
},
|
||||||
|
"VolumeMounts": [{
|
||||||
|
"Volume": "site-data",
|
||||||
|
"Destination": "/alloc/data"
|
||||||
|
}],
|
||||||
|
"Resources": {
|
||||||
|
"CPU": 200,
|
||||||
|
"MemoryMB": 256
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"Env": {
|
{
|
||||||
"SERVER_ROOT": "/var/www",
|
"Name": "server",
|
||||||
"SERVER_LOG_LEVEL": "info"
|
"Driver": "docker",
|
||||||
},
|
"Config": {
|
||||||
"Resources": {
|
"image": "joseluisq/static-web-server:2",
|
||||||
"CPU": 100,
|
"ports": ["http"]
|
||||||
"MemoryMB": 64
|
},
|
||||||
|
"Env": {
|
||||||
|
"SERVER_ROOT": "/var/www",
|
||||||
|
"SERVER_LOG_LEVEL": "info"
|
||||||
|
},
|
||||||
|
"VolumeMounts": [{
|
||||||
|
"Volume": "site-data",
|
||||||
|
"Destination": "/var/www",
|
||||||
|
"ReadOnly": true
|
||||||
|
}],
|
||||||
|
"Resources": {
|
||||||
|
"CPU": 100,
|
||||||
|
"MemoryMB": 64
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}]
|
]
|
||||||
}]
|
}]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue