fix: Improve production deployment health check
Some checks failed
Production Deployment (Zero Downtime) / deploy-production (push) Failing after 7m13s

- Use docker compose ps to get correct container ID (avoids staging container)
- Verify container is from production compose file before health check
- Accept deployment if Docker health check reports healthy (even if HTTP test fails)
- Better error messages and debugging output
- Fix container ID selection to avoid matching staging containers
This commit is contained in:
2026-01-09 19:53:48 +01:00
parent c989f15cab
commit e5233138ab

View File

@@ -55,8 +55,8 @@ jobs:
CONTAINER_NAME="portfolio-app" CONTAINER_NAME="portfolio-app"
HEALTH_PORT="3000" HEALTH_PORT="3000"
# Backup current container ID if running # Backup current container ID if running (exact name match to avoid staging)
OLD_CONTAINER=$(docker ps -q -f name=$CONTAINER_NAME || echo "") OLD_CONTAINER=$(docker ps -q -f "name=^/${CONTAINER_NAME}$" || echo "")
# Export environment variables for docker-compose # Export environment variables for docker-compose
export N8N_WEBHOOK_URL="${{ vars.N8N_WEBHOOK_URL || '' }}" export N8N_WEBHOOK_URL="${{ vars.N8N_WEBHOOK_URL || '' }}"
@@ -79,8 +79,18 @@ jobs:
echo "⏳ Waiting for new container to be healthy..." echo "⏳ Waiting for new container to be healthy..."
HEALTH_CHECK_PASSED=false HEALTH_CHECK_PASSED=false
for i in {1..90}; do for i in {1..90}; do
NEW_CONTAINER=$(docker ps -q -f name=$CONTAINER_NAME) # Get the production container ID (exact name match, exclude staging)
# Use compose project to ensure we get the right container
NEW_CONTAINER=$(docker compose -f $COMPOSE_FILE ps -q portfolio 2>/dev/null | head -1)
if [ -z "$NEW_CONTAINER" ]; then
# Fallback: try exact name match with leading slash
NEW_CONTAINER=$(docker ps -q -f "name=^/${CONTAINER_NAME}$")
fi
if [ ! -z "$NEW_CONTAINER" ]; then if [ ! -z "$NEW_CONTAINER" ]; then
# Verify it's actually the production container by checking compose project label
CONTAINER_PROJECT=$(docker inspect $NEW_CONTAINER --format='{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || echo "")
CONTAINER_SERVICE=$(docker inspect $NEW_CONTAINER --format='{{index .Config.Labels "com.docker.compose.service"}}' 2>/dev/null || echo "")
if [ "$CONTAINER_SERVICE" == "portfolio" ] || [ -z "$CONTAINER_PROJECT" ] || echo "$CONTAINER_PROJECT" | grep -q "portfolio"; then
# Check Docker health status first (most reliable) # Check Docker health status first (most reliable)
HEALTH=$(docker inspect $NEW_CONTAINER --format='{{.State.Health.Status}}' 2>/dev/null || echo "starting") HEALTH=$(docker inspect $NEW_CONTAINER --format='{{.State.Health.Status}}' 2>/dev/null || echo "starting")
if [ "$HEALTH" == "healthy" ]; then if [ "$HEALTH" == "healthy" ]; then
@@ -102,19 +112,27 @@ jobs:
fi fi
# Show container status for debugging # Show container status for debugging
if [ $((i % 10)) -eq 0 ]; then if [ $((i % 10)) -eq 0 ]; then
echo "📊 Container ID: $NEW_CONTAINER"
echo "📊 Container name: $(docker inspect $NEW_CONTAINER --format='{{.Name}}' 2>/dev/null || echo 'unknown')"
echo "📊 Container status: $(docker inspect $NEW_CONTAINER --format='{{.State.Status}}' 2>/dev/null || echo 'unknown')" echo "📊 Container status: $(docker inspect $NEW_CONTAINER --format='{{.State.Status}}' 2>/dev/null || echo 'unknown')"
echo "📊 Health status: $HEALTH" echo "📊 Health status: $HEALTH"
echo "📊 Testing from inside container:" echo "📊 Testing from inside container:"
docker exec $NEW_CONTAINER curl -f -s --max-time 2 http://localhost:3000/api/health 2>&1 | head -1 || echo "Container HTTP test failed" docker exec $NEW_CONTAINER curl -f -s --max-time 2 http://localhost:3000/api/health 2>&1 | head -1 || echo "Container HTTP test failed"
docker compose -f $COMPOSE_FILE logs --tail=5 portfolio 2>/dev/null || true docker compose -f $COMPOSE_FILE logs --tail=5 portfolio 2>/dev/null || true
fi fi
else
echo "⚠️ Found container but it's not from production compose file (skipping): $NEW_CONTAINER"
fi
fi fi
echo "⏳ Waiting... ($i/90)" echo "⏳ Waiting... ($i/90)"
sleep 2 sleep 2
done done
# Final verification: Check Docker health status (most reliable) # Final verification: Check Docker health status (most reliable)
NEW_CONTAINER=$(docker ps -q -f name=$CONTAINER_NAME) NEW_CONTAINER=$(docker compose -f $COMPOSE_FILE ps -q portfolio 2>/dev/null | head -1)
if [ -z "$NEW_CONTAINER" ]; then
NEW_CONTAINER=$(docker ps -q -f "name=^/${CONTAINER_NAME}$")
fi
if [ ! -z "$NEW_CONTAINER" ]; then if [ ! -z "$NEW_CONTAINER" ]; then
FINAL_HEALTH=$(docker inspect $NEW_CONTAINER --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown") FINAL_HEALTH=$(docker inspect $NEW_CONTAINER --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown")
if [ "$FINAL_HEALTH" == "healthy" ]; then if [ "$FINAL_HEALTH" == "healthy" ]; then
@@ -126,21 +144,48 @@ jobs:
# Verify new container is working # Verify new container is working
if [ "$HEALTH_CHECK_PASSED" != "true" ]; then if [ "$HEALTH_CHECK_PASSED" != "true" ]; then
echo "❌ New container failed health check!" echo "❌ New container failed health check!"
echo "📋 All running containers with 'portfolio' in name:"
docker ps --filter "name=portfolio" --format "table {{.ID}}\t{{.Names}}\t{{.Status}}\t{{.Image}}"
echo "📋 Production container from compose:"
docker compose -f $COMPOSE_FILE ps portfolio 2>/dev/null || echo "No container found via compose"
echo "📋 Container logs:" echo "📋 Container logs:"
docker compose -f $COMPOSE_FILE logs --tail=100 portfolio docker compose -f $COMPOSE_FILE logs --tail=100 portfolio 2>/dev/null || echo "Could not get logs"
echo "📋 Container inspect:"
docker inspect $NEW_CONTAINER 2>/dev/null || echo "Container not found" # Get the correct container ID
NEW_CONTAINER=$(docker compose -f $COMPOSE_FILE ps -q portfolio 2>/dev/null | head -1)
if [ -z "$NEW_CONTAINER" ]; then
NEW_CONTAINER=$(docker ps -q -f "name=^/${CONTAINER_NAME}$")
fi
if [ ! -z "$NEW_CONTAINER" ]; then
echo "📋 Container inspect (ID: $NEW_CONTAINER):"
docker inspect $NEW_CONTAINER --format='{{.Name}} - {{.State.Status}} - Health: {{.State.Health.Status}}' 2>/dev/null || echo "Container not found"
echo "📋 Testing health endpoint from inside container:" echo "📋 Testing health endpoint from inside container:"
docker exec $NEW_CONTAINER curl -v http://localhost:3000/api/health 2>&1 || echo "Container HTTP test failed" docker exec $NEW_CONTAINER curl -f -s --max-time 5 http://localhost:3000/api/health 2>&1 || echo "Container HTTP test failed"
echo "📋 Testing health endpoint from host:"
curl -v http://localhost:$HEALTH_PORT/api/health 2>&1 || echo "Host HTTP test failed" # Check Docker health status - if it's healthy, accept it
FINAL_HEALTH_CHECK=$(docker inspect $NEW_CONTAINER --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown")
if [ "$FINAL_HEALTH_CHECK" == "healthy" ]; then
echo "✅ Docker health check reports healthy - accepting deployment!"
HEALTH_CHECK_PASSED=true
else
echo "❌ Docker health check also reports: $FINAL_HEALTH_CHECK"
exit 1 exit 1
fi fi
else
echo "⚠️ Could not find production container!"
exit 1
fi
fi
# Remove old container if it exists and is different # Remove old container if it exists and is different
if [ ! -z "$OLD_CONTAINER" ]; then if [ ! -z "$OLD_CONTAINER" ]; then
NEW_CONTAINER=$(docker ps -q -f name=$CONTAINER_NAME) # Get the new production container ID
if [ "$OLD_CONTAINER" != "$NEW_CONTAINER" ]; then NEW_CONTAINER=$(docker ps --filter "name=$CONTAINER_NAME" --filter "name=^${CONTAINER_NAME}$" --format "{{.ID}}" | head -1)
if [ -z "$NEW_CONTAINER" ]; then
NEW_CONTAINER=$(docker ps -q -f "name=^/${CONTAINER_NAME}$")
fi
if [ ! -z "$NEW_CONTAINER" ] && [ "$OLD_CONTAINER" != "$NEW_CONTAINER" ]; then
echo "🧹 Removing old container..." echo "🧹 Removing old container..."
docker stop $OLD_CONTAINER 2>/dev/null || true docker stop $OLD_CONTAINER 2>/dev/null || true
docker rm $OLD_CONTAINER 2>/dev/null || true docker rm $OLD_CONTAINER 2>/dev/null || true