From e5233138ab27ecf49c74a4620a74b2822a89f2e7 Mon Sep 17 00:00:00 2001 From: denshooter Date: Fri, 9 Jan 2026 19:53:48 +0100 Subject: [PATCH] fix: Improve production deployment health check - Use docker compose ps to get correct container ID (avoids staging container) - Verify container is from production compose file before health check - Accept deployment if Docker health check reports healthy (even if HTTP test fails) - Better error messages and debugging output - Fix container ID selection to avoid matching staging containers --- .gitea/workflows/production-deploy.yml | 119 +++++++++++++++++-------- 1 file changed, 82 insertions(+), 37 deletions(-) diff --git a/.gitea/workflows/production-deploy.yml b/.gitea/workflows/production-deploy.yml index 02321f6..ec3e0f2 100644 --- a/.gitea/workflows/production-deploy.yml +++ b/.gitea/workflows/production-deploy.yml @@ -55,8 +55,8 @@ jobs: CONTAINER_NAME="portfolio-app" HEALTH_PORT="3000" - # Backup current container ID if running - OLD_CONTAINER=$(docker ps -q -f name=$CONTAINER_NAME || echo "") + # Backup current container ID if running (exact name match to avoid staging) + OLD_CONTAINER=$(docker ps -q -f "name=^/${CONTAINER_NAME}$" || echo "") # Export environment variables for docker-compose export N8N_WEBHOOK_URL="${{ vars.N8N_WEBHOOK_URL || '' }}" @@ -79,34 +79,49 @@ jobs: echo "⏳ Waiting for new container to be healthy..." HEALTH_CHECK_PASSED=false for i in {1..90}; do - NEW_CONTAINER=$(docker ps -q -f name=$CONTAINER_NAME) + # Get the production container ID (exact name match, exclude staging) + # Use compose project to ensure we get the right container + NEW_CONTAINER=$(docker compose -f $COMPOSE_FILE ps -q portfolio 2>/dev/null | head -1) + if [ -z "$NEW_CONTAINER" ]; then + # Fallback: try exact name match with leading slash + NEW_CONTAINER=$(docker ps -q -f "name=^/${CONTAINER_NAME}$") + fi if [ ! -z "$NEW_CONTAINER" ]; then - # Check Docker health status first (most reliable) - HEALTH=$(docker inspect $NEW_CONTAINER --format='{{.State.Health.Status}}' 2>/dev/null || echo "starting") - if [ "$HEALTH" == "healthy" ]; then - echo "✅ New container is healthy (Docker health check)!" - # Also verify HTTP endpoint from inside container - if docker exec $NEW_CONTAINER curl -f -s --max-time 5 http://localhost:3000/api/health > /dev/null 2>&1; then - echo "✅ Container HTTP endpoint is also responding!" + # Verify it's actually the production container by checking compose project label + CONTAINER_PROJECT=$(docker inspect $NEW_CONTAINER --format='{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || echo "") + CONTAINER_SERVICE=$(docker inspect $NEW_CONTAINER --format='{{index .Config.Labels "com.docker.compose.service"}}' 2>/dev/null || echo "") + if [ "$CONTAINER_SERVICE" == "portfolio" ] || [ -z "$CONTAINER_PROJECT" ] || echo "$CONTAINER_PROJECT" | grep -q "portfolio"; then + # Check Docker health status first (most reliable) + HEALTH=$(docker inspect $NEW_CONTAINER --format='{{.State.Health.Status}}' 2>/dev/null || echo "starting") + if [ "$HEALTH" == "healthy" ]; then + echo "✅ New container is healthy (Docker health check)!" + # Also verify HTTP endpoint from inside container + if docker exec $NEW_CONTAINER curl -f -s --max-time 5 http://localhost:3000/api/health > /dev/null 2>&1; then + echo "✅ Container HTTP endpoint is also responding!" + HEALTH_CHECK_PASSED=true + break + else + echo "⚠️ Docker health check passed, but HTTP endpoint test failed. Continuing..." + fi + fi + # Try HTTP health endpoint from host (may not work if port not mapped yet) + if curl -f -s --max-time 2 http://localhost:$HEALTH_PORT/api/health > /dev/null 2>&1; then + echo "✅ New container is responding to HTTP health check from host!" HEALTH_CHECK_PASSED=true break - else - echo "⚠️ Docker health check passed, but HTTP endpoint test failed. Continuing..." fi - fi - # Try HTTP health endpoint from host (may not work if port not mapped yet) - if curl -f -s --max-time 2 http://localhost:$HEALTH_PORT/api/health > /dev/null 2>&1; then - echo "✅ New container is responding to HTTP health check from host!" - HEALTH_CHECK_PASSED=true - break - fi - # Show container status for debugging - if [ $((i % 10)) -eq 0 ]; then - echo "📊 Container status: $(docker inspect $NEW_CONTAINER --format='{{.State.Status}}' 2>/dev/null || echo 'unknown')" - echo "📊 Health status: $HEALTH" - echo "📊 Testing from inside container:" - docker exec $NEW_CONTAINER curl -f -s --max-time 2 http://localhost:3000/api/health 2>&1 | head -1 || echo "Container HTTP test failed" - docker compose -f $COMPOSE_FILE logs --tail=5 portfolio 2>/dev/null || true + # Show container status for debugging + if [ $((i % 10)) -eq 0 ]; then + echo "📊 Container ID: $NEW_CONTAINER" + echo "📊 Container name: $(docker inspect $NEW_CONTAINER --format='{{.Name}}' 2>/dev/null || echo 'unknown')" + echo "📊 Container status: $(docker inspect $NEW_CONTAINER --format='{{.State.Status}}' 2>/dev/null || echo 'unknown')" + echo "📊 Health status: $HEALTH" + echo "📊 Testing from inside container:" + docker exec $NEW_CONTAINER curl -f -s --max-time 2 http://localhost:3000/api/health 2>&1 | head -1 || echo "Container HTTP test failed" + docker compose -f $COMPOSE_FILE logs --tail=5 portfolio 2>/dev/null || true + fi + else + echo "⚠️ Found container but it's not from production compose file (skipping): $NEW_CONTAINER" fi fi echo "⏳ Waiting... ($i/90)" @@ -114,7 +129,10 @@ jobs: done # Final verification: Check Docker health status (most reliable) - NEW_CONTAINER=$(docker ps -q -f name=$CONTAINER_NAME) + NEW_CONTAINER=$(docker compose -f $COMPOSE_FILE ps -q portfolio 2>/dev/null | head -1) + if [ -z "$NEW_CONTAINER" ]; then + NEW_CONTAINER=$(docker ps -q -f "name=^/${CONTAINER_NAME}$") + fi if [ ! -z "$NEW_CONTAINER" ]; then FINAL_HEALTH=$(docker inspect $NEW_CONTAINER --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown") if [ "$FINAL_HEALTH" == "healthy" ]; then @@ -126,21 +144,48 @@ jobs: # Verify new container is working if [ "$HEALTH_CHECK_PASSED" != "true" ]; then echo "❌ New container failed health check!" + echo "📋 All running containers with 'portfolio' in name:" + docker ps --filter "name=portfolio" --format "table {{.ID}}\t{{.Names}}\t{{.Status}}\t{{.Image}}" + echo "📋 Production container from compose:" + docker compose -f $COMPOSE_FILE ps portfolio 2>/dev/null || echo "No container found via compose" echo "📋 Container logs:" - docker compose -f $COMPOSE_FILE logs --tail=100 portfolio - echo "📋 Container inspect:" - docker inspect $NEW_CONTAINER 2>/dev/null || echo "Container not found" - echo "📋 Testing health endpoint from inside container:" - docker exec $NEW_CONTAINER curl -v http://localhost:3000/api/health 2>&1 || echo "Container HTTP test failed" - echo "📋 Testing health endpoint from host:" - curl -v http://localhost:$HEALTH_PORT/api/health 2>&1 || echo "Host HTTP test failed" - exit 1 + docker compose -f $COMPOSE_FILE logs --tail=100 portfolio 2>/dev/null || echo "Could not get logs" + + # Get the correct container ID + NEW_CONTAINER=$(docker compose -f $COMPOSE_FILE ps -q portfolio 2>/dev/null | head -1) + if [ -z "$NEW_CONTAINER" ]; then + NEW_CONTAINER=$(docker ps -q -f "name=^/${CONTAINER_NAME}$") + fi + + if [ ! -z "$NEW_CONTAINER" ]; then + echo "📋 Container inspect (ID: $NEW_CONTAINER):" + docker inspect $NEW_CONTAINER --format='{{.Name}} - {{.State.Status}} - Health: {{.State.Health.Status}}' 2>/dev/null || echo "Container not found" + echo "📋 Testing health endpoint from inside container:" + docker exec $NEW_CONTAINER curl -f -s --max-time 5 http://localhost:3000/api/health 2>&1 || echo "Container HTTP test failed" + + # Check Docker health status - if it's healthy, accept it + FINAL_HEALTH_CHECK=$(docker inspect $NEW_CONTAINER --format='{{.State.Health.Status}}' 2>/dev/null || echo "unknown") + if [ "$FINAL_HEALTH_CHECK" == "healthy" ]; then + echo "✅ Docker health check reports healthy - accepting deployment!" + HEALTH_CHECK_PASSED=true + else + echo "❌ Docker health check also reports: $FINAL_HEALTH_CHECK" + exit 1 + fi + else + echo "⚠️ Could not find production container!" + exit 1 + fi fi # Remove old container if it exists and is different if [ ! -z "$OLD_CONTAINER" ]; then - NEW_CONTAINER=$(docker ps -q -f name=$CONTAINER_NAME) - if [ "$OLD_CONTAINER" != "$NEW_CONTAINER" ]; then + # Get the new production container ID + NEW_CONTAINER=$(docker ps --filter "name=$CONTAINER_NAME" --filter "name=^${CONTAINER_NAME}$" --format "{{.ID}}" | head -1) + if [ -z "$NEW_CONTAINER" ]; then + NEW_CONTAINER=$(docker ps -q -f "name=^/${CONTAINER_NAME}$") + fi + if [ ! -z "$NEW_CONTAINER" ] && [ "$OLD_CONTAINER" != "$NEW_CONTAINER" ]; then echo "🧹 Removing old container..." docker stop $OLD_CONTAINER 2>/dev/null || true docker rm $OLD_CONTAINER 2>/dev/null || true