Deploying AI Applications

server {
    listen 443 ssl http2;
    server_name api.neurochat.io;

    ssl_certificate /etc/letsencrypt/live/api.neurochat.io/fullchain.pem;
    ssl_certificate_key /etc/letsencrypt/live/api.neurochat.io/privkey.pem;

    location /v1/chat/stream {
        proxy_pass http://fastapi_upstream;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        # ── CRITICAL FOR AI SSE STREAMING ──
        # Disable response buffering so tokens stream immediately to client
        proxy_buffering off;
        proxy_cache off;
        proxy_set_header Connection '';
        chunked_transfer_encoding on;

        # ── EXTENDED TIMEOUTS FOR DEEP REASONING MODELS (e.g. o1 / Claude) ──
        proxy_connect_timeout 180s;
        proxy_send_timeout    300s;
        proxy_read_timeout    300s;
        keepalive_timeout     300s;
    }
}

import asyncio
from contextlib import asynccontextmanager
from fastapi import FastAPI, Response, status
import asyncpg
import httpx

state = {}

@asynccontextmanager
async def lifespan(app: FastAPI):
    # ── 1. STARTUP: Warm pools & verify upstream credentials ──
    print("[STARTUP] Initializing async PostgreSQL connection pool...")
    state["db_pool"] = await asyncpg.create_pool("postgresql://app_user:prod_pass@pg-cluster:5432/neurochat_db")
    state["http_client"] = httpx.AsyncClient(timeout=60.0)
    state["ready"] = True
    print("[STARTUP] All pools initialized. Ready for traffic.")

    yield

    # ── 2. SHUTDOWN (SIGTERM received): Graceful drain ──
    print("[SHUTDOWN] SIGTERM caught. Draining in-flight AI streams...")
    state["ready"] = False  # Immediately fail readiness so LB stops sending new requests
    
    # Allow 15-second drain window for active LLM streaming responses to finish
    await asyncio.sleep(15)
    
    # Safely close persistent connection pools
    await state["db_pool"].close()
    await state["http_client"].aclose()
    print("[SHUTDOWN] Clean exit completed.")

app = FastAPI(lifespan=lifespan)

@app.get("/health/live", tags=["Probes"])
async def liveness():
    """Lightweight check: Is process event loop responsive?"""
    return {"status": "alive"}

@app.get("/health/ready", tags=["Probes"])
async def readiness(response: Response):
    """Deep check: Are DB pool and resources ready to accept traffic?"""
    if not state.get("ready"):
        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
        return {"status": "draining_or_unready"}
    try:
        async with state["db_pool"].acquire() as conn:
            await conn.execute("SELECT 1")
        return {"status": "ready", "database": "connected"}
    except Exception as e:
        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
        return {"status": "database_error", "detail": str(e)}

version: "3.9"

services:
  # ── 1. REVERSE PROXY & TLS INGRESS ──
  ingress:
    image: nginx:1.27-alpine
    restart: unless-stopped
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - ./nginx.prod.conf:/etc/nginx/conf.d/default.conf:ro
      - /etc/letsencrypt:/etc/letsencrypt:ro
    depends_on:
      api:
        condition: service_healthy

  # ── 2. STATELESS FASTAPI AI BACKEND (SCALABLE REPLICAS) ──
  api:
    image: ghcr.io/acme-org/neurochat-backend:v2.4.0
    restart: unless-stopped
    deploy:
      replicas: 3
    environment:
      - ENVIRONMENT=production
      - DATABASE_URL=postgresql://app_user:${DB_PASSWORD}@postgres:5432/neurochat_db
      - REDIS_URL=redis://redis:6379/0
      - S3_BUCKET=neurochat-prod-uploads
      - OPENAI_API_KEY=${OPENAI_API_KEY}
    resources:
      limits:
        cpus: '2.0'
        memory: 2048M
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health/ready')"]
      interval: 15s
      timeout: 5s
      retries: 3
      start_period: 20s

  # ── 3. ASYNC BACKGROUND WORKER (CHUNKING & EMBEDDINGS) ──
  worker:
    image: ghcr.io/acme-org/neurochat-backend:v2.4.0
    command: ["arq", "app.worker.WorkerSettings"]
    restart: unless-stopped
    environment:
      - ENVIRONMENT=production
      - DATABASE_URL=postgresql://app_user:${DB_PASSWORD}@postgres:5432/neurochat_db
      - REDIS_URL=redis://redis:6379/0
      - OPENAI_API_KEY=${OPENAI_API_KEY}
    resources:
      limits:
        cpus: '2.0'
        memory: 3072M

  # ── 4. PERSISTENT STORAGE: POSTGRESQL + PGVECTOR ──
  postgres:
    image: pgvector/pgvector:pg16
    restart: unless-stopped
    environment:
      POSTGRES_DB: neurochat_db
      POSTGRES_USER: app_user
      POSTGRES_PASSWORD: ${DB_PASSWORD}
    volumes:
      - pgdata:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U app_user -d neurochat_db"]
      interval: 10s
      timeout: 5s
      retries: 5

  # ── 5. ASYNC TASK QUEUE & CACHE ──
  redis:
    image: redis:7.2-alpine
    restart: unless-stopped
    command: ["redis-server", "--appendonly", "yes", "--maxmemory", "512mb", "--maxmemory-policy", "allkeys-lru"]
    volumes:
      - redisdata:/data

volumes:
  pgdata:
  redisdata: