-- Add new columns for improved crawl progress tracking
ALTER TABLE crawl_sessions 
ADD COLUMN IF NOT EXISTS total_urls INTEGER DEFAULT 0,
ADD COLUMN IF NOT EXISTS processed_urls INTEGER DEFAULT 0,
ADD COLUMN IF NOT EXISTS progress_percentage INTEGER DEFAULT 0,
ADD COLUMN IF NOT EXISTS current_url TEXT;

-- Update existing sessions to use new columns (map old data)
UPDATE crawl_sessions 
SET 
  total_urls = pages_discovered,
  processed_urls = pages_processed,
  progress_percentage = CASE 
    WHEN pages_discovered > 0 THEN ROUND((pages_processed::float / pages_discovered::float) * 100)
    ELSE 0 
  END
WHERE total_urls IS NULL OR total_urls = 0;

-- Add index for better performance
CREATE INDEX IF NOT EXISTS idx_crawl_sessions_status ON crawl_sessions(status);
CREATE INDEX IF NOT EXISTS idx_crawl_sessions_website_status ON crawl_sessions(website_id, status);