import { NextResponse } from "next/server"; import { getSupabaseAdmin } from "@/lib/admin"; import { NewCrawlerService } from "@/services/newCrawlerService"; export async function POST(request: Request) { try { // Parse request body let websiteId; try { const body = await request.json(); websiteId = body.websiteId; // Normalize the websiteId to ensure consistent format websiteId = String(websiteId).trim().toLowerCase(); console.log("Processing website ID:", websiteId); } catch (parseError) { console.error("Error parsing request body:", parseError); return NextResponse.json( { error: "Invalid request format" }, { status: 400 }, ); } // Validate input if (!websiteId) { return NextResponse.json( { error: "Website ID is required" }, { status: 400 }, ); } // Fetch website details using admin client (bypasses RLS) const { data: websites, error: websiteError } = await getSupabaseAdmin() .from("websites") .select("id, name, base_url") .eq("id", websiteId); if (websiteError) { console.error("Website query error:", websiteError); return NextResponse.json( { error: `Database error: ${websiteError.message}` }, { status: 500 }, ); } // Check if website exists if (!websites || websites.length === 0) { console.error("Website not found with ID:", websiteId); // Try to find similar websites for debugging const { data: allWebsites } = await getSupabaseAdmin() .from("websites") .select("id, name, base_url"); console.log(`Found ${allWebsites?.length || 0} total websites`); // Log available IDs for comparison const availableIds = allWebsites?.map((w) => String(w.id).toLowerCase()) || []; console.log("Available website IDs:", availableIds); return NextResponse.json({ error: "Website not found", debug: { requestedId: websiteId, availableIds: availableIds, totalWebsites: allWebsites?.length || 0 } }, { status: 404 }); } // Website found, proceed with crawl const website = websites[0]; console.log("Found website:", website.name, website.base_url); // Create a new crawl session (using admin client) const { data: sessions, error: sessionError } = await getSupabaseAdmin() .from("crawl_sessions") .insert([ { website_id: website.id, // Use the ID from the database status: "pending", start_url: website.base_url, total_urls: 0, processed_urls: 0, progress_percentage: 0, pages_discovered: 0, pages_processed: 0, }, ]) .select(); if (sessionError) { console.error("Failed to create crawl session:", sessionError); console.error("Session error details:", { message: sessionError.message, details: sessionError.details, hint: sessionError.hint, code: sessionError.code }); return NextResponse.json( { error: "Failed to create crawl session", details: sessionError.message, code: sessionError.code }, { status: 500 }, ); } if (!sessions || sessions.length === 0) { return NextResponse.json( { error: "Session was created but not returned" }, { status: 500 }, ); } const session = sessions[0] as { id: string }; console.log("Created crawl session:", session.id); // Start crawler in background const crawler = new NewCrawlerService(String(website.id), String(session.id)); crawler.startCrawl().catch((err) => { console.error("Crawler error:", err); }); // Return successful response return NextResponse.json({ success: true, message: "Crawl started", sessionId: session.id, }); } catch (error) { // Catch-all error handler console.error("Crawl initialization error:", error); return NextResponse.json( { error: "Failed to start crawl: " + (error) }, { status: 500 }, ); } }