feat: initialize monorepo with full dev team best practices

- Unified monorepo with backend (Express), frontend (Next.js), and devops - Backend: ESLint, Prettier, Jest tests (3 passing), health endpoint, .env.example - Frontend: Fixed build errors, fixed all lint errors (0 remaining), tests passing - DevOps: Docker Compose with PostgreSQL, backend, frontend + healthchecks - CI/CD: 3 GitHub Actions workflows (backend, frontend, docker integration) - DX: Husky pre-commit hooks with smart change detection - Docs: Root README with architecture, CONTRIBUTING.md, PR template Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-03-06 00:05:50 +01:00
commit 14a32bdc0d
241 changed files with 71273 additions and 0 deletions
@@ -0,0 +1,141 @@
+import { NextResponse } from "next/server";
+import { getSupabaseAdmin } from "@/lib/admin";
+import { NewCrawlerService } from "@/services/newCrawlerService";
+
+export async function POST(request: Request) {
+  try {
+    // Parse request body
+    let websiteId;
+    try {
+      const body = await request.json();
+      websiteId = body.websiteId;
+
+      // Normalize the websiteId to ensure consistent format
+      websiteId = String(websiteId).trim().toLowerCase();
+      console.log("Processing website ID:", websiteId);
+    } catch (parseError) {
+      console.error("Error parsing request body:", parseError);
+      return NextResponse.json(
+        { error: "Invalid request format" },
+        { status: 400 },
+      );
+    }
+
+    // Validate input
+    if (!websiteId) {
+      return NextResponse.json(
+        { error: "Website ID is required" },
+        { status: 400 },
+      );
+    }
+
+    // Fetch website details using admin client (bypasses RLS)
+    const { data: websites, error: websiteError } = await getSupabaseAdmin()
+      .from("websites")
+      .select("id, name, base_url")
+      .eq("id", websiteId);
+
+    if (websiteError) {
+      console.error("Website query error:", websiteError);
+      return NextResponse.json(
+        { error: `Database error: ${websiteError.message}` },
+        { status: 500 },
+      );
+    }
+
+    // Check if website exists
+    if (!websites || websites.length === 0) {
+      console.error("Website not found with ID:", websiteId);
+
+      // Try to find similar websites for debugging
+      const { data: allWebsites } = await getSupabaseAdmin()
+        .from("websites")
+        .select("id, name, base_url");
+
+      console.log(`Found ${allWebsites?.length || 0} total websites`);
+
+      // Log available IDs for comparison
+      const availableIds =
+        allWebsites?.map((w) => String(w.id).toLowerCase()) || [];
+      console.log("Available website IDs:", availableIds);
+
+      return NextResponse.json({ 
+        error: "Website not found",
+        debug: {
+          requestedId: websiteId,
+          availableIds: availableIds,
+          totalWebsites: allWebsites?.length || 0
+        }
+      }, { status: 404 });
+    }
+
+    // Website found, proceed with crawl
+    const website = websites[0];
+    console.log("Found website:", website.name, website.base_url);
+
+    // Create a new crawl session (using admin client)
+    const { data: sessions, error: sessionError } = await getSupabaseAdmin()
+      .from("crawl_sessions")
+      .insert([
+        {
+          website_id: website.id, // Use the ID from the database
+          status: "pending",
+          start_url: website.base_url,
+          total_urls: 0,
+          processed_urls: 0,
+          progress_percentage: 0,
+          pages_discovered: 0,
+          pages_processed: 0,
+        },
+      ])
+      .select();
+
+    if (sessionError) {
+      console.error("Failed to create crawl session:", sessionError);
+      console.error("Session error details:", {
+        message: sessionError.message,
+        details: sessionError.details,
+        hint: sessionError.hint,
+        code: sessionError.code
+      });
+      return NextResponse.json(
+        { 
+          error: "Failed to create crawl session",
+          details: sessionError.message,
+          code: sessionError.code
+        },
+        { status: 500 },
+      );
+    }
+
+    if (!sessions || sessions.length === 0) {
+      return NextResponse.json(
+        { error: "Session was created but not returned" },
+        { status: 500 },
+      );
+    }
+
+    const session = sessions[0] as { id: string };
+    console.log("Created crawl session:", session.id);
+
+    // Start crawler in background
+    const crawler = new NewCrawlerService(String(website.id), String(session.id));
+    crawler.startCrawl().catch((err) => {
+      console.error("Crawler error:", err);
+    });
+
+    // Return successful response
+    return NextResponse.json({
+      success: true,
+      message: "Crawl started",
+      sessionId: session.id,
+    });
+  } catch (error) {
+    // Catch-all error handler
+    console.error("Crawl initialization error:", error);
+    return NextResponse.json(
+      { error: "Failed to start crawl: " + (error) },
+      { status: 500 },
+    );
+  }
+}