Files
cloudlense/website-monitoring-frontend/src/app/api/cron/scan/route.ts
T
Dennis 1c545c93b4 feat: production hardening + smart subpage scanning with layout dedup
Security:
- Add CRON_SECRET auth to /api/cron/* endpoints
- Add admin role verification to /api/admin/* routes
- Add org membership check to /api/billing/usage
- Add security headers (HSTS, X-Frame-Options, CSP, etc.)
- Add env variable validation at startup
- Add rate limiting to backend API (30 req/min per IP)

Infrastructure:
- Multi-stage Dockerfiles with non-root user + healthchecks
- Updated cron workflow to pass CRON_SECRET header
- Updated .env.example with all optional vars

Smart subpage scanning:
- Crawler now computes template_hash (DOM structure without content)
- Scanner scans ALL unique-layout pages, not just main page
- Pages with same layout (e.g. product pages) scanned only once
- Deduplication by template_hash, fallback to content_hash
- Main page always scanned with high priority
- Re-checks subscription limits before each page scan

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-03-06 07:44:32 +01:00

264 lines
8.5 KiB
TypeScript

import { NextResponse } from "next/server";
import { scanScheduler } from "@/services/scanScheduler";
import { lighthouseScanner } from "@/services/lighthouseScanner";
import { logError } from "@/utils/errorUtils";
import { verifyCronSecret } from "@/lib/apiAuth";
export async function GET(request: Request) {
const authError = verifyCronSecret(request);
if (authError) return authError;
try {
const url = new URL(request.url);
const mode = url.searchParams.get("mode") || "all"; // "scheduled", "change_detection", "all"
const organizationId = url.searchParams.get("organizationId"); // Optional: limit to specific org
console.info(JSON.stringify({ level: 'info', event: 'scan_process_start', mode, timestamp: new Date().toISOString() }));
const results = {
scheduledScans: 0,
changeDetectionScans: 0,
errors: [] as string[],
startTime: new Date().toISOString(),
};
// Process scheduled scans
if (mode === "scheduled" || mode === "all") {
try {
console.info(JSON.stringify({ level: 'info', event: 'processing_scheduled_scans', timestamp: new Date().toISOString() }));
await scanScheduler.processScheduledScans();
// Get count of processed scans
const scheduledScans = await scanScheduler.getScheduledScans();
results.scheduledScans = scheduledScans.length;
console.info(JSON.stringify({ level: 'info', event: 'scheduled_scans_processed', count: results.scheduledScans, timestamp: new Date().toISOString() }));
} catch (error) {
const errorMsg = `Error processing scheduled scans: ${error instanceof Error ? error.message : 'Unknown error'}`;
logError(errorMsg, error);
results.errors.push(errorMsg);
}
}
// Process change detection
if (mode === "change_detection" || mode === "all") {
try {
console.info(JSON.stringify({ level: 'info', event: 'processing_change_detection', timestamp: new Date().toISOString() }));
await scanScheduler.processChangeDetection();
// Note: Change detection count is harder to track since it's based on actual changes
// We'll just indicate it was processed
results.changeDetectionScans = -1; // -1 indicates processed but count unknown
console.info(JSON.stringify({ level: 'info', event: 'change_detection_processed', timestamp: new Date().toISOString() }));
} catch (error) {
const errorMsg = `Error processing change detection: ${error instanceof Error ? error.message : 'Unknown error'}`;
logError(errorMsg, error);
results.errors.push(errorMsg);
}
}
// Get overall statistics
const stats = await getScanStatistics(organizationId ?? undefined);
const response = {
success: results.errors.length === 0,
message: `Automatic scan process completed - ${results.scheduledScans} scheduled scans, change detection processed`,
results,
statistics: stats,
timestamp: new Date().toISOString(),
};
console.info(JSON.stringify({ level: 'info', event: 'scan_process_completed', success: response.success, timestamp: new Date().toISOString() }));
return NextResponse.json(response);
} catch (error) {
const errorMsg = `Critical error in automatic scan process: ${error instanceof Error ? error.message : 'Unknown error'}`;
logError(errorMsg, error);
return NextResponse.json(
{
success: false,
error: errorMsg,
timestamp: new Date().toISOString(),
},
{ status: 500 }
);
}
}
/**
* Get scan statistics for monitoring
*/
async function getScanStatistics(organizationId?: string) {
try {
const { getSupabaseAdmin } = await import("@/lib/admin");
const supabase = getSupabaseAdmin();
const now = new Date();
const startOfDay = new Date(now.getFullYear(), now.getMonth(), now.getDate());
const startOfMonth = new Date(now.getFullYear(), now.getMonth(), 1);
// Build query
let query = supabase
.from('scans')
.select('id, status, created_at, triggered_by');
if (organizationId) {
const { data: websitesForOrg, error: orgErr } = await supabase
.from('websites')
.select('id')
.eq('organization_id', organizationId);
if (orgErr) {
throw orgErr;
}
const websiteIds = (websitesForOrg || []).map((w: any) => w.id);
if (websiteIds.length === 0) {
return {
today: { total: 0, byStatus: {}, byTrigger: {} },
thisMonth: { total: 0 },
last24Hours: { total: 0 },
};
}
query = query.in('website_id', websiteIds as any[]);
}
// Get today's scans
const { data: todayScans } = await query
.gte('created_at', startOfDay.toISOString());
// Get this month's scans
const { data: monthScans } = await query
.gte('created_at', startOfMonth.toISOString());
// Get scans by status
const { data: statusCounts } = await query
.select('status') as unknown as { data: Array<{ status: string }> };
const statusBreakdown = (statusCounts?.reduce((acc: Record<string, number>, scan: { status: string }) => {
const key = String(scan.status || 'unknown');
acc[key] = (acc[key] || 0) + 1;
return acc;
}, {} as Record<string, number>)) || {};
// Get scans by trigger type
const { data: triggerCounts } = await query
.select('triggered_by') as unknown as { data: Array<{ triggered_by: string | null }> };
const triggerBreakdown = (triggerCounts?.reduce((acc: Record<string, number>, scan: { triggered_by: string | null }) => {
const trigger = String(scan.triggered_by || 'unknown');
acc[trigger] = (acc[trigger] || 0) + 1;
return acc;
}, {} as Record<string, number>)) || {};
return {
today: {
total: todayScans?.length || 0,
byStatus: statusBreakdown,
byTrigger: triggerBreakdown,
},
thisMonth: {
total: monthScans?.length || 0,
},
last24Hours: {
total: todayScans?.length || 0,
},
};
} catch (error) {
logError('Error getting scan statistics', error);
return {
today: { total: 0, byStatus: {}, byTrigger: {} },
thisMonth: { total: 0 },
last24Hours: { total: 0 },
};
}
}
/**
* Manual scan trigger endpoint
*/
export async function POST(request: Request) {
try {
const body = await request.json();
const { websiteId, pageId, deviceType = 'desktop', categories, priority = 'medium' } = body;
if (!websiteId || !pageId) {
return NextResponse.json(
{ error: "Website ID and Page ID are required" },
{ status: 400 }
);
}
console.info(JSON.stringify({ level: 'info', event: 'manual_scan_triggered', websiteId, pageId, timestamp: new Date().toISOString() }));
// Check subscription limits
const { data: website } = await (await import("@/lib/admin")).getSupabaseAdmin()
.from('websites')
.select('organization_id')
.eq('id', websiteId)
.single();
if (!website) {
return NextResponse.json(
{ error: "Website not found" },
{ status: 404 }
);
}
const { canScan, limits, currentUsage } = await lighthouseScanner.checkSubscriptionLimits(
String(website.organization_id)
);
if (!canScan) {
return NextResponse.json(
{
error: "Subscription limit exceeded",
limits,
currentUsage,
},
{ status: 429 }
);
}
// Perform the scan
const scanConfig = {
websiteId,
pageId,
deviceType: deviceType as 'desktop' | 'mobile',
categories: categories || ['performance', 'accessibility', 'seo', 'best_practices'],
priority: priority as 'low' | 'medium' | 'high',
triggeredBy: 'manual' as const,
};
const result = await lighthouseScanner.performScan(scanConfig);
if (result.success) {
return NextResponse.json({
success: true,
scanId: result.scanId,
message: "Scan completed successfully",
metrics: result.metrics,
});
} else {
return NextResponse.json(
{
success: false,
error: result.error,
},
{ status: 500 }
);
}
} catch (error) {
const errorMsg = `Error in manual scan: ${error instanceof Error ? error.message : 'Unknown error'}`;
logError(errorMsg, error);
return NextResponse.json(
{
success: false,
error: errorMsg,
},
{ status: 500 }
);
}
}