denshooter/cloudlense
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
1c545c93b4f15659de8fdd0093f9bfe5b54e9078
cloudlense/website-monitoring-frontend/setup-database.sql
T
Dennis 1c545c93b4 feat: production hardening + smart subpage scanning with layout dedup 
Security:
- Add CRON_SECRET auth to /api/cron/* endpoints
- Add admin role verification to /api/admin/* routes
- Add org membership check to /api/billing/usage
- Add security headers (HSTS, X-Frame-Options, CSP, etc.)
- Add env variable validation at startup
- Add rate limiting to backend API (30 req/min per IP)

Infrastructure:
- Multi-stage Dockerfiles with non-root user + healthchecks
- Updated cron workflow to pass CRON_SECRET header
- Updated .env.example with all optional vars

Smart subpage scanning:
- Crawler now computes template_hash (DOM structure without content)
- Scanner scans ALL unique-layout pages, not just main page
- Pages with same layout (e.g. product pages) scanned only once
- Deduplication by template_hash, fallback to content_hash
- Main page always scanned with high priority
- Re-checks subscription limits before each page scan

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-03-06 07:44:32 +01:00

547 lines
19 KiB
SQL
Raw Blame History

-- Website Monitoring Frontend - Database Setup Script
-- Run this in your Supabase SQL editor to create all required tables
-- Enable necessary extensions
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
CREATE EXTENSION IF NOT EXISTS "pg_stat_statements";
CREATE EXTENSION IF NOT EXISTS "pg_trgm";
------ ENUMS ------
-- Core enums for status and types
DO $$ BEGIN
CREATE TYPE scan_status AS ENUM (
'pending',
'queued',
'running',
'completed',
'failed',
'cancelled'
);
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE severity_level AS ENUM (
'critical',
'high',
'medium',
'low',
'info'
);
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE comparison_operator AS ENUM (
'less_than',
'less_than_equal',
'greater_than',
'greater_than_equal',
'equal_to',
'not_equal_to'
);
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE metric_category AS ENUM (
'performance',
'seo',
'accessibility',
'best_practices',
'security',
'pwa'
);
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE resource_type AS ENUM (
'script',
'stylesheet',
'image',
'font',
'document',
'media',
'other'
);
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE notification_channel AS ENUM (
'email',
'slack',
'webhook',
'in_app'
);
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE subscription_tier AS ENUM (
'free',
'starter',
'professional',
'enterprise'
);
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
DO $$ BEGIN
CREATE TYPE user_role AS ENUM (
'owner',
'admin',
'editor',
'viewer'
);
EXCEPTION
WHEN duplicate_object THEN null;
END $$;
------ CORE TABLES ------
-- Organizations table (if not exists)
CREATE TABLE IF NOT EXISTS organizations (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
name VARCHAR NOT NULL,
subscription_tier subscription_tier DEFAULT 'free',
subscription_status VARCHAR DEFAULT 'active',
billing_email VARCHAR,
max_websites INTEGER DEFAULT 5,
max_users INTEGER DEFAULT 3,
scan_frequency_minutes INTEGER DEFAULT 60,
settings JSONB DEFAULT '{
"alert_email_digest": "daily",
"default_scan_depth": 3,
"retention_days": 90,
"enable_competitor_analysis": false
}'::jsonb,
metadata JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
-- Users table (if not exists)
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
email VARCHAR UNIQUE NOT NULL,
name VARCHAR,
organization_id UUID REFERENCES organizations(id),
role user_role DEFAULT 'viewer',
is_active BOOLEAN DEFAULT true,
last_login_at TIMESTAMPTZ,
settings JSONB DEFAULT '{
"email_notifications": true,
"notification_frequency": "instant",
"dashboard_layout": "default"
}'::jsonb,
metadata JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
-- Websites table (if not exists)
CREATE TABLE IF NOT EXISTS websites (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
organization_id UUID REFERENCES organizations(id) NOT NULL,
base_url VARCHAR NOT NULL,
name VARCHAR NOT NULL,
is_active BOOLEAN DEFAULT true,
crawl_settings JSONB DEFAULT '{
"max_pages": 100,
"max_depth": 3,
"exclude_patterns": [
"/admin/*",
"/api/*",
"*.pdf",
"*.jpg",
"*.png"
],
"include_patterns": ["/*"],
"respect_robots_txt": true,
"crawl_frequency": "daily",
"crawl_timing": "off_peak"
}'::jsonb,
scan_schedule JSONB DEFAULT '{
"frequency": "hourly",
"time_windows": ["0-6", "20-23"],
"days": ["monday", "tuesday", "wednesday", "thursday", "friday"]
}'::jsonb,
performance_budgets JSONB DEFAULT '{
"page_weight_kb": 1000,
"max_requests": 100,
"time_to_interactive_ms": 3000,
"first_contentful_paint_ms": 1000
}'::jsonb,
notifications JSONB DEFAULT '{
"channels": ["email"],
"thresholds": {
"performance": 90,
"accessibility": 90,
"seo": 90,
"best_practices": 90
}
}'::jsonb,
last_crawl_at TIMESTAMPTZ,
last_scan_at TIMESTAMPTZ,
metadata JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(organization_id, base_url)
);
-- Pages table (MISSING - this is causing the 400 errors)
CREATE TABLE IF NOT EXISTS pages (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
website_id UUID REFERENCES websites(id) NOT NULL,
url VARCHAR NOT NULL,
path VARCHAR NOT NULL,
title VARCHAR,
description TEXT,
content_hash VARCHAR,
template_hash VARCHAR,
content_type VARCHAR,
status_code INTEGER,
is_active BOOLEAN DEFAULT true,
priority INTEGER DEFAULT 1,
depth INTEGER DEFAULT 0,
parent_page_id UUID REFERENCES pages(id),
discovery_method VARCHAR DEFAULT 'crawl',
last_seen_at TIMESTAMPTZ,
metadata JSONB DEFAULT '{
"inbound_links": 0,
"outbound_links": 0,
"word_count": 0,
"has_canonical": false,
"is_indexable": true
}'::jsonb,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(website_id, url)
);
------ METRIC DEFINITIONS ------
CREATE TABLE IF NOT EXISTS metric_definitions (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
key VARCHAR NOT NULL UNIQUE,
name VARCHAR NOT NULL,
description TEXT NOT NULL,
category metric_category NOT NULL,
unit VARCHAR,
is_core_metric BOOLEAN DEFAULT false,
default_threshold NUMERIC,
warning_threshold NUMERIC,
critical_threshold NUMERIC,
direction VARCHAR NOT NULL DEFAULT 'higher_is_better',
weight NUMERIC DEFAULT 1.0,
documentation_url TEXT,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Populate core metrics (if not already populated)
INSERT INTO metric_definitions
(key, name, description, category, unit, is_core_metric, default_threshold, warning_threshold, critical_threshold, direction)
VALUES
-- Core Web Vitals
('performance', 'Performance Score', 'Overall performance score of the website', 'performance', '%', true, 90, 80, 70, 'higher_is_better'),
('accessibility', 'Accessibility Score', 'Overall accessibility score of the website', 'accessibility', '%', true, 90, 80, 70, 'higher_is_better'),
('seo', 'SEO Score', 'Overall SEO score of the website', 'seo', '%', true, 90, 80, 70, 'higher_is_better'),
('bestPractices', 'Best Practices Score', 'Overall best practices score', 'best_practices', '%', true, 90, 80, 70, 'higher_is_better'),
-- Performance Metrics
('firstContentfulPaint', 'First Contentful Paint', 'Time when the first text or image is painted', 'performance', 'ms', true, 1800, 2500, 4000, 'lower_is_better'),
('largestContentfulPaint', 'Largest Contentful Paint', 'Time when the largest text or image is painted', 'performance', 'ms', true, 2500, 4000, 6000, 'lower_is_better'),
('totalBlockingTime', 'Total Blocking Time', 'Sum of all time periods between FCP and Time to Interactive', 'performance', 'ms', true, 200, 400, 600, 'lower_is_better'),
('cumulativeLayoutShift', 'Cumulative Layout Shift', 'Measures visual stability', 'performance', 'score', true, 0.1, 0.25, 0.4, 'lower_is_better'),
('speedIndex', 'Speed Index', 'How quickly content is visually displayed', 'performance', 'ms', true, 3400, 5800, 8800, 'lower_is_better'),
('interactive', 'Time to Interactive', 'Time to fully interactive', 'performance', 'ms', true, 3800, 7300, 12700, 'lower_is_better'),
-- Resource Metrics
('totalByteWeight', 'Total Byte Weight', 'Total size of all resources', 'performance', 'bytes', false, 1600000, 2400000, 3200000, 'lower_is_better'),
('serverResponseTime', 'Server Response Time', 'Time for server to respond to main document request', 'performance', 'ms', false, 100, 200, 400, 'lower_is_better'),
('networkRtt', 'Network Round Trip Time', 'Network round trip time', 'performance', 'ms', false, 40, 100, 150, 'lower_is_better'),
('networkServerLatency', 'Network Server Latency', 'Server latency in network requests', 'performance', 'ms', false, 30, 100, 150, 'lower_is_better')
ON CONFLICT (key) DO NOTHING;
------ SCANS AND RESULTS (MISSING - this is causing the 400 errors) ------
-- Scans table
CREATE TABLE IF NOT EXISTS scans (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
website_id UUID REFERENCES websites(id) NOT NULL,
page_id UUID REFERENCES pages(id) NOT NULL,
triggered_by UUID REFERENCES users(id),
scan_type VARCHAR NOT NULL DEFAULT 'full',
status scan_status DEFAULT 'pending',
priority INTEGER DEFAULT 1,
categories metric_category[] DEFAULT ARRAY['performance', 'seo', 'accessibility', 'best_practices'],
device_type VARCHAR DEFAULT 'desktop',
user_agent VARCHAR,
lighthouse_version VARCHAR,
chrome_version VARCHAR,
environment JSONB DEFAULT '{}'::jsonb,
scheduled_at TIMESTAMPTZ,
started_at TIMESTAMPTZ DEFAULT NOW(),
completed_at TIMESTAMPTZ,
duration_ms INTEGER,
error_message TEXT,
retry_count INTEGER DEFAULT 0,
metadata JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
-- Scan results table (MISSING - this is causing the 400 errors)
CREATE TABLE IF NOT EXISTS scan_results (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
scan_id UUID REFERENCES scans(id) NOT NULL,
category metric_category NOT NULL,
score NUMERIC,
raw_data JSONB,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Metric values table
CREATE TABLE IF NOT EXISTS metric_values (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
scan_id UUID REFERENCES scans(id) NOT NULL,
metric_id UUID REFERENCES metric_definitions(id) NOT NULL,
value NUMERIC NOT NULL,
raw_value VARCHAR,
unit VARCHAR,
is_passing BOOLEAN,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Resource analysis table
CREATE TABLE IF NOT EXISTS resource_analysis (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
scan_id UUID REFERENCES scans(id) NOT NULL,
resource_type resource_type NOT NULL,
url VARCHAR NOT NULL,
size_bytes INTEGER NOT NULL,
transfer_size_bytes INTEGER,
duration_ms INTEGER,
is_third_party BOOLEAN DEFAULT false,
is_cached BOOLEAN,
compression_ratio NUMERIC,
mime_type VARCHAR,
protocol VARCHAR,
priority VARCHAR,
status_code INTEGER,
metadata JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ DEFAULT NOW()
);
------ ALERTS (MISSING - this is causing the 400 errors) ------
-- Alert configurations
CREATE TABLE IF NOT EXISTS alert_configurations (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
website_id UUID REFERENCES websites(id) NOT NULL,
name VARCHAR NOT NULL,
description TEXT,
metric_id UUID REFERENCES metric_definitions(id) NOT NULL,
threshold NUMERIC NOT NULL,
comparison comparison_operator DEFAULT 'less_than',
severity severity_level DEFAULT 'medium',
consecutive_count INTEGER DEFAULT 1,
cooldown_minutes INTEGER DEFAULT 60,
notification_channels notification_channel[] DEFAULT ARRAY['email'],
notification_template TEXT,
is_active BOOLEAN DEFAULT true,
last_triggered_at TIMESTAMPTZ,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
-- Alerts table
CREATE TABLE IF NOT EXISTS alerts (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
website_id UUID REFERENCES websites(id) NOT NULL,
page_id UUID REFERENCES pages(id),
config_id UUID REFERENCES alert_configurations(id),
metric_id UUID REFERENCES metric_definitions(id),
severity severity_level DEFAULT 'medium',
title VARCHAR NOT NULL,
message TEXT NOT NULL,
details JSONB DEFAULT '{}'::jsonb,
status VARCHAR DEFAULT 'open',
acknowledged_by UUID REFERENCES users(id),
acknowledged_at TIMESTAMPTZ,
resolved_at TIMESTAMPTZ,
resolution_note TEXT,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
------ CRAWL MANAGEMENT ------
-- Crawl queue
CREATE TABLE IF NOT EXISTS crawl_queue (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
website_id UUID REFERENCES websites(id) NOT NULL,
url VARCHAR NOT NULL,
priority INTEGER DEFAULT 1,
status VARCHAR DEFAULT 'pending',
parent_url VARCHAR,
discovery_depth INTEGER DEFAULT 0,
attempts INTEGER DEFAULT 0,
last_attempt_at TIMESTAMPTZ,
next_attempt_at TIMESTAMPTZ,
error_message TEXT,
metadata JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
-- Crawl sessions
CREATE TABLE IF NOT EXISTS crawl_sessions (
id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
website_id UUID REFERENCES websites(id) NOT NULL,
status VARCHAR DEFAULT 'running',
pages_discovered INTEGER DEFAULT 0,
pages_processed INTEGER DEFAULT 0,
start_url VARCHAR NOT NULL,
max_depth INTEGER,
started_at TIMESTAMPTZ DEFAULT NOW(),
completed_at TIMESTAMPTZ,
error_message TEXT,
metadata JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ DEFAULT NOW()
);
------ INDEXES ------
-- Performance indexes
CREATE INDEX IF NOT EXISTS idx_scans_website_status ON scans(website_id, status);
CREATE INDEX IF NOT EXISTS idx_scans_created_at ON scans(created_at);
CREATE INDEX IF NOT EXISTS idx_scan_results_scan_id ON scan_results(scan_id);
CREATE INDEX IF NOT EXISTS idx_metric_values_scan_metric ON metric_values(scan_id, metric_id);
CREATE INDEX IF NOT EXISTS idx_pages_website_active ON pages(website_id, is_active);
CREATE INDEX IF NOT EXISTS idx_crawl_queue_status_priority ON crawl_queue(status, priority);
CREATE INDEX IF NOT EXISTS idx_alerts_website_status ON alerts(website_id, status);
CREATE INDEX IF NOT EXISTS idx_resource_analysis_scan ON resource_analysis(scan_id);
CREATE INDEX IF NOT EXISTS idx_metric_values_created_at ON metric_values(created_at);
CREATE INDEX IF NOT EXISTS idx_pages_url_trgm ON pages USING gin (url gin_trgm_ops);
------ SAMPLE DATA FOR TESTING ------
-- Insert a sample organization if none exists
INSERT INTO organizations (id, name, subscription_tier, subscription_status)
VALUES (
'00000000-0000-0000-0000-000000000001',
'Demo Organization',
'free',
'active'
)
ON CONFLICT (id) DO NOTHING;
-- Insert a sample website if none exists
INSERT INTO websites (id, organization_id, base_url, name, is_active)
VALUES (
'00000000-0000-0000-0000-000000000002',
'00000000-0000-0000-0000-000000000001',
'https://example.com',
'Example Website',
true
)
ON CONFLICT (id) DO NOTHING;
-- Insert a sample page if none exists
INSERT INTO pages (id, website_id, url, path, title, is_active)
VALUES (
'00000000-0000-0000-0000-000000000003',
'00000000-0000-0000-0000-000000000002',
'https://example.com',
'/',
'Example Homepage',
true
)
ON CONFLICT (id) DO NOTHING;
-- Insert a sample scan if none exists
INSERT INTO scans (id, website_id, page_id, status, scan_type, device_type)
VALUES (
'00000000-0000-0000-0000-000000000004',
'00000000-0000-0000-0000-000000000002',
'00000000-0000-0000-0000-000000000003',
'completed',
'full',
'desktop'
)
ON CONFLICT (id) DO NOTHING;
-- Insert sample scan results
INSERT INTO scan_results (scan_id, category, score, raw_data)
VALUES
('00000000-0000-0000-0000-000000000004', 'performance', 85, '{"firstContentfulPaint": 1200, "largestContentfulPaint": 2100}'),
('00000000-0000-0000-0000-000000000004', 'seo', 92, '{"metaDescription": true, "titleTag": true}'),
('00000000-0000-0000-0000-000000000004', 'accessibility', 88, '{"ariaLabels": 5, "contrastRatio": "4.5:1"}'),
('00000000-0000-0000-0000-000000000004', 'best_practices', 95, '{"usesHttps": true, "noConsoleErrors": true}')
ON CONFLICT DO NOTHING;
-- Insert sample metric values
INSERT INTO metric_values (scan_id, metric_id, value, unit, is_passing)
SELECT
'00000000-0000-0000-0000-000000000004',
md.id,
CASE md.key
WHEN 'performance' THEN 85
WHEN 'seo' THEN 92
WHEN 'accessibility' THEN 88
WHEN 'bestPractices' THEN 95
WHEN 'firstContentfulPaint' THEN 1200
WHEN 'largestContentfulPaint' THEN 2100
WHEN 'totalBlockingTime' THEN 150
WHEN 'cumulativeLayoutShift' THEN 0.05
ELSE 80
END,
md.unit,
CASE md.key
WHEN 'performance' THEN true
WHEN 'seo' THEN true
WHEN 'accessibility' THEN true
WHEN 'bestPractices' THEN true
WHEN 'firstContentfulPaint' THEN true
WHEN 'largestContentfulPaint' THEN true
WHEN 'totalBlockingTime' THEN true
WHEN 'cumulativeLayoutShift' THEN true
ELSE true
END
FROM metric_definitions md
WHERE md.key IN ('performance', 'seo', 'accessibility', 'bestPractices', 'firstContentfulPaint', 'largestContentfulPaint', 'totalBlockingTime', 'cumulativeLayoutShift')
ON CONFLICT DO NOTHING;
------ ROW LEVEL SECURITY ------
-- Enable RLS on all tables
ALTER TABLE organizations ENABLE ROW LEVEL SECURITY;
ALTER TABLE users ENABLE ROW LEVEL SECURITY;
ALTER TABLE websites ENABLE ROW LEVEL SECURITY;
ALTER TABLE pages ENABLE ROW LEVEL SECURITY;
ALTER TABLE scans ENABLE ROW LEVEL SECURITY;
ALTER TABLE scan_results ENABLE ROW LEVEL SECURITY;
ALTER TABLE metric_values ENABLE ROW LEVEL SECURITY;
ALTER TABLE alerts ENABLE ROW LEVEL SECURITY;
ALTER TABLE alert_configurations ENABLE ROW LEVEL SECURITY;
ALTER TABLE crawl_queue ENABLE ROW LEVEL SECURITY;
ALTER TABLE crawl_sessions ENABLE ROW LEVEL SECURITY;
-- Basic RLS policies (you may need to adjust these based on your auth setup)
CREATE POLICY "Enable read access for authenticated users" ON organizations FOR SELECT USING (true);
CREATE POLICY "Enable read access for authenticated users" ON users FOR SELECT USING (true);
CREATE POLICY "Enable read access for authenticated users" ON websites FOR SELECT USING (true);
CREATE POLICY "Enable read access for authenticated users" ON pages FOR SELECT USING (true);
CREATE POLICY "Enable read access for authenticated users" ON scans FOR SELECT USING (true);
CREATE POLICY "Enable read access for authenticated users" ON scan_results FOR SELECT USING (true);
CREATE POLICY "Enable read access for authenticated users" ON metric_values FOR SELECT USING (true);
CREATE POLICY "Enable read access for authenticated users" ON alerts FOR SELECT USING (true);
CREATE POLICY "Enable read access for authenticated users" ON alert_configurations FOR SELECT USING (true);
CREATE POLICY "Enable read access for authenticated users" ON crawl_queue FOR SELECT USING (true);
CREATE POLICY "Enable read access for authenticated users" ON crawl_sessions FOR SELECT USING (true);
-- Success message
SELECT 'Database setup completed successfully! All required tables have been created.' as status;
Reference in New Issue View Git Blame Copy Permalink