cloudlense/website-monitoring-frontend/databaseStructure.txt

-- Enable necessary extensions
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
CREATE EXTENSION IF NOT EXISTS "pgcrypto";
CREATE EXTENSION IF NOT EXISTS "pg_stat_statements";
CREATE EXTENSION IF NOT EXISTS "pg_trgm";

------ ENUMS ------
-- Core enums for status and types
CREATE TYPE scan_status AS ENUM (
    'pending',
    'queued',
    'running',
    'completed',
    'failed',
    'cancelled'
);

CREATE TYPE severity_level AS ENUM (
    'critical',
    'high',
    'medium',
    'low',
    'info'
);

CREATE TYPE comparison_operator AS ENUM (
    'less_than',
    'less_than_equal',
    'greater_than',
    'greater_than_equal',
    'equal_to',
    'not_equal_to'
);

CREATE TYPE metric_category AS ENUM (
    'performance',
    'seo',
    'accessibility',
    'best_practices',
    'security',
    'pwa'
);

CREATE TYPE resource_type AS ENUM (
    'script',
    'stylesheet',
    'image',
    'font',
    'document',
    'media',
    'other'
);

CREATE TYPE notification_channel AS ENUM (
    'email',
    'slack',
    'webhook',
    'in_app'
);

CREATE TYPE subscription_tier AS ENUM (
    'free',
    'starter',
    'professional',
    'enterprise'
);

CREATE TYPE user_role AS ENUM (
    'owner',
    'admin',
    'editor',
    'viewer'
);

------ CORE TABLES ------
-- Organizations table
CREATE TABLE organizations (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    name VARCHAR NOT NULL,
    subscription_tier subscription_tier DEFAULT 'free',
    subscription_status VARCHAR DEFAULT 'active',
    billing_email VARCHAR,
    max_websites INTEGER DEFAULT 5,
    max_users INTEGER DEFAULT 3,
    scan_frequency_minutes INTEGER DEFAULT 60,
    settings JSONB DEFAULT '{
        "alert_email_digest": "daily",
        "default_scan_depth": 3,
        "retention_days": 90,
        "enable_competitor_analysis": false
    }'::jsonb,
    metadata JSONB DEFAULT '{}'::jsonb,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Users table
CREATE TABLE users (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    email VARCHAR UNIQUE NOT NULL,
    name VARCHAR,
    organization_id UUID REFERENCES organizations(id),
    role user_role DEFAULT 'viewer',
    is_active BOOLEAN DEFAULT true,
    last_login_at TIMESTAMPTZ,
    settings JSONB DEFAULT '{
        "email_notifications": true,
        "notification_frequency": "instant",
        "dashboard_layout": "default"
    }'::jsonb,
    metadata JSONB DEFAULT '{}'::jsonb,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Websites table
CREATE TABLE websites (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    organization_id UUID REFERENCES organizations(id) NOT NULL,
    base_url VARCHAR NOT NULL,
    name VARCHAR NOT NULL,
    is_active BOOLEAN DEFAULT true,
    crawl_settings JSONB DEFAULT '{
        "max_pages": 100,
        "max_depth": 3,
        "exclude_patterns": [
            "/admin/*",
            "/api/*",
            "*.pdf",
            "*.jpg",
            "*.png"
        ],
        "include_patterns": ["/*"],
        "respect_robots_txt": true,
        "crawl_frequency": "daily",
        "crawl_timing": "off_peak"
    }'::jsonb,
    scan_schedule JSONB DEFAULT '{
        "frequency": "hourly",
        "time_windows": ["0-6", "20-23"],
        "days": ["monday", "tuesday", "wednesday", "thursday", "friday"]
    }'::jsonb,
    performance_budgets JSONB DEFAULT '{
        "page_weight_kb": 1000,
        "max_requests": 100,
        "time_to_interactive_ms": 3000,
        "first_contentful_paint_ms": 1000
    }'::jsonb,
    notifications JSONB DEFAULT '{
        "channels": ["email"],
        "thresholds": {
            "performance": 90,
            "accessibility": 90,
            "seo": 90,
            "best_practices": 90
        }
    }'::jsonb,
    last_crawl_at TIMESTAMPTZ,
    last_scan_at TIMESTAMPTZ,
    metadata JSONB DEFAULT '{}'::jsonb,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW(),
    UNIQUE(organization_id, base_url)
);

-- Competitor tracking
CREATE TABLE competitor_websites (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    website_id UUID REFERENCES websites(id) NOT NULL,
    competitor_url VARCHAR NOT NULL,
    name VARCHAR NOT NULL,
    is_active BOOLEAN DEFAULT true,
    scan_frequency VARCHAR DEFAULT 'daily',
    metrics_to_track VARCHAR[] DEFAULT ARRAY[
        'performance',
        'seo',
        'accessibility',
        'best_practices'
    ],
    last_scan_at TIMESTAMPTZ,
    metadata JSONB DEFAULT '{}'::jsonb,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW(),
    UNIQUE(website_id, competitor_url)
);

-- Pages table
CREATE TABLE pages (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    website_id UUID REFERENCES websites(id) NOT NULL,
    url VARCHAR NOT NULL,
    path VARCHAR NOT NULL,
    title VARCHAR,
    description TEXT,
    content_hash VARCHAR,
    content_type VARCHAR,
    status_code INTEGER,
    is_active BOOLEAN DEFAULT true,
    priority INTEGER DEFAULT 1,
    depth INTEGER DEFAULT 0,
    parent_page_id UUID REFERENCES pages(id),
    discovery_method VARCHAR DEFAULT 'crawl',
    last_seen_at TIMESTAMPTZ,
    metadata JSONB DEFAULT '{
        "inbound_links": 0,
        "outbound_links": 0,
        "word_count": 0,
        "has_canonical": false,
        "is_indexable": true
    }'::jsonb,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW(),
    UNIQUE(website_id, url)
);

------ METRIC DEFINITIONS AND THRESHOLDS ------
CREATE TABLE metric_definitions (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    key VARCHAR NOT NULL UNIQUE,
    name VARCHAR NOT NULL,
    description TEXT NOT NULL,
    category metric_category NOT NULL,
    unit VARCHAR,
    is_core_metric BOOLEAN DEFAULT false,
    default_threshold NUMERIC,
    warning_threshold NUMERIC,
    critical_threshold NUMERIC,
    direction VARCHAR NOT NULL DEFAULT 'higher_is_better',
    weight NUMERIC DEFAULT 1.0,
    documentation_url TEXT,
    created_at TIMESTAMPTZ DEFAULT NOW()
);

-- Populate core metrics
INSERT INTO metric_definitions
(key, name, description, category, unit, is_core_metric, default_threshold, warning_threshold, critical_threshold, direction)
VALUES
-- Core Web Vitals
('performance', 'Performance Score', 'Overall performance score of the website', 'performance', '%', true, 90, 80, 70, 'higher_is_better'),
('accessibility', 'Accessibility Score', 'Overall accessibility score of the website', 'accessibility', '%', true, 90, 80, 70, 'higher_is_better'),
('seo', 'SEO Score', 'Overall SEO score of the website', 'seo', '%', true, 90, 80, 70, 'higher_is_better'),
('bestPractices', 'Best Practices Score', 'Overall best practices score', 'best_practices', '%', true, 90, 80, 70, 'higher_is_better'),

-- Performance Metrics
('firstContentfulPaint', 'First Contentful Paint', 'Time when the first text or image is painted', 'performance', 'ms', true, 1800, 2500, 4000, 'lower_is_better'),
('largestContentfulPaint', 'Largest Contentful Paint', 'Time when the largest text or image is painted', 'performance', 'ms', true, 2500, 4000, 6000, 'lower_is_better'),
('totalBlockingTime', 'Total Blocking Time', 'Sum of all time periods between FCP and Time to Interactive', 'performance', 'ms', true, 200, 400, 600, 'lower_is_better'),
('cumulativeLayoutShift', 'Cumulative Layout Shift', 'Measures visual stability', 'performance', 'score', true, 0.1, 0.25, 0.4, 'lower_is_better'),
('speedIndex', 'Speed Index', 'How quickly content is visually displayed', 'performance', 'ms', true, 3400, 5800, 8800, 'lower_is_better'),
('interactive', 'Time to Interactive', 'Time to fully interactive', 'performance', 'ms', true, 3800, 7300, 12700, 'lower_is_better'),

-- Resource Metrics
('totalByteWeight', 'Total Byte Weight', 'Total size of all resources', 'performance', 'bytes', false, 1600000, 2400000, 3200000, 'lower_is_better'),
('serverResponseTime', 'Server Response Time', 'Time for server to respond to main document request', 'performance', 'ms', false, 100, 200, 400, 'lower_is_better'),
('networkRtt', 'Network Round Trip Time', 'Network round trip time', 'performance', 'ms', false, 40, 100, 150, 'lower_is_better'),
('networkServerLatency', 'Network Server Latency', 'Server latency in network requests', 'performance', 'ms', false, 30, 100, 150, 'lower_is_better');

------ SCANS AND RESULTS ------
-- Scans table
CREATE TABLE scans (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    website_id UUID REFERENCES websites(id) NOT NULL,
    page_id UUID REFERENCES pages(id) NOT NULL,
    triggered_by UUID REFERENCES users(id),
    scan_type VARCHAR NOT NULL DEFAULT 'full',
    status scan_status DEFAULT 'pending',
    priority INTEGER DEFAULT 1,
    categories metric_category[] DEFAULT ARRAY['performance', 'seo', 'accessibility', 'best_practices'],
    device_type VARCHAR DEFAULT 'desktop',
    user_agent VARCHAR,
    lighthouse_version VARCHAR,
    chrome_version VARCHAR,
    environment JSONB DEFAULT '{}'::jsonb,
    started_at TIMESTAMPTZ DEFAULT NOW(),
    completed_at TIMESTAMPTZ,
    duration_ms INTEGER,
    error_message TEXT,
    retry_count INTEGER DEFAULT 0,
    metadata JSONB DEFAULT '{}'::jsonb,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Scan results
CREATE TABLE scan_results (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    scan_id UUID REFERENCES scans(id) NOT NULL,
    category metric_category NOT NULL,
    score NUMERIC,
    raw_data JSONB,
    created_at TIMESTAMPTZ DEFAULT NOW()
);

-- Metric values
CREATE TABLE metric_values (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    scan_id UUID REFERENCES scans(id) NOT NULL,
    metric_id UUID REFERENCES metric_definitions(id) NOT NULL,
    value NUMERIC NOT NULL,
    raw_value VARCHAR,
    unit VARCHAR,
    is_passing BOOLEAN,
    created_at TIMESTAMPTZ DEFAULT NOW()
);

-- Resource analysis
CREATE TABLE resource_analysis (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    scan_id UUID REFERENCES scans(id) NOT NULL,
    resource_type resource_type NOT NULL,
    url VARCHAR NOT NULL,
    size_bytes INTEGER NOT NULL,
    transfer_size_bytes INTEGER,
    duration_ms INTEGER,
    is_third_party BOOLEAN DEFAULT false,
    is_cached BOOLEAN,
    compression_ratio NUMERIC,
    mime_type VARCHAR,
    protocol VARCHAR,
    priority VARCHAR,
    status_code INTEGER,
    metadata JSONB DEFAULT '{}'::jsonb,
    created_at TIMESTAMPTZ DEFAULT NOW()
);

------ MONITORING AND ALERTS ------
-- Alert configurations
CREATE TABLE alert_configurations (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    website_id UUID REFERENCES websites(id) NOT NULL,
    name VARCHAR NOT NULL,
    description TEXT,
    metric_id UUID REFERENCES metric_definitions(id) NOT NULL,
    threshold NUMERIC NOT NULL,
    comparison comparison_operator DEFAULT 'less_than',
    severity severity_level DEFAULT 'medium',
    consecutive_count INTEGER DEFAULT 1,
    cooldown_minutes INTEGER DEFAULT 60,
    notification_channels notification_channel[] DEFAULT ARRAY['email'],
    notification_template TEXT,
    is_active BOOLEAN DEFAULT true,
    last_triggered_at TIMESTAMPTZ,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Alerts
CREATE TABLE alerts (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    website_id UUID REFERENCES websites(id) NOT NULL,
    page_id UUID REFERENCES pages(id),
    config_id UUID REFERENCES alert_configurations(id),
    metric_id UUID REFERENCES metric_definitions(id),
    severity severity_level DEFAULT 'medium',
    title VARCHAR NOT NULL,
    message TEXT NOT NULL,
    details JSONB DEFAULT '{}'::jsonb,
    status VARCHAR DEFAULT 'open',
    acknowledged_by UUID REFERENCES users(id),
    acknowledged_at TIMESTAMPTZ,
    resolved_at TIMESTAMPTZ,
    resolution_note TEXT,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Alert history
CREATE TABLE alert_history (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    alert_id UUID REFERENCES alerts(id) NOT NULL,
    event_type VARCHAR NOT NULL,
    event_data JSONB NOT NULL,
    created_by UUID REFERENCES users(id),
    created_at TIMESTAMPTZ DEFAULT NOW()
);

-- Notification delivery
CREATE TABLE notification_deliveries (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    alert_id UUID REFERENCES alerts(id) NOT NULL,
    channel notification_channel NOT NULL,
    recipient VARCHAR NOT NULL,
    content TEXT NOT NULL,
    status VARCHAR DEFAULT 'pending',
    sent_at TIMESTAMPTZ,
    error_message TEXT,
    retry_count INTEGER DEFAULT 0,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

------ CRAWL MANAGEMENT ------
-- Crawl queue
CREATE TABLE crawl_queue (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    website_id UUID REFERENCES websites(id) NOT NULL,
    url VARCHAR NOT NULL,
    priority INTEGER DEFAULT 1,
    status VARCHAR DEFAULT 'pending',
    parent_url VARCHAR,
    discovery_depth INTEGER DEFAULT 0,
    attempts INTEGER DEFAULT 0,
    last_attempt_at TIMESTAMPTZ,
    next_attempt_at TIMESTAMPTZ,
    error_message TEXT,
    metadata JSONB DEFAULT '{}'::jsonb,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Crawl sessions
CREATE TABLE crawl_sessions (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    website_id UUID REFERENCES websites(id) NOT NULL,
    status VARCHAR DEFAULT 'running',
    pages_discovered INTEGER DEFAULT 0,
    pages_processed INTEGER DEFAULT 0,
    start_url VARCHAR NOT NULL,
    max_depth INTEGER,
    started_at TIMESTAMPTZ DEFAULT NOW(),
    completed_at TIMESTAMPTZ,
    error_message TEXT,
    metadata JSONB DEFAULT '{}'::jsonb,
    created_at TIMESTAMPTZ DEFAULT NOW()
);

-- URL patterns
CREATE TABLE url_patterns (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    website_id UUID REFERENCES websites(id) NOT NULL,
    pattern VARCHAR NOT NULL,
    pattern_type VARCHAR NOT NULL, -- 'include' or 'exclude'
    description TEXT,
    is_regex BOOLEAN DEFAULT false,
    is_active BOOLEAN DEFAULT true,
    priority INTEGER DEFAULT 1,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

------ PERFORMANCE BUDGETS ------
CREATE TABLE performance_budgets (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    website_id UUID REFERENCES websites(id) NOT NULL,
    name VARCHAR NOT NULL,
    description TEXT,
    metric_id UUID REFERENCES metric_definitions(id),
    budget_type VARCHAR NOT NULL, -- 'size', 'timing', 'count'
    threshold NUMERIC NOT NULL,
    applies_to JSONB DEFAULT '{
        "resource_types": ["all"],
        "paths": ["/*"]
    }'::jsonb,
    is_active BOOLEAN DEFAULT true,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Budget violations
CREATE TABLE budget_violations (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    budget_id UUID REFERENCES performance_budgets(id) NOT NULL,
    scan_id UUID REFERENCES scans(id) NOT NULL,
    actual_value NUMERIC NOT NULL,
    threshold_value NUMERIC NOT NULL,
    percentage_over NUMERIC,
    created_at TIMESTAMPTZ DEFAULT NOW()
);

------ CUSTOM DASHBOARDS AND REPORTS ------
-- Dashboard definitions
CREATE TABLE dashboards (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    organization_id UUID REFERENCES organizations(id) NOT NULL,
    name VARCHAR NOT NULL,
    description TEXT,
    layout JSONB DEFAULT '[]'::jsonb,
    is_default BOOLEAN DEFAULT false,
    created_by UUID REFERENCES users(id),
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Dashboard widgets
CREATE TABLE dashboard_widgets (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    dashboard_id UUID REFERENCES dashboards(id) NOT NULL,
    widget_type VARCHAR NOT NULL,
    name VARCHAR NOT NULL,
    config JSONB NOT NULL,
    position JSONB NOT NULL,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Report templates
CREATE TABLE report_templates (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    organization_id UUID REFERENCES organizations(id) NOT NULL,
    name VARCHAR NOT NULL,
    description TEXT,
    template_type VARCHAR NOT NULL,
    content JSONB NOT NULL,
    schedule JSONB DEFAULT NULL,
    is_active BOOLEAN DEFAULT true,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Generated reports
CREATE TABLE generated_reports (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    template_id UUID REFERENCES report_templates(id) NOT NULL,
    website_id UUID REFERENCES websites(id),
    generated_by UUID REFERENCES users(id),
    report_data JSONB NOT NULL,
    format VARCHAR DEFAULT 'pdf',
    file_url VARCHAR,
    created_at TIMESTAMPTZ DEFAULT NOW()
);

------ API ACCESS AND RATE LIMITING ------
-- API keys
CREATE TABLE api_keys (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    organization_id UUID REFERENCES organizations(id) NOT NULL,
    name VARCHAR NOT NULL,
    key_hash VARCHAR NOT NULL,
    scopes VARCHAR[] DEFAULT ARRAY['read'],
    rate_limit_per_minute INTEGER DEFAULT 60,
    is_active BOOLEAN DEFAULT true,
    last_used_at TIMESTAMPTZ,
    expires_at TIMESTAMPTZ,
    created_by UUID REFERENCES users(id),
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

-- Rate limiting
CREATE TABLE rate_limits (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    key_type VARCHAR NOT NULL, -- 'api_key', 'ip_address'
    key_value VARCHAR NOT NULL,
    window_start TIMESTAMPTZ NOT NULL,
    request_count INTEGER DEFAULT 1,
    created_at TIMESTAMPTZ DEFAULT NOW(),
    updated_at TIMESTAMPTZ DEFAULT NOW()
);

------ AUDIT LOGGING ------
CREATE TABLE audit_logs (
    id UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
    organization_id UUID REFERENCES organizations(id),
    user_id UUID REFERENCES users(id),
    action VARCHAR NOT NULL,
    entity_type VARCHAR NOT NULL,
    entity_id UUID,
    changes JSONB,
    ip_address VARCHAR,
    user_agent VARCHAR,
    created_at TIMESTAMPTZ DEFAULT NOW()
);

------ VIEWS ------
-- Performance trend view
CREATE VIEW performance_trends AS
SELECT
    w.id AS website_id,
    w.name AS website_name,
    p.url AS page_url,
    m.key AS metric_key,
    mv.value AS metric_value,
    s.created_at AS scan_date,
    AVG(mv.value) OVER (
        PARTITION BY w.id, p.id, m.id
        ORDER BY s.created_at
        ROWS BETWEEN 6 PRECEDING AND CURRENT ROW
    ) AS rolling_average
FROM websites w
JOIN pages p ON p.website_id = w.id
JOIN scans s ON s.page_id = p.id
JOIN metric_values mv ON mv.scan_id = s.id
JOIN metric_definitions m ON m.id = mv.metric_id
WHERE s.created_at >= NOW() - INTERVAL '30 days';

-- Resource usage summary view
CREATE VIEW resource_usage_summary AS
SELECT
    w.id AS website_id,
    w.name AS website_name,
    ra.resource_type,
    COUNT(*) AS resource_count,
    AVG(ra.size_bytes) AS avg_size,
    SUM(ra.size_bytes) AS total_size,
    AVG(ra.duration_ms) AS avg_duration
FROM websites w
JOIN scans s ON s.website_id = w.id
JOIN resource_analysis ra ON ra.scan_id = s.id
WHERE s.created_at >= NOW() - INTERVAL '24 hours'
GROUP BY w.id, w.name, ra.resource_type;

------ FUNCTIONS ------
-- Calculate health score
CREATE OR REPLACE FUNCTION calculate_health_score(website_id UUID)
RETURNS NUMERIC AS $$
DECLARE
    score NUMERIC;
BEGIN
    SELECT
        AVG(
            CASE
                WHEN m.direction = 'higher_is_better' THEN
                    LEAST(mv.value / NULLIF(m.default_threshold, 0) * 100, 100)
                ELSE
                    LEAST(m.default_threshold / NULLIF(mv.value, 0) * 100, 100)
            END
        )
    INTO score
    FROM scans s
    JOIN metric_values mv ON mv.scan_id = s.id
    JOIN metric_definitions m ON m.id = mv.metric_id
    WHERE s.website_id = calculate_health_score.website_id
    AND s.created_at >= NOW() - INTERVAL '24 hours'
    AND m.is_core_metric = true;

    RETURN COALESCE(score, 0);
END;
$$ LANGUAGE plpgsql;

-- Update scan status
CREATE OR REPLACE FUNCTION update_scan_status()
RETURNS TRIGGER AS $$
BEGIN
    IF NEW.status = 'completed' THEN
        -- Update website's last scan timestamp
        UPDATE websites
        SET last_scan_at = NOW()
        WHERE id = NEW.website_id;

        -- Check for alerts
        INSERT INTO alerts (website_id, page_id, severity, title, message)
        SELECT
            NEW.website_id,
            NEW.page_id,
            ac.severity,
            'Metric threshold exceeded',
            format('%s is %s threshold of %s', m.name, ac.comparison, ac.threshold)
        FROM metric_values mv
        JOIN metric_definitions m ON m.id = mv.metric_id
        JOIN alert_configurations ac ON ac.metric_id = m.id
        WHERE mv.scan_id = NEW.id
        AND ac.website_id = NEW.website_id
        AND (
            CASE ac.comparison
                WHEN 'less_than' THEN mv.value < ac.threshold
                WHEN 'greater_than' THEN mv.value > ac.threshold
                WHEN 'equal_to' THEN mv.value = ac.threshold
            END
        );
    END IF;

    RETURN NEW;
END;
$$ LANGUAGE plpgsql;

-- Create trigger for scan status updates
CREATE TRIGGER scan_status_update
    AFTER UPDATE OF status ON scans
    FOR EACH ROW
    WHEN (OLD.status IS DISTINCT FROM NEW.status)
    EXECUTE FUNCTION update_scan_status();

    ------ INDEXES ------
-- Performance indexes
CREATE INDEX idx_scans_website_status ON scans(website_id, status);
CREATE INDEX idx_scans_created_at ON scans(created_at);
CREATE INDEX idx_metric_values_scan_metric ON metric_values(scan_id, metric_id);
CREATE INDEX idx_pages_website_active ON pages(website_id, is_active);
CREATE INDEX idx_crawl_queue_status_priority ON crawl_queue(status, priority);
CREATE INDEX idx_alerts_website_status ON alerts(website_id, status);
CREATE INDEX idx_resource_analysis_scan ON resource_analysis(scan_id);
CREATE INDEX idx_audit_logs_organization ON audit_logs(organization_id, created_at);
CREATE INDEX idx_metric_values_created_at ON metric_values(created_at);
CREATE INDEX idx_pages_url_trgm ON pages USING gin (url gin_trgm_ops);

------ SECURITY POLICIES ------
-- RLS Policies
ALTER TABLE organizations ENABLE ROW LEVEL SECURITY;
ALTER TABLE users ENABLE ROW LEVEL SECURITY;
ALTER TABLE websites ENABLE ROW LEVEL SECURITY;
ALTER TABLE pages ENABLE ROW LEVEL SECURITY;
ALTER TABLE scans ENABLE ROW LEVEL SECURITY;
ALTER TABLE metric_values ENABLE ROW LEVEL SECURITY;
ALTER TABLE alerts ENABLE ROW LEVEL SECURITY;
ALTER TABLE dashboards ENABLE ROW LEVEL SECURITY;
ALTER TABLE api_keys ENABLE ROW LEVEL SECURITY;

-- Organization access
CREATE POLICY "Users can view their organization"
    ON organizations
    FOR SELECT
    USING (id IN (
        SELECT organization_id FROM users WHERE id = auth.uid()
    ));

-- Website access
CREATE POLICY "Users can view their organization's websites"
    ON websites
    FOR SELECT
    USING (organization_id IN (
        SELECT organization_id FROM users WHERE id = auth.uid()
    ));

CREATE POLICY "Admins can manage their organization's websites"
    ON websites
    FOR ALL
    USING (
        organization_id IN (
            SELECT organization_id
            FROM users
            WHERE id = auth.uid() AND role IN ('admin', 'owner')
        )
    );

------ DATA RETENTION ------
-- Create retention policy function
CREATE OR REPLACE FUNCTION apply_data_retention()
RETURNS void AS $$
DECLARE
    org RECORD;
BEGIN
    -- Loop through organizations
    FOR org IN SELECT id, (settings->>'retention_days')::integer AS retention_days
               FROM organizations
               WHERE settings->>'retention_days' IS NOT NULL
    LOOP
        -- Delete old scan data
        DELETE FROM metric_values
        WHERE scan_id IN (
            SELECT id FROM scans
            WHERE website_id IN (
                SELECT id FROM websites WHERE organization_id = org.id
            )
            AND created_at < NOW() - (org.retention_days || ' days')::interval
        );

        -- Delete old resource analysis
        DELETE FROM resource_analysis
        WHERE scan_id IN (
            SELECT id FROM scans
            WHERE website_id IN (
                SELECT id FROM websites WHERE organization_id = org.id
            )
            AND created_at < NOW() - (org.retention_days || ' days')::interval
        );

        -- Delete old scans
        DELETE FROM scans
        WHERE website_id IN (
            SELECT id FROM websites WHERE organization_id = org.id
        )
        AND created_at < NOW() - (org.retention_days || ' days')::interval;

        -- Archive resolved alerts
        UPDATE alerts
        SET status = 'archived'
        WHERE website_id IN (
            SELECT id FROM websites WHERE organization_id = org.id
        )
        AND status = 'resolved'
        AND resolved_at < NOW() - (org.retention_days || ' days')::interval;
    END LOOP;
END;
$$ LANGUAGE plpgsql;

------ ADDITIONAL FUNCTIONS ------
-- Calculate competitor comparison
CREATE OR REPLACE FUNCTION calculate_competitor_comparison(website_id UUID)
RETURNS TABLE (
    metric_key VARCHAR,
    your_score NUMERIC,
    competitor_avg NUMERIC,
    competitor_best NUMERIC,
    percentile NUMERIC
) AS $$
BEGIN
    RETURN QUERY
    WITH your_metrics AS (
        SELECT
            m.key,
            mv.value
        FROM scans s
        JOIN metric_values mv ON mv.scan_id = s.id
        JOIN metric_definitions m ON m.id = mv.metric_id
        WHERE s.website_id = calculate_competitor_comparison.website_id
        AND s.created_at = (
            SELECT MAX(created_at)
            FROM scans
            WHERE website_id = calculate_competitor_comparison.website_id
        )
    ),
    competitor_metrics AS (
        SELECT
            m.key,
            mv.value,
            PERCENT_RANK() OVER (PARTITION BY m.key ORDER BY mv.value) as percentile
        FROM scans s
        JOIN metric_values mv ON mv.scan_id = s.id
        JOIN metric_definitions m ON m.id = mv.metric_id
        WHERE s.website_id IN (
            SELECT competitor_url_id
            FROM competitor_websites
            WHERE website_id = calculate_competitor_comparison.website_id
        )
        AND s.created_at >= NOW() - INTERVAL '30 days'
    )
    SELECT
        ym.key,
        ym.value as your_score,
        AVG(cm.value) as competitor_avg,
        MAX(cm.value) as competitor_best,
        MAX(cm.percentile) * 100 as percentile
    FROM your_metrics ym
    LEFT JOIN competitor_metrics cm ON cm.key = ym.key
    GROUP BY ym.key, ym.value;
END;
$$ LANGUAGE plpgsql;

-- Generate performance report
CREATE OR REPLACE FUNCTION generate_performance_report(website_id UUID, days INTEGER)
RETURNS JSONB AS $$
DECLARE
    report JSONB;
BEGIN
    SELECT jsonb_build_object(
        'website_info', (
            SELECT jsonb_build_object(
                'name', name,
                'url', base_url,
                'report_period', jsonb_build_object(
                    'start', NOW() - (days || ' days')::interval,
                    'end', NOW()
                )
            )
            FROM websites
            WHERE id = website_id
        ),
        'performance_summary', (
            SELECT jsonb_build_object(
                'average_performance_score', AVG(mv.value),
                'best_performance_score', MAX(mv.value),
                'worst_performance_score', MIN(mv.value),
                'trend', jsonb_agg(
                    jsonb_build_object(
                        'date', DATE(s.created_at),
                        'score', mv.value
                    ) ORDER BY s.created_at
                )
            )
            FROM scans s
            JOIN metric_values mv ON mv.scan_id = s.id
            JOIN metric_definitions md ON md.id = mv.metric_id
            WHERE s.website_id = generate_performance_report.website_id
            AND md.key = 'performance'
            AND s.created_at >= NOW() - (days || ' days')::interval
        ),
        'core_metrics', (
            SELECT jsonb_object_agg(
                md.key,
                jsonb_build_object(
                    'average', AVG(mv.value),
                    'best', MAX(mv.value),
                    'worst', MIN(mv.value)
                )
            )
            FROM scans s
            JOIN metric_values mv ON mv.scan_id = s.id
            JOIN metric_definitions md ON md.id = mv.metric_id
            WHERE s.website_id = generate_performance_report.website_id
            AND md.is_core_metric = true
            AND s.created_at >= NOW() - (days || ' days')::interval
        ),
        'resource_summary', (
            SELECT jsonb_object_agg(
                resource_type,
                jsonb_build_object(
                    'count', COUNT(*),
                    'total_size', SUM(size_bytes),
                    'average_duration', AVG(duration_ms)
                )
            )
            FROM scans s
            JOIN resource_analysis ra ON ra.scan_id = s.id
            WHERE s.website_id = generate_performance_report.website_id
            AND s.created_at >= NOW() - (days || ' days')::interval
        ),
        'alerts', (
            SELECT jsonb_agg(
                jsonb_build_object(
                    'severity', severity,
                    'message', message,
                    'created_at', created_at
                )
            )
            FROM alerts
            WHERE website_id = generate_performance_report.website_id
            AND created_at >= NOW() - (days || ' days')::interval
        )
    ) INTO report;

    RETURN report;
END;
$$ LANGUAGE plpgsql;

------ NOTIFICATIONS ------
-- Create notification function
CREATE OR REPLACE FUNCTION process_alert_notifications()
RETURNS trigger AS $$
DECLARE
    website_record RECORD;
    user_record RECORD;
BEGIN
    -- Get website details
    SELECT * INTO website_record
    FROM websites
    WHERE id = NEW.website_id;

    -- Insert notification for each user in the organization
    FOR user_record IN
        SELECT u.*
        FROM users u
        WHERE u.organization_id = website_record.organization_id
        AND (u.settings->>'email_notifications')::boolean = true
    LOOP
        INSERT INTO notification_deliveries (
            alert_id,
            channel,
            recipient,
            content
        ) VALUES (
            NEW.id,
            'email',
            user_record.email,
            format(
                'Alert for %s: %s. Severity: %s',
                website_record.name,
                NEW.message,
                NEW.severity
            )
        );
    END LOOP;

    RETURN NEW;
END;
$$ LANGUAGE plpgsql;

-- Create trigger for alert notifications
CREATE TRIGGER alert_notification_trigger
    AFTER INSERT ON alerts
    FOR EACH ROW
    EXECUTE FUNCTION process_alert_notifications();

------ MAINTENANCE PROCEDURES ------
-- Create maintenance function
CREATE OR REPLACE FUNCTION perform_maintenance()
RETURNS void AS $$
BEGIN
    -- Clean up old rate limit records
    DELETE FROM rate_limits
    WHERE window_start < NOW() - INTERVAL '1 day';

    -- Archive old notifications
    UPDATE notification_deliveries
    SET status = 'archived'
    WHERE created_at < NOW() - INTERVAL '30 days';

    -- Clean up expired API keys
    UPDATE api_keys
    SET is_active = false
    WHERE expires_at < NOW();

    -- Update statistics
    ANALYZE websites;
    ANALYZE scans;
    ANALYZE metric_values;
    ANALYZE resource_analysis;

    -- Vacuum analyze for better query planning
    VACUUM ANALYZE websites;
    VACUUM ANALYZE scans;
    VACUUM ANALYZE metric_values;
    VACUUM ANALYZE resource_analysis;
END;
$$ LANGUAGE plpgsql;