import requests
import time
import csv
import re
import os

# Define file paths
INPUT_CSV = "/home/staging67/public_html/domains_list.csv"
OUTPUT_CSV = "/home/staging67/public_html/security_headers_results.csv"
BACKUP_CSV = "/home/staging67/public_html/security_headers_results_backup.csv"

# Load domains from CSV
def load_domains():
    domains = []
    with open(INPUT_CSV, "r", encoding="utf-8") as file:
        reader = csv.reader(file)
        next(reader, None)  # Skip header if present
        for row in reader:
            if row:
                domains.append(row[0])  # Assuming the domain is in the first column
    return domains

# Check which domains are already processed
def load_previous_results():
    processed_domains = set()
    if os.path.exists(OUTPUT_CSV):
        with open(OUTPUT_CSV, "r", encoding="utf-8") as file:
            reader = csv.reader(file)
            next(reader, None)  # Skip header
            for row in reader:
                if len(row) >= 2:
                    processed_domains.add(row[0])  # Store completed domains
    return processed_domains

# Fetch Security Headers grade
def get_security_headers_grade(domain):
    url = f"https://securityheaders.com/?q={domain}&followRedirects=on"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"
    }

    try:
        response = requests.get(url, headers=headers, timeout=10)
        response.raise_for_status()
        
        # Extract grade from meta description
        match = re.search(r"scored the grade ([A-F])", response.text)
        if match:
            return match.group(1)
        else:
            return "Error"
    except requests.exceptions.RequestException as e:
        print(f"❌ Failed to fetch {domain}: {e}")
        return "Failed"

# Process domains with automatic saving and resume functionality
def process_domains():
    domains = load_domains()
    processed_domains = load_previous_results()
    results = []

    for index, domain in enumerate(domains):
        if domain in processed_domains:
            print(f"⏭️ Skipping {domain} (already processed)")
            continue  # Skip domains that were already completed

        print(f"🌐 Checking {domain} ({index+1}/{len(domains)})...")
        grade = get_security_headers_grade(domain)
        print(f"✅ Checked {domain} – Grade: {grade}")
        results.append([domain, grade])

        # Save progress every 100 records
        if (index + 1) % 100 == 0:
            print("💾 Saving progress (Checkpoint)...")
            save_results(results)
            results.clear()  # Clear memory after saving

        # Rate limiting: Wait 5 seconds after every 10 requests
        if (index + 1) % 10 == 0:
            print("⏳ Pausing to avoid rate limits...")
            time.sleep(5)

    # Final save after processing all records
    print("\n✅ Final save...")
    save_results(results)
    print(f"✅ Done! Results saved to {OUTPUT_CSV}")

# Save results to CSV (Checkpoint system)
def save_results(data):
    write_mode = "a" if os.path.exists(OUTPUT_CSV) else "w"
    
    with open(OUTPUT_CSV, write_mode, newline="", encoding="utf-8") as file:
        writer = csv.writer(file)
        if write_mode == "w":
            writer.writerow(["Domain", "Security Header Grade"])  # Write header if new file
        writer.writerows(data)

    # Also save a backup
    with open(BACKUP_CSV, "w", newline="", encoding="utf-8") as backup_file:
        writer = csv.writer(backup_file)
        writer.writerow(["Domain", "Security Header Grade"])
        writer.writerows(data)

# Run the script
if __name__ == "__main__":
    process_domains()