secret-protection-custom-pattern-automation/secret_protection.ts

A Playwright-based tool to automate GitHub secret scanning custom pattern management.

import { chromium, BrowserContext, Page, Locator } from 'playwright';
import minimist from 'minimist';
import { promises as fs } from 'fs';
import * as yaml from 'js-yaml';
import * as path from 'path';
import chalk from 'chalk';
import Table from 'cli-table3';
import inquirer from 'inquirer';
import cliProgress from 'cli-progress';
import { exit } from 'process';
import { PatternValidator } from './validator.js';
import { HELP_TEXT } from './help.js';
export interface Pattern {
    name: string;
    type?: string;
    experimental?: boolean;
    regex: {
        version: number;
        pattern: string;
        start?: string;
        end?: string;
        additional_match?: string[];
        additional_not_match?: string[];
    };
    test?: {
        data: string;
        start_offset?: number;
        end_offset?: number;
    };
    expected?: Array<{
        name: string;
        start_offset: number;
        end_offset: number;
    }>;
    push_protection?: boolean;
    comments?: string[];
}

export interface PatternFile {
    name: string;
    patterns: Pattern[];
}

interface BrowserStorageState {
    cookies: Array<{
        name: string;
        value: string;
        domain: string;
        path: string;
        expires: number;
        httpOnly: boolean;
        secure: boolean;
        sameSite: "Strict" | "Lax" | "None";
    }>;
    origins: Array<{
        origin: string;
        localStorage: Array<{
            name: string;
            value: string;
        }>;
    }>;
}

interface DryRunMatch {
    match: string | undefined;
    repository_location: string | undefined;
    link: string | null;
}

interface DryRunResult {
    id: string;
    name: string;
    hits: number;
    results: DryRunMatch[];
    completed: boolean;
}

export interface Config {
    server: string;
    target: string;
    scope: 'repo' | 'org' | 'enterprise';
    patterns?: string[];
    patternsToInclude?: string[];
    patternsToExclude?: string[];
    dryRunThreshold: number;
    enablePushProtection?: boolean;
    noChangePushProtection?: boolean;
    disablePushProtection?: boolean;
    headless?: boolean;
    downloadExisting?: boolean;
    deleteExisting?: boolean;
    validate?: boolean;
    validateOnly?: boolean;
    debug?: boolean;
    dryRunAllRepos?: boolean;
    dryRunRepoList?: string[];
    forceSubmission?: boolean;
    maxTestTries: number;
}

let state: BrowserStorageState | null = null;

export async function main() {
    const config = parseArgs();

    if (!config) {
        console.error(chalk.red('✖ Invalid configuration. Please check your command line arguments.'));
        console.log(HELP_TEXT);
        process.exit(1);
    }

    console.log(chalk.bold.blue(`🔐 Secret Scanning Custom Pattern Automation Tool`));
    if (config.server !== 'https://github.com') {
        console.log(chalk.gray(`Using server: ${config.server}`));
    }
    console.log(chalk.gray(`Target: ${config.target} (${config.scope})\n`));

    // Handle validation-only mode
    if (config.validateOnly) {
        if (!config.patterns || config.patterns.length === 0) {
            console.error(chalk.red('✖ No pattern files specified for validation'));
            process.exit(1);
        }

        console.log(chalk.yellow('🔍 Running validation-only mode (no upload)'));

        for (const patternPath of config.patterns) {
            try {
                console.log(chalk.blue(`\n📁 Loading pattern file: ${patternPath}`));
                const patternFile = await loadPatternFile(patternPath);
                validatePatterns(patternFile, config);
            } catch (error) {
                console.error(chalk.red(`✖ Validation failed for ${patternPath}:`), error);
                process.exit(1);
            }
        }

        console.log(chalk.green('\n✓ All pattern files passed validation'));
        process.exit(0);
    }

    try {
        await login(config.server, config);
    } catch (error) {
        console.error(chalk.red('✖ Login failed:'), error);
        process.exit(1);
    }

    if (state === null) {
        console.error(chalk.red('✖ Authentication error.'));
        process.exit(1);
    }

    const browser = await chromium.launch({ headless: config.headless });
    const context = await browser.newContext({ storageState: state || undefined });

    try {
        if (config.downloadExisting) {
            await downloadExistingPatterns(context, config);
        }

        if (config.deleteExisting) {
            await deleteExistingPatterns(context, config);
        }

        if (config.patterns && config.patterns.length > 0) {
            await uploadPatterns(context, config);
        }
    } finally {
        browser.close();
    }
}

export function parseArgs(): Config | undefined {
    const args = minimist(process.argv.slice(2));

    const target: string | undefined = args._.pop();

    const patterns = args.pattern ? (Array.isArray(args.pattern) ? args.pattern : [args.pattern]) : undefined;
    const include_patterns = args['include-pattern-name'] ? (Array.isArray(args['include-pattern-name']) ? args['include-pattern-name'] : [args['include-pattern-name']]) : undefined;
    const exclude_patterns = args['exclude-pattern-name'] ? (Array.isArray(args['exclude-pattern-name']) ? args['exclude-pattern-name'] : [args['exclude-pattern-name']]) : undefined;

    // For validate-only mode, target can be a placeholder
    if (args['validate-only']) {
        console.log(chalk.yellow('ℹ️  Running validation-only mode without target specification'));
        return {
            server: 'https://github.com',
            target: 'validation-only',
            scope: 'repo',
            patterns: patterns,
            patternsToInclude: include_patterns,
            patternsToExclude: exclude_patterns,
            validateOnly: true,
            validate: true,
            dryRunAllRepos: true,
            dryRunThreshold: 0,
            maxTestTries: 25,
        };
    }

    if (!target) {
        console.error(chalk.red('✖ Please provide a target repository, organization, or enterprise.'));
        process.exit(1);
    }

    // auto-detect scope based on target
    let scope: 'repo' | 'org' | 'enterprise';
    if (target.includes('/')) {
        scope = 'repo';
    } else if (args.scope === undefined) {
        scope = 'org';
    } else {
        scope = args.scope;
    }

    // dry run threshold, from ENV or args - args win
    let dryRunThreshold = 0;
    if (process.env.DRY_RUN_THRESHOLD !== undefined) {
        dryRunThreshold = parseInt(process.env.DRY_RUN_THRESHOLD, 10);
    }
    if (args['dry-run-threshold'] !== undefined) {
        dryRunThreshold = parseInt(args['dry-run-threshold'], 10);
    }

    const dryRunRepoList = args['dry-run-repo'] ? (Array.isArray(args['dry-run-repo']) ? args['dry-run-repo'] : [args['dry-run-repo']]) : []

    let maxTestTries = 25;
    if (args['max-test-tries'] !== undefined) {
        maxTestTries = parseInt(args['max-test-tries'], 10);
    }

    const config: Config = {
        server: args.server ?? process.env.GITHUB_SERVER ?? 'https://github.com',
        target,
        scope,
        patterns: patterns,
        patternsToInclude: include_patterns,
        patternsToExclude: exclude_patterns,
        dryRunThreshold: dryRunThreshold,
        enablePushProtection: args['enable-push-protection'] ?? false,
        noChangePushProtection: args['keep-push-protection'] ?? false,
        disablePushProtection: args['disable-push-protection'] ?? false,
        headless: args.headless ?? true,
        downloadExisting: args['download-existing'] ?? false,
        deleteExisting: args['delete-existing'] ?? false,
        validateOnly: args['validate-only'] ?? false,
        validate: args.validate ?? true,
        debug: args.debug ?? false,
        dryRunAllRepos: args['dry-run-all-repos'] ?? false,
        dryRunRepoList: dryRunRepoList,
        forceSubmission: args['force-submission'] ?? false,
        maxTestTries: maxTestTries,
    };

    if (config.debug) {
        console.log(chalk.blue('🐛 Debug mode enabled'));
        console.log(chalk.gray(`Config: \n${JSON.stringify(config, null, 2)}`));
        console.log(chalk.grey(`Args: \n${JSON.stringify(args, null, 2)}`));
    }

    // check scope is valid
    const validScopes = ['repo', 'org', 'enterprise'];
    if (!validScopes.includes(config.scope)) {
        console.error(chalk.red(`✖ Invalid scope: ${config.scope}. Valid scopes are: ${validScopes.join(', ')}`));
        process.exit(1);
    }

    if (isNaN(config.dryRunThreshold) || config.dryRunThreshold < 0) {
        config.dryRunThreshold = 0;
    }

    if ((!config.patterns || config.patterns.length === 0) && !config.downloadExisting && !config.deleteExisting) {
        console.warn(chalk.yellow('ℹ️  No patterns specified for upload. You can use --pattern to specify one or more pattern files.'));
        return undefined;
    }

    if (config.enablePushProtection && config.noChangePushProtection) {
        console.warn(chalk.yellow('⚠️ Both --enable-push-protection and --no-change-push-protection are set. Choose one of them only.'));
        return undefined;
    }

    if (config.enablePushProtection && config.disablePushProtection) {
        console.warn(chalk.yellow('⚠️ Both --enable-push-protection and --disable-push-protection are set. Choose one of them only.'));
        return undefined;
    }

    if (config.disablePushProtection && config.noChangePushProtection) {
        console.warn(chalk.yellow('⚠️ Both --disable-push-protection and --no-change-push-protection are set. Choose one of them only.'));
        return undefined;
    }

    if (config.scope === 'org' && (!config.validateOnly && config.patterns !== undefined && config.patterns?.length > 0) && !config.dryRunAllRepos && dryRunRepoList.length === 0) {
        console.error(chalk.red('✖ No specific repositories provided for dry-run. To run dry-run on all repositories, use --dry-run-all-repos'));
        return undefined;
    }

    if (config.scope === 'enterprise' && (!config.validateOnly && config.patterns !== undefined && config.patterns?.length > 0)) {
        if (config.dryRunAllRepos) {
            console.error(chalk.red('✖ Dry-run on all repositories is not supported for enterprise scope. Please specify individual repositories using --dry-run-repo'));
        }

        if (dryRunRepoList.length === 0) {
            console.error(chalk.red('✖ No repositories provided for dry-run. Please specify individual repositories using --dry-run-repo'));
            return undefined;
        }
    }

    return config;
}

async function goto(page: Page, url: string, config: Config): Promise<boolean> {
    while (true) {
        try {
            const result = await page.goto(url);

            if (!result || !result.ok()) {
                console.warn(`Failed to load page: ${result?.status() || 'unknown error'}`);
                return false;
            }

            break;
        } catch (err) {
            const error = err as Error;
            if (error.message.startsWith('page.goto: net::ERR')) {
                if (error.message.startsWith('page.goto: net::ERR_ABORTED')) {
                    if (config.debug) {
                        console.warn(chalk.yellow(`⚠️ Network error occurred while loading page: ${error.message}`));
                    }
                    continue;
                }
            }
            console.error(chalk.red(`⨯ Error loading page: ${error.message}`));
            return false;
        }
    }
    return true;
}

async function reload(page: Page, config: Config): Promise<boolean> {
    while (true) {
        try {
            const result = await page.reload({ waitUntil: 'load' });

            if (!result || !result.ok()) {
                console.warn(`⚠️  Failed to reload page: ${result?.status() || 'unknown error'}`);
                return false;
            }

            break;
        } catch (err) {
            const error = err as Error;
            if (error.message.startsWith('page.reload: net::ERR')) {
                if (error.message.startsWith('page.reload: net::ERR_ABORTED')) {
                    if (config.debug) {
                        console.warn(chalk.yellow(`⚠️  Network error occurred while reloading page: ${error.message}`));
                    }
                    continue;
                }
            }
            console.error(chalk.red(`⨯ Error reloading page: ${error.message}`));
            return false;
        }
    }
    return true;
}

async function login(server: string, config: Config) {
    // look for existing state stored in .state file locally
    const stateFilePath = path.join(process.cwd(), '.state');
    try {
        state = JSON.parse(await fs.readFile(stateFilePath, 'utf-8'));
        console.log(chalk.gray('🔑 Using existing authentication from .state file'));
        return;
    } catch {
        console.log(chalk.blue('🔑 No existing authentication found, doing browser login'));
    }

    const browser = await chromium.launch({ headless: false });
    const context = await browser.newContext();
    const page = await context.newPage();

    // Wait for user to log in
    const url = `${server}/login`;

    const success = await goto(page, url, config);
    if (!success) {
        console.error(chalk.red('✖ Failed to load login page. Please check your server URL.'));
        await browser.close();
        return;
    }

    console.log(chalk.blue(`🖥️ Please log in manually to GitHub on ${server}`));

    console.log(chalk.blue('⌨ Waiting for manual login... Press Enter once logged in'));
    // Wait for user input
    await new Promise<void>((resolve) => {
        process.stdin.once('data', () => resolve());
    });

    // Save browser state
    state = await context.storageState();

    await fs.writeFile(stateFilePath, JSON.stringify(state, null, 2));

    console.log(chalk.green('✓ Login successful, state saved'));
    await browser.close();
}

async function deleteExistingPatterns(context: BrowserContext, config: Config): Promise<void> {
    console.log('Deleting existing patterns...');
    const page = await context.newPage();

    try {
        const url_path = config.scope !== 'enterprise' ? 'settings/security_analysis' : 'settings/security_analysis_policies/security_features';
        const url = buildUrl(config, url_path);

        const success = await goto(page, url, config);

        if (!success) {
            console.error(chalk.red(`⨯ Failed to load existing patterns`));
            return;
        }

        const existingPatterns = await findExistingPatterns(context, config);

        if (existingPatterns === null) {
            console.error(chalk.red('✖ Failed to find existing patterns'));
            return;
        }

        if (Array.from(existingPatterns.keys()).length === 0) {
            return;
        }

        const deletedPatternNames: Set<string> = new Set();

        const patternsToDelete: Array<[string, string]> = Array.from(existingPatterns.entries()).filter(([name, _url]) => {
            return (config.patternsToInclude ? config.patternsToInclude.includes(name) : true) &&
                !(config.patternsToExclude && config.patternsToExclude.includes(name));
        });

        if (patternsToDelete.length === 0) {
            console.log(chalk.blue('ℹ No patterns to delete based on include/exclude filters'));
            return;
        }

        const deleteCount = patternsToDelete.length;

        // confirm deletion of N patterns
        const confirmDelete = await inquirer.prompt({
            type: 'confirm',
            name: 'confirm',
            message: `Are you sure you want to delete ${deleteCount} existing pattern${deleteCount === 1 ? '' : 's'}?`,
            default: false,
        });

        if (!confirmDelete.confirm) {
            console.log(chalk.yellow('⚠️  Deletion cancelled by user'));
            return;
        }

        // progress bar
        const progressBar = new cliProgress.MultiBar({}, cliProgress.Presets.shades_classic);
        const progressBarSimple = progressBar.create(deleteCount, 0);

        for (const [name, url] of patternsToDelete) {
            try {
                const id = url.split('/').pop()?.split('?')[0] || '';

                // now get the content of the URL, by loading it and extracting it from the page
                const patternPage = await context.newPage();
                const success = await goto(patternPage, `${config.server}${url}`, config);
                if (!success) {
                    progressBar.log(chalk.red(`⨯ Failed to load pattern page`));
                    progressBarSimple.increment();
                    continue;
                }

                await patternPage.waitForLoadState('load');

                // show delete dialog - Playwright has trouble clicking the button
                const confirmDialog = await showDialog(patternPage, `remove-pattern-dialog-pattern-${id}`);

                if (!confirmDialog) {
                    progressBar.log(chalk.yellow(`⚠️ No confirmation dialog found for pattern "${name}"`));
                    progressBarSimple.increment();
                    continue;
                }

                // TODO: pick between deleting and closing alerts, and by default confirm delete

                const confirmDeleteSelector = `button[data-close-dialog-id="remove-pattern-dialog-pattern-${id}"][type="submit"]`;
                const confirmDeleteButton = confirmDialog.locator(confirmDeleteSelector);

                if (!confirmDeleteButton) {
                    progressBar.log(chalk.yellow(`⚠️ No confirm delete button found for pattern "${name}"`));
                    progressBarSimple.increment();
                    continue;
                }

                await confirmDeleteButton.click();

                deletedPatternNames.add(name);

                progressBarSimple.increment();
            } catch (err) {
                const error = err as Error;
                console.error(chalk.red(`✖ Error when deleting existing pattern: ${error.message}`));
                progressBar.stop();
                return;
            }
        }
        progressBar.stop();
        // wait for a bit for the backend to catch up with the deletes
        await page.waitForTimeout(1000);
    } finally {
        await page.close();
    }
}

async function downloadExistingPatterns(context: BrowserContext, config: Config): Promise<void> {
    console.log('Downloading existing patterns...');
    const page = await context.newPage();

    try {
        const url_path = config.scope !== 'enterprise' ? 'settings/security_analysis' : 'settings/security_analysis_policies/security_features';
        const url = buildUrl(config, url_path);

        const success = await goto(page, url, config);

        if (!success) {
            console.error(chalk.red(`⨯ Failed to load existing patterns`));
            return;
        }

        let keepGoing = true;
        const extractedPatterns: Pattern[] = [];
        let count = 0;
        let firstPage = true;

        // progress bar
        const progressBar = new cliProgress.MultiBar({}, cliProgress.Presets.shades_classic);
        const progressBarSimple = progressBar.create(count, 0);

        while (keepGoing) {
            await page.waitForLoadState('load');

            const customPatternList = page.locator('.js-custom-pattern-list').first();

            let busy = true;

            while (busy) {
                busy = await customPatternList.getAttribute('busy') !== null;
            }

            // wait a little longer to ensure the table is fully loaded
            await page.waitForTimeout(200);

            if (!customPatternList) {
                progressBar.stop();
                console.warn(chalk.yellow('⚠️ No custom patterns found on the page'));
                return;
            }

            if ((await customPatternList.textContent())?.includes('There are no custom patterns for this')) {
                progressBar.stop();
                console.log(chalk.blue('ℹ No custom patterns exist on this repository'));
                return;
            }

            const customPatternCount = await customPatternList.locator('.js-custom-pattern-total-count').first().textContent();

            if (!customPatternCount) {
                progressBar.stop();
                console.warn(chalk.yellow('⚠️ No custom pattern count found on the page'));
                return;
            }

            // put out value from text
            if (firstPage) {
                firstPage = false;
                count = parseInt(customPatternCount.match(/\d+/)?.[0] ?? '0', 10);
                progressBarSimple.setTotal(count);
                progressBarSimple.update(0);
            }

            const patternRows = await customPatternList.locator('li[class="Box-row"]').all();

            if (!patternRows || patternRows.length === 0) {
                progressBar.stop();
                return;
            }

            for (const row of patternRows) {
                const link = row.locator('.js-navigation-open').first();

                if (link) {
                    const name = await link.textContent();
                    const url = await link.getAttribute('href');
                    const id = url?.split('/').pop()?.split('?')[0] || '';

                    progressBarSimple.increment();

                    // now get the content of the URL, by loading it and extracting it from the page
                    const patternPage = await context.newPage();
                    const success = await goto(patternPage, `${config.server}${url}`, config);
                    if (!success) {
                        console.error(chalk.red(`⨯ Failed to load pattern page`));
                        continue;
                    }

                    await patternPage.waitForLoadState('load');

                    // the data is in HTML content of the page, so we need to use the right locators to get it out
                    const patternName = await patternPage.locator('#display_name').getAttribute('value');
                    const secretFormat = await patternPage.locator('#secret_format').getAttribute('value');
                    const beforeSecret = await patternPage.locator('#before_secret').getAttribute('value');
                    const afterSecret = await patternPage.locator('#after_secret').getAttribute('value');
                    const additionalMatches = await patternPage.locator('.js-additional-secret-format').all();

                    if (patternName !== name) {
                        console.warn(chalk.yellow(`⚠️ Pattern name mismatch: expected "${name}", found "${patternName}"`));
                    }

                    // record if it is published or not
                    const subHead = await patternPage.locator('Subhead-heading').first().textContent();
                    const isPublished = subHead?.includes('Update pattern');

                    // pull out additional matches, and if they are Must match or Must not Match
                    const additionalMatchRules: Map<string, Array<string>> = new Map();

                    for await (const match of additionalMatches) {
                        // skip if it has 'has-removed-contents' set in the class
                        const className = await match.getAttribute('class');
                        if (className?.includes('has-removed-contents')) {
                            continue;
                        }

                        const additionalSecretFormat = await match.locator('input[type="text"]').getAttribute('value');

                        if (!additionalSecretFormat) {
                            console.warn(chalk.yellow('⚠️ No additional secret format found, skipping this match'));
                            continue;
                        }

                        // Get the radio button with value='must_match'
                        const mustMatchRadio = match.locator('input[type="radio"][value="must_match"]');

                        if (!mustMatchRadio) {
                            console.warn(chalk.yellow('⚠️ No must match radio button found, skipping this match'));
                            continue;
                        }

                        const isMustMatch = await mustMatchRadio.isChecked();

                        // the matchType is a radio button with value 'must_match' or 'must_not_match'
                        const matchType = isMustMatch ? 'must_match' : 'must_not_match';

                        if (!additionalMatchRules.has(matchType)) {
                            additionalMatchRules.set(matchType, []);
                        }
                        additionalMatchRules.get(matchType)?.push(additionalSecretFormat);
                    }

                    // Convert to the Pattern interface format
                    const pattern: Pattern = {
                        name: patternName || `Pattern_${id}`,
                        regex: {
                            version: 1,
                            pattern: secretFormat || '',
                            ...(beforeSecret && { start: beforeSecret }),
                            ...(afterSecret && { end: afterSecret }),
                            ...(additionalMatchRules.get('must_match') && { additional_match: additionalMatchRules.get('must_match') }),
                            ...(additionalMatchRules.get('must_not_match') && { additional_not_match: additionalMatchRules.get('must_not_match') })
                        },
                        comments: [
                            `Downloaded from ${config.scope}: ${config.target} (${config.server})`,
                            `Original ID: ${id}`,
                            `Published: ${isPublished ? 'Yes' : 'No'}`
                        ]
                    };

                    extractedPatterns.push(pattern);
                }
            }

            // record how many we found on the page
            count -= patternRows.length;

            // find the Next> button
            const nextButton = customPatternList.locator('button[id="next_cursor_button_udp"]');
            if (await nextButton.isVisible() && await nextButton.isEnabled()) {
                await nextButton.click();
            } else {
                keepGoing = false;
            }
        }

        progressBar.stop();

        // Create PatternFile structure matching the import format
        const patternFile: PatternFile = {
            name: `Downloaded patterns from ${config.target}`,
            patterns: extractedPatterns
        };

        // Save patterns to file
        const outputPath = path.join(process.cwd(), 'existing-patterns.yml');
        await fs.writeFile(outputPath, yaml.dump(patternFile));
        console.log(chalk.blue(`⬇️  Saved to: ${outputPath}`));

    } finally {
        await page.close();
    }
}

async function uploadPatterns(context: BrowserContext, config: Config): Promise<void> {
    if (!config.patterns || config.patterns.length === 0) {
        console.log('No patterns specified for upload');
        return;
    }

    console.log(`Uploading ${config.patterns.length} pattern file(s)...`);

    const unprocessedPatterns: Map<string, [string, string][]> = new Map();

    const existingPatterns = await findExistingPatterns(context, config);

    if (existingPatterns === null) {
        console.error(chalk.red('✖ Failed to find existing patterns'));
        return;
    }

    if (config.debug) {
        console.log(chalk.blue(`Debug: Found ${Array.from(existingPatterns.entries()).length} existing patterns`));
    }

    for (const patternPath of config.patterns) {
        try {
            console.log(`Processing pattern file: ${patternPath}`);
            const patternFile = await loadPatternFile(patternPath);

            if (config.validate) {
                validatePatterns(patternFile, config);
            }

            for (const pattern of patternFile.patterns) {
                if (config.patternsToInclude && !config.patternsToInclude.includes(pattern.name)) {
                    if (config.debug) {
                        console.log(chalk.blue(`Skipping pattern '${pattern.name}' not in the include list`));
                    }
                    continue;
                }

                if (config.patternsToExclude && config.patternsToExclude.includes(pattern.name)) {
                    if (config.debug) {
                        console.log(chalk.blue(`Skipping pattern '${pattern.name}' in the exclude list`));
                    }
                    continue;
                }

                try {
                    await processPattern(context, config, pattern, existingPatterns);
                } catch (err) {
                    const error = err as Error;
                    console.error(chalk.red(`✖ Failed to process pattern '${pattern.name ?? "**unnamed pattern**"}' in file ${patternPath}:`, error.message));
                    unprocessedPatterns.set(patternPath, unprocessedPatterns.get(patternPath) || []);
                    unprocessedPatterns.get(patternPath)?.push([pattern?.name ?? "**unnamed pattern**", error.message]);

                    if (err instanceof Error && err.message.includes('Target page, context or browser has been closed')) {
                        console.error(chalk.red('⚠️  Browser context or page was closed unexpectedly.'));
                        exit(1);
                    }
                }
            }
        } catch (err) {
            const error = err as Error;
            console.error(chalk.red(`✖ Failed to fully process pattern file ${patternPath}:`, error.message));
        }
    }

    if (unprocessedPatterns.size > 0) {
        console.log(chalk.yellow('\n⚠️  Some patterns could not be processed:'));
        for (const [filePath, patterns] of unprocessedPatterns.entries()) {
            console.log(chalk.yellow(`\nFile: ${filePath}`));
            for (const [patternName, errorMessage] of patterns) {
                console.log(chalk.red(`  Pattern: ${patternName} - Error: ${errorMessage}`));
            }
        }
    } else {
        console.log(chalk.green('✓ All patterns processed successfully'));
    }
}

export async function loadPatternFile(filePath: string): Promise<PatternFile> {
    const content = await fs.readFile(filePath, 'utf-8');

    try {
        return yaml.load(content) as PatternFile;
    } catch {
        try {
            return JSON.parse(content) as PatternFile;
        } catch (jsonError) {
            throw new Error(`Failed to parse file as YAML or JSON: ${jsonError}`);
        }
    }
}

function validatePatterns(patternFile: PatternFile, config: Config): void {
    const fileResult = PatternValidator.validatePatternFile(patternFile, config);

    if (fileResult.isValid && !config.validateOnly) {
        console.log(chalk.green('✔ All patterns passed validation'));
    } else {
        PatternValidator.printValidationReport(fileResult);
    }

    // Individual pattern validation for summary reporting
    const patternResults = patternFile.patterns.filter(pattern => {
        // Filter out patterns that are not in the include list or are in the exclude list
        if (config.patternsToInclude && !config.patternsToInclude.includes(pattern.name)) {
            return false;
        }
        if (config.patternsToExclude && config.patternsToExclude.includes(pattern.name)) {
            return false;
        }
        return true;
    }).map(pattern => ({
        name: pattern.name,
        result: PatternValidator.validatePattern(pattern)
    }));

    // Print summary table
    const summaryTable = PatternValidator.createSummaryTable(patternResults);
    console.log('\n📊 Validation Summary:');
    console.log(summaryTable);

    if (!fileResult.isValid) {
        throw new Error('Pattern validation failed');
    }
}

async function expandMoreOptions(page: Page): Promise<void> {
    const optionsData = page.locator('div.Details-content--shown').first();
    if (await optionsData.isVisible()) {
        return;
    }
    const moreOptions = page.locator('div.js-more-options').first();
    const moreOptionsButton = await moreOptions.locator('button.js-details-target:text-is("More options")').first();
    const isExpanded = await moreOptionsButton.getAttribute('aria-expanded');
    if (isExpanded !== 'true') {
        await moreOptionsButton.click();
        const beforeSecretInput = page.locator('input#before_secret');
        await beforeSecretInput.waitFor({ state: 'visible' });
    }
}

export function comparePatterns(patternA: string | undefined | null, patternB: string | undefined | null): boolean {
    if (patternA === null || patternA === undefined || patternB === null || patternB === undefined) {
        return false;
    }
    return patternA?.trim() === patternB?.trim();
}

async function fillInPattern(page: Page, pattern: Pattern, isExisting: boolean = false, config: Config): Promise<boolean> {

    // If this is an existing pattern, clear the fields first, if they are different to what we are uploading
    if (isExisting) {
        let changed: Boolean = false;
        const removeExistingAdditionalMatchesSelector = 'button.js-remove-secret-format-button';

        await expandMoreOptions(page);

        const currentSecretFormat = page.locator('input[name="secret_format"]');
        const secretFormatContent = await currentSecretFormat.getAttribute('value');
        if (secretFormatContent && !comparePatterns(secretFormatContent, pattern.regex.pattern)) {
            console.log(secretFormatContent);
            console.log(pattern.regex.pattern);
            await currentSecretFormat.click();
            await currentSecretFormat.clear();
            changed = true;
        }

        // Clear before/after secret fields if they exist
        const beforeSecretInput = page.locator('input[name="before_secret"]');
        const beforeSecretContent = await beforeSecretInput.getAttribute('value');
        if (beforeSecretContent && !comparePatterns(beforeSecretContent, pattern.regex.start)) {
            if (await beforeSecretInput.isVisible()) {
                await beforeSecretInput.click();
                await beforeSecretInput.clear();
                changed = true;
            }
        }

        const afterSecretInput = page.locator('input[name="after_secret"]');
        const afterSecretContent = await afterSecretInput.getAttribute('value');
        if (afterSecretContent && !comparePatterns(afterSecretContent, pattern.regex.end)) {
            if (await afterSecretInput.isVisible()) {
                await afterSecretInput.click();
                await afterSecretInput.clear();
                changed = true;
            }
        }

        // Clear existing additional rules by removing them
        try {
            const removeExistingAdditionalMatches = await page.locator(removeExistingAdditionalMatchesSelector).all();

            if (pattern.regex.additional_match === undefined) {
                pattern.regex.additional_match = [];
            }

            if (pattern.regex.additional_not_match === undefined) {
                pattern.regex.additional_not_match = [];
            }

            // if there are no additional matches, and no buttons to remove them, there's no change
            if (removeExistingAdditionalMatches.length === 0 && pattern.regex.additional_match.length === 0 && pattern.regex.additional_not_match.length === 0) {
                if (config.debug) {
                    console.log(chalk.blue(`✓ No existing additional matches to clear, none to add`));
                }
            } else {
                // check if the additional matches are already present on the page - if they all are, and there are no extra ones, then we didn't change anything
                const existingAdditionalMatchCount = parseInt(await page.locator('div.js-post-processing-expression-count').textContent() || '0', 10);
                const newAdditionalMatchCount = pattern.regex.additional_match.length + pattern.regex.additional_not_match.length;

                if (config.debug) {
                    console.log(chalk.blue(`Found ${existingAdditionalMatchCount} existing additional matches`));
                    console.log(chalk.blue(`Adding ${newAdditionalMatchCount} additional matches`));
                }

                if (existingAdditionalMatchCount === newAdditionalMatchCount) {
                    const existingAdditionalMatches = (await page.locator('div.js-additional-secret-format').all()).filter(
                        async match => (await match.locator('input[type="radio"]').count()) > 0
                    );

                    if (config.debug) {
                        console.log(chalk.blue(`Found ${existingAdditionalMatches.length}`));
                    }

                    // Check if all existing matches are the same as the new ones
                    const existingMustMatches: Locator[] = [];
                    const existingMustNotMatches: Locator[] = [];

                    for (let i = 0; i < existingAdditionalMatchCount && i < existingAdditionalMatches.length; i++) {
                        const existingMatch = existingAdditionalMatches[i];
                        const radioButton = existingAdditionalMatches[i].locator('input[type="radio"][value="must_match"]');
                        const isMustMatch = await radioButton.isChecked();
                        if (isMustMatch) {
                            existingMustMatches.push(existingMatch);
                        } else {
                            existingMustNotMatches.push(existingMatch);
                        }
                    }

                    for (let i = 0; i < existingMustMatches.length; i++) {
                        const existingMustMatch = existingMustMatches[i];
                        const existingMatchValue = await existingMustMatch.locator('input[type="text"]').inputValue();
                        const newMatchValue = pattern.regex.additional_match[i];
                        if (!comparePatterns(existingMatchValue, newMatchValue)) {
                            changed = true;
                            if (config.debug) {
                                console.log(chalk.blue(`Old value and new value differ: ${existingMatchValue} !== ${newMatchValue}`))
                            }
                            break;
                        }
                    }

                    if (!changed) {
                        for (let i = 0; i < existingMustNotMatches.length; i++) {
                            const existingMustNotMatch = existingMustNotMatches[i];
                            const existingMatchValue = await existingMustNotMatch.locator('input[type="text"]').inputValue();
                            const newMatchValue = pattern.regex.additional_not_match[i];
                            if (!comparePatterns(existingMatchValue, newMatchValue)) {
                                changed = true;
                                if (config.debug) {
                                    console.log(chalk.blue(`Old value and new value differ: ${existingMatchValue} !== ${newMatchValue}`))
                                }
                                break;
                            }
                        }
                    }
                } else {
                    if (config.debug) {
                        console.log(chalk.blue(`✓ Existing additional matches count (${existingAdditionalMatchCount}) does not match new count (${newAdditionalMatchCount}), will clear all additional matches`));
                    }
                    changed = true;
                }

                if (changed || config.forceSubmission) {
                    if (config.debug) {
                        console.log(chalk.blue(`Removing ${await page.locator(removeExistingAdditionalMatchesSelector).count()} existing additional matches`));
                    }

                    while (await page.locator(removeExistingAdditionalMatchesSelector).count() > 0) {
                        const removeButton = page.locator(removeExistingAdditionalMatchesSelector).last();
                        await removeButton.click();
                    }
                }
            }
        } catch (error) {
            console.log(chalk.gray(`Note: Could not check/clear all existing additional rules: ${error}`));
        }

        if (!changed && !config.forceSubmission) {
            console.log(chalk.yellow(`⏩ No changes detected against existing pattern, skipping submission`));
            return false;
        }

        // wait a bit
        await page.waitForTimeout(200);

        if (config.debug) {
            console.log(chalk.blue(`✓ Cleared existing pattern fields`));
            //take screenshot of the cleared pattern
            await page.setViewportSize({ width: 1920, height: 2000 });
            const screenshotPath = path.join(process.cwd(), `debug-cleared_pattern_${pattern.name.replace(/[^a-zA-Z0-9]/g, '_')}_${Date.now()}.png`);
            await page.screenshot({ path: screenshotPath });
            console.log(`📸 Screenshot of cleared pattern saved to ${screenshotPath}`);
        }

    } else {
        const nameField = page.locator('input[name="display_name"]');
        await nameField.fill(pattern.name);
    }

    if (pattern.regex.start || pattern.regex.end || pattern.regex.additional_match || pattern.regex.additional_not_match) {
        await expandMoreOptions(page);
    }

    const secretFormat = page.locator('input[name="secret_format"]');
    await secretFormat.clear();
    await secretFormat.fill(pattern.regex.pattern);

    if (pattern.regex.start) {
        const beforeSecretInput = page.locator('input[name="before_secret"]');
        await beforeSecretInput.clear();
        await beforeSecretInput.fill(pattern.regex.start);
    }

    if (pattern.regex.end) {
        const afterSecretInput = page.locator('input[name="after_secret"]');
        await afterSecretInput.clear();
        await afterSecretInput.fill(pattern.regex.end);
    }

    if (pattern.regex.additional_match && pattern.regex.additional_match.length > 0) {
        for (const [index, rule] of pattern.regex.additional_match.entries()) {
            const success = await addAdditionalRule(page, rule, 'must_match', index, config);
            if (!success) {
                console.error(chalk.red(`⨯ Failed to add additional match rule ${index} for ${rule}`));
                return false;
            } else {
                if (config.debug) {
                    console.log(chalk.blue(`✓ Added additional match rule ${index}: ${rule}`));
                }
            }
        }
    }

    if (pattern.regex.additional_not_match && pattern.regex.additional_not_match.length > 0) {
        for (const [index, rule] of pattern.regex.additional_not_match.entries()) {
            const offset = pattern.regex.additional_match?.length || 0;
            const success = await addAdditionalRule(page, rule, 'must_not_match', index + offset, config);
            if (!success) {
                console.error(chalk.red(`⨯ Failed to add additional not match rule ${index + offset} for ${rule}`));
                return false;
            } else {
                if (config.debug) {
                    console.log(chalk.blue(`✓ Added additional not match rule ${index + offset}: ${rule}`));
                }
            }
        }
    }

    console.log(chalk.green(`✓ Pattern filled in - checking for test result and looking for errors`));

    return true;
}

async function findExistingPatterns(context: BrowserContext, config: Config): Promise<Map<string, string> | null> {
    const page = await context.newPage();
    const existingPatterns: Map<string, string> = new Map();

    try {
        const url_path = config.scope !== 'enterprise' ? 'settings/security_analysis' : 'settings/security_analysis_policies/security_features';
        const url = buildUrl(config, url_path);

        const success = await goto(page, url, config);

        if (!success) {
            console.error(chalk.red(`⨯ Failed to load security analysis page`));
            if (config.debug) {
                console.log(chalk.gray(`Debug: Failed to load URL ${url}`));
            }
            return null;
        }

        let keepGoing = true;

        while (keepGoing) {
            await page.waitForLoadState('load');

            const customPatternList = page.locator('.js-custom-pattern-list').first();

            let busy = true;
            while (busy) {
                busy = await customPatternList.getAttribute('busy') !== null;
            }

            // wait a little to ensure the table is fully loaded
            await page.waitForTimeout(200);

            if (!customPatternList) {
                console.warn(chalk.yellow('⚠️ No custom pattern list found on the page'));
                return null;
            }

            if ((await customPatternList.textContent())?.includes('There are no custom patterns for this repository')) {
                console.log(chalk.blue('ℹ No custom patterns exist on this repository'));
                return existingPatterns;
            }

            let patternRows: Locator[] = [];

            try {
                patternRows = await customPatternList.locator('li[class="Box-row"]').all();
            } catch (error) {
                if (config.debug) {
                    console.log(chalk.blue(`Waiting for custom pattern list to be stable, trying again: ${error}`));
                }
                // did page reload? try again
                continue;
            }

            if (!patternRows || patternRows.length === 0) {
                return existingPatterns;
            }

            // Check each pattern on this page
            for (const row of patternRows) {
                const link = row.locator('.js-navigation-open').first();
                if (link) {
                    const name = await link.textContent();
                    const url = await link.getAttribute('href');

                    if (!name || !url) {
                        console.warn(chalk.yellow('⚠️ No name or URL found for pattern'));
                        continue;
                    }

                    existingPatterns.set(name, url);
                }
            }

            // Check for next page
            const nextButton = customPatternList.locator('button[id="next_cursor_button_udp"]');
            if (await nextButton.isVisible() && await nextButton.isEnabled()) {
                await nextButton.click();
            } else {
                keepGoing = false;
            }
        }

        return existingPatterns;

    } catch (error) {
        console.error(chalk.red(`⨯ Error checking for existing patterns: ${error}`));
        if (config.debug) {
            // take screenshot
            const screenshotPath = path.join(process.cwd(), `debug-check_existing_patterns_error_screenshot_${Date.now()}.png`);
            await page.screenshot({ path: screenshotPath });
            console.log(`📸 Screenshot saved to ${screenshotPath}`);
        }
        return null;
    } finally {
        await page.close();
    }
}

// TODO: catch errors/warnings after each step and log them, or stop on error
async function processPattern(context: BrowserContext, config: Config, pattern: Pattern, existingPatterns: Map<string, string>): Promise<void> {
    console.log(chalk.bold(`\n🔄 Processing pattern: ${pattern.name}`));

    // fill in some defaults if parts are missing
    if (pattern.regex.start === undefined) {
        pattern.regex.start = '\\A|[^0-9A-Za-z]';
    }
    if (pattern.regex.end === undefined) {
        pattern.regex.end = '\\z|[^0-9A-Za-z]';
    }

    const page = await context.newPage();

    try {
        // Look at existing patterns to see if one matches this pattern name
        const existingPatternUrl = existingPatterns.get(pattern.name);

        let url: string;
        if (existingPatternUrl) {
            url = `${config.server}${existingPatternUrl}`;
            const id = existingPatternUrl.split('/').pop()?.split('?')[0] || '';
            console.log(chalk.blue(`🔍 Found existing pattern: ${id}`));
        } else {
            const url_path = config.scope !== 'enterprise' ? 'settings/security_analysis/custom_patterns/new' : 'settings/advanced_security/custom_patterns/new';
            url = buildUrl(config, url_path);
        }

        // Navigate to pattern page (new or existing)
        const success = await goto(page, url, config);

        if (!success) {
            console.warn(`Failed to load custom pattern page`);
            return;
        }

        await page.waitForLoadState('load');

        const needToSubmit = await fillInPattern(page, pattern, !!existingPatternUrl, config);

        if (needToSubmit) {
            // Test the pattern
            const testResult = await testPattern(page, pattern, config);

            if (!testResult) {
                throw new Error(`Pattern test failed for '${pattern.name}'`);
            }

            // Perform dry run
            const dryRunResult = await performDryRun(page, pattern, config, !existingPatternUrl);

            // Interactive confirmation based on results
            const shouldProceed = await confirmPatternAction(pattern, dryRunResult, config);

            if (!shouldProceed) {
                console.log(chalk.yellow(`⏭️  Skipped pattern`));
                return;
            }

            // Publish the pattern
            awa