ctrlshiftleft
Version:
AI-powered toolkit for embedding QA and security testing into development workflows
428 lines (415 loc) • 18.8 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.LLMService = void 0;
const openai_1 = __importDefault(require("openai"));
class LLMService {
constructor() {
// Initialize OpenAI client with API key from environment variable
const apiKey = process.env.OPENAI_API_KEY;
if (!apiKey) {
console.warn('OPENAI_API_KEY environment variable not set. LLM features will not work.');
}
this.openai = new openai_1.default({
apiKey: process.env.OPENAI_API_KEY,
});
}
/**
* Extract test scenarios from source code using LLM
* @param sourceCode Source code to analyze
* @param filePath Path to the source file (for context)
* @returns Array of test scenarios
*/
async extractTestScenarios(sourceCode, filePath) {
try {
if (!process.env.OPENAI_API_KEY) {
throw new Error('OPENAI_API_KEY environment variable not set');
}
const prompt = this.buildTestGenerationPrompt(sourceCode, filePath);
// Call OpenAI API with enhanced security-focused system message
const response = await this.openai.chat.completions.create({
model: 'gpt-4-turbo-preview',
messages: [
{
role: 'system',
content: 'You are a specialized test engineer for the ctrl.shift.left toolkit. Your purpose is to analyze source code and generate comprehensive test scenarios that cover three equally important areas: (1) UI/UX functionality testing, (2) API and integration testing, and (3) security vulnerability testing. You should aim for balanced coverage across these three areas, with approximately equal emphasis on each. Your output must be precise, actionable test scenarios in valid JSON format that can be directly executed by automated testing frameworks.'
},
{
role: 'user',
content: prompt
}
],
temperature: 0.1, // Lower temperature for more focused security testing results
response_format: { type: 'json_object' }
});
// Parse and validate the response
const content = response.choices[0]?.message.content;
if (!content) {
throw new Error('Empty response from OpenAI API');
}
const parsedResponse = JSON.parse(content);
if (!Array.isArray(parsedResponse.scenarios)) {
throw new Error('Invalid response format: expected scenarios array');
}
return parsedResponse.scenarios;
}
catch (error) {
console.error('Failed to extract test scenarios:', error);
return [];
}
}
/**
* Build prompt for test generation
*/
buildTestGenerationPrompt(sourceCode, filePath) {
return `
# Comprehensive Test Generation for ctrl.shift.left
## CONTEXT
You are a specialized security and QA test generator for the Ctrl.shift.left toolkit. Your mission is to shift security testing and quality assurance earlier in the development lifecycle by generating comprehensive test scenarios that detect vulnerabilities, validate functionality, and ensure robust implementation.
## SOURCE CODE TO ANALYZE
File path: ${filePath}
\`\`\`
${sourceCode}
\`\`\`
## TEST GENERATION PRIORITIES
### 1. UI/UX TESTING (HIGH PRIORITY)
Identify and create tests for these UI/UX aspects:
- Responsive design across device sizes
- Accessibility compliance (ARIA, keyboard navigation, screen readers)
- Form validation and error message clarity
- UI state management (loading, success, error states)
- Navigation flows and user journeys
- Component interactions and event handling
- Visual consistency and design implementation
- Animation and transition behavior
### 2. API & INTEGRATION TESTING (HIGH PRIORITY)
Identify and create tests for these API & integration aspects:
- API request/response validation
- Data processing and transformation
- Error handling and recovery
- State management between components
- Event propagation and handling
- Network error scenarios
- Performance considerations
- Integration with external services
### 3. SECURITY VULNERABILITIES (HIGH PRIORITY)
Identify and create tests for these security risks:
- Input validation & sanitization flaws
- Authentication bypass opportunities
- Authorization control weaknesses
- Data exposure vulnerabilities (PII, credentials)
- Injection attack vectors (XSS, SQL, command injection)
- Mishandled sensitive data
- Missing security headers or protections
- Excessive permissions or privileges
### 2. EDGE CASES & ERROR HANDLING
- Test boundary conditions and limits
- Verify proper error capture and messaging
- Check graceful failure modes
- Test with invalid/malformed/unexpected inputs
- Verify recovery from error states
### 3. FUNCTIONAL CORRECTNESS
- Validate core user flows
- Test state transitions and side effects
- Verify all component interactions
- Test accessibility compliance
## OUTPUT FORMAT: TEST SCENARIOS ARRAY
Generate detailed test scenarios in JSON format. Each test scenario should include:
1. A descriptive title
2. A numbered list of test steps
3. Expected results or assertions
4. Security or functional concerns being tested
IMPORTANT:
- Prioritize security-related tests
- Include at least one security test for each vulnerability type found
- Cover edge cases like invalid input handling
- Ensure comprehensive test coverage of all functionality
The response should be strictly JSON with this structure:
{
"scenarios": [
{
"title": "Test Title",
"description": "What the test verifies",
"type": "security|functionality|edge-case",
"steps": ["Step 1", "Step 2", ...],
"assertions": ["Assertion 1", "Assertion 2", ...],
"concerns": ["Security concern or risk being tested"]
}
]
}
`;
}
/**
* Generate QA and security checklist from source code
* @param sourceCode Source code to analyze
* @param filePath Path to the source file (for context)
* @param type Type of checklist to generate (security or quality)
* @returns Checklist in JSON format
*/
async generateChecklist(sourceCode, filePath, type) {
try {
if (!process.env.OPENAI_API_KEY) {
throw new Error('OPENAI_API_KEY environment variable not set');
}
const prompt = this.buildChecklistGenerationPrompt(sourceCode, filePath, type);
// Call OpenAI API with specialized security system message
const response = await this.openai.chat.completions.create({
model: 'gpt-4-turbo-preview',
messages: [
{
role: 'system',
content: 'You are a specialized security and quality assurance analyst for the ctrl.shift.left toolkit. Your purpose is to analyze source code and generate comprehensive security or quality assurance checklists that identify potential issues, vulnerabilities, and best practice violations. For security issues, provide detailed risk scoring using CVSS methodology, specific remediation guidance with code examples, and references to security standards like CWE and OWASP. Your output must be in valid JSON format.'
},
{
role: 'user',
content: prompt
}
],
temperature: 0.1, // Lower temperature for more focused security results
response_format: { type: 'json_object' }
});
// Parse and validate the response
const content = response.choices[0]?.message.content;
if (!content) {
throw new Error('Empty response from OpenAI API');
}
const result = JSON.parse(content);
// Add timestamp to each checklist item
const timestamp = new Date().toISOString();
if (Array.isArray(result.items)) {
result.items.forEach((item) => {
item.createdAt = timestamp;
item.updatedAt = timestamp;
// Add file reference if not present
if (!item.file) {
item.file = filePath;
}
});
}
return result;
}
catch (error) {
console.error('Failed to generate checklist:', error);
return {
categories: [],
items: []
};
}
}
/**
* Build a prompt for generating security checklist from code
* @param sourceCode Source code content to analyze
* @param filePath Path to the source file (for context)
* @param type Type of checklist to generate (security or quality)
* @returns Prompt for generating checklists
*/
buildChecklistGenerationPrompt(sourceCode, filePath, type) {
return [
`You are a security expert analyzing the following ${type === 'security' ? 'security' : 'quality'} issues in ${filePath}.`,
"",
"As part of the Ctrl.shift.left toolkit, you specialize in shifting security testing left in the development lifecycle by identifying issues early.",
"",
"Create a detailed checklist covering these categories:",
"",
"1. INPUT VALIDATION & SANITIZATION",
" - XSS vulnerabilities",
" - SQL Injection points",
" - Command Injection risks",
" - Input boundary testing",
" - Missing sanitization",
"",
"2. AUTHENTICATION & AUTHORIZATION",
" - Improper authentication flows",
" - Missing authorization checks",
" - Hard-coded credentials",
" - Weak password policies",
" - Authentication bypass risks",
"",
"3. DATA PROTECTION & PRIVACY",
" - Sensitive data exposure",
" - Missing encryption",
" - Insecure data storage",
" - PII handling issues",
" - Improper access controls",
"",
"4. ERROR HANDLING & LOGGING",
" - Information disclosure in errors",
" - Insufficient error logging",
" - Missing exception handling",
" - Stack traces exposed to users",
"",
"5. SESSION MANAGEMENT",
" - Insecure session handling",
" - Missing session timeouts",
" - Session fixation risks",
" - CSRF vulnerabilities",
"",
"6. API SECURITY",
" - Missing rate limiting",
" - Insecure API endpoints",
" - Improper CORS configuration",
" - Missing API authentication",
"",
"7. FRONTEND SECURITY",
" - DOM-based vulnerabilities",
" - Client-side validation only issues",
" - Insecure state management",
" - Insecure local storage usage",
"",
"## SECURITY FOCUS AREAS",
"- Input validation and sanitization",
"- Authentication and authorization",
"- Session management",
"- Data protection and privacy",
"- Cryptographic vulnerabilities",
"- Error handling and logging",
"- Secure configuration",
"- Secure communication (TLS, CORS, CSP)",
"- Third-party dependencies",
"- API security",
"- Business logic vulnerabilities",
"- Client-side security (XSS, CSRF)",
"- Supply chain security",
"- Code injection vulnerabilities",
"",
"## RISK ASSESSMENT",
"For each security issue, provide a risk assessment with:",
"- CVSS score (0.0-10.0) with calculation explanation",
"- Impact level (critical, high, medium, low, info)",
"- Likelihood of exploitation (very-high, high, medium, low, very-low)",
"- References to security standards (CWE, OWASP)",
"",
"For each identified issue, include:",
"- A clear description of the issue with line number references",
"- Severity (Critical/High/Medium/Low/Info) with explanation of impact",
"- Specific verification steps",
"- Category classification",
"- Detailed remediation guidance with code examples",
"- Security standards references (CWE, OWASP)",
"- Risk scoring (CVSS or similar methodology)",
"",
"Return your checklist in this JSON format:",
"{",
' "categories": ["InputValidation", "Authentication", "DataProtection", ...],',
' "items": [',
' {',
' "id": "SEC-1",',
' "title": "[Issue type]: [Specific instance]",',
' "description": "Detailed explanation with line references",',
' "category": "One of the categories listed above",',
' "severity": "critical|high|medium|low|info",',
' "verification": "Concrete steps to verify the issue is fixed",',
' "lineNumber": 42,',
' "codeSnippet": "const password = \\\'myPassword\\\'",',
' "references": [',
' {',
' "cwe": "CWE-259",',
' "owasp": "A07:2021-Identification and Authentication Failures",',
' "description": "Hard-coded password",',
' "url": "https://cwe.mitre.org/data/definitions/259.html"',
' }',
' ],',
' "remediation": {',
' "description": "Replace hardcoded credentials with environment variables",',
' "codeExample": "const password = process.env.PASSWORD",',
' "effort": "low",',
' "priority": "high"',
' },',
' "riskScore": {',
' "score": 7.5,',
' "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N",',
' "impact": "high",',
' "likelihood": "medium",',
' "calculator": "CVSS-3.1"',
' }',
' }',
' ]',
'}',
"",
"Source code to analyze:",
"",
"```",
sourceCode,
"```"
].join("\n");
}
/**
* Perform semantic search on source code to find security issues
* @param sourceCode Source code to analyze
* @param query Search query for semantic matching
* @returns Array of matches with relevance scores
*/
async performSemanticSearch(sourceCode, query) {
try {
if (!process.env.OPENAI_API_KEY) {
throw new Error('OPENAI_API_KEY environment variable not set');
}
// Split code into lines for better context
const lines = sourceCode.split('\n');
// Build a prompt for semantic search
const prompt = [
`Query: "${query}"`,
"",
"Source code to analyze for security issues related to the query:",
"",
"```",
sourceCode,
"```",
"",
"Instructions:",
"1. Identify code sections that match the security query semantically",
"2. For each match, provide:",
" - Line number (approximate)",
" - A relevance score (0.0-1.0) indicating how well the match relates to the query",
" - A code snippet showing the relevant section",
" - Context explaining the security implications, risks, and potential mitigations",
"",
"Return results in this JSON format:",
"{",
" \"matches\": [",
" {",
" \"line\": 42,",
" \"score\": 0.85,",
" \"snippet\": \"const userInput = req.params.id;\",",
" \"securityContext\": \"This line accepts user input without validation, creating potential for SQL injection...\"",
" }",
" ]",
"}"
].join('\n');
// Call OpenAI API with security-focused system message
const response = await this.openai.chat.completions.create({
model: 'gpt-4-turbo-preview',
messages: [
{
role: 'system',
content: 'You are a specialized security code analyzer for the Ctrl.shift.left toolkit. Your purpose is to identify security vulnerabilities, code patterns, and implementation details that match specific security queries. You excel at understanding the semantic meaning behind security-related queries and finding relevant code sections, even when they don\'t contain the exact keywords. Focus exclusively on security implications with high precision.'
},
{
role: 'user',
content: prompt
}
],
temperature: 0.1, // Lower temperature for more precise results
response_format: { type: 'json_object' }
});
// Parse and validate the response
const content = response.choices[0]?.message.content;
if (!content) {
throw new Error('Empty response from OpenAI API');
}
const parsedResponse = JSON.parse(content);
if (!Array.isArray(parsedResponse.matches)) {
throw new Error('Invalid response format: expected matches array');
}
return parsedResponse.matches;
}
catch (error) {
console.error('Failed to perform semantic search:', error);
return [];
}
}
}
exports.LLMService = LLMService;
//# sourceMappingURL=llmService.js.map