@dawans/promptshield
Version:
Secure your LLM stack with enterprise-grade RulePacks for AI safety scanning
292 lines (291 loc) • 11.1 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.JsonProcessor = void 0;
const Result_1 = require("../../../../shared/types/Result");
const StreamJson = __importStar(require("stream-json"));
const StreamArray_1 = require("stream-json/streamers/StreamArray");
/**
* Processes JSON and NDJSON content
*/
class JsonProcessor {
/**
* Checks if this processor can handle the given file type
*/
canProcess(filePath) {
const extensions = this.getSupportedExtensions();
return extensions.some((ext) => filePath.toLowerCase().endsWith(ext));
}
/**
* Gets the supported file extensions
*/
getSupportedExtensions() {
return ['.json', '.ndjson', '.jsonl'];
}
/**
* Processes JSON content and returns structured data
*/
async process(content, context) {
try {
const isNdjson = context.isNdjsonMode();
if (isNdjson) {
return this.processNdjson(content, context);
}
else {
return this.processRegularJson(content, context);
}
}
catch (error) {
return (0, Result_1.err)(new Error(`Failed to process JSON content: ${error}`));
}
}
/**
* Processes regular JSON (array or single object)
*/
async processRegularJson(content, context) {
try {
const data = JSON.parse(content);
// Handle both arrays and single objects
const items = Array.isArray(data) ? data : [data];
const results = [];
const maxObjects = context.getMaxObjects();
const fieldsToScan = context.getFieldsToScan();
for (let i = 0; i < items.length; i++) {
if (maxObjects && i >= maxObjects)
break;
const item = items[i];
const fields = {};
// Extract specified fields
for (const field of fieldsToScan) {
if (item[field] !== undefined) {
fields[field] = String(item[field]);
}
}
// Optionally scan entire object
if (context.shouldScanEntireObject()) {
fields['_entire_object'] = JSON.stringify(item);
}
results.push({
data: item,
fields,
metadata: {
index: i,
source: 'json',
type: 'object',
},
});
}
return (0, Result_1.ok)(results);
}
catch (error) {
return (0, Result_1.err)(new Error(`Failed to parse JSON: ${error}`));
}
}
/**
* Processes NDJSON (newline-delimited JSON)
*/
async processNdjson(content, context) {
try {
const lines = content.split('\n').filter((line) => line.trim());
const results = [];
const maxObjects = context.getMaxObjects();
const fieldsToScan = context.getFieldsToScan();
for (let i = 0; i < lines.length; i++) {
if (maxObjects && i >= maxObjects)
break;
const line = lines[i].trim();
if (!line)
continue;
try {
const item = JSON.parse(line);
const fields = {};
// Extract specified fields
for (const field of fieldsToScan) {
if (item[field] !== undefined) {
fields[field] = String(item[field]);
}
}
// Optionally scan entire object
if (context.shouldScanEntireObject()) {
fields['_entire_object'] = JSON.stringify(item);
}
results.push({
data: item,
fields,
metadata: {
index: i,
source: 'ndjson',
type: 'object',
},
});
}
catch (parseError) {
// Skip malformed lines
if (context.config.debug) {
console.error(`Failed to parse line ${i + 1}: ${parseError}`);
}
}
}
return (0, Result_1.ok)(results);
}
catch (error) {
return (0, Result_1.err)(new Error(`Failed to process NDJSON: ${error}`));
}
}
/**
* Processes content in streaming mode for large files
*/
async processStream(content, context, onItem) {
try {
const isNdjson = context.isNdjsonMode();
if (isNdjson) {
return this.processNdjsonStream(content, context, onItem);
}
else {
return this.processJsonStream(content, context, onItem);
}
}
catch (error) {
return (0, Result_1.err)(new Error(`Failed to process stream: ${error}`));
}
}
/**
* Processes JSON array in streaming mode
*/
async processJsonStream(content, context, onItem) {
return new Promise((resolve) => {
try {
const fieldsToScan = context.getFieldsToScan();
const maxObjects = context.getMaxObjects();
let processedCount = 0;
const pipeline = StreamJson.parser().pipe((0, StreamArray_1.streamArray)());
pipeline.on('data', async ({ value }) => {
if (maxObjects && processedCount >= maxObjects) {
pipeline.destroy();
return;
}
const fields = {};
// Extract specified fields
for (const field of fieldsToScan) {
if (value[field] !== undefined) {
fields[field] = String(value[field]);
}
}
// Optionally scan entire object
if (context.shouldScanEntireObject()) {
fields['_entire_object'] = JSON.stringify(value);
}
await onItem({
data: value,
fields,
metadata: {
index: processedCount,
source: 'json-stream',
type: 'object',
},
});
processedCount++;
});
pipeline.on('end', () => resolve((0, Result_1.ok)(undefined)));
pipeline.on('error', (error) => resolve((0, Result_1.err)(new Error(`Stream error: ${error}`))));
// Write content to pipeline
pipeline.write(content);
pipeline.end();
}
catch (error) {
resolve((0, Result_1.err)(new Error(`Failed to create stream: ${error}`)));
}
});
}
/**
* Processes NDJSON in streaming mode
*/
async processNdjsonStream(content, context, onItem) {
try {
const lines = content.split('\n');
const fieldsToScan = context.getFieldsToScan();
const maxObjects = context.getMaxObjects();
let processedCount = 0;
for (const line of lines) {
if (maxObjects && processedCount >= maxObjects)
break;
const trimmedLine = line.trim();
if (!trimmedLine)
continue;
try {
const value = JSON.parse(trimmedLine);
const fields = {};
// Extract specified fields
for (const field of fieldsToScan) {
if (value[field] !== undefined) {
fields[field] = String(value[field]);
}
}
// Optionally scan entire object
if (context.shouldScanEntireObject()) {
fields['_entire_object'] = JSON.stringify(value);
}
await onItem({
data: value,
fields,
metadata: {
index: processedCount,
source: 'ndjson-stream',
type: 'object',
},
});
processedCount++;
}
catch (parseError) {
// Skip malformed lines
if (context.config.debug) {
console.error(`Failed to parse line: ${parseError}`);
}
}
}
return (0, Result_1.ok)(undefined);
}
catch (error) {
return (0, Result_1.err)(new Error(`Failed to process NDJSON stream: ${error}`));
}
}
/**
* Determines if streaming should be used based on content size
*/
shouldUseStreaming(contentSize, threshold) {
return contentSize > threshold * 1024 * 1024; // Convert MB to bytes
}
}
exports.JsonProcessor = JsonProcessor;