@dawans/promptshield
Version:
Secure your LLM stack with enterprise-grade RulePacks for AI safety scanning
105 lines (104 loc) • 3.3 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.TextProcessor = void 0;
const Result_1 = require("../../../../shared/types/Result");
/**
* Processes plain text content
*/
class TextProcessor {
/**
* Checks if this processor can handle the given file type
*/
canProcess(filePath) {
const extensions = this.getSupportedExtensions();
return extensions.some((ext) => filePath.toLowerCase().endsWith(ext));
}
/**
* Gets the supported file extensions
*/
getSupportedExtensions() {
return ['.txt', '.text', '.log', '.md'];
}
/**
* Processes text content and returns structured data
*/
async process(content, context) {
try {
const maxObjects = context.getMaxObjects();
// For text files, treat the entire content as one object
const results = [
{
data: content,
fields: {
content: content,
},
metadata: {
index: 0,
source: 'text',
type: 'text',
},
},
];
// If maxObjects is set to 0, return empty array
if (maxObjects === 0) {
return (0, Result_1.ok)([]);
}
return (0, Result_1.ok)(results);
}
catch (error) {
return (0, Result_1.err)(new Error(`Failed to process text content: ${error}`));
}
}
/**
* Splits text into chunks if needed (for large text files)
*/
splitIntoChunks(content, chunkSize = 10000) {
const chunks = [];
const lines = content.split('\n');
let currentChunk = '';
for (const line of lines) {
if (currentChunk.length + line.length > chunkSize &&
currentChunk.length > 0) {
chunks.push(currentChunk);
currentChunk = line;
}
else {
currentChunk += (currentChunk ? '\n' : '') + line;
}
}
if (currentChunk) {
chunks.push(currentChunk);
}
return chunks;
}
/**
* Process text with chunking for large files
*/
async processWithChunking(content, context) {
try {
const maxObjects = context.getMaxObjects();
const chunks = this.splitIntoChunks(content);
const results = [];
for (let i = 0; i < chunks.length; i++) {
if (maxObjects && i >= maxObjects)
break;
results.push({
data: chunks[i],
fields: {
content: chunks[i],
},
metadata: {
index: i,
source: 'text-chunk',
type: 'text',
},
});
}
return (0, Result_1.ok)(results);
}
catch (error) {
return (0, Result_1.err)(new Error(`Failed to process text with chunking: ${error}`));
}
}
}
exports.TextProcessor = TextProcessor;