project-indexer/src/projectIndexer.js

Local project indexer with browser trigger support

// import dotenv from 'dotenv';
import axios from 'axios';
import FormData from 'form-data';
import fs from 'fs';
import path from 'path';
import {
  DEFAULT_EXTENSIONS,
  getFocusedFiles
} from './fileValidator.js';
// dotenv.config();

/**
 * Indexes the project files and uploads them to the server
 * @param {string} projectPath - Path to the project directory
 * @param {string} serverUrl - URL of the server
 * @param {string} projectId - Project ID
 * @param {string} token - Authorization token
 * @param {string} deviceId - Device ID
 * @param {string[]} extensions - Array of file extensions to include
 * @returns {Promise<Object>} - Server response
 */
export async function indexProject(
  projectPath,
  serverUrl,
  projectId,
  projectName,
  token,
  deviceId,
  extensions = DEFAULT_EXTENSIONS
) {
  const baseDir = path.resolve(projectPath);
  console.log(`[Indexing] Starting project indexing from ${baseDir}`);
  
  try {
    // Get files to be indexed using the validator module
    const files = await getFocusedFiles(baseDir, extensions);
    console.log(`[Indexing] Found ${files.length} core project files.`);
    
    if (files.length === 0) {
      console.warn('[Warning] No files found to index. Check your ignore patterns and extensions.');
      return { success: false, message: 'No files found to index' };
    }
    
    // Upload the files
    const result = await uploadFiles(baseDir, files, serverUrl, projectId, projectName, token, deviceId);
    return result;
  } catch (error) {
    console.error(`[Error] Failed to index project: ${error.message}`);
    throw error;
  }
}

/**
 * Uploads files to the server
 * @param {string} baseDir - Base directory of the project
 * @param {string[]} files - Array of absolute file paths
 * @param {string} serverUrl - URL of the server
 * @param {string} projectId - Project ID
 * @param {string} token - Authorization token
 * @param {string} deviceId - Device ID
 * @returns {Promise<Object>} - Server response
 */
async function uploadFiles(baseDir, files, serverUrl, projectId, projectName, token, deviceId) {
  // Prepare form data for upload
  const form = new FormData();
  const filePaths = [];
  
  // Add each file to the form data
  files.forEach((absPath) => {
    try {
      const relativePath = path.relative(baseDir, absPath).replace(/\\/g, '/');
      const fileStream = fs.createReadStream(absPath);
      form.append('files', fileStream, { filename: path.basename(absPath) });
      filePaths.push(projectName + '/' + relativePath);
    } catch (error) {
      console.warn(`[Warning] Could not read file ${absPath}: ${error.message}`);
    }
  });
  
  // Add file paths to the form data
  filePaths.forEach(fp => form.append('filePaths', fp));
  
  try {
    console.log(`[Uploading] Uploading ${filePaths.length} files to server...`);

    filePaths.forEach(fp=>console.log(fp));
    
    // Make the request to the server
    const response = await axios.post(
      `${serverUrl}`,
      form,
      {
        headers: {
          ...form.getHeaders(),
          'X-Device-Id': deviceId,
          Authorization: `Bearer ${token}`,
        },
        maxBodyLength: Infinity,
        maxContentLength: Infinity,
        timeout: 60000, // 60 seconds timeout
      }
    );
    
    console.log(`[Success] Successfully uploaded ${filePaths.length} files.`);
    return response.data;
  } catch (err) {
    handleUploadError(err);
    throw err;
  }
}

/**
 * Handles upload errors with detailed logging
 * @param {Error} err - The error object
 */
function handleUploadError(err) {
  console.error(`[Upload failed] ${err.message}`);
  
  if (err.response) {
    // The request was made and the server responded with a status code
    // that falls out of the range of 2xx
    console.error(`[Server response] Status: ${err.response.status}`);
    console.error(`[Server response] Headers:`, err.response.headers);
    console.error(`[Server response] Data:`, err.response.data);
  } else if (err.request) {
    // The request was made but no response was received
    console.error('[Network error] No response received from server');
    console.error(`[Request details]:`, err.request);
  } else {
    // Something happened in setting up the request that triggered an Error
    console.error('[Request setup error]', err.message);
  }
  
  if (err.code === 'ECONNABORTED') {
    console.error('[Timeout] The request timed out. Consider increasing the timeout value or reducing the number of files.');
  }
}

// /**
//  * Estimates the total size of files to be uploaded
//  * @param {string[]} files - Array of absolute file paths
//  * @returns {Promise<number>} - Total size in bytes
//  */
// export async function estimateUploadSize(files) {
//   let totalSize = 0;
  
//   for (const file of files) {
//     try {
//       const stats = await fs.promises.stat(file);
//       totalSize += stats.size;
//     } catch (error) {
//       console.warn(`[Warning] Could not get size of file ${file}: ${error.message}`);
//     }
//   }
  
//   return totalSize;
// }

// /**
//  * Validates server connection before attempting upload
//  * @param {string} serverUrl - URL of the server
//  * @param {string} token - Authorization token
//  * @returns {Promise<boolean>} - True if connection is valid
//  */
// export async function validateServerConnection(serverUrl, token) {
//   try {
//     const response = await axios.get(
//       `${serverUrl}/health`,
//       {
//         headers: {
//           Authorization: `Bearer ${token}`
//         },
//         timeout: 5000 // 5 seconds timeout
//       }
//     );
    
//     return response.status === 200;
//   } catch (error) {
//     console.error(`[Connection validation failed] ${error.message}`);
//     return false;
//   }
// }



// /**
//  * Entry point function that handles the entire indexing process
//  * @param {Object} options - Options for indexing
//  * @returns {Promise<Object>} - Result of the indexing process
//  */
// export async function runIndexing(options) {
//   const {
//     projectPath,
//     serverUrl,
//     token,
//     deviceId,
//     projectId,
//     projectName,
//     extensions
//   } = options;
  
//   try {
//     // Validate server connection
//     const isConnected = await validateServerConnection(serverUrl, token);
//     if (!isConnected) {
//       console.error('[Error] Could not connect to the server. Please check your server URL and token.');
//       return { success: false, message: 'Server connection failed' };
//     }
    
//     // Get or create project ID
//     let finalProjectId = projectId;
//     if (!finalProjectId && projectName) {
//       const project = await createProject(serverUrl, token, deviceId, { name: projectName });
//       finalProjectId = project.projectId;
//     }
    
//     if (!finalProjectId) {
//       console.error('[Error] Project ID is required. Either provide a project ID or a project name to create a new project.');
//       return { success: false, message: 'Project ID is required' };
//     }
    
//     // Run the indexing
//     const result = await indexProject(
//       projectPath,
//       serverUrl,
//       finalProjectId,
//       token,
//       deviceId,
//       extensions
//     );
    
//     return { success: true, ...result, projectId: finalProjectId };
//   } catch (error) {
//     console.error(`[Indexing failed] ${error.message}`);
//     return { success: false, message: error.message };
//   }
// }