discord-media-server/lib/scanner.js

Self-hosted Discord bot for scanning and serving information on local media files.

// lib/scanner.js (unified with database.js + supports SQLite/MySQL)
import fs from 'fs';
import path from 'path';
import axios from 'axios';
import dotenv from 'dotenv';
dotenv.config();

import { CONFIG_DIR, DATA_DIR } from './setupConfig.js';
import { writeFile, readFile } from 'fs/promises';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

const envPath = path.join(CONFIG_DIR, '.env');
if (!fs.existsSync(envPath)) {
  console.warn('No .env file found. Using environment variables only.');
}

//if (loaded.error) {
//  console.error('\nFailed to load .env\nPlease run setup first\n\n', loaded.error);
//  process.exit(1);
//}

const API_KEY = process.env.TMDB_API_KEY;
if (!API_KEY) {
  console.error('Missing TMDB_API_KEY in .env');
  //console.error(`Looked in: ${envPath}`);
  //process.exit(1);
}

const MEDIA_DIR = process.env.MEDIA_DIR;
const DB_NAME = process.env.DB_NAME || 'media';

if (!MEDIA_DIR) {
  console.error('\nMissing MEDIA_DIR in .env');
  process.exit(1);
}

import { query, close, isSQLite, initDatabase, backupSQLite } from './database.js';

const CACHE_FILE = `${DB_NAME}.json`;
const cachePath = path.join(DATA_DIR, CACHE_FILE);
if (!fs.existsSync(cachePath)) {
  console.warn('No .json cache file found. Creating one.');
}

const VIDEO_EXTENSIONS = ['.mp4', '.mkv', '.avi'];

function walk(dir, fileList = []) {
  //let folder;
  const files = fs.readdirSync(dir);
  for (const file of files) {
    const fullPath = path.join(dir, file);
    const stat = fs.statSync(fullPath);
    if (stat.isDirectory()) {
      /*
      folder = fullPath.replace(/\/$/, "").split("\\").pop();
      if (!folder.startsWith('.')) {
        console.log(' \\'+folder);
      }
      */
      walk(fullPath, fileList);
    } else if (VIDEO_EXTENSIONS.includes(path.extname(file).toLowerCase())) {
      console.log(' '+file);
      fileList.push(fullPath);
    }
  }
  return fileList;
}

function parseFileName(filename) {
  const name = path.basename(filename, path.extname(filename));
  const format = path.extname(filename).substring(1).toUpperCase();

  // Normalize separators to spaces
  const normalized = name.replace(/[\.\_]/g, ' ').replace(/\s+/g, ' ').trim();
  const yearMatch = normalized.match(/\b(19\d{2}|20\d{2})\b/);

  let year = '';
  let title = '';

  if (yearMatch) {
    year = yearMatch[yearMatch.length - 1];

    // Everything before the release year = title
    title = normalized.split(year)[0].trim();

    //year = yearMatch[0]; // Store the extracted year
    //const yearIndex = normalized.indexOf(year); // Find the starting index of the year

    // Extract the title as everything before the year
    //title = normalized.substring(0, yearIndex).trim();

    // Remove any trailing spaces or punctuation from the title
    //title = title.replace(/[\s\(\)\.\-]+$/, ''); 
  } else {
    // If no year is found, consider the entire normalized string as the title
    title = normalized;
  }
  title = title.replace("(", "").trim();

  return { title, year, format };
}


async function fetchFromTMDb(title, year = null) {
  try {
    let release_year = '';
    const params = {
      api_key: API_KEY || '',
      query: title || '',
      ...(year ? { year } : {})
    };

    const res = await axios.get('https://api.themoviedb.org/3/search/movie', { params });
    const movie = res.data.results?.[0];
    
    if (!movie) return null;

    if (res.data.results.length == 1 && !year) {
      year = movie.release_date?.substring(0, 4);
      release_year = movie.release_date?.substring(0, 4) || '';
    }

    // Step 2: Get full details using the TMDB movie ID
    const detailsRes = await axios.get(`https://api.themoviedb.org/3/movie/${movie.id}`, {
      params: { api_key: API_KEY }
    });

    const details = detailsRes.data;

    return {
      tmdb: movie.id?.toString() || '',
      imdb: details.imdb_id || '',
      poster_fallback: movie.poster_path ? `https://image.tmdb.org/t/p/w500${movie.poster_path}` : '',
      rating: movie.vote_average?.toFixed(1) || '',
      runtime: details.runtime ? `${details.runtime} mins` : '',
      overview: movie.overview || '',
      year: year || '',
      release: release_year || '',
      backdrop: details.backdrop_path ? `https://image.tmdb.org/t/p/original${details.backdrop_path}` : '',
      popularity: details.popularity || '',
      realTitle: movie.title || ''
    };
  } catch (err) {
    console.error(`TMDb error for "${title}" (${year || ''}):`, err.message);
    return null;
  }
}

// main scan function
export default async function runScanner(mediaDir = process.env.MEDIA_DIR, dbName = process.env.DB_NAME || 'media') {
  if (!mediaDir) {
    throw new Error('Missing MEDIA_DIR');
  }

  const startTime = Date.now(); // scan time start

  await initDatabase(); // This is crucial for db to work!

  await backupSQLite(); // backup the current db before scan

  console.log(`\n${new Date().toLocaleString()} Run Scanner triggered\n`,"\x1b[1m");
  console.log(`Scanning Directory: ${mediaDir}`,"\x1b[0m");

  const files = walk(mediaDir);
  const cached = await loadCache();
  const currentList = [];
  const added = [];
  const updated = [];
  const unchanged = [];
  const removed = [];
  const duplicates = [];
  const possibleDuplicates = [];
  const seenTitle = new Set();
  const seenTitleYear = new Set();
  const currentMap = new Map();
  const realTitleClusters = new Map(); // realTitle => [filename1, filename2, ...]
  
  console.log("\n","\x1b[1m");
  console.log(`File Compare:`,"\x1b[0m");
  for (const file of files) {
    const filename = path.basename(file);
    let { title, year, format } = parseFileName(filename);
    if (!title) {
      console.log(`* SKIPPING: ${title} - ${year} - ${filename}`);
      continue;
    }
    if (!format) {
      //console.log(`* SKIPPING: ${title} - ${year} - ${filename}`);
      format = path.extname(filename).slice(1).toUpperCase();
    }

    const stat = fs.statSync(file);
    //const mtime = fs.statSync(file).mtimeMs;
    const relativePath = path.relative(mediaDir, file).replace(/\\/g, '/');
    const folder = path.dirname(relativePath);
    const baseName = path.basename(file, path.extname(file));
    const relativePoster = `${folder}/${baseName}-poster.jpg`.replace(/\\/g, '/');

    if (!year) {
      year = '0000';
    }

    let meta = {};
    if (API_KEY) {
      meta = await fetchFromTMDb(title, year) || {};
    }

    let popularity = meta.popularity || '';
    let backdrop = meta.backdrop || '';

    // duplicate file check
    const key = `${title}|${year}`;
    if (seenTitleYear.has(key)) {
      duplicates.push({ title, year: year, filename });
      //console.log("\x1b[1;41m",`* DUPLICATE: ${title} (${year}) [${filename}]`,"\x1b[0m");
      console.log(`* DUPLICATE: ${title} (${year}) [${filename}]`);
      continue;
    }
    seenTitleYear.add(key);

    // possible duplicates with same title
    if (seenTitle.has(meta.realTitle || title)) {
      if (!meta.release || year === '0000') {
        possibleDuplicates.push({ title, filename });
        //console.log("\x1b[0;41m",`* POSSIBLE DUPLICATE: ${title} [${filename}]`,"\x1b[0m");
        console.log(`* POSSIBLE DUPLICATE: ${title} [${filename}]`);
      }
    }
    seenTitle.add(meta.realTitle);
    seenTitle.add(title);

    // group possible duplicates
    if (!realTitleClusters.has(title)) {
      realTitleClusters.set(title, []);
    }
    realTitleClusters.get(title).push({
      title,
      year,
      filename
    });

    // file data
    const fileEntry = {
      title: title,
      year: year,
      filename: filename,
      filepath: relativePath,
      poster: relativePoster,
      format: format,
      filesize: stat.size,
      poster_fallback: meta.poster_fallback || '',
      imdb: meta.imdb || '', 
      runtime: meta.runtime || '', 
      rating: meta.rating || '', 
      overview: meta.overview || ''
    };

    currentMap.set(key, fileEntry);
    currentList.push(fileEntry);

    const cachedEntry = cached.find(f => f.title === title && f.year === year);

    if (!cachedEntry) {
      added.push(fileEntry);
      console.log("\x1b[42m",`ADDED: ${fileEntry.filename}`,"\x1b[0m");
    } else if (
      cachedEntry.filename !== filename ||
      cachedEntry.filepath !== relativePath ||
      cachedEntry.format !== format ||
      cachedEntry.filesize !== stat.size
    ) {
      updated.push(fileEntry);
      console.log("\x1b[43m",`UPDATED: ${fileEntry.filename}`,"\x1b[0m");
    } else {
      unchanged.push(fileEntry);
      console.log(`UNCHANGED: ${fileEntry.filename}`);
    }

/*
    // download image
    if (meta.poster && !fs.existsSync(relativePoster)) {
      console.log(`Downloading poster to ${relativePoster}`);
      await downloadImage(meta.poster, relativePoster);
    }
*/

  } // end of movie for loop

  // handle removed/missing files
  for (const cachedEntry of cached) {
    const key = `${cachedEntry.title}|${cachedEntry.year}`;
    if (!currentMap.has(key)) {
      removed.push(cachedEntry);
      console.log("\x1b[101m", `REMOVED: ${cachedEntry.filename}`, "\x1b[0m");
    }
  }

  console.log("\n","\x1b[1m");
  console.log(`Database Operations:`,"\x1b[0m");
  const scanTime = Date.now();
  const scanDuration = (scanTime - startTime) / 1000; // file scan Time in seconds

  // === Perform DB operations ===
  for (const movie of added) {
    try {
      const result = await query(
      `INSERT OR IGNORE INTO Movie_Info
        (title, year, filename, poster, poster_fallback, filepath, filesize, imdb, format, runtime, rating, overview)
        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
        [movie.title, movie.year, movie.filename, movie.poster, movie.poster_fallback, movie.filepath, movie.filesize, movie.imdb, movie.format, movie.runtime, movie.rating, movie.overview]
      );
      if (result.changes === 1) {
        console.log(`INSERTED: ${movie.filename}`);
      //} else {
      //  console.log(`  IGNORED: ${movie.filename}`);
      }
      
    } catch(err) {
      console.log(`FAILED to insert: ${movie.filename}`, err.message);
    }
    
  }

  for (const movie of updated) {
    await query(
      `UPDATE Movie_Info 
        SET filename = ?, format = ?, filesize = ?, filepath = ?, poster = ? 
        WHERE title = ? AND year = ?`,
        [movie.filename, movie.format, movie.filesize, movie.filepath, movie.poster, movie.title, movie.year]
    );
    console.log(`UPDATED: ${movie.filename}`);
  }

  for (const movie of removed) {
    await query(
      `DELETE FROM Movie_Info WHERE filename = ?`,
      [movie.filename]
    );
    console.log(`REMOVED: ${movie.filename}`);
  }

  await saveCache(currentList); // update cache file

  const endTime = Date.now(); // Capture the end time and calculate the scan/db duration
  const dbDuration = (endTime - scanTime) / 1000; // db actions time
  const duration = (endTime - startTime) / 1000; // Time in seconds

  console.log(`\nFile Scan ${scanDuration.toFixed(2)} seconds.`);
  console.log(`DB Actions ${dbDuration.toFixed(2)} seconds.`);
  console.log(`Completed in ${duration.toFixed(2)} seconds.`);

  console.log("\n","\x1b[1;4m");
  console.log(`Scan Summary:`,"\x1b[0m");
  console.log(`  Total scanned: ${files.length}`);
  console.log(`  Added: ${added.length}`);
  console.log(`  Updated: ${updated.length}`);
  console.log(`  Unchanged: ${unchanged.length}`);
  console.log(`  Removed: ${removed.length}`);
  console.log(`  Duplicates: ${duplicates.length}`);
  console.log(`  Possible Duplicates: ${possibleDuplicates.length}`);

  try {
    if (duplicates.length > 0) {
      console.log("\n","\x1b[1;4m");
      console.log('Duplicate Files Skipped:',"\x1b[0m");
      duplicates.forEach(d => {
        console.log(`  - ${d.title} (${d.year}) [${d.filename}]`);
      });
    }
    
    if (removed.length > 0) {
      console.log("\n","\x1b[1;4m");
      console.log('Removed Files:',"\x1b[0m");
      removed.forEach(d => {
        console.log(`  - ${d.title} (${d.year}) - [${d.filename}]`);
      });
    }

    if (possibleDuplicates.length > 0) {
      console.log("\n","\x1b[1;4m");
      console.log('Possible Duplicate Files:',"\x1b[0m");
      possibleDuplicates.forEach(d => {
        console.log(`  - ${d.title} [${d.filename}]`);
      });
      console.log("\n","\x1b[4m");
      console.log('Grouped by Real Title:',"\x1b[0m");
      for (const [realTitle, files] of realTitleClusters.entries()) {
        if (files.length < 2) continue; // only show clusters with >1

        console.log(`"${realTitle}" (${files.length} files)`);
        for (const file of files) {
          console.log(`  - ${file.title} (${file.year}) [${file.filename}]`);
        }
        console.log('');
      }
    }
    
  } catch(err) {
    console.log("Results error: ", err);
  }
  
  console.log(`\n${new Date().toLocaleString()} Scanner Finished\n`);

  return { 
    totalFiles: files.length,
    newMovies: added.length,
    updatedMovies: updated.length,
    removedMovies: removed.length,
    addedList: added
  };
}


// ========== JSON cache helpers ==========
async function loadCache() {
  try {
    const raw = await readFile(cachePath, 'utf8');
    return JSON.parse(raw);
  } catch {
    return [];
  }
}

async function saveCache(data) {
  await writeFile(cachePath, JSON.stringify(data, null, 2), { encoding: 'utf8' });
}


const downloadImage = async (url, destPath) => {
  const writer = fs.createWriteStream(destPath);
  const response = await axios.get(url, { responseType: 'stream' });
  response.data.pipe(writer);
  return new Promise((resolve, reject) => {
    writer.on('finish', resolve);
    writer.on('error', reject);
  });
};


// universal scan call (CLI only)
if (process.argv[1] === __filename) {
  const MEDIA_DIR = process.env.MEDIA_DIR;
  const DB_NAME = process.env.DB_NAME || 'media';

  if (!MEDIA_DIR) {
    console.error('\nMissing MEDIA_DIR in .env');
    process.exit(1);
  }

  runScanner().catch(err => console.error('Scanner error:', err));
}