twl-generator/src/utils/usfm-alignment-remover.js

Generate term-to-article lists from unfoldingWord en_tw archive for Bible books. Works in both Node.js (CLI) and React.js (browser) environments.

/* eslint-disable no-async-promise-executor, no-throw-literal */

import { BibleBookData } from '../common/books.js';
import { removeAlignments } from 'usfm-alignment-remover';

// Environment detection
const isNode = typeof window === 'undefined' && typeof process !== 'undefined' && process.versions?.node;

// Get appropriate fetch implementation
async function getFetch() {
  // Both Node.js 18+ and browsers have native fetch
  return globalThis.fetch;
}

// Get appropriate base64 decoder
function decodeBase64(base64String) {
  if (isNode) {
    return Buffer.from(base64String, 'base64').toString('utf-8');
  }
  // Browser implementation - properly handle UTF-8 characters (e.g., smart quotes)
  const binaryString = atob(base64String);
  const bytes = new Uint8Array(binaryString.length);
  for (let i = 0; i < binaryString.length; i++) {
    bytes[i] = binaryString.charCodeAt(i);
  }
  const decoder = new TextDecoder('utf-8');
  return decoder.decode(bytes);
}

// Note: This version doesn't use usfm-js to avoid external dependencies
// It implements a simple USFM alignment remover for the specific case

export const removeAllTagsExceptChapterVerse = (usfmContent) => {
  if (!usfmContent) return '';

  let cleanContent = removeAlignments(usfmContent);

  // Remove empty lines that might result from marker removal
  cleanContent = cleanContent.replace(/\n\s*\n\s*\n/g, '\n\n');

  // Clean up any remaining alignment syntax patterns
  cleanContent = cleanContent.replace(/\|[^\\]*(?=\\)/g, '');

  cleanContent = cleanContent.replace(/\n/g, ' ');
  cleanContent = cleanContent.replace(/ +\\v +/g, '\n\\v ');
  cleanContent = cleanContent.replace(/ +\\c +/g, '\n\\c ');
  cleanContent = cleanContent.replace(/ *(\\q\d*|\\p|\\ts\\\*) */g, ' ');
  // Strip section headings (\s, \s1, \sr, \sp, etc.) but preserve \d (chapter
  // descriptions / psalm superscriptions) so front-matter TWLs can be generated.
  cleanContent = cleanContent.replace(/\\s.*?(\\|\n)/g, '$1');
  cleanContent = cleanContent.replace(/ +/g, ' ');
  cleanContent = cleanContent.replace(/^ +$/g, '');
  cleanContent = cleanContent.replace(/\\f .*?\\f\*/g, ' ');
  // NOTE: Curly braces ({ }) wrap "supplied" words/morphemes in the ULT (e.g.
  // "creature{s}") that the matcher must see through but must NOT discard. They
  // are preserved here so the matcher can match the brace-free reading
  // ("creatures") yet retain the braces in OrigWords ("creature{s}"), which is
  // what tsv-quote-converters needs to align the quote back to the original
  // language. Brace-transparency lives in twl-matcher.js (findMatches).

  // Remove all lines before the first \c marker, keeping the \c line
  const lines = cleanContent.split('\n');
  const firstCIndex = lines.findIndex(line => line.includes('\\c'));
  if (firstCIndex > 0) {
    cleanContent = lines.slice(firstCIndex).join('\n');
  }

  return cleanContent.trim();
};

/**
 * Download and process USFM file for a given book
 * @param {string} book - The book identifier
 * @return {Promise<Object>} - Object with chapters and verses
 */
export async function processUsfmForBook(book, dcsHost = 'https://git.door43.org') {
  // Normalize book key to lowercase to match BibleBookData keys
  const key = String(book || '').toLowerCase();
  if (!BibleBookData[key]) throw new Error(`Unknown book: ${book}`);

  const fetch = await getFetch();
  const usfmUrl = `${dcsHost}/api/v1/repos/unfoldingWord/en_ult/contents/${BibleBookData[key].usfm}.usfm?ref=master`;
  const usfmRes = await fetch(usfmUrl);
  if (!usfmRes.ok) throw new Error(`Failed to download USFM file for ${book}`);
  const usfmData = await usfmRes.json();
  const usfmContent = decodeBase64(usfmData.content);

  // Remove alignments from USFM
  const cleanUsfm = removeAllTagsExceptChapterVerse(usfmContent);

  // Parse USFM into chapters and verses
  return parseUsfmToVerses(cleanUsfm);
}

/**
 * Parse clean USFM content into a chapters/verses object
 * @param {string} usfm - Clean USFM content
 * @return {Object} - Object keyed by chapter number, then verse number
 */
export function parseUsfmToVerses(usfm) {
  const versesObj = {};
  let currentChapter = 1;

  // Split by chapters and verses
  const parts = usfm.split(/\\([cv])\s*(\d+)/);

  for (let i = 1; i < parts.length; i += 3) {
    const tag = parts[i]; // 'c' or 'v'
    const number = parseInt(parts[i + 1]);
    const text = parts[i + 2] || '';

    if (tag === 'c') {
      currentChapter = number;
      if (!versesObj[currentChapter]) {
        versesObj[currentChapter] = {};
      }
      // Capture chapter front matter (\d description / psalm superscription) so it
      // can produce `<chapter>:front` TWL rows. Other pre-verse markers (\s, \q, \p)
      // have already been stripped, leaving the \d text in the chapter head.
      const frontMatch = text.match(/\\d\s+([^\\]*)/);
      if (frontMatch) {
        const frontText = frontMatch[1].replace(/\s+/g, ' ').trim();
        if (frontText) {
          versesObj[currentChapter].front = frontText;
        }
      }
    } else if (tag === 'v') {
      if (!versesObj[currentChapter]) {
        versesObj[currentChapter] = {};
      }
      // Clean up the text: remove extra whitespace and newlines
      const cleanText = text.replace(/\s+/g, ' ').trim();
      if (cleanText) {
        versesObj[currentChapter][number] = cleanText;
      }
    }
  }

  return versesObj;
}