@lg-tools/crawler/src/utils/createClusterNameFromURL.ts

Crawler for MongoDB documentation and other sites

/**
 * If we're crawling a new URL, we need to create a cluster name from it.
 */
export const createClusterNameFromURL = (url: string): string => {
  const urlObj = new URL(url);
  const hostname = urlObj.hostname;
  // Remove the TLD (everything after the last dot) and any 'www.' prefix
  const newCollectionName = hostname
    .replace(/^www\./, '') // Remove www. prefix if present
    .replace(/\.[^.]+$/, '') // Remove the TLD (last dot and everything after)
    .replace(/\./g, '-'); // Replace any remaining dots with hyphens

  return newCollectionName;
};