site-validator-cli/lib/get-urls.js

A command line tool that takes a URL or a file, then uses html-validator (a wrapper for https://validator.w3.org/nu/) to validate each page.

'use strict'

module.exports = async (options) => {
  const urlsFromFileOnline = require('./helpers-get-urls/get-urls-from-file-online')
  const urlsFromCrawler = require('./helpers-get-urls/get-urls-from-crawler')
  const urlsFromFile = require('./helpers-get-urls/get-urls-from-file')
  const getPathType = require('./helpers-get-urls/get-path-type')
  const normalizer = require('normalize-url')
  const { yellow } = require('./clc')

  var pathType = getPathType(options.path)
  var path = options.path
  var data
  if (pathType === 'file') {
    data = await urlsFromFile(options.path)
  } else {
    if (options.isLocal) {
      if (/^(localhost)/.test(options.path)) {
        path = normalizer(options.path, { forceHttp: true })
      }
    } else {
      if (!/^(http)/.test(options.path)) {
        path = normalizer(options.path, { forceHttps: true })
      }
    }
    if (pathType === 'online-file') {
      data = await urlsFromFileOnline(path, options.cacheTime, options.clearCache)
      if (data.length === 0) {
        console.log(yellow('No Urls Found, retrying with HTTP...'))
        path = normalizer(options.path, { forceHttp: true })
        data = await urlsFromFileOnline(path, options.cacheTime, options.clearCache)
      }
    } else {
      if (options.singlePage === true) {
        data = [path]
      } else {
        data = await urlsFromCrawler(path, options.cacheTime, options.clearCache)
        if (data.length === 0) {
          if (options.isLocal) {
            data = []
          } else {
            console.log(yellow('No Urls Found, retrying with HTTP...'))
            path = normalizer(options.path, { forceHttp: true })
            data = await urlsFromCrawler(path, options.cacheTime, options.clearCache)
          }
        }
      }
    }
  }
  options.path = path

  if (data.length === 0) {
    throw new Error('No Urls found')
  }

  return data
}