url-metadata/test/robots.test.js

Request a url and scrape the metadata from its HTML using Node.js or the browser.

const urlMetadata = require('./../index')

test('retrieves robots meta directives', async () => {
  const url = 'https://moz.com/learn/seo/robots-meta-directives'
  try {
    const metadata = await urlMetadata(url)
    expect(metadata.robots).toBe('all')
  } catch (err) {
    expect(err).toBe(undefined)
  }
})