website-to-json/test2.js

Converts all websites to JSON data

var wtj = require('website-to-json')
var wtj = require('./index')

var Nightmare = require('nightmare');
var nightmare = Nightmare({
  show: true,
}).useragent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36')

//wtj.extractUrl('http://itemsapi.com/', {
wtj.extractUrl('https://www.google.com/?gfe_rd=cr&ei=#safe=off&q=reddit+bitcoin', {
  fields: ['data'],
  nightmare: nightmare,
  recipes: [{
    //pattern: 'https://en.wikipedia.org/wiki/(.*)',
    // you can also catch all pages
    pattern: '(.*)',
    parse: function($) {
      return {
        name: $('h1').text(),
        results_count: $('#resultStats').text()
      }
    }
  }]
})
.then(function(res) {
  console.log(res)
})

/*Promise.all(['onet.pl', 'wp.pl', 'interia.pl', 'gazeta.pl'])
.then(function(val) {
  return val
})
.map(function(val) {
  return wtj.extractUrl(val, {
    nightmare: nightmare,
    fields: ['data', 'meta', 'social'],
  })
  .then(function(res) {
    console.log(res);
  })
}, {concurrency: 1})*/