gram-stat-scraper/app.js

Scrape instagram for post and user data

const puppeteer = require('puppeteer')
const request = require('request')
const defaultSettings = {
  updateRoute: process.env.BG_UPDATE_ROUTE || '',
  baseUrl: process.env.BG_BASE_URL || 'https://nowhere.com',
  queueOptions: JSON.parse(process.env.BG_QUEUE_OPTIONS) || null,
  queueRoute: process.env.BG_QUEUE_ROUTE || '',
  updateOptions: JSON.parse(process.env.BG_UPDATE_OPTIONS) || null,
  updateUsers: true,
  getQueue: true,
  includePhotos: false
}

module.exports = {
  runScrape
}
function runScrape(users, settings){
  settings = settings || defaultSettings
  if(settings.getQueue){ return getQueue(settings)}
  if(users){return scrape(users, settings)}
  if(!users){return new Error("UsersEmptyError: No users to scrape")}
}
function getQueue(settings){
  settings = settings || defaultSettings
  var queueOptions
  request(`${settings.baseUrl}${settings.queueRoute}`, settings.queueOptions, function(err, res, body){
    if(err){ return err}
    if(!body.users){ return }
    scrape(body.users, settings)
  }).on('error', function(err){
    console.error(err)
    return err  
  })
}

function scrape(users, settings){
  settings = settings || defaultSettings
  var q = {}
  q.users = []
  q.rejectedUsers = []
  const startTime = new Date().getTime()
  return puppeteer.launch({headless: true}).then((browser) => {
     q.browser = browser
     return q.browser.newPage()
  }).then((page) => {

      const run = async () => {
        var count = 0
        for (user of users){
          count += 1
          if(!user.gramHandle) continue 
          await page.goto(`https://instagram.com/${user.gramHandle.replace('@','').replace(' ', '')}`, {waitUntil: 'networkidle2'})
          const scriptsArr = await page.evaluate(
            () => [...document.querySelectorAll('script')].map(elem => elem.innerText)
          )

          const origText = scriptsArr.filter((s) => s.includes('window._sharedData'))[0] || null
          if(!origText) { q.rejectedUsers.push(user);  continue}
          const splitText = origText.split('window._sharedData = ')
          
          if(splitText.length == 1){ q.rejectedUsers.push(user) ; continue}
          const jsonString = splitText[1].replace(';', '') 
          
          var jsonObj 
          try {
            jsonObj = JSON.parse(jsonString)
          } catch (err){
            q.rejectedUsers.push(user)
            continue
          }
          if (!jsonObj.entry_data.ProfilePage || !jsonObj.entry_data.ProfilePage[0].graphql.user){ q.rejectedUsers.push(user); continue}
   
          const cleanedData = cleanInstaData(jsonObj.entry_data.ProfilePage[0].graphql.user, user, settings.includePhotos)
          if(!cleanedData) { q.rejectedUsers.push(user); continue}
          q.users.push(cleanedData)
        }
      }
      return run()
  }).then(() => {
     return q.browser.close()
  }).then(() => {
     delete q.browser
     const endTime = new Date().getTime()
     q.loadTime = endTime - startTime
     if( settings.updateUsers){
        return updateUsers(q.users, endTime - startTime, q.rejectedUsers, settings)
     } 
     return
  }).then((r) => { return q }).catch((err) => {
    console.error(err)
    return err
  })
}

function cleanInstaData(instaData, user, includePhotos){
  var q = {...user}
  q.photos = instaData.edge_owner_to_timeline_media.edges.map((node) => {
    var imgs = {}
    let post = node.node
    post.thumbnail_resources.map((img) => {
      switch(img.config_width){
        case 150:
          imgs.thumbnail = img.src || ''
        case 480:
          imgs.lowres = img.src || ''
        case 640: 
          imgs.standardres = img.src || ''
      }
    })
    imgs.highres = post.display_url || ''
    return {
      comments: post.edge_media_to_comment.count || 0,
      likes: post.edge_liked_by.count || 0,
      createdAt: post.taken_at_timestamp || Date(),
      images: imgs
    }
  })
  instaData.full_name ? q.fullName = instaData.full_name : null
  q.following = instaData.edge_follow.count || 0
  q.followers = instaData.edge_followed_by.count || 0
  q.postCount = q.photos.length || 0
  q.avgLikes = 0  
  q.avgComments = 0
  q.photos.forEach((p) => {
    q.avgLikes    += p.likes || 0
    q.avgComments += p.comments || 0
  })
  q.avgLikes    /= q.postCount 
  q.avgComments /= q.postCount 
  q.postFrequency = q.photos.length ? postFrequency(
    q.photos.length,
    q.photos[q.photos.length - 1].createdAt * 1000
  ) : 0
  includePhotos ? null : delete q.photos  
  return q
}
function postFrequency(postCount, lastPostTimestamp) {
  const now = +new Date
  const lastPostTime = new Date(lastPostTimestamp)
  const msecPerDay = 1000 * 60 * 60 * 24
  const days = Math.max(Math.round((now - lastPostTime)/ msecPerDay), 1)
  return postCount / days
}

function updateUsers(users, loadTime, rejectedGramHandles, settings){
  settings = settings || settings.defaultSettings
  request(`${settings.baseUrl}${settings.updateRoute}`,{
    ...settings.updateOptions,
    body: { users: users, rejectedGramHandles: rejectedGramHandles, loadTime: loadTime } ,
  },function(err, res, body){
    if(err) { console.warn(err); return err }
    return res
  })
}