@ssb-graphql/whakapapa/test/queries/load-whakapapa-links.bench.js

GraphQL types and resolvers for the ssb-whakapapa plugin

const test = require('tape')
const { promisify: p } = require('util')
const { join } = require('path')
const { createWhakapapaTree } = require('ahau-fixtures')

const pull = require('pull-stream')
const Pushable = require('pull-pushable')
const paraMap = require('pull-paramap')
const flatMap = require('pull-flatmap')

const TestBot = require('../test-bot')

const SIZE = 5000 // how many profiles!
const DIVISOR = 5000 / 100
const LOG_PROGESS = true

test('loadWhakapapaLinks', async t => {
  const { ssb, apollo } = await TestBot({
    path: join(__dirname, '../db'),
    startUnclean: true
  })

  const view = await findOrCreateWhakapapa(ssb, SIZE)
    .catch(err => {
      console.error('error in findOrCreateWhakapapa')
      throw err
    })

  console.log({ view })
  let width = 20
  // write link lookup for children via existing API
  console.time(`Load iteratively (width ${width})`)
  const links1 = await loadWhakapapaIteratively(apollo, view.focus, width)
  console.timeEnd(`Load iteratively (width ${width})`)
  console.log('----')

  // width = 10
  // console.time(`Load iteratively (width ${width})`)
  // const links1b = await loadWhakapapaIteratively(apollo, view.focus, width)
  // console.timeEnd(`Load iteratively (width ${width})`)
  // console.log('----')

  // t.deepEqual(
  //   links1.sort(childLinkComparator),
  //   links1b.sort(childLinkComparator),
  //   'links are same regardless of how width parallel process is'
  // )

  width = 20
  console.time(`Load parallel (width ${width})`)
  const links2 = await loadWhakapapaParallel(apollo, view.focus, width)
  console.timeEnd(`Load parallel (width ${width})`)
  console.log('----')

  console.time('Load in one go')
  const links3 = await loadWhakapapaInOneGo(apollo, view.key)
  console.timeEnd('Load in one go')
  console.log('----')

  console.time('Load in one go (now cached!)')
  const links4 = await loadWhakapapaInOneGo(apollo, view.key)
  console.timeEnd('Load in one go (now cached!)')
  console.log('----')

  t.deepEqual(
    links1.sort(childLinkComparator),
    links2.sort(childLinkComparator),
    'whakapapaView.links.childLinks are the same'
  )
  t.deepEqual(
    links2.sort(childLinkComparator),
    links3.sort(childLinkComparator),
    'whakapapaView.links.childLinks are the same'
  )
  t.deepEqual(
    links3.sort(childLinkComparator),
    links4.sort(childLinkComparator),
    'whakapapaView.links.childLinks are the same'
  )

  ssb.close()
  t.end()
})

async function findOrCreateWhakapapa (ssb, size) {
  const views = await ssb.whakapapa.view.list({})
    .catch(err => {
      console.error('error in whakapapa.view.list')
      throw err
    })
  console.log(views.map(view => view.name))

  let view = views.find(view => view.name === `bulk-${size}`)
  if (view) return view

  console.log('no existing view...')
  const groupId = await findOrCreateTribe(ssb)
    .catch(err => {
      console.error('error in findOrCreateTribe')
      throw err
    })

  console.log({ groupId })

  // build a large whakapapa tree, then measure link lookup
  // 1. with recurssive loadFamilyOfPerson
  // 2. with single backend lookup!

  console.log('starting whakapapa build')
  const viewId = await createWhakapapaTree(ssb, groupId, SIZE)
    .catch(err => {
      console.error('error in createWhakapapaTree')
      throw err
    })
  console.log({ viewId })

  while (!view) {
    view = await ssb.whakapapa.view.get(viewId)
      .catch(err => console.error(err.message))
    if (view) continue

    console.log('not ready')
    await new Promise(resolve => setTimeout(resolve, 1000))
  }

  console.log('here', view)
  return view
}

async function findOrCreateTribe (ssb) {
  const [groupId] = await p(ssb.tribes.list)({})
    .catch(err => { throw err })

  if (groupId) return groupId

  const tribe = await p(ssb.tribes.create)({})
    .catch(err => { throw err })

  return tribe.groupId
}

async function loadWhakapapaIteratively (apollo, focus, width = 5) {
  let links = []
  let queue = [focus]

  const QUERY = `query ($profileId: String!, $extended: Boolean) {
    loadFamilyOfPerson(id: $profileId, extended: $extended) {
      childLinks {
        parent
        child
        relationshipType
      }
    }
  }`

  while (queue.length) {
    const parentIds = queue.slice(0, width)
    queue = queue.slice(width)

    const parentChildLinks = await Promise.all(
      parentIds.map(profileId => apollo.query({
        query: QUERY,
        variables: {
          profileId,
          extended: true
        }
      }))
    )
    const childLinks = parentChildLinks
      .map(result => result.data.loadFamilyOfPerson.childLinks)
      .filter((link, _, i) => link.parent === queue[i])
      .flatMap(arr => arr)

    links = links.concat(childLinks)
    queue = queue.concat(childLinks.map(link => link.child))

    if (LOG_PROGESS) logProgress(links, queue)
  }

  return links
}

async function loadWhakapapaParallel (apollo, focus, width = 10) {
  const links = []
  const queue = Pushable()
  queue.push(focus)

  const QUERY = `query ($profileId: String!, $extended: Boolean) {
    loadFamilyOfPerson(id: $profileId, extended: $extended) {
      childLinks {
        parent
        child
        relationshipType
      }
    }
  }`
  let count = 0

  return new Promise((resolve, reject) => {
    pull(
      queue,
      paraMap(
        (parentId, cb) => {
          apollo.query({
            query: QUERY,
            variables: {
              profileId: parentId,
              extended: true
            }
          })
            .catch(err => cb(err))
            .then(res => {
              const childLinks = res.data.loadFamilyOfPerson.childLinks
                .filter(link => link.parent === parentId)

              if (LOG_PROGESS && ++count % width === 0) logProgress(links, queue.buffer)

              cb(null, childLinks)
            })
        },
        width
      ),
      flatMap(arr => arr),
      pull.through(link => queue.push(link.child)),
      pull.take(SIZE - 1),
      pull.drain(
        link => {
          links.push(link)
        },
        err => {
          if (err) reject(err)
          else resolve(links)
        }
      )
    )
  })
}

async function loadWhakapapaInOneGo (apollo, viewId) {
  const QUERY = `query ($viewId: String!) {
    whakapapaView(id: $viewId) {
      links {
        childLinks {
          parent
          child
          relationshipType
        }
      }
    }
  }`
  const result = await apollo.query({
    query: QUERY,
    variables: {
      viewId
    }
  })
    .catch(console.error)

  return result.data.whakapapaView.links.childLinks
}

function logProgress (links, queue) {
  const barWidth = Math.ceil(queue.length / DIVISOR)

  console.log([
    'links: ' + leftPad(links.length),
    'queue: ' + leftPad(queue.length) + ' ' + new Array(barWidth).fill('▦').join('')
  ].join(', '))
}

function leftPad (str, width = 3) {
  let output = String(str)
  while (output.length < width) {
    output = ' ' + output
  }
  return output
}

function childLinkComparator (a, b) {
  const A = (a.parent + a.child)
  const B = (b.parent + b.child)
  return A < B ? -1 : 1
}