ipbcrawler/miners/topics.js

Package to mine forum data using the Invision Power Board platform.

const { accessElementInformation, asyncSendElementToAccessInformation } = require('./../utils/extractor-access-element-information')

const paginationService = require('./../services/pagination.service')
const postService = require('./../services/post.service')
const listTopicService = require('./../services/list-topics.service')

const flat = require('./../utils/flat')
const waitFourSecs = require('./../utils/wait-time')(4000)


const accessPost = async topic => {
	const { id, url } = topic
	
	return await waitFourSecs(async () => {
		try {
			const postInformation = await postService(url)

			return { id, ...postInformation, url }
		} catch(e) {
			console.log(e)
		}
	})
}
/**
 * [description]
 * @param  {[type]} url [description]
 * @return {[type]}     [description]
 */
const accessListTopic = async pagination => {
	const { url } = pagination

	/**
	 * [description]
	 * @param  {[type]} async (             [description]
	 * @return {[type]}       [description]
	 */
	return await waitFourSecs(async () => await listTopicService(url))
}


module.exports = async (url) => {
	/**
	 * [pagination description]
	 * @type {[type]}
	 */
	const pagination = await paginationService(url)

	/**
	 * [listTopics description]
	 * @type {[type]}
	 */
	const listTopics = await asyncSendElementToAccessInformation({ elements: pagination, extractor: accessListTopic })

	/**
	 * [listTopicsFlatten description]
	 * @type {[type]}
	 */
	const listTopicsFlatten = flat(listTopics)

	/**
	 * [posts description]
	 * @type {[type]}
	 */
	const posts = await asyncSendElementToAccessInformation({ elements: listTopicsFlatten, extractor: accessPost })

	return posts
}