ipbcrawler/actions/find-posts.js

Package to mine forum data using the Invision Power Board platform.

const topics = require('./../miners/topics')
const flat = require('./../utils/flat')

const { asyncSendElementToAccessInformation, sendElementToAccessInformation } = require('./../utils/extractor-access-element-information')

const processCategoryLinks = async (category) => {
	const { id, url } = category
	
	const posts = await asyncSendElementToAccessInformation({
		elements: url,
		extractor: topics
	})

	const postsFlatten = flat(posts)
	
	try {
		return { category: id, posts: postsFlatten }
	} catch(e) {
		console.log(e)
	}
}


module.exports = async (categories) => {
	try {
		const t = await asyncSendElementToAccessInformation({
			elements: categories,
			extractor: processCategoryLinks
		})

		return t
	} catch(e) {
		console.log(e)
	}
}