ipbcrawler/extractions/list-topics.js

Package to mine forum data using the Invision Power Board platform.

const { accessElementInformation, sendElementToAccessInformation } = require('./../utils/extractor-access-element-information')

/**
 * [description]
 * @param  {[type]} $ [description]
 * @return {[type]}   [description]
 */
const accessTopicInformation = accessElementInformation((element, $) => {
	/**
	 * [aElement description]
	 * @type {[type]}
	 */
	const aElement = element.find('.ipsDataItem_main span.ipsType_break.ipsContained a[data-ipshover-target]')

	return {
		/**
		 * [id description]
		 * @type {[type]}
		 */
		id: element.attr('data-rowid'),

		/**
		 * [url description]
		 * @type {[type]}
		 */
		url: aElement.attr('href'),

		/**
		 * [title description]
		 * @type {[type]}
		 */
		title: aElement.attr('title')
	}
})

/**
 * [description]
 * @param  {[type]} $ [description]
 * @return {[type]}   [description]
 */
const accessPageInformation = accessElementInformation((element, $) => {
	const aElement = $(element).find('a')

	const url = aElement.attr('href')
	const pageNumber = aElement.attr('data-page')

	return { url, pageNumber }
})


/**
 * [description]
 * @param  {[type]} $ [description]
 * @return {[type]}   [description]
 */
const accessPagination = ($, url) => {
	const pagination = $(".ipsPagination")[0]

	/**
	 * [pages description]
	 * @type {[type]}
	 */
	const pages = $(pagination).find(".ipsPagination_last a")
	const total = pages.attr('data-page')

	if (! total) {
		return [{
			url,
			page: 1
		}]
	}

	return Array
			.from({ length: total })
			.map((v, key) => {
				const k = key + 1
				
				return {
					url: `${url}/page/${k}`,
					page: k
				}
			})
			.reverse()
}

/**
 * [description]
 * @param  {[type]} $ [description]
 * @return {[type]}   [description]
 */
module.exports = ($) => {
	return {
		/**
		 * [listOfTopics description]
		 * @type {[type]}
		 */
		topics: sendElementToAccessInformation({
			elements: $('ol li.ipsDataItem[data-rowid]'),
			extractor: accessTopicInformation($)
		}),
	}
}

module.exports.pagination = ($, url) => {
	return {
		pagination: accessPagination($, url)
	}
}