ptt-scr/dist/scraper.js

ＰＴＴ文章列表

"use strict";

var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");

Object.defineProperty(exports, "__esModule", {
  value: true
});
exports.default = void 0;

var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));

var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));

var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));

var _request = _interopRequireDefault(require("./request"));

var _getUrlContent = _interopRequireDefault(require("./getUrlContent"));

var _list = _interopRequireDefault(require("./list"));

var _content = _interopRequireDefault(require("./content"));

var mainUrl = 'https://www.ptt.cc';
/**
 *  @param boardName {string}
 *  @param pageCounts {number} 往前走幾頁 default:3
 *  @param startPage {number} 開始的頁數  default:''
 *  @param categoryPattern {RegExp} 發文分類 正則表達式規則 default: /\[(.+)\]/
 */

var _default =
/*#__PURE__*/
function () {
  var _ref = (0, _asyncToGenerator2.default)(
  /*#__PURE__*/
  _regenerator.default.mark(function _callee() {
    var _ref2,
        _ref2$boardName,
        boardName,
        _ref2$pageCounts,
        pageCounts,
        _ref2$startPage,
        startPage,
        _ref2$categoryPattern,
        categoryPattern,
        _ref2$isScrapContent,
        isScrapContent,
        boardUrl,
        prePageNumber,
        items,
        session,
        i,
        html,
        titleList,
        contentItems,
        itemsLength,
        batch,
        _i,
        steps,
        _args = arguments;

    return _regenerator.default.wrap(function _callee$(_context) {
      while (1) {
        switch (_context.prev = _context.next) {
          case 0:
            _ref2 = _args.length > 0 && _args[0] !== undefined ? _args[0] : {}, _ref2$boardName = _ref2.boardName, boardName = _ref2$boardName === void 0 ? 'Gossiping' : _ref2$boardName, _ref2$pageCounts = _ref2.pageCounts, pageCounts = _ref2$pageCounts === void 0 ? 3 : _ref2$pageCounts, _ref2$startPage = _ref2.startPage, startPage = _ref2$startPage === void 0 ? 0 : _ref2$startPage, _ref2$categoryPattern = _ref2.categoryPattern, categoryPattern = _ref2$categoryPattern === void 0 ? /\[(.+)\]/ : _ref2$categoryPattern, _ref2$isScrapContent = _ref2.isScrapContent, isScrapContent = _ref2$isScrapContent === void 0 ? false : _ref2$isScrapContent;
            boardUrl = "".concat(mainUrl, "/bbs/").concat(boardName, "/index").concat(startPage || '', ".html");
            prePageNumber = 0;
            items = [];
            _context.next = 6;
            return (0, _request.default)();

          case 6:
            session = _context.sent;
            i = 0;

          case 8:
            if (!(i < pageCounts)) {
              _context.next = 19;
              break;
            }

            _context.next = 11;
            return (0, _getUrlContent.default)(session, boardUrl);

          case 11:
            html = _context.sent;
            titleList = (0, _list.default)(html, categoryPattern);
            boardUrl = "".concat(mainUrl, "/bbs/").concat(boardName, "/index").concat(titleList.prePageNumber, ".html");
            prePageNumber = titleList.prePageNumber;
            items.push.apply(items, (0, _toConsumableArray2.default)(titleList.items));

          case 16:
            i++;
            _context.next = 8;
            break;

          case 19:
            if (!isScrapContent) {
              _context.next = 39;
              break;
            }

            contentItems = [];
            itemsLength = items.length;
            batch = itemsLength / pageCounts; // console.log(`總共：${itemsLength}`);

            _i = 0;

          case 24:
            if (!(_i < itemsLength / batch + 1)) {
              _context.next = 38;
              break;
            }

            steps = items.splice(0, batch).map(function (step) {
              return (0, _content.default)(step);
            });

            if (!steps.length) {
              _context.next = 35;
              break;
            }

            _context.t0 = contentItems.push;
            _context.t1 = contentItems;
            _context.t2 = _toConsumableArray2.default;
            _context.next = 32;
            return Promise.all(steps);

          case 32:
            _context.t3 = _context.sent;
            _context.t4 = (0, _context.t2)(_context.t3);

            _context.t0.apply.call(_context.t0, _context.t1, _context.t4);

          case 35:
            _i++;
            _context.next = 24;
            break;

          case 38:
            return _context.abrupt("return", {
              prePage: boardUrl,
              prePageNumber: prePageNumber,
              items: contentItems
            });

          case 39:
            return _context.abrupt("return", {
              prePage: boardUrl,
              prePageNumber: prePageNumber,
              items: items
            });

          case 40:
          case "end":
            return _context.stop();
        }
      }
    }, _callee, this);
  }));

  return function () {
    return _ref.apply(this, arguments);
  };
}();

exports.default = _default;