oddsportal/scrapeMatches.js

oddsportal scrape for www.keydobot.com

const puppeteer = require("puppeteer");
const fetch = require("node-fetch");
const { asyncForEach, waitFor, waitPageClose } = require("./helpers/helpers");
const { pageSetting } = require("./helpers/puppetterSettings");
const { convertMn, convertOna, activeList } = require("./helpers/oddsportal");

const { addBookmakers } = require("./POST/addBookmakers");

const scrapeMatches = async (browser) => {
  const openM = process.env.OPEN_M.split("|");
  const closeM = process.env.CLOSE_M.split("|");
  while (1) {
    let matchesLinks = await fetch(`${process.env.EXPRESS_IP}helpers/matches`)
      .then((response) => response.json())
      .then((data) => data.data)
      .catch((err) => {
        console.log(new Error("/helpers/matches error "));
      });
    if (!matchesLinks) {
      return;
    }
    await asyncForEach(matchesLinks, async (el, key) => {
      console.log(1)
      const { sid, eid, iddaaEn, oddsportalUrl, enabled, ed } = el;
      if (!enabled || ed < Date.now() / 1000) return;
      console.log(eid, iddaaEn, browser._connection._sessions.size);

      scrapeIddaa(sid, eid, openM, closeM);

      if (
        Number(browser._connection._sessions.size) + 1 >
        process.env.MAX_PAGE
      ) {
        await waitPageClose(browser);
      }
      scrapeEvents(sid, eid, oddsportalUrl, browser);

      await waitFor(2000);
    });
  }
};

const scrapeIddaa = async (sid, eid, openM, closeM) => {

  const m = await fetch(`${process.env.IDDAA_API_MARKETS}${sid}/${eid}`)
    .then((res) => res.json())
    .then((data) => data.data.event.m)
    .catch((err) => console.log(err));

  if (m) {
    await asyncForEach(m, async (e) => {
      if (openM.includes(e.mn) && !closeM.includes(e.mn) || 1) {
        const { mn, mno} = e;
        const o=[];
        e.o.forEach(el=>{
          let {odd,ona} = el;
          o.push({odd,ona})
        })
        const bname = "Iddaa";
        let bookmaker = {
          mn,
          mno,
          bname,
          lastUpdate: Date.now(),
          o,
        };

        bookmaker.contact = {
          iddaa: "https://www.nesine.com/iddaa/detail/" + eid + "/0",
        };

        //
        await fetch(`${process.env.EXPRESS_IP}add/bookmaker/${sid}/${eid}`, {
          method: "POST", // or 'PUT'
          headers: {
            "Content-Type": "application/json",
          },
          body: JSON.stringify(bookmaker),
        })
          .then((response) => response.json())
          .then((res) => {
            // console.log(res);
          })
          .catch((error) => {
            console.error("Error:", error);
          });
      }
    });
  }
};

async function scrapeEvents(sid, eid, url, browser) {
  return new Promise(async (resolve, reject) => {
    const page = await browser.newPage();
    await page.goto(url, pageSetting());

    await page.exposeFunction("convertMn", convertMn);
    await page.exposeFunction("activeList", activeList);
    await page.exposeFunction("convertOna", convertOna);
    await page.exposeFunction("waitFor", waitFor);
    await page.exposeFunction("asyncForEach", asyncForEach);
    await page.exposeFunction("addBookmakers", addBookmakers);
    try {
      await page.evaluate(
        async (sid, eid, url) => {
          const inWaitTime = 6500;

          await waitFor(6500);

          async function asyncForEach(array, callback) {
            for (let index = 0; index < array.length; index++) {
              await callback(array[index], index, array);
            }
          }

          const scrape = (sid, eid) => {
            return new Promise(async (resolve, reject) => {
              let thead = [];
              let tbody = [];
              activeMn = document
                .querySelector("ul.ul-nav li.active")
                .textContent.trim();

              activeMn = await convertMn(sid, activeMn);
              const trheads = document.querySelector(
                "table.table-main.detail-odds.sortable thead"
              );

              if (!trheads) resolve();

              trheads.querySelectorAll("th").forEach((el) => {
                if (el.textContent.trim() === "Bookmakers") {
                  thead.push("bname");
                } else thead.push(el.textContent.trim());
              });
              // console.log(thead);
              let trs = document.querySelectorAll(
                "table.table-main.detail-odds.sortable tbody tr.lo"
              );
              // .querySelectorAll("tr.lo");
              await asyncForEach(trs, async (tr, key) => {
                let xparamElement = tr.querySelector("td:last-child") || null;
                if (xparamElement) {
                  if (
                    xparamElement.getAttribute("xparam") ===
                    "FRESH ODDS<br />(last update %t ago)~2"
                  ) {
                    let Bookmaker = { mn: activeMn, o: [] };
                    for (const [index, td] of tr
                      .querySelectorAll("td")
                      .entries()) {
                      if (thead[index] == "" || thead[index] == "Payout")
                        continue;

                      if (thead[index] == "Total") {
                        Bookmaker.mn +=
                          " " +
                          td.textContent
                            .trim()
                            .replace("+", "")
                            .replace(".", ",");
                        continue;
                      }

                      if (thead[index] == "Hand.") {
                        td.textContent.trim() == "-1"
                          ? (Bookmaker.mn += " 0:1")
                          : "";
                        td.textContent.trim() == "+1"
                          ? (Bookmaker.mn += " 1:0")
                          : "";
                        continue;
                      }

                      if (thead[index] === "bname") {
                        Bookmaker[thead[index]] = td.textContent.trim();
                      } else {
                        if (activeMn === "Karşılıklı Gol") {
                          ona = (await thead[index]) === "Yes" ? "Var" : "Yok";
                        } else {
                          ona = await convertOna(thead[index]);
                        }

                        Bookmaker.o.push({
                          ona,
                          odd: Number(td.textContent.trim()),
                        });
                      }
                    }
                    Bookmaker.lastUpdate = Date.now();
                    if (trs.length === key + 1) {
                      Bookmaker.last = true;
                    }

                    await addBookmakers(Bookmaker, sid, eid, url);
                  }
                }
              });

              resolve();
            });
          };

          let active = await document.querySelector("li.first.active");
          if (!active) return [];
          active = await activeList(sid, active.textContent.trim());

          if (active) {
            scrape(sid, eid);
          }

          const ulnav = await document.querySelectorAll("ul.ul-nav li a");

          await asyncForEach(ulnav, async (ul, key) => {
            active = await activeList(sid, ul.textContent.trim());
            if (key === ulnav.length - 1) return;
            if (!active) return;
            if (ul.onmousedown) await ul.onmousedown();
            if (ul.onclick) await ul.onclick();

            await waitFor(inWaitTime);
            if (!active.clicks) {
              await scrape(sid, eid);
              return;
            }
            if (active.clicks) {
              let tds = await document.querySelectorAll(
                "div.table-container strong a"
              );

              for (let i = 0; i < tds.length; i++) {
                if (
                  (activein = active.clicks.find(
                    (el) => el.click === tds[i].textContent.trim()
                  ))
                ) {
                  await tds[i].click();
                  await waitFor(inWaitTime);
                  await scrape(sid, eid);
                  await tds[i].click();
                }
              }
            }
          });

          return;
        },
        sid,
        eid,
        url
      );
      await page.close().then((data) => resolve());
    } catch (err) {
      console.log("Hata Alındı");
      await page.close().then((data) => resolve());
    }
  });
}

module.exports = scrapeMatches;