edx-archive/index.js

Simple tool to download edX course pages (including excercises with answers, course materials, errata, etc).

#!/usr/bin/env node

const path = require('path');
const fs = require('fs');
const sanitize = require("sanitize-filename");
const puppeteer = require('puppeteer');
const prompt = require('async-prompt');
const program = require('commander');
const retry = require('async-retry');
const PromisePool = require('es6-promise-pool');


function parseFormat(value, previous) {
  if (!["pdf", "png"].includes(value)) {
    console.log(`invalid format: ${value}`);
    process.exit(1);
  }
  return value;
}

async function getConfiguration() {
  const courseUrlPattern = /^https:\/\/learning.edx.org\/course\/.*$/
  function parseInteger(v) { return parseInt(v); }

  program
    .name("edx-archive")
    .arguments('<course_url>')
    .option('-u, --user <email>', 'edx login (email)')
    .option('-p, --password <password>', 'edx password')
    .option('-o, --output <directory>', 'output directory', 'Archive')
    .option('-f, --format <format>', 'save pages as pdf or png', parseFormat, 'pdf')
    .option('-r, --retries <retries>', 'number of retry attempts in case of failure', parseInteger, 3)
    .option('-d, --delay <seconds>', 'delay before saving page', parseInteger, 1)
    .option('-c, --concurrency <number>', 'number of pages to save in parallel', parseInteger, 4)
    .option('--debug', 'output extra debugging', false)
    .parse(process.argv);

  if (program.args.length !== 1) {
    program.help();
  }

  if (!program.args[0].match(courseUrlPattern)) {
    console.log("Invalid course url.\nCourse url should look like: https://learning.edx.org/course/<course_id>");
    process.exit(1);
  }

  const configuration = Object.assign({}, program.opts());

  configuration.courseUrl = program.args[0];

  if (!configuration.user) {
    configuration.user = await prompt('User: ');
  }

  if (!configuration.password) {
    configuration.password = await prompt.password('Password: ');
  }

  return configuration;
}

async function openPage(url, browser, configuration) {
  const page = await browser.newPage();
  const navigationPromise = page.waitForNavigation({waitUntil: "domcontentloaded"});
  await page.goto(url);
  await navigationPromise;
  console.log('did I get this far?');
  return page;
}

async function loginBrowser(browser, configuration) {
  const page = await openPage('https://courses.edx.org/login', browser, configuration);
  // await page.waitForNavigation({waitUntil: "domcontentloaded"});
  console.log("test1");
  // page.evaluate("")
  // console.log(console.dir(page));
  // var x = page.document.querySelectorAll("");
  // console.log(x);
  await page.waitForSelector("#emailOrUsername");
  await page.waitForSelector("#password");
  console.log("test2");
  await page.type('#emailOrUsername', configuration.user);
  await page.type('#password', configuration.password);
  // await page.click('#password');
  // await page.keyboard.type(configuration.password);
  console.log("test3");
  await page.waitForSelector(".pgn__stateful-btn");
  console.log("test4");
  // await page.keyboard.press("Enter");
  const loginEndpoints = [
    "https://courses.edx.org/api/user/v2/account/login_session/",
    // "https://courses.edx.org/login_ajax"
  ];
  const [response] = await Promise.all([
    page.waitForResponse(r => loginEndpoints.includes(r.url())),
    page.click('.pgn__stateful-btn')
    // page.waitForNavigation({ waitUntil: 'networkidle0' })
    // page.click('#password'),
    // page.keyboard.type(configuration.password),
  ]);

  if (configuration.debug) {
    console.log(`Login response status code: ${response.status()}`);
  }

  if (response.status() !== 200) {
    console.log("Login failed.");
    process.exit(1);
  }
}

async function getPages(browser, configuration) {
  const page = await openPage(configuration.courseUrl, browser, configuration);

  const pages = await page.evaluate(() => {
    return $("a.outline-item").map(function(i, e) {
      return { "index": i, "url": e.href };
    }).toArray();
  });

  await page.close();

  if (configuration.debug) {
    console.log("Fetched pages:");
    console.log(pages);
  }

  return pages;
}

async function savePage(pageData, page, configuration) {
  const filename = path.join(configuration.output, `${pageData.index + 1} - ${pageData.title}`);

  if (configuration.debug) {
    console.log(`Saving page: ${pageData.url} as: ${filename}`);
  }

  if (!fs.existsSync(configuration.output)) {
      fs.mkdirSync(configuration.output);
  }

  if (configuration.format === "png") {
    await page.screenshot({ path: filename + '.png', fullPage: true });
  }
  if (configuration.format === "pdf") {
    await page.pdf({ path: filename + '.pdf' });
  }
}

function prettifyPage() {
  $(".show").trigger("click");
  $(".hideshowbottom").trigger("click");
  $(".discussion-show.shown").trigger("click");
  $(".discussion-module").hide();
  $("header").hide();
  $("#footer-edx-v3").hide();
  $(".course-tabs").hide();
  $(".course-expiration-message").hide();
  $(".verification-sock").hide();
  $("#frontend-component-cookie-policy-banner").hide();
  $(".sequence-bottom").hide();
  $(".sequence-nav").hide();
  $(".nav-utilities").hide();
  $(".course-license").hide();
  $(".bookmark-button-wrapper").hide();
  $(".subtitles").hide();
  $(".video-wrapper").hide();
}

function buildTitle(breadcumbs) {
  return breadcumbs.split(/\n/).map((part) => {
    return sanitize(part.trim())
      .replace(/\s+/g, " ")
      .replace(/^(Course)$/, "");
  }).filter((Boolean)).join(" - ");
}

function waitForMathJax() {
  return new Promise(function(resolve, reject) {
    try {
      // TODO this might be unreliable. Try to find a better way to detect
      // whether math has been processed
      MathJax.Hub.Queue(() => { resolve(); });
    } catch (e) {
      reject(e);
    }
  });
}

async function processPage(pageData, browser, configuration) {
  const page = await openPage(pageData.url, browser, configuration);

  pageData.title = buildTitle(await page.evaluate(() => {
    return $(".breadcrumbs").first().text();
  }));

  await page.evaluate(prettifyPage);

  await page.evaluate(waitForMathJax)

  await page.waitFor(configuration.delay * 1000);

  await savePage(pageData, page, configuration);

  await page.close();
}

async function main() {
  try {
    // build configuration
    const configuration = await getConfiguration();
    if (configuration.debug) {
      console.log("Configuration:");
      console.log(Object.assign(Object.assign({}, configuration), { user: "<censored>", password: "<censored>" }));
    }

    // log in browser
    const browser = await puppeteer.launch();
    await loginBrowser(browser, configuration);

    // build list of pages that should be saved
    const pages = await retry(async () => {
      return await getPages(browser, configuration);
    }, {
      retries: configuration.retries,
      onRetry: () => { console.log("Failed to fetch pages. Retrying."); }
    });

    // process pages
    const jobGenerator = function * () {
      for (const pageData of pages) {
        console.log(`Processing page: ${pageData.url}.`);
        yield retry(async () => {
          return await processPage(pageData, browser, configuration);
        }, {
          retries: configuration.retries,
          onRetry: () => { console.log(`Failed to process page: ${pageData.url}. Retrying.`); }
        });
      }
    }
    const pool = new PromisePool(jobGenerator, configuration.concurrency);
    await pool.start();

    //
    await browser.close();
    console.log(`\nSaved ${pages.length} pages to: ${path.resolve(configuration.output)}`);
    console.log("Done.");

  } catch (e) {
    console.error(e);
    process.exit(1);
  }
}


main();