tcdown/lib/helpers/worker.js

Downloader and scraper for teachable.com for members

const { parentPort } = require('worker_threads')
const Promise = require('bluebird')
const youtubedl = require('youtube-dl-wrap')
const path = require('path')
const { formatBytes } = require('./writeWaitingInfo')
const FileChecker = require('./fileChecker')
const logger = require('./logger')

const download = (url, dest, localSizeInBytes, remoteSizeInBytes, downFolder, index, logger, ms, resourceUrl) => {
    return new Promise(async (resolve, reject) => {

        // ms.update(dest, {
        //     text : `to be processed by youtube-dl... ${dest.split('/').pop()} Found:${localSizeInBytes}/${remoteSizeInBytes}`,
        //     color: 'blue'
        // });
        // console.log(`to be processed by youtube-dl... ${dest.split('/').pop()} Found:${localSizeInBytes}/${remoteSizeInBytes}`)

        // return await retry(async () => {//return
        const youtubeDlWrap = new youtubedl()
        let youtubeDlEventEmitter = youtubeDlWrap
            // .exec([url, "-o", path.toNamespacedPath(dest)])
            .exec([
                url,
                // "--write-subs",
                // "--write-auto-sub",
                // '--referer', 'https://example.com/',
                '-o', path.resolve(dest),
                '--socket-timeout', '5'
            ])
            .on('progress', (progress) => {
                // ms.update(dest, { text: `${index}. Downloading: ${progress.percent}% of ${progress.totalSize} at ${progress.currentSpeed} in ${progress.eta} | ${dest.split('/').pop()} Found:${localSizeInBytes}/${remoteSizeInBytes}` })
            })
            // .on("youtubeDlEvent", (eventType, eventData) => console.log(eventType, eventData))
            .on('error', (error) => {
                // ms.remove(dest, { text: error })
                // console.log('error--', error)
                // ms.remove(dest)
                /*fs.unlink(dest, (err) => {
                    reject(error);
                });*/
                reject(error)

            })
            .on('close', () => {
                // ms.succeed(dest, { text: `${index}. End download ytdl: ${dest} Found:${localSizeInBytes}/${remoteSizeInBytes} - Size:${formatBytes(getFilesizeInBytes(dest))}` })//.split('/').pop()
                // ms.remove(dest);
                // console.log(`${index}. End download ytdl: ${dest} Found:${localSizeInBytes}/${remoteSizeInBytes} - Size:${formatBytes(getFilesizeInBytes(dest))}`.green);
                // videoLogger.write(`${dest} Size:${getFilesizeInBytes(dest)}\n`);
                // FileChecker.writeWithOutSize(downFolder, dest)
                // FileChecker.writeResourceUrlWithOutSize(downFolder, resourceUrl)
                resolve(`-done- ${resourceUrl}`)
            })

        // }, 6, 2e3, true)

    })
}
parentPort.on('message', async (m) => {
    try {

        const {
                  url,
                  dest,
                  localSizeInBytes,
                  remoteSizeInBytes,
                  downFolder,
                  index,
                  logger,
                  ms,
                  resourceUrl
              } = m
        // parentPort.postMessage(`recieved: ${resourceUrl} = ${url}`)
        // const result = await download(url, dest, localSizeInBytes, remoteSizeInBytes, downFolder, index, logger, ms, resourceUrl)
        // parentPort.postMessage(result)

        const youtubeDlWrap = new youtubedl()
        let youtubeDlEventEmitter = youtubeDlWrap
            // .exec([url, "-o", path.toNamespacedPath(dest)])
            .exec([
                url,
                // "--write-subs",
                // "--write-auto-sub",
                // '--referer', 'https://example.com/',
                '-o', path.resolve(dest),
                '--socket-timeout', '10'
            ])
            .on('progress', (progress) => {
                // parentPort.postMessage(`${index}. Downloading: ${progress.percent}% of ${progress.totalSize} at ${progress.currentSpeed} in ${progress.eta} | ${dest.split('/').pop()} Found:${localSizeInBytes}/${remoteSizeInBytes}`)
                // ms.update(dest, { text: `${index}. Downloading: ${progress.percent}% of ${progress.totalSize} at ${progress.currentSpeed} in ${progress.eta} | ${dest.split('/').pop()} Found:${localSizeInBytes}/${remoteSizeInBytes}` })
            })
            // .on("youtubeDlEvent", (eventType, eventData) => console.log(eventType, eventData))
            .on('error', (error) => {
                // ms.remove(dest, { text: error })
                // console.log('error--', error)
                // ms.remove(dest)
                /*fs.unlink(dest, (err) => {
                    reject(error);
                });*/
                //reject(error)
                throw new Error(error)

            })
            .on('close', () => {
                // ms.succeed(dest, { text: `${index}. End download ytdl: ${dest} Found:${localSizeInBytes}/${remoteSizeInBytes} - Size:${formatBytes(getFilesizeInBytes(dest))}` })//.split('/').pop()
                // ms.remove(dest);
                // console.log(`${index}. End download ytdl: ${dest} Found:${localSizeInBytes}/${remoteSizeInBytes} - Size:${formatBytes(getFilesizeInBytes(dest))}`.green);
                // videoLogger.write(`${dest} Size:${getFilesizeInBytes(dest)}\n`);
                // FileChecker.writeWithOutSize(downFolder, dest)
                // FileChecker.writeResourceUrlWithOutSize(downFolder, resourceUrl)
                //resolve(`-done- ${resourceUrl}`)
                parentPort.postMessage(`End download ytdl:: ${dest} -- ${resourceUrl}`)
            })

    } catch (error) {
        parentPort.postMessage(`Error: ${error.message}`)
    }
})