jul11co-mangadl/handlers/h/hbrowse.js

Jul11co Web Manga Downloader

var path = require('path');
var urlutil = require('url');

var utils = require('jul11co-wdt').Utils;

module.exports = {
  
  name: 'HBrowse',
  website: 'http://www.hbrowse.com',

  match: function(link, options) {
    return /www\.hbrowse\.com\//g.test(link);
  },

  dispatch: function(saver, $, page, options, callback) {

    if (options.group_by_site && !options.orig_output_dir) {
      options.orig_output_dir = options.output_dir;
      options.output_dir = path.join(options.output_dir, 'HBrowse');
      saver.setMangaOutputDir(options.output_dir);
    }

    // http://www.hbrowse.com/16585/c00001
    // http://www.hbrowse.com/16585/c00001/00002
    var url_path = page.url.replace('http://www.hbrowse.com/', '');
    var url_parts = url_path.split('/');
    var manga_url = '';
    var chapter_url = '';

    if (url_parts.length >= 3) {
      chapter_url = 'http://www.hbrowse.com/' + url_parts[0] + '/' + url_parts[1];
      manga_url = 'http://www.hbrowse.com/' + url_parts[0];
    } else if (url_parts.length >= 2) {
      chapter_url = page.url;
      manga_url = 'http://www.hbrowse.com/' + url_parts[0];
    } else {
      manga_url = page.url;
    }

    var updateDownloadState = function(cb) {

      if (options.chapter_url == '' || typeof options.chapter_url == 'undefined') {
        return cb();
      }

      // Check if all chapter pages are visited
      var all_chapter_pages_visited = true;
      for (var prop in options.chapter_pages) {
        if (options.chapter_pages[prop].visited == false) {
          all_chapter_pages_visited = false;
          break;
        }
      }

      if (all_chapter_pages_visited) {

        var chapter_images = [];
        for (var chapter_page_url in options.chapter_pages) {
          // var chapter_page = options.chapter_pages[chapter_page_url];
          // chapter_images = chapter_images.concat(chapter_page.images);
          var chapter_page = saver.getStateData(chapter_page_url);
          if (chapter_page && chapter_page.images) {
            chapter_images = chapter_images.concat(chapter_page.images);
          }
        }
        if (options.debug) console.log(chapter_images);

        if (saver.isDone(options.chapter_url)) {
          options.chapter_pages = {};
          options.chapter_url = '';
          return cb();
        }

        saver.downloadMangaChapter({
          chapter_url: chapter_url,
          chapter_title: options.chapter_title,
          chapter_images: chapter_images,
          output_dir: options.chapter_output_dir
        }, options, function(err) {
          
          // Reset
          options.chapter_pages = {};
          options.chapter_url = '';

          cb(err);
        });
      } else {

        if (options.debug) console.log(options.chapter_pages);

        // Get next unprocessed page in options.chapter_pages
        var next_page_url = '';
        for (var prop in options.chapter_pages) {
          if (options.chapter_pages[prop].visited == false) {
            next_page_url = prop;
            break;
          }
        }

        var all_chapter_pages_count = 0;
        var visited_chapter_pages_count = 0;
        for (var chapter_page_url in options.chapter_pages) {
          all_chapter_pages_count++;
          if (options.chapter_pages[chapter_page_url].visited) visited_chapter_pages_count++;
        }

        console.log('Chapter page:', (visited_chapter_pages_count+1) + '/' + all_chapter_pages_count, next_page_url);

        // Process next page
        saver.processPage(next_page_url, options, cb);
      }
    }

    var extractChapterPages = function(cb) {
      if (chapter_url && $('.pageImage').length) {

        if (saver.isDone(options.chapter_url)) {
          options.chapter_url = '';
          return cb();
        }

        if (options.verbose) {
          console.log('Chapter page:', page.url);
        }

        if (!options.chapter_url) {
          options.chapter_url = chapter_url;
          options.chapter_title = path.basename(chapter_url);
          options.chapter_output_dir = path.join(saver.getOutputDir(), path.basename(chapter_url));
        }

        var page_script = '';
        $('script').each(function() {
          page_script += $(this).html();
        });

        var chapter_images_info = null;
        var image_dir_str = '';
        if (page_script) {
          var chapter_pages_str = utils.extractSubstring(page_script,'list = [', '];');
          var image_dir_str = utils.extractSubstring(page_script,'imageDir = "', '";');

          if (chapter_pages_str && chapter_pages_str != '') {
            chapter_pages_str = '[' + chapter_pages_str + ']';
            try {
              chapter_images_info = JSON.parse(chapter_pages_str);
            } catch(e) {
              console.log(e);
            }
          }
        }

        if (image_dir_str && chapter_images_info && chapter_images_info.length) {
          var chapter_images = [];
          chapter_images_info.forEach(function(page_src,idx) {
            if (!page_src || page_src == '' || page_src == 'zzz') return;

            chapter_images.push({
              src: 'http://www.hbrowse.com' + image_dir_str + page_src
            });
          });
          if (options.debug) console.log(chapter_images);

          saver.downloadMangaChapter({
            chapter_url: options.chapter_url,
            chapter_title: options.chapter_title,
            chapter_images: chapter_images,
            output_dir: options.chapter_output_dir
          }, options, function(err) {

            options.chapter_url = '';
            cb(err);
          });

        } else {
          var images = saver.getImages($, page, '.pageImage');
          if (options.debug) console.log(images);

          if (images.length == 0) {
            return updateDownloadState(cb);
          }

          saver.updateStateData(page.url, {
            images: images
          });

          var chapter_page_links = saver.getLinks($, page, 'td.pageList');
          if (options.debug) console.log(chapter_page_links);

          if (typeof options.chapter_pages == 'undefined') {
            // Init data holder for chapter pages in options (passed through all callback)
            options.chapter_pages = {};
          }

          for (var i = 0; i < chapter_page_links.length; i++) {
            var chapter_page_link = chapter_page_links[i];
            if (typeof options.chapter_pages[chapter_page_link] == 'undefined') {
              var chapter_page_cache = saver.getStateData(chapter_page_link);
              if (chapter_page_cache && chapter_page_cache.visited && chapter_page_cache.images) {
                options.chapter_pages[chapter_page_link] = {
                  visited: true,
                  // images: chapter_page_cache.images
                };
              }
              else {
                options.chapter_pages[chapter_page_link] = {
                  visited: false,
                  // images: []
                };
              }
            }
          }

          if (typeof options.chapter_pages != 'undefined' 
            && typeof options.chapter_pages[page.url] != 'undefined') {
            options.chapter_pages[page.url].visited = true;
          } else {
            // console.log('Wrong state! This page not included in chapter links');
            // return callback();
            options.chapter_pages[page.url] = {
              visited: true,
              // images: images
            };
          }

          // Save to options
          // options.chapter_pages[page.url].images = options.chapter_pages[page.url].images.concat(images);

          updateDownloadState(cb);
        }
      } else {
        cb();
      }
    }

    options.manga_info_extracted = options.manga_info_extracted || {};
    
    if (manga_url && !options.manga_info_extracted[manga_url] && $('table.listTable').length) {
      console.log('----');
      console.log('Manga page: ' + manga_url);

      var manga_info = this.getMangaInfo($, page);
      var manga_title = manga_info['title'];
      console.log('Manga title: ' + manga_title);

      if (options.auto_manga_dir && !options.batch_download) {
        options.output_dir = path.join(options.output_dir, manga_title);
        saver.setMangaOutputDir(options.output_dir);
      } else if (/*options.auto_manga_dir && */options.batch_download) {
        var manga_output_dir = path.join(options.batch_output_dir, manga_title);
        saver.deleteStateData(page.url);
        saver.setMangaOutputDir(manga_output_dir, {save_old_state: true});
        options.output_dir = manga_output_dir;
      }

      if (saver.isDone(page.url) && !options.refresh_info) {
        return callback();
      }

      options.manga_info_extracted[manga_url] = true;

      if (manga_info.chapters && manga_info.chapters.length) {
        manga_info.chapters.forEach(function(chapter) {
          saver.updateStateData(chapter.url, {
            url: chapter.url,
            chapter_title: chapter.title
          });
        });
      }

      saver.setStateData('url', page.url);
      if (options.save_index_html) {
        saver.saveHtmlSync(path.join(options.output_dir, 'index.html'), $.html());
      }
      if (manga_info && manga_info.url) {
        saver.saveJsonSync(path.join(options.output_dir, 'manga.json'), manga_info);
      }

      var chapter_links = manga_info.chapters.map(function(chapter) {
        return chapter.url;
      });
      chapter_links = chapter_links.filter(function(chapter_link) {
        return !saver.isVisited(chapter_link);
      });
      // chapter_links.unshift(page.url);
      // if (options.debug) console.log(chapter_links);

      saver.downloadMangaChapters(chapter_links, options, function(err) {
        if (err) return callback(err);
        extractChapterPages(callback);
      });

    } else if ($('table.thumbTable').length) {

      options.paging_links = options.paging_links || {};
      if (options.paging_links[page.url]) {
        return callback();
      }

      options.paging_links[page.url] = 1;

      console.log('--');
      console.log('Manga list: ' + page.url);

      options.batch_download = true;

      if (!options.batch_url) {
        options.batch_url = page.url;
      }
      if (!options.batch_output_dir) {
        options.batch_output_dir = options.output_dir;
      }
      if (options.batch_output_dir != options.output_dir) {
        saver.deleteStateData(page.url);
        saver.setMangaOutputDir(options.batch_output_dir, {save_old_state: true});
        options.output_dir = options.batch_output_dir;
      }

      if (!saver.getStateData('url')) {
        saver.updateStateData('url', page.url);
      }

      var manga_links = saver.getLinks($, page, 'table.thumbTable .thumbInfoTd', {
        blacklist: [
          'http://www.hbrowse.com/browse/'
        ]
      });

      var paging_links = [];
      $('#main p a').each(function() {
        var link_title = $(this).attr('title');
        if (link_title && link_title.indexOf('jump to page') != -1) {
          var link_url = $(this).attr('href');
          if (paging_links.indexOf(link_url) == -1) {
            paging_links.push(link_url);
          }
        }
      });

      saver.downloadMangaList(manga_links, options, function(err) {
        if (err) return callback(err);

        saver.setMangaOutputDir(options.batch_output_dir, {save_old_state: true});
        options.output_dir = options.batch_output_dir;
        manga_links.forEach(function(link) {
          if (!saver.isVisited(link)) saver.setVisited(link);
        });
        saver.setStateData('last_update', new Date());

        if (options.all_pages && paging_links.length) {
          if (options.refresh_info) {
            paging_links = paging_links.filter(function(link) {
              return !options.paging_links[link];
            });
            paging_links.forEach(function(link) {
              saver.updateStateData(link, {visited: 0});
            });
          } else {
            paging_links = paging_links.filter(function(link) {
              return !saver.isVisited(link);
            });
          }

          if (paging_links.length) {
            saver.processPages(paging_links, options, callback);
          } else {
            callback();
          }
        } else {
          callback();
        }
      });
    } else {
      extractChapterPages(callback);
    }
  },

  getMangaInfo: function($, page, options) {
    var manga_info = {};

    manga_info.url = page.url;

    var extractListItems = function(item_objs) {
      var items = [];
      item_objs.each(function() {
        var item_name = $(this).text().trim();
        if (item_name) {
          items.push({
            name: item_name,
            url: $(this).attr('href'),
            title: $(this).attr('title')
          })
        }
      });
      return items;
    }

    var addTagItems = function(items, item_type) {
      manga_info.tags = manga_info.tags || [];
      items.forEach(function(item) {
        manga_info.tags.push({
          type: item_type,
          name: item.name,
          url: item.url,
          title: item.title
        });
      });
    }

    $('table.listTable tr td.listShort').each(function() {
      var $item = $(this).parent();
      var item_name = $(this).text().trim();
      var item_value = $item.find('td.listLong').text().trim();

      if (item_name == 'Title') {
        manga_info['title'] = item_value;
      } else if (item_name == 'Artist') {
        var artists = extractListItems($item.find('td.listLong').find('a'));
        if (artists.length) manga_info.artists = artists;
      } else if (item_name == 'Length') {
        manga_info['length'] = item_value;
      } else if (item_name == 'Origin') {
        // manga_info['origin'] = item_value;
        var parodies = extractListItems($item.find('td.listLong').find('a'));
        if (parodies.length) manga_info.parodies = parodies;
      } else if (item_name == 'Genre') {
        var genres = extractListItems($item.find('td.listLong').find('a'));
        if (genres.length) manga_info.genres = genres;
      } else if (item_name == 'Type') {
        var types = extractListItems($item.find('td.listLong').find('a'));
        if (types.length) manga_info.types = types;
      } else if (item_name == 'Scene') {
        var scenes = extractListItems($item.find('td.listLong').find('a'));
        if (scenes.length) addTagItems(scenes, 'scene');
      } else if (item_name == 'Setting') {
        var settings = extractListItems($item.find('td.listLong').find('a'));
        if (settings.length) addTagItems(settings, 'setting');
      } else if (item_name == 'Fetish') {
        var fetishes = extractListItems($item.find('td.listLong').find('a'));
        if (fetishes.length) addTagItems(fetishes, 'fetish');
      } else if (item_name == 'Role') {
        var roles = extractListItems($item.find('td.listLong').find('a'));
        if (roles.length) addTagItems(roles, 'role');
      } else if (item_name == 'Relationship') {
        var relationships = extractListItems($item.find('td.listLong').find('a'));
        if (relationships.length) addTagItems(relationships, 'relationship');
      } else if (item_name == 'Male Body') {
        var males = extractListItems($item.find('td.listLong').find('a'));
        if (males.length) addTagItems(males, 'male_body');
      } else if (item_name == 'Female Body') {
        var females = extractListItems($item.find('td.listLong').find('a'));
        if (females.length) addTagItems(females, 'female_body');
      } else if (item_name == 'Grouping') {
        var grouping = extractListItems($item.find('td.listLong').find('a'));
        if (grouping.length) addTagItems(grouping, 'grouping');
      } else if (item_name == 'Position') {
        var positions = extractListItems($item.find('td.listLong').find('a'));
        if (positions.length) addTagItems(positions, 'position');
      } else {
        // manga_info[item_name] = item_value;
      }
    });

    if ($('table.listTable tr td.listMiddle').length) {
      var manga_chapters = [];

      $('table.listTable tr td.listMiddle').each(function() {
        var $item = $(this).parent();
        var item_name = $item.find('td.listLeft').text().trim();
        // if (item_name.indexOf('Chapter') == 0) {
          var $item_link = $item.find('td.listMiddle').find('a.listLink');
          if ($item_link) {
            var chapter_link = $item_link.attr('href');
            if (chapter_link) {
              manga_chapters.push({
                url: chapter_link,
                title: item_name
              });
            }
          }
        // }
      });

      manga_info.chapter_count = manga_chapters.length;
      manga_info.chapters = manga_chapters;
    }

    return manga_info;
  }

}