image-dataset/dist/analysis.js

Tool to build image dataset: collect, classify, review

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.analysis = analysis;
const fs_1 = require("fs");
const config_1 = require("./config");
const db_1 = require("./db");
const csv_1 = require("@beenotung/tslib/csv");
const path_1 = require("path");
const format_1 = require("@beenotung/tslib/format");
const fs_2 = require("@beenotung/tslib/fs");
let select_counts = db_1.db.prepare(/* sql */ `
select
  keyword_id as id
, keyword
, count(distinct image.id) as image_count
, count(distinct page_id) as page_count
, count(distinct domain_id) as site_count
from image
inner join keyword on keyword.id = keyword_id
inner join page on page.id = page_id
group by keyword_id
order by keyword_id asc
`);
function scanImageDir(dir) {
    let filenames = (0, fs_2.getDirFilenamesSync)(dir);
    let downloaded = filenames.length;
    let removed = 0;
    let removeDirnames = ['remove', 'removed', 'delete', 'deleted'];
    for (let removeDirname of removeDirnames) {
        downloaded--;
        if (filenames.includes(removeDirname)) {
            removed += (0, fs_2.getDirFilenamesSync)((0, path_1.join)(dir, removeDirname)).length;
        }
    }
    let passed = downloaded - removed;
    return {
        downloaded,
        passed: `${passed} (${(0, format_1.format_percentage)(passed / downloaded)})`,
        removed: `${removed} (${(0, format_1.format_percentage)(removed / downloaded)})`,
    };
}
function analysis() {
    let rows = select_counts.all();
    if (rows.length == 0 && (0, fs_1.existsSync)(config_1.config.downloadedRootDir)) {
        let keywords = (0, fs_2.getDirFilenamesSync)(config_1.config.downloadedRootDir);
        rows = keywords.map(keyword => ({ keyword }));
    }
    if (rows.length == 0) {
        console.error('Error: no images downloaded yet');
        process.exit(1);
    }
    for (let row of rows) {
        let dir = (0, path_1.join)(config_1.config.downloadedRootDir, row.keyword);
        if (!(0, fs_1.existsSync)(dir)) {
            continue;
        }
        let extra = scanImageDir(dir);
        Object.assign(row, extra);
    }
    showTable(rows);
}
function showTable(rows) {
    let csv = (0, csv_1.json_to_csv)(rows);
    let text = (0, csv_1.csv_to_table_text)(csv);
    console.log(text);
}
if (process.argv[1] == __filename) {
    analysis();
}