image-dataset
Version:
Tool to build image dataset: collect, classify, review
869 lines (830 loc) • 26.9 kB
HTML
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Image Dataset</title>
<style>
body {
font-family: Arial, Helvetica, sans-serif;
}
section {
margin-bottom: 1rem;
}
.section-header {
font-size: 1.25rem;
margin: 0.25rem 0;
}
#actionList {
display: flex;
gap: 0.5rem;
}
#actionList button {
padding: 0.25rem 0.5rem;
}
#statsList {
display: flex;
flex-direction: row;
gap: 0.5rem;
padding: 0.5rem 0;
}
#statsList > div {
display: flex;
flex-direction: column;
gap: 0.5rem;
padding: 0.5rem 0;
}
.stats-title {
font-weight: bold;
display: inline-block;
background-color: #faebd780;
padding: 0.25rem 0.5rem;
}
#statsList table {
margin-block-start: 0.25rem;
border-collapse: collapse;
}
#statsList table th,
#statsList table td {
border: 1px solid black;
padding: 0.25rem 0.5rem;
}
#classifiedList [data-text='className'],
#unclassifiedList [data-text='className'] {
font-weight: bold;
margin: 0.25rem 0;
}
.img-list {
display: flex;
flex-wrap: wrap;
}
#classifiedList .img-list {
line-height: 0;
}
.img-list img {
width: 200px;
height: 200px;
object-fit: contain;
padding: 5px;
--border: 5px;
--top: transparent;
--bottom: transparent;
background-size: var(--border) 100%;
background-position: 0 0, 100% 0;
background-repeat: no-repeat;
border-top: var(--border) solid var(--top);
border-bottom: var(--border) solid var(--bottom);
background-image: linear-gradient(var(--top), var(--bottom)),
linear-gradient(var(--top), var(--bottom));
}
.img-list img.selected {
--top: #47c465;
--bottom: #00deff;
}
.heatmap-container.overlap {
position: relative;
}
.heatmap-container.overlap .heatmap-canvas {
position: absolute;
inset: 0;
}
.class-list {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.class-item {
}
#correctClassDialog {
position: sticky;
bottom: 1rem;
margin: auto;
padding: 1rem;
border-radius: 1rem;
border: 1px solid black;
background-color: #fffa;
backdrop-filter: blue(3px);
width: fit-content;
}
.fabs.bottom.right {
position: fixed;
bottom: 1.5rem;
right: 1.5rem;
font-size: 2rem;
display: flex;
flex-direction: column;
}
button key {
text-decoration: underline;
}
.text-center {
text-align: center;
}
.text-muted {
color: #666;
font-size: 0.9em;
}
#modelSettingsDialog table {
margin: 0.5rem 0;
}
#modelSettingsDialog th {
text-align: right;
padding-right: 1rem;
vertical-align: top;
padding-top: 0.5rem;
white-space: nowrap;
}
#modelSettingsDialog td {
min-width: 200px;
padding-top: 0.5rem;
}
#modelSettingsDialog input {
margin-bottom: 0.25rem;
}
#modelSettingsDialog .text-muted {
line-height: 1.4;
}
</style>
<script
type="module"
src="/ionicons/dist/ionicons/ionicons.esm.js"
></script>
<script nomodule src="/ionicons/dist/ionicons/ionicons.js"></script>
<script src="/heatmap-helpers/bundle.js"></script>
</head>
<body>
<div class="fabs bottom right">
<ion-icon
onclick="scrollToTop()"
name="chevron-up-circle-outline"
></ion-icon>
<ion-icon
onclick="scrollToBottom()"
name="chevron-down-circle-outline"
></ion-icon>
</div>
<template data-name="button">
<button
data-text="text"
data-value="className"
data-onclick="onclick"
></button>
</template>
<section>
<div class="section-header">Actions</div>
<div id="actionList" data-template="button" data-bind="actions">
loading...
</div>
<div id="actionResult"></div>
</section>
<section id="modelSettingsDialog" hidden>
<div class="section-header">Model Settings</div>
<table>
<tr>
<th>Model Complexity:</th>
<td>
<input
type="number"
id="newComplexity"
min="1"
max="5"
style="width: 4em"
/>
<div class="text-muted">
1 for easy tasks (default)<br />
2-3 for medium difficulty tasks<br />
4-5 for complex tasks<br />
Higher values give larger hidden layer size but may result in
over-fitting
</div>
</td>
</tr>
<tr>
<th>Current Labels:</th>
<td id="currentLabels"></td>
</tr>
<tr>
<th>
<label for="newLabels">New Labels:</label>
</th>
<td>
<input
type="text"
id="newLabels"
placeholder="e.g. others, cat, dog"
style="width: 100%"
onchange="trimNewLabels()"
/>
<div class="text-muted">comma-separated, at least 2 labels</div>
</td>
</tr>
</table>
<div style="margin-top: 0.5rem">
<button onclick="applyModelSettings()">Apply</button>
<button onclick="hideModelSettingsDialog()">Cancel</button>
<span id="modelSettingsResult"></span>
</div>
</section>
<section>
<div class="section-header">Stats</div>
<button onclick="loadStats()">reload</button>
<div id="statsList" hidden>
<div>
<div>
<div class="stats-title">downloaded:</div>
<table>
<thead>
<th>count</th>
<th>search term</th>
</thead>
<tbody>
<tr data-value="downloaded">
<td data-text="count"></td>
<td data-text="dirname"></td>
<td>
<button onclick="unclassifyDir(event, 'downloaded')">
unclassify
</button>
</td>
</tr>
</tbody>
</table>
</div>
</div>
<div>
<div>
<div class="stats-title">dataset:</div>
<table>
<thead>
<th>count</th>
<th>label</th>
</thead>
<tbody>
<tr data-value="dataset">
<td data-text="count"></td>
<td data-text="dirname"></td>
<td>
<button onclick="unclassifyDir(event, 'dataset')">
unclassify
</button>
</td>
</tr>
</tbody>
</table>
</div>
<div>
<div class="stats-title">classified:</div>
<table>
<thead>
<th>count</th>
<th>label</th>
</thead>
<tbody>
<tr data-value="classified">
<td data-text="count"></td>
<td data-text="dirname"></td>
<td>
<button onclick="unclassifyDir(event, 'classified')">
unclassify
</button>
</td>
</tr>
</tbody>
</table>
</div>
<div>
<div class="stats-title">unclassified:</div>
<span data-text="unclassified" id="unclassifiedTotal"
>loading...</span
>
<button
onclick="restoreUnclassified()"
title="move back to downloaded directories"
>
restore
</button>
<div id="restoreResult"></div>
</div>
</div>
</div>
</section>
<section>
<div class="section-header">Unclassified</div>
<button onclick="loadUnclassified()">reload</button>
<div
id="unclassifiedList"
data-template="unclassified"
data-bind="classes"
>
(not auto loaded)
</div>
<template data-name="unclassified">
<div class="unclassified">
<div data-text="className"></div>
<details>
<summary>
<span data-text="count"></span> images
<button onclick="toggleAllImage(event)">select all</button>
</summary>
<div class="img-list">
<div data-value="images" class="image-item">
<img
data-src="src"
loading="lazy"
onclick="toggleImage(event)"
/>
<div class="text-center">
<div>
<button onclick="showHeatmap(event)">heatmap</button>
<button
hidden
class="overlap-button"
onclick="overlapHeatmap(event)"
>
overlap
</button>
<button
hidden
class="separate-button"
onclick="separateHeatmap(event)"
>
separate
</button>
</div>
<div class="heatmap-container overlap" hidden>
<canvas class="image-canvas"></canvas>
<canvas class="heatmap-canvas"></canvas>
</div>
</div>
<div class="class-list">
<div data-value="results" class="class-item">
<div data-text="label"></div>
<div data-text="confidence"></div>
</div>
</div>
</div>
</div>
</details>
</div>
</template>
</section>
<section>
<div class="section-header">Classified</div>
<button onclick="loadClassified()">reload</button>
<div id="classifiedList" data-template="classified" data-bind="classes">
(not auto loaded)
</div>
<template data-name="classified">
<div class="classified">
<div data-text="className"></div>
<details>
<summary>
<span data-text="count"></span> images
<button onclick="toggleAllImage(event)">select all</button>
</summary>
<div class="img-list">
<div data-value="images">
<img
data-src="src"
loading="lazy"
onclick="toggleImage(event)"
/>
</div>
</div>
</details>
</div>
</template>
</section>
<div id="correctClassDialog">
<div class="section-header">
Correct Label (<span id="correctCount">0</span>)
<button id="undoButton" onclick="undo()" disabled>
<key>u</key>ndo<span id="undoStatus"></span>
</button>
</div>
<div id="correctList" data-template="button" data-bind="buttons"></div>
<div id="correctResult"></div>
</div>
<script src="/data-template/base.js"></script>
<script src="/bundle.js"></script>
<script>
async function get(url, doneFn) {
let res = await fetch(url)
let json = await res.json()
doneFn(json)
}
async function post(url, body) {
try {
let res = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
})
let json = await res.json()
return json
} catch (error) {
return {
error: String(error),
}
}
}
get('/actions', json => {
actionList.textContent = ''
renderTemplate(actionList, {
actions: json.actions.map(action => {
let name = action[0].toUpperCase() + action.slice(1)
return {
text: name,
async onclick(event) {
if (name == 'UpdateModelSettings') {
event.preventDefault()
showModelSettingsDialog()
return
}
let button = event.currentTarget
button.textContent = name + ' (running...)'
let res = await fetch('/' + action, { method: 'POST' })
let json = await res.json()
if (json.error) {
button.textContent = name + ' (error)'
actionResult.textContent = json.error
return
}
button.textContent = name + ' (done)'
actionResult.textContent = ''
loadStats()
},
}
}),
})
})
function loadStats() {
statsList.querySelectorAll('tbody').forEach(tbody => {
for (;;) {
let row = tbody.rows[1]
if (row) {
row.remove()
} else {
break
}
}
tbody.rows[0].cells[0].textContent = 'loading...'
tbody.rows[0].cells[1].textContent = ''
})
get('/stats', json => {
renderData(statsList, json.stats)
statsList.hidden = false
renderTemplate(correctList, {
buttons: json.stats.classNames.map((className, index) => {
return {
text: `[${index + 1}] ${className}`,
className,
onclick(event) {
let button = event.currentTarget
correctLabel(button)
},
}
}),
})
})
}
loadStats()
async function correctLabel(button) {
let className = button.value
if (!className) return
let index = Array.from(button.parentElement.children).indexOf(button)
button.textContent = `[${index + 1}] ${className} (correcting...)`
let selectedImgs = document.querySelectorAll(
'#classifiedList .selected, #unclassifiedList .selected',
)
let images = Array.from(selectedImgs, img => img.getAttribute('src'))
undoStack.push({ className, images })
undoButton.disabled = false
let json = await post('/correct', {
className,
images,
})
if (json.error) {
button.textContent = `[${index + 1}] ${className} (error)`
correctResult.textContent = json.error
return
}
button.textContent = `[${index + 1}] ${className} (done)`
correctResult.textContent = ''
correctCount.textContent = 0
for (let img of selectedImgs) {
img.parentElement.remove()
}
document
.querySelectorAll(
'#classifiedList .classified, #unclassifiedList .unclassified',
)
.forEach(div => {
div.querySelector('[data-text="count"]').textContent =
div.querySelectorAll('img').length
})
}
function toggleImage(event) {
let img = event.currentTarget
if (!img.classList.contains('selected')) {
img.classList.add('selected')
correctCount.textContent++
} else {
img.classList.remove('selected')
correctCount.textContent--
}
}
function toggleAllImage(event) {
let button = event.currentTarget
let imgList = button.closest('details').querySelector('.img-list')
if (button.textContent == 'select all') {
imgList.querySelectorAll('img').forEach(img => {
if (!img.classList.contains('selected')) {
img.classList.add('selected')
correctCount.textContent++
}
})
button.textContent = 'unselect all'
} else {
imgList.querySelectorAll('img.selected').forEach(img => {
img.classList.remove('selected')
correctCount.textContent--
})
button.textContent = 'select all'
}
}
async function unclassifyDir(event, dir) {
let tr = event.currentTarget.closest('tr')
let dirname = tr.cells[1].textContent
let json = await post(
'/unclassify?' +
new URLSearchParams({
dir: dir + '/' + dirname,
}),
)
if (json.error) {
alert(json.error)
return
}
tr.cells[0].textContent = 0
unclassifiedTotal.textContent = json.total
}
async function restoreUnclassified() {
restoreResult.textContent = 'restoring...'
let json = await post('/unclassified/restore')
if (json.error) {
restoreResult.textContent = json.error
return
}
let message = `${json.restoredCount} restored`
if (json.unknownCount > 0) {
message += `, ${json.unknownCount} unknown`
}
restoreResult.textContent = message
await loadStats()
}
function loadUnclassified() {
unclassifiedList.textContent = 'loading...'
get('/unclassified', json => {
if (json.error) {
unclassifiedList.textContent = json.error
return
}
renderTemplate(unclassifiedList, {
classes: json.classes.map(({ className, images }) => ({
className,
count: images.length,
images: images
.sort((a, b) => a.confidence - b.confidence)
.map(image => ({
...image,
src: `/unclassified/${image.filename}`,
results: image.results
.sort((a, b) => b.confidence - a.confidence)
.map(result => ({
label: result.label,
confidence: (result.confidence * 100).toFixed(2) + '%',
})),
})),
})),
})
})
}
function loadClassified() {
classifiedList.textContent = 'loading...'
get('/classified', json => {
if (json.error) {
classifiedList.textContent = json.error
return
}
if (json.classes.length == 0) {
classifiedList.textContent = 'no classified images'
return
}
renderTemplate(classifiedList, {
classes: json.classes.map(({ className, filenames }) => {
return {
className,
count: filenames.length,
images: filenames.map(filename => ({
src: `/classified/${className}/${filename}`,
})),
}
}),
})
})
}
let undoStack = []
async function undo() {
if (undoStack.length == 0) {
return
}
let { className, images } = undoStack.pop()
try {
undoStatus.textContent = 'ing...'
let res = await fetch('/correct', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
images,
}),
})
let json = await post('/undo', { className, images })
if (json.error) {
throw json.error
}
undoStatus.textContent = ' (done)'
undoButton.disabled = undoStack.length == 0
correctResult.textContent = ''
} catch (error) {
undoStack.push({ className, images })
undoStatus.textContent = ' (error)'
correctResult.textContent = String(error)
}
}
window.addEventListener('keypress', event => {
if (event.key == 'u') {
undo()
event.preventDefault()
return
}
let index = event.key - 1
let buttons = correctList.querySelectorAll('button[data-value]')
let button = buttons[index]
if (!button) return
correctLabel(button)
event.preventDefault()
})
function scrollToTop() {
document.body.scrollIntoView({
behavior: 'smooth',
block: 'start',
})
}
function scrollToBottom() {
document.body.scrollIntoView({
behavior: 'smooth',
block: 'end',
})
}
async function showHeatmap(event) {
let item = event.currentTarget.closest('.image-item')
let heatmap_container = item.querySelector('.heatmap-container')
let img = item.querySelector('img')
let image_canvas = item.querySelector('.image-canvas')
let heatmap_canvas = item.querySelector('.heatmap-canvas')
let overlap_button = item.querySelector('.overlap-button')
let separate_button = item.querySelector('.separate-button')
overlap_button.hidden = true
separate_button.hidden = false
heatmap_container.hidden = false
let classNames = Array.from(
correctList.querySelectorAll('[data-value]'),
button => button.value,
)
let models = await loadModels({ classNames })
let { width, height } = models.baseModel.spec
image_canvas.width = width
image_canvas.height = height
heatmap_canvas.width = width
heatmap_canvas.height = height
// let context = image_canvas.getContext('2d')
// context.drawImage(img, 0, 0, width, height)
let image_tensor = await tf.browser.fromPixels(img)
let cropped_image_tensor = cropAndResizeImageTensor({
imageTensor: image_tensor,
width,
height,
})
tf.browser.toPixels(toTensor3D(cropped_image_tensor), image_canvas)
let results = await models.classifyImage(img)
results.sort((a, b) => b.confidence - a.confidence)
console.log(results)
let label = results[0].label
let base_confidence = results[0].confidence
console.log({ label, base_confidence })
await heatmap_helpers.build_heatmap({
image_canvas,
heatmap_canvas,
slide_ratio: 0.5,
calc_score: async context => {
let { image_data } = context
let result = await models.classifyImage(image_data)
let confidence = result.find(item => item.label == label).confidence
let diff = confidence - base_confidence
if (diff == 0) {
return 0.5
}
console.log({
confidence,
base_confidence,
diff,
})
if (confidence < base_confidence) {
let diff = base_confidence - confidence
return diff / base_confidence / 2 + 0.5
} else {
let diff = confidence - base_confidence
return (1 - diff / (1 - base_confidence)) / 2
}
},
should_zoom: score => {
return score <= 0.2 || score >= 0.8
},
})
}
function overlapHeatmap(event) {
let item = event.currentTarget.closest('.image-item')
let overlap_button = item.querySelector('.overlap-button')
let separate_button = item.querySelector('.separate-button')
let heatmap_container = item.querySelector('.heatmap-container')
overlap_button.hidden = true
separate_button.hidden = false
heatmap_container.classList.add('overlap')
}
function separateHeatmap(event) {
let item = event.currentTarget.closest('.image-item')
let overlap_button = item.querySelector('.overlap-button')
let separate_button = item.querySelector('.separate-button')
let heatmap_container = item.querySelector('.heatmap-container')
overlap_button.hidden = false
separate_button.hidden = true
heatmap_container.classList.remove('overlap')
}
function showModelSettingsDialog() {
modelSettingsDialog.hidden = false
newLabels.value = ''
modelSettingsResult.textContent = ''
get('/getClassLabelsInfo', json => {
if (json.error) {
modelSettingsResult.style.color = 'red'
modelSettingsResult.textContent = json.error
return
}
newComplexity.value = json.complexity
currentLabels.textContent = json.classNames.join(', ')
})
}
function trimNewLabels() {
let classNames = newLabels.value
.split(',')
.map(s => s.trim())
.filter(s => s)
newLabels.value = classNames.join(', ')
return classNames
}
async function applyModelSettings() {
modelSettingsResult.style.color = 'red'
let complexity = +newComplexity.value
if (!(complexity >= 1 && complexity <= 5)) {
modelSettingsResult.textContent = 'Complexity must be between 1 and 5'
return
}
let classNames = trimNewLabels()
if (classNames.length < 2) {
modelSettingsResult.textContent =
'At least two class names are required'
return
}
modelSettingsResult.style.color = 'gray'
modelSettingsResult.textContent = 'Saving...'
let json = await post('/updateModelSettings', {
complexity,
classNames,
})
if (json.error) {
modelSettingsResult.style.color = 'red'
modelSettingsResult.textContent = json.error
return
}
modelSettingsResult.style.color = 'green'
modelSettingsResult.textContent = 'Settings saved successfully!'
loadStats()
setTimeout(hideModelSettingsDialog, 3000)
}
function hideModelSettingsDialog() {
modelSettingsDialog.hidden = true
}
</script>
</body>
</html>