poto-siril
Version:
Automatization around Siril (<https://siril.org/>) for deep sky astrophotography.
560 lines (559 loc) • 26.1 kB
JavaScript
import fs from "fs-extra";
import path from "path";
import Enquirer from "enquirer";
import { formatMessage, logger } from "../utils/logger.js";
import { copyFileToProject, getFitsFromDirectory, matchSetFile, getImageSpecFromSetName, isAsiAirDirectoryF, } from "../utils/utils.js";
import { POTO_JSON, POTO_VERSION } from "../utils/const.js";
const enquirer = new Enquirer();
const prepare = async ({ inputDirectories, projectDirectory, }) => {
// TODO. logger.command("prepare"); to introduce the command in the logs.
const continueProcess = await ensureProjectDirectoryExists(projectDirectory);
if (!continueProcess) {
logger.warning("Aborted.");
return;
}
logger.step("Reading input directories");
const inputFiles = [];
for (const inputDirectory of inputDirectories) {
logger.debug(`Reading input directory ${inputDirectory}`);
const files = await getAllFitsInInputDirectory(inputDirectory, projectDirectory);
inputFiles.push(...files);
}
let allLights = inputFiles.filter(file => file.type === "Light");
const allFlats = inputFiles.filter(file => file.type === "Flat");
const allDarksBiases = inputFiles.filter(file => file.type === "Dark" || file.type === "Bias");
logger.info(`Found ${inputFiles.length} .fit files in input directories 🌋.`);
logger.info(`Including ${allLights.length} lights 🌟.`);
logger.space();
logger.info("Light sequences:");
logger.info(`${[
...new Set(allLights
.sort((a, b) => a.sequenceId.localeCompare(b.sequenceId))
.map(light => `${light.setName} ${light.sequenceId}`)),
]
.map(x => `- ${x}`)
.join("\n")}`);
logger.step("Matching lights and flats (early stage)");
const allFlatsMatchingLights = getFlatsMatchingLightsNaive({
lights: allLights,
flats: allFlats,
});
const notMatchedLights = getLightsMatchingLightNaive({
lights: allLights,
flats: allFlats,
});
allLights = allLights.filter(light => !notMatchedLights.includes(light));
logger.step("Matching lights and flats (final stage)");
const lightsFlatsMatches = await matchLightsToFlats(allFlatsMatchingLights, allLights);
let layerSets = initLayerSetsWithLightsnFlats(lightsFlatsMatches, allLights, allFlatsMatchingLights, projectDirectory);
logger.step("Tagging darks and biases");
logger.info(`Found ${allDarksBiases.length} darks+baises files (without temperature filtering).`);
layerSets = await AssignDarksBiasesToLayerSets(layerSets, allDarksBiases);
logger.step("Preview before dispatching");
const { go, metrics } = await previewBeforeDispatching(layerSets);
if (!go) {
logger.warning("Aborted.");
return;
}
logger.step("Dispatching");
const potoProject = {
generatedAt: new Date(),
potoVersion: POTO_VERSION,
metrics,
layerSets,
};
await dispatchProject(potoProject, projectDirectory);
logger.success("Dispatch complete.");
// TODO. custom sort if LRGBSHO filter names to ease reading.
};
export default prepare;
/**
* Ensure that the project directory exists.
* If it does not exist, ask the user if they want to create it.
* If it already exists, warn the user that the project will be overwritten (but may have extra files in it).
*
* @param projectDirectory - The directory of the current project.
* @returns A boolean indicating if we can continue.
*/
const ensureProjectDirectoryExists = async (projectDirectory) => {
if (!fs.existsSync(projectDirectory)) {
const { createProjectDirectory } = (await enquirer.prompt({
type: "confirm",
name: "createProjectDirectory",
message: `Directory ${projectDirectory} does not exist. Do you want to create it?`,
initial: true,
}));
if (createProjectDirectory) {
fs.mkdirSync(projectDirectory, { recursive: true });
return true;
}
else {
return false;
}
}
else {
if (fs.existsSync(path.join(projectDirectory, POTO_JSON))) {
const { continueEvenIfProjectAlreadyExists } = (await enquirer.prompt({
type: "confirm",
name: "continueEvenIfProjectAlreadyExists",
message: `Directory ${projectDirectory} already have a ${POTO_JSON}. Continue? (will overwrite the project, but may contains extra files that were previously imported.)`,
initial: true,
}));
return continueEvenIfProjectAlreadyExists;
}
else {
return true;
}
}
};
export const selectedInputSubDirectoryChoices = ["Use Autorun directory", "Use Plan directory", "Use both directory"];
/**
* Retrieve all FITS files from the input directory.
*
* @param inputDirectory - The directory to scan.
* @param projectDirectory - The directory of the current project.
* @returns An array of FileImageSpec objects representing the FITS files.
*/
const getAllFitsInInputDirectory = async (inputDirectory, projectDirectory) => {
// if ASIAIR used, may stores the lights+flats files in either the Autorun or Plan directories.
const { isAsiAirDirectory, autorunDirectory, planDirectory } = isAsiAirDirectoryF(inputDirectory);
const logFiles = (files) => {
logger.info(`Found ${files.length} FITS in input dir ${inputDirectory}.`);
};
if (!fs.existsSync(inputDirectory)) {
logger.errorThrow(`Input directory ${inputDirectory} does not exists.`);
}
if (!isAsiAirDirectory) {
const allFiles = getFitsFromDirectory({
directory: inputDirectory,
projectDirectory,
});
if (allFiles.length === 0) {
logger.errorThrow(`No FITS files found in input dir ${inputDirectory}.`);
}
logFiles(allFiles);
return allFiles;
}
logger.debug(`ASIAIR dump detected in input dir ${inputDirectory}.`);
const autorunFiles = fs.existsSync(autorunDirectory)
? getFitsFromDirectory({
directory: autorunDirectory,
projectDirectory,
})
: [];
const planFiles = fs.existsSync(planDirectory)
? getFitsFromDirectory({
directory: planDirectory,
projectDirectory,
})
: [];
if (autorunFiles.length === 0 && planFiles.length === 0) {
logger.errorThrow("No FITS files found in Autorun nor Plan folders.");
}
if (autorunFiles.length > 0 && planFiles.length === 0) {
logFiles(autorunFiles);
return autorunFiles;
}
if (autorunFiles.length === 0 && planFiles.length > 0) {
logFiles(planFiles);
return planFiles;
}
if (autorunFiles.length > 0 && planFiles.length > 0) {
const { selectedInputSubDirectory } = (await enquirer.prompt({
type: "select",
name: "selectedInputSubDirectory",
message: "Files found in both Autorun and Plan directories. How do we proceed?",
choices: selectedInputSubDirectoryChoices,
}));
switch (selectedInputSubDirectory) {
case "Use Autorun directory":
logFiles(autorunFiles);
return autorunFiles;
case "Use Plan directory":
logFiles(planFiles);
return planFiles;
case "Use both directory":
logFiles([...autorunFiles, ...planFiles]);
return [...autorunFiles, ...planFiles];
}
}
};
/**
* Naive matching of flats with lights.
* Sufficient enough to skip the flats that do not have a matching light at all.
*
* @param flats - The input files to process. Searching for the flats in this list.
* @param lights - The lights to match with the flats.
*/
const getFlatsMatchingLightsNaive = ({ flats, lights, }) => {
const notMatchedFlatsLog = [];
const matchingFlats = flats.filter(flat => {
if (lights.find(light => matchSetFile(light, flat))) {
return flat;
}
else {
notMatchedFlatsLog.push(`No matching lights for ${flat.setName} (seq ${flat.sequenceId}). The flat sequence won't be used.`);
}
});
const sequences = [...new Set(matchingFlats.map(file => file.sequenceId))];
for (const skip of [...new Set(notMatchedFlatsLog)]) {
logger.warning(skip);
}
const sequencesLightsCount = [
...new Set(matchingFlats.map(file => file.sequenceId)),
].length;
logger.info(`Pre-selected ${matchingFlats.length} flats (${sequences.length} sequences) that matches with ${lights.length} lights (${sequencesLightsCount} sequences).`);
return matchingFlats;
};
/**
* Reverse of `getFlatsMatchingLightsNaive`.
* @returns The lights that have not matched with a flat.
*/
const getLightsMatchingLightNaive = ({ lights, flats, }) => {
const notMatchedLightsLog = [];
const notMatchedLights = [];
const matchingLights = lights.filter(light => {
if (flats.find(flat => matchSetFile(light, flat))) {
return light;
}
else {
notMatchedLightsLog.push(`No matching flats for ${light.setName} (seq ${light.sequenceId}). The light sequence won't be used.`);
notMatchedLights.push(light);
}
});
const sequences = [...new Set(matchingLights.map(file => file.sequenceId))];
for (const skip of [...new Set(notMatchedLightsLog)]) {
logger.warning(skip);
}
const sequencesLightsCount = [
...new Set(matchingLights.map(file => file.sequenceId)),
].length;
logger.info(`Pre-selected ${matchingLights.length} lights (${sequences.length} sequences) that matches with ${flats.length} flats (${sequencesLightsCount} sequences).`);
return notMatchedLights;
};
/**
* Process the light and flat matching.
* If multiple sequences are found for a flat, ask the user to select the flat sequence to use for each light sequence.
* NOTE. We expect that all flats have a matching light at this point, thanks to the `getFlatsMatchingLightsNaive` function.
*
* @param matchingFlats - The flats that have a matching light.
* @param lights - The lights to match with the flats.
*/
const matchLightsToFlats = async (matchingFlats, lights) => {
const flatSets = [...new Set(matchingFlats.map(file => file.setName))];
const LightFlatMatches = [];
let introManualMatchingDisplayed = false;
// TODO. Review the archi overall. We should walk light by light instead of flat by flat. This will be easier to discover and setup the project.
for (const flatSet of flatSets) {
const flatSetSpecs = getImageSpecFromSetName(flatSet);
// Search for sequences that are similar.
const flatSetNameSequenceIds = [
...new Set(matchingFlats
.filter(flat =>
// https://pixinsight.com/forum/index.php?threads/can-flats-be-different-iso-than-lights.23686/
flat.bin === flatSetSpecs.bin &&
flat.filter === flatSetSpecs.filter)
.sort((a, b) => a.sequenceId.localeCompare(b.sequenceId))
.map(flat => `${flat.setName}__${flat.sequenceId}`)),
];
if (flatSetNameSequenceIds.length === 0) {
throw new Error(`❓❓❓❗️ No sequences found for flat ${flatSet}.`);
}
if (flatSetNameSequenceIds.length === 1) {
// Auto match the flat to the light. Nothing to ask from the user.
const flatSetNameSequenceId = flatSetNameSequenceIds[0];
const matchingLights = lights.filter(light => matchSetFile(light, flatSetSpecs));
const matchingLightSetNameSequenceIds = [
...new Set(matchingLights.map(light => `${light.setName}__${light.sequenceId}`)),
];
for (const lightSetSequence of matchingLightSetNameSequenceIds) {
LightFlatMatches.push({
lightSetName: lightSetSequence.split("__")[0],
lightSequenceId: lightSetSequence.split("__")[1],
flatSetName: flatSetNameSequenceId.split("__")[0],
flatSequenceId: flatSetNameSequenceId.split("__")[1],
isManualMatch: false,
});
}
continue;
}
const lightsConcerned = [
...new Set(lights
.filter(light => matchSetFile(light, getImageSpecFromSetName(flatSet)))
.map(light => ({
setName: light.setName,
sequenceId: light.sequenceId,
}))
.sort((a, b) => {
return a.sequenceId.localeCompare(b.sequenceId);
})
.map(light => `${light.setName}__${light.sequenceId}`)),
];
const allAlreadyMatched = lightsConcerned.filter(lightConcerned => LightFlatMatches.map(x => `${x.lightSetName}__${x.lightSequenceId}`).includes(lightConcerned)).length === lightsConcerned.length;
// To avoid re-asking for a light sequence that has already been matched.
if (allAlreadyMatched) {
continue;
}
if (introManualMatchingDisplayed) {
logger.space();
}
logger.info(`🤚 Several sequences of flats are compatible with ${flatSetSpecs.filter
? `${flatSetSpecs.bin} Filter ${flatSetSpecs.filter}`
: flatSetSpecs.bin}:`);
logger.info(`${flatSetNameSequenceIds.map(x => `- ${x}`).join("\n")}`);
if (!introManualMatchingDisplayed) {
logger.debug("We assume that multiple sequences of the same flat kind indicate multiple night sessions where the flats had to be re-shot in between (e.g., a significant date gap between shooting sessions and the lights were not collected with the same collimation and/or same dust in the optical train).");
logger.debug("We will ask you to tag each concerned light sequence to the right flat sequence (this disclaimer won't be displayed again 🤓).");
introManualMatchingDisplayed = true;
}
for (const lightConcerned of lightsConcerned) {
if (LightFlatMatches.map(x => `${x.lightSetName}__${x.lightSequenceId}`).includes(lightConcerned)) {
// Skip if the light sequence has already been matched to a flat sequence.
continue;
}
logger.space();
const lightConcernedSetName = lightConcerned.split("__")[0];
const lightConcernedSequenceId = lightConcerned.split("__")[1];
const { selectedFlatSequence } = (await enquirer.prompt({
type: "select",
name: "selectedFlatSequence",
message: formatMessage(`${lightConcernedSetName} ${lightConcernedSequenceId} will use`),
choices: flatSetNameSequenceIds.map(x => ({
name: x,
message: formatMessage(x.replace("__", " ")), // TODO. Print sequence length.
})),
}));
if (!selectedFlatSequence) {
logger.errorThrow("No flat sequence selected.", {
lightConcernedSetName,
lightConcernedSequenceId,
choices: flatSetNameSequenceIds,
});
}
LightFlatMatches.push({
lightSetName: lightConcernedSetName,
lightSequenceId: lightConcernedSequenceId,
flatSetName: selectedFlatSequence.split("__")[0],
flatSequenceId: selectedFlatSequence.split("__")[1],
isManualMatch: true,
});
}
}
logger.space();
logger.info("👉 Light - Flat matching summary:");
for (const pair of LightFlatMatches) {
logger.debug(`- ${pair.lightSetName} ${pair.lightSequenceId} 🏹 ${pair.flatSetName} ${pair.flatSequenceId}`);
}
return LightFlatMatches;
};
/**
* Initialize the layer sets with lights and flats matches.
*
* @param lightsFlatsMatches - The lights and flats matches.
* @param allLights - All the lights.
* @param allFlatsMatchingLights - All the flats matching lights.
*/
const initLayerSetsWithLightsnFlats = (lightsFlatsMatches, allLights, allFlatsMatchingLights, projectDirectory) => {
const layerSets = [];
for (const lightsFlatsMatch of lightsFlatsMatches) {
const lights = lightsFlatsMatch.isManualMatch
? allLights.filter(light => light.sequenceId === lightsFlatsMatch.lightSequenceId &&
light.setName === lightsFlatsMatch.lightSetName)
: allLights.filter(light => light.setName === lightsFlatsMatch.lightSetName);
if (!lights) {
throw new Error(`❓❓❓❗️ Light ${lightsFlatsMatch.lightSetName} ${lightsFlatsMatch.lightSequenceId} not found.`);
}
const flats = allFlatsMatchingLights.filter(flat => flat.sequenceId === lightsFlatsMatch.flatSequenceId);
if (!flats) {
throw new Error(`❓❓❓❗️ Flat ${lightsFlatsMatch.lightSetName} ${lightsFlatsMatch.flatSequenceId} not found.`);
}
const layerSetId = lightsFlatsMatch.isManualMatch
? `${lights[0].setName}__${lights[0].sequenceId}`
: lights[0].setName;
const lightSequenceIds = new Set([...lights].map(light => light.sequenceId));
const lightSequences = [];
for (const lightSequenceId of lightSequenceIds) {
const lightsOfSequence = lights.filter(light => light.sequenceId === lightSequenceId);
const count = lightsOfSequence.length;
const integrationMs = lightsOfSequence.reduce((total, light) => total + light.bulbMs, 0);
lightSequences.push({
sequenceId: lightSequenceId,
count,
integrationMinutes: integrationMs / 1000 / 60,
});
}
const lightTotalIntegrationMs = lights.reduce((total, light) => total + light.bulbMs, 0);
const layerSet = {
layerSetId,
filter: lights[0].filter,
lightSequences,
lightTotalCount: lights.length,
lightTotalIntegrationMinutes: lightTotalIntegrationMs / 1000 / 60,
lights,
flatSet: flats[0].setName,
flatSequenceId: flats[0].sequenceId,
flatsCount: flats.length,
flats,
};
// Backfill lights `projectFileDirectory` and `projectFilePath`.
for (const light of lights) {
light.projectFileDirectory = light.filter
? `${projectDirectory}/${light.filter}/${layerSetId}`
: `${projectDirectory}/${layerSetId}`;
light.projectFilePath = `${light.projectFileDirectory}/${light.fileName}`;
}
layerSets.push(layerSet);
}
return layerSets;
};
/**
* Assign darks and biases to the layer sets.
* We darks are assigned to lights, and biases are assigned to flats.
*
* @param layerSets - The layer sets with lights and flats filled in.
* @param bankFiles - The bank files available.
*/
const AssignDarksBiasesToLayerSets = async (layerSets, bankFiles) => {
const { darkTemperatureTolerance } = (await enquirer.prompt({
type: "input",
name: "darkTemperatureTolerance",
message: "Select the temperature tolerance for lights-darks matching (+/-0.5°C to +/-10°C).",
initial: 3,
validate: (value) => {
const number = parseFloat(value);
return number >= 0.2 ? true : "Please enter something >=0.2";
},
}));
for (const layerSet of layerSets) {
layerSet.darkSet = "No darks matched";
layerSet.darksCount = 0;
layerSet.darkTotalIntegrationMinutes = 0;
layerSet.darks = [];
const darksAllTemperature = bankFiles.filter(dark => dark.type === "Dark" && matchSetFile(layerSet.lights[0], dark));
if (darksAllTemperature.length === 0) {
logger.error(`No darks matching light set ${layerSet.lights[0].setName} (regardless of temperature filtering).`);
}
else {
const darks = darksAllTemperature.filter(dark => {
const temperatureDiff = Math.abs(dark.temperature - layerSet.lights[0].temperature);
return temperatureDiff <= darkTemperatureTolerance;
});
if (darks.length === 0) {
logger.error(`No darks available for ${layerSet.lights[0].setName} with temperature window +-${darkTemperatureTolerance}.`);
logger.info(`There are ${darksAllTemperature.length} darks for ${layerSet.lights[0].setName} if we ignore temperature.`);
}
else {
layerSet.darkSet = darks[0].setName;
layerSet.darksCount = darks.length;
const darkTotalIntegrationMs = darks.reduce((total, dark) => total + dark.bulbMs, 0);
layerSet.darkTotalIntegrationMinutes =
darkTotalIntegrationMs / 1000 / 60;
layerSet.darks = darks;
}
// Warn if there are multiple sequences for the same layer set
const darkSequences = new Set(darks.map(dark => dark.sequenceId));
if (darkSequences.size > 1) {
logger.warning(`Multiple dark sequences found for ${layerSet.lights[0].setName}: ${[
...darkSequences,
].join(", ")}`);
logger.warning("Gathering them all for the master dark. Make sure that's what you wanted.");
}
}
layerSet.biasSet = "No biases matched";
layerSet.biasesCount = 0;
layerSet.biases = [];
const biases = bankFiles.filter(bias => bias.type === "Bias" && matchSetFile(layerSet.flats[0], bias));
if (biases.length === 0) {
logger.error(`No biases found for ${layerSet.flats[0].setName}.`);
}
else {
layerSet.biasSet = biases[0].setName;
layerSet.biasesCount = biases.length;
layerSet.biases = biases;
}
const biasSequences = new Set(biases.map(bias => bias.sequenceId));
if (biasSequences.size > 1) {
logger.warning(`Multiple bias sequences found for ${layerSet.flats[0].setName}: ${[
...biasSequences,
].join(", ")}`);
logger.warning("Gathering them all for the master bias. Make sure that's what you wanted.");
}
}
logger.space();
logger.info("👉 Light - Dark matching summary:");
for (const layerSet of layerSets) {
for (const lightSequence of layerSet.lightSequences) {
if (layerSet.layerSetId.includes("__20")) {
logger.debug(`- ${layerSet.layerSetId} 🏹 ${layerSet.darkSet} (${layerSet.darksCount} files / ${layerSet.darkTotalIntegrationMinutes} minutes)`);
}
else {
logger.debug(`- ${layerSet.layerSetId} ${lightSequence.sequenceId} 🏹 ${layerSet.darkSet} (${layerSet.darksCount} files / ${layerSet.darkTotalIntegrationMinutes} minutes)`);
}
}
}
logger.info("👉 Flat - Bias matching summary:");
for (const layerSet of layerSets) {
logger.debug(`- ${layerSet.flatSet} ${layerSet.flatSequenceId} 🏹 ${layerSet.biasSet} (${layerSet.biasesCount} files)`);
}
return layerSets;
};
/**
* Preview the project composition before dispatching.
*
* @param layerSets - The layer sets to preview.
*/
const previewBeforeDispatching = async (layerSets) => {
const metrics = {
cumulatedLightIntegrationMinutes: layerSets.reduce((total, layerSet) => total + layerSet.lightTotalIntegrationMinutes, 0),
cumulatedDarksIntegrationMinutes: layerSets.reduce((total, layerSet) => total + layerSet.darkTotalIntegrationMinutes, 0),
totalLights: layerSets.reduce((total, layerSet) => total + layerSet.lightTotalCount, 0),
totalDarks: layerSets.reduce((total, layerSet) => total + layerSet.darksCount, 0),
totalFlats: layerSets.reduce((total, layerSet) => total + layerSet.flatsCount, 0),
totalBiases: layerSets.reduce((total, layerSet) => total + layerSet.biasesCount, 0),
};
logger.info(`🔭 Cumulated light integration: ${metrics.cumulatedLightIntegrationMinutes} minutes.`);
logger.info(`📦 Project size: ${metrics.totalLights} lights, ${metrics.totalDarks} darks, ${metrics.totalFlats} flats, ${metrics.totalBiases} biases.`);
logger.space();
logger.info("🌌 Layer sets:");
for (const layerSet of layerSets) {
const log = `- ${layerSet.layerSetId} has ${layerSet.lights.length} lights, ${layerSet.darks.length} darks, ${layerSet.flats.length} flats, ${layerSet.biases.length} biases.`;
if (layerSet.lights.length > 0 &&
layerSet.flats.length > 0 &&
layerSet.darks.length > 0 &&
layerSet.biases.length > 0) {
logger.debug(log);
}
else {
logger.warning(log);
}
}
logger.space();
const { go } = (await enquirer.prompt({
type: "confirm",
name: "go",
message: "Do you want to proceed with the dispatch?",
initial: true,
}));
return { go, metrics };
};
/**
* Dispatch the project json and the files to the poto project directory.
*
* @param potoProject - The POTO project to prepare.
* @param projectDirectory - The project directory.
*/
const dispatchProject = (potoProject, projectDirectory) => {
const potoJsonPath = `${projectDirectory}/${POTO_JSON}`;
fs.writeFileSync(potoJsonPath, JSON.stringify(potoProject, null, 2));
let alreadyImported = [];
for (const layerSet of potoProject.layerSets) {
for (const file of [
...layerSet.lights,
...layerSet.darks,
...layerSet.flats,
...layerSet.biases,
]) {
alreadyImported = copyFileToProject(file, alreadyImported);
}
}
};