@momsfriendlydevco/bom
Version:
Scraper for the Australian Bureau of Meteorology (BOM) - including radar images
559 lines (513 loc) • 15.7 kB
JavaScript
// COMMAND TO CONVERT TO GIF: convert -delay 50 -loop 0 -draw 'image DstOver 0,0,0,0 background.gif' -dispose background IDR032.T.20180321*.png out.gif
var _ = require('lodash');
var async = require('async-chainable');
var debug = require('debug')('bom');
var events = require('events');
var ftp = require('ftp');
var fs = require('fs');
var fspath = require('path');
var glob = require('glob');
var im = require('imagemagick');
var mkdirp = require('mkdirp');
var os = require('os');
function BomRadar(options) {
var bom = this;
/**
* Settings to use for this BomRadar instance
* @var Object
*/
bom.settings = _.defaults(options, {
id: '032',
host: 'ftp.bom.gov.au',
framePath: '/anon/gen/radar',
backgroundsPath: '/anon/gen/radar_transparencies',
cachePath: fspath.join(os.tmpdir(), '/bom-cache'),
getThreads: 1,
fetch: {
frames: true,
backgrounds: true,
},
backgrounds: {
background: true,
catchments: false,
locations: true,
rail: false,
range: false,
riverBasins: false,
roads: false,
topography: true,
waterWays: false,
wthrDistricts: false,
},
clean: {
olderThan: 60*60*24 * 1000, //= 24 hours
},
composite: {
format: 'gif',
method: 'path',
cache: true,
delay: 50,
cacheFile: b => `IDR${b.settings.id}.composite.${b.settings.composite.format}`,
cacheFileExpiry: 60*60*1 * 1000, //= 1 hour
removeAttribution: false,
autoRefresh: true,
autoClean: true,
arguments: [ // Any argument functions are evaluated as ({frames,backgrounds,settings})
// Set the animation delay + loop parameters
'-delay', b => b.settings.composite.delay,
'-loop', '0',
// Add background layers
b => _(b.backgrounds)
.pickBy((v, k) =>
v // Has a value
&& b.settings.backgrounds[k] // User has enabled the background
)
.map((v, k) => [
'-draw',
`image DstOver 0,0,0,0 '${b.backgrounds[k]}'`,
])
.sortBy(v => /\.background\.png/.test(v) ? `AAA${v}` : v) // Sort main background layer first
.value(),
// Clear the foreground for each frame
'-dispose', 'background',
// List the animation frames
b => b.frames,
// Optionally remove the attribution - assuming the user has credited the BOM elsewhere
b => b.settings.composite.removeAttribution ? ['-chop', 'x15'] : undefined,
// Append the output file
b => fspath.join(b.settings.cachePath, _.isFunction(b.settings.composite.cacheFile) ? b.settings.composite.cacheFile(b) : b.settings.composite.cacheFile),
],
},
});
/**
* Convenience function to quickly set settings
* @param {string|array} key The settings path to set, dotted or array notation is supported
* @param {*} value The value to set the setting to
* @returns {BomRadar} This chainable object
*/
bom.set = function(key, value) {
_.set(bom.settings, key, value);
return bom;
};
/**
* Attempt to refresh BOM radar data from the FTP site
* @param {Object} [options] Additional options to use (overrides bom.settings)
* @param {function} cb Callback to call as (error, {backgrounds, frames})
* @returns {BomRadar} This chainable object
*/
bom.refresh = function(options, cb) {
// Argument mangling {{{
if (_.isFunction(options)) { // Called as (cb)
cb = options;
options = {};
}
// }}}
var settings = _.defaults(options, bom.settings);
async()
// Ensure cache directory exists {{{
.then(function(next) {
mkdirp(settings.cachePath, next);
})
// }}}
// Connect {{{
.then('client', function(next) {
debug('FTP> Connecting...');
var ftpClient = new ftp()
.on('ready', ()=> next(null, ftpClient));
ftpClient.connect({host: settings.host}, next)
})
// }}}
// Fetch backgrounds {{{
.then('backgrounds', function(next) {
if (!settings.fetch.backgrounds) return next(); // All backgrounds disabled - then skip
async()
.set('client', this.client)
// Change directory {{{
.then(function(next) {
debug('FTP (backgrounds)> cd', settings.backgroundsPath);
this.client.cwd(settings.backgroundsPath, next);
})
// }}}
// List files {{{
.then('files', function(next) {
debug('FTP (backgrounds)> ls');
this.client.list(next);
})
// }}}
// Filter background images into the ones we want {{{
.then('files', function(next) {
next(null,
this.files
.filter(i => i.name.startsWith(`IDR${settings.id}`))
.filter(i => i.name.endsWith('.png'))
.filter(i => {
var [,type] = /^IDR.*\.([a-z]+)\.png$/.exec(i.name) || [];
if (!type) return false;
if (!settings.backgrounds[type]) return false; // Dont fetch this type
return true;
})
);
})
// }}}
// Grab background images as async {{{
.limit(settings.getThreads)
.forEach('files', function(next, file) {
file.cachePath = fspath.join(settings.cachePath, file.name);
async()
.set('client', this.client)
// Lookup size of existing file (if any) {{{
.then('stat', function(next) {
fs.stat(file.cachePath, function(err, stat) {
if (err) {
return next(); // No file - continue to grab it
} else if (stat && stat.size == file.size) { // Found file and the size matches
debug('FTP (backgrounds)> # skip', file.name);
return next('SKIP');
} else { // Everything else - grab the file
return next();
}
});
})
// }}}
// Grab the file {{{
.then(function(next) {
debug('FTP (backgrounds)> get', file.name, '->', file.cachePath);
this.client.get(file.name, function(err, getStream) {
getStream
.pipe(fs.createWriteStream(file.cachePath))
.on('close', ()=> next())
});
})
// }}}
// End {{{
.end(function(err) {
if (err && err == 'SKIP') { // Ignore skips
return next();
} else {
next(err);
}
})
// }}}
})
.end(function(err) {
if (err) return next(err);
next(null, this.files.map(f => f.cachePath));
});
})
// }}}
// }}}
// Fetch frames {{{
.then('frames', function(next) {
if (!settings.fetch.frames) return next();
async()
.set('client', this.client)
// Change directory {{{
.then(function(next) {
debug('FTP (frames)> cd', settings.framePath);
this.client.cwd(settings.framePath, next);
})
// }}}
// List files {{{
.then('files', function(next) {
debug('FTP (frames)> ls');
this.client.list(next);
})
// }}}
// Filter radar images into the ones we want {{{
.then('files', function(next) {
next(null,
this.files
.filter(i => i.name.startsWith(`IDR${settings.id}`))
.filter(i => i.name.endsWith('.png'))
);
})
// }}}
// Grab radar images as async {{{
.limit(settings.getThreads)
.forEach('files', function(next, file) {
file.cachePath = fspath.join(settings.cachePath, file.name);
async()
.set('client', this.client)
// Lookup size of existing file (if any) {{{
.then('stat', function(next) {
fs.stat(file.cachePath, function(err, stat) {
if (err) {
return next(); // No file - continue to grab it
} else if (stat && stat.size == file.size) { // Found file and the size matches
debug('FTP (frames)> # skip', file.name);
return next('SKIP');
} else { // Everything else - grab the file
return next();
}
});
})
// }}}
// Grab the file {{{
.then(function(next) {
debug('FTP (frames)> get', file.name, '->', file.cachePath);
this.client.get(file.name, function(err, getStream) {
getStream
.pipe(fs.createWriteStream(file.cachePath))
.on('close', ()=> next())
});
})
// }}}
// End {{{
.end(function(err) {
if (err && err == 'SKIP') { // Ignore skips
return next();
} else {
next(err);
}
})
// }}}
})
.end(function(err) {
if (err) return next(err);
next(null, this.files.map(f => f.cachePath));
});
})
// }}}
// }}}
// Close the FTP connection {{{
.then(function(next) {
debug('FTP> exit');
this.client.end();
next();
})
// }}}
// End {{{
.end(function(err) {
if (err) return cb(err);
cb(null, {
backgrounds: this.backgrounds,
frames: this.frames,
});
})
// }}}
return bom;
};
/**
* Retrieve the files we have stored locally on disk
* This operates the same as bom.refresh() but does not connect to the FTP - only using local data
* @param {Object} [options] Additional options to use (overrides bom.settings)
* @param {function} cb Callback to call as (error, {backgrounds, frames})
* @returns {BomRadar} This chainable object
*/
bom.cached = function(options, cb) {
// Argument mangling {{{
if (_.isFunction(options)) { // Called as (cb)
cb = options;
options = {};
}
// }}}
var settings = _.defaults(options, bom.settings);
async()
// Ensure cache directory exists {{{
.then(function(next) {
mkdirp(settings.cachePath, next);
})
// }}}
// Glob the directory {{{
.then('files', function(next) {
glob(fspath.join(settings.cachePath, '*.{gif,png}'), next);
})
// }}}
// Map files into basepaths only {{{
.then('files', function(next) {
next(null, this.files.map(f => fspath.basename(f)));
})
// }}}
// Break files into their correct sections {{{
.then('files', function(next) {
next(null, {
frames: this.files
.filter(f => f.startsWith(`IDR${settings.id}.T.`))
.filter(f => f.endsWith('.png'))
.map(f => fspath.join(settings.cachePath, f)),
backgrounds: _(settings.backgrounds)
.mapValues((f, k) => _(this.files)
.filter(f => f.startsWith(`IDR${settings.id}.`))
.filter(f => f.endsWith('.png'))
.filter(f => (new RegExp(`^IDR.*${k}\.png$`)).test(f))
.first()
)
.pickBy((v, k) => !!v)
.mapValues(v => fspath.join(settings.cachePath, v))
.value(),
});
})
// }}}
// End {{{
.end(function(err) {
if (err) return cb(err);
cb(null, this.files);
});
// }}}
return bom;
};
/**
* Cleans out older radar images
* @param {Object} [options] Additional options to use (overrides bom.settings)
* @param {function} cb Callback to call as (error, {backgrounds, frames})
* @returns {BomRadar} This chainable object
*/
bom.clean = function(options, cb) {
// Argument mangling {{{
if (_.isFunction(options)) { // Called as (cb)
cb = options;
options = {};
}
// }}}
var settings = _.defaults(options, bom.settings);
async()
// Fetch local data {{{
.then('files', function(next) {
bom.cached(settings, next);
})
// }}}
// Filter down to files we need to remove {{{
.then('files', function(next) {
var expiry = new Date(Date.now() - settings.clean.olderThan);
next(null, this.files.frames
.filter(f => bom.utils.nameToDate(f) < expiry)
);
})
// }}}
// Remove marked files {{{
.forEach('files', function(next, file) {
debug('rm', file);
fs.unlink(file, next);
next();
})
// }}}
// End {{{
.end(function(err) {
if (err) return cb(err);
cb(null, this.files);
})
// }}}
return bom;
};
/**
* Create a composite image based on the cached data from bom.refresh()
* This function creates a single GIF / MP4 file which has a background + animated radar layers
* @param {Object} [options] Additional options to use (overrides bom.settings)
* @param {function} cb Callback to call as (error, buffer|path|stream)
* @returns {BomRadar} This chainable object
*/
bom.composite = function(options, cb) {
// Argument mangling {{{
if (_.isFunction(options)) { // Called as (cb)
cb = options;
options = {};
}
// }}}
var settings = _.defaults(options, bom.settings);
async()
.set('cacheFile', fspath.join(settings.cachePath, _.isFunction(settings.composite.cacheFile) ? settings.composite.cacheFile({settings}) : settings.composite.cacheFile))
// Check the cached version hasn't expired (If settings.composite.cache) {{{
.then(function(next) {
if (!settings.composite.cache) return next(); // Forcing no cache
// Calculate the cache file value
var cacheFileExpiry = new Date(Date.now() - settings.composite.cacheFileExpiry);
fs.stat(this.cacheFile, (err, stat) => {
if (err) {
debug(`Radar ${settings.id} composite "${this.cacheFile}" does not exist - creating`);
next(); // No cached radar - continue on to generate it
} else if (stat.mtime < cacheFileExpiry) {
debug(`Radar ${settings.id} composite "${this.cacheFile}" expired! - Recreating`);
next(); // File exists but it has expired
} else {
debug(`Radar ${settings.id} composite "${this.cacheFile}" is still within cache range - Using cached version`);
next('SKIP');
}
});
})
// }}}
// Refresh data (if settings.composite.autoRefresh) {{{
.then(function(next) {
bom.refresh(settings, next);
})
// }}}
// Clean data (if settings.composite.autoClean) {{{
.then(function(next) {
bom.clean(settings, next);
})
// }}}
// Fetch cached file list {{{
.then('files', function(next) {
bom.cached(next);
})
// }}}
// Compute ImageMagick arguments {{{
.then('arguments', function(next) {
next(null,
_(settings.composite.arguments)
.map(a =>
_.isFunction(a) ? a({
settings,
...this.files,
})
: a
)
.filter()
.flattenDeep()
.value()
);
})
// }}}
// Run ImageMagick {{{
.then(function(next) {
if (debug.enabled) {
debug('ImageMagic> Run as -',
'convert '
+ this.arguments
.map(a => / /.test(a) ? `"${a}"` : a)
.join(' ')
);
}
im.convert(this.arguments, next);
})
// }}}
// End {{{
.end(function(err) {
if (err && err != 'SKIP') {
cb(err);
} else if (settings.composite.method == 'path') {
cb(null, this.cacheFile);
} else if (settings.composite.method == 'buffer') {
fs.readFile(this.cacheFile, cb);
} else if (settings.composite.method == 'stream') {
cb(null, fs.createReadStream(this.cacheFile));
} else {
cb(`Unknown composite return method: "${settings.composite.method}"`);
}
})
// }}}
return bom;
};
/**
* General utilities storage for this module
* @var Object
*/
bom.utils = {};
/**
* Utility function to convert a BOM filename to a JavaScript Date object
* @param {string} path The file path to translate (basename will be computed automatically)
* @returns {Date|null} Either a JavaScript date if one could be extracted or null
*/
bom.utils.nameToDate = function(path) {
var name = fspath.basename(path);
// ID... YEAR MONTH DAY Hour (24) Minute
// IDR... 2018 03 21 05 00 .png
var [, year, month, day, hour, minute] = /^IDR.+\.([0-9]{4})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})\./.exec(name) || [];
return (
year
? new Date(year, month - 1, day, hour, minute)
: null
);
};
return bom;
};
module.exports = BomRadar;