balena-preload
Version:
Preload balena OS images with a user application container
794 lines • 30.4 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.Preloader = exports.applicationExpandOptions = exports.CONTAINER_NAME = void 0;
const _ = require("lodash");
const EventEmitter = require("events");
const dockerProgress = require("docker-progress");
const Docker = require("dockerode");
const path = require("path");
const streamModule = require("stream");
const tarfs = require("tar-fs");
const fs_1 = require("fs");
const getPort = require("get-port");
const os = require("os");
const compareVersions = require("compare-versions");
const { R_OK, W_OK } = fs_1.constants;
const DOCKER_TMPDIR = '/docker_tmpdir';
const DOCKER_IMAGE_TAG = 'balena/balena-preload';
const DISK_IMAGE_PATH_IN_DOCKER = '/img/balena.img';
const SPLASH_IMAGE_PATH_IN_DOCKER = '/img/balena-logo.png';
const DOCKER_STEP_RE = /Step (\d+)\/(\d+)/;
const CONCURRENT_REQUESTS_TO_REGISTRY = 10;
const limitedMap = (arr, fn, { concurrency = CONCURRENT_REQUESTS_TO_REGISTRY, } = {}) => {
if (concurrency >= arr.length) {
return Promise.all(arr.map(fn));
}
return new Promise((resolve, reject) => {
const result = new Array(arr.length);
let inFlight = 0;
let idx = 0;
const runNext = async () => {
const i = idx;
idx++;
if (i >= arr.length) {
return;
}
try {
inFlight++;
result[i] = await fn(arr[i], i, arr);
void runNext();
}
catch (err) {
idx = arr.length;
result.length = 0;
reject(err);
}
finally {
inFlight--;
if (inFlight === 0) {
resolve(result);
}
}
};
while (inFlight < concurrency) {
void runNext();
}
});
};
const GRAPHDRIVER_ERROR = 'Error starting daemon: error initializing graphdriver: driver not supported';
const OVERLAY_MODULE_MESSAGE = 'You need to load the "overlay" module to be able to preload this image: run "sudo modprobe overlay".';
const DOCKERD_USES_OVERLAY = '--storage-driver=overlay2';
const SUPERVISOR_USER_AGENT = 'Supervisor/v6.6.0 (Linux; Resin OS 2.12.3; prod)';
const MISSING_APP_INFO_ERROR_MSG = 'Could not fetch the target state because of missing application info';
class BufferBackedWritableStream extends streamModule.Writable {
constructor() {
super(...arguments);
this.chunks = [];
}
_write(chunk, _enc, next) {
this.chunks.push(chunk);
next();
}
getData() {
return Buffer.concat(this.chunks);
}
}
function setBindMount(hostConfig, mounts, dockerApiVersion) {
if (compareVersions(dockerApiVersion, '1.25') >= 0) {
hostConfig.Mounts = mounts.map(([source, target]) => ({
Source: path.resolve(source),
Target: target,
Type: 'bind',
Consistency: 'delegated',
}));
}
else {
hostConfig.Binds = mounts.map(([source, target]) => `${path.resolve(source)}:${target}`);
}
}
exports.CONTAINER_NAME = 'balena-image-preloader';
exports.applicationExpandOptions = {
owns__release: {
$select: ['id', 'commit', 'end_timestamp', 'composition'],
$expand: {
release_image: {
$select: ['id'],
$expand: {
image: {
$select: ['image_size', 'is_stored_at__image_location'],
},
},
},
},
$filter: {
status: 'success',
},
$orderby: [{ end_timestamp: 'desc' }, { id: 'desc' }],
},
};
const createContainer = async (docker, image, splashImage, dockerPort, proxy) => {
const mounts = [];
const version = await docker.version();
if (os.platform() === 'linux') {
mounts.push(['/dev', '/dev']);
}
if (splashImage) {
mounts.push([splashImage, SPLASH_IMAGE_PATH_IN_DOCKER]);
}
const env = [
`HTTP_PROXY=${proxy || ''}`,
`HTTPS_PROXY=${proxy || ''}`,
`DOCKER_PORT=${dockerPort || ''}`,
`DOCKER_TMPDIR=${DOCKER_TMPDIR}`,
];
mounts.push([image, DISK_IMAGE_PATH_IN_DOCKER]);
const containerOptions = {
Image: DOCKER_IMAGE_TAG,
name: exports.CONTAINER_NAME,
AttachStdout: true,
AttachStderr: true,
OpenStdin: true,
Env: env,
Volumes: {
[DOCKER_TMPDIR]: {},
},
HostConfig: {
Privileged: true,
},
};
if (containerOptions.HostConfig !== undefined) {
setBindMount(containerOptions.HostConfig, mounts, version.ApiVersion);
if (os.platform() === 'linux') {
containerOptions.HostConfig.NetworkMode = 'host';
}
else {
containerOptions.HostConfig.NetworkMode = 'bridge';
containerOptions.ExposedPorts = {};
containerOptions.ExposedPorts[`${dockerPort}/tcp`] = {};
containerOptions.HostConfig.PortBindings = {};
containerOptions.HostConfig.PortBindings[`${dockerPort}/tcp`] = [
{
HostPort: `${dockerPort}`,
HostIp: '',
},
];
}
}
return await docker.createContainer(containerOptions);
};
const isReadWriteAccessibleFile = async (image) => {
try {
const [, stats] = await Promise.all([
fs_1.promises.access(image, R_OK | W_OK),
fs_1.promises.stat(image),
]);
return stats.isFile();
}
catch {
return false;
}
};
const deviceTypeQuery = {
$select: 'slug',
$expand: {
is_of__cpu_architecture: {
$select: 'slug',
},
},
};
const getApplicationQuery = (releaseFilter) => {
return {
$expand: {
should_be_running__release: {
$select: 'commit',
},
is_for__device_type: {
$select: 'slug',
$expand: {
is_of__cpu_architecture: {
$select: 'slug',
},
},
},
owns__release: {
$select: ['id', 'commit', 'end_timestamp', 'composition'],
$expand: {
release_image: {
$select: ['id'],
$expand: {
image: {
$select: ['image_size', 'is_stored_at__image_location'],
},
},
},
},
$filter: releaseFilter,
$orderby: [{ end_timestamp: 'desc' }, { id: 'desc' }],
},
},
};
};
class Preloader extends EventEmitter {
constructor(balena, docker, appId, commit, image, splashImage, proxy, dontCheckArch, pinDevice = false, certificates = [], additionalSpace = null) {
super();
this.docker = docker;
this.appId = appId;
this.commit = commit;
this.image = image;
this.splashImage = splashImage;
this.proxy = proxy;
this.dontCheckArch = dontCheckArch;
this.pinDevice = pinDevice;
this.certificates = certificates;
this.additionalSpace = additionalSpace;
this.stdout = new streamModule.PassThrough();
this.stderr = new streamModule.PassThrough();
this.bufferedStderr = new BufferBackedWritableStream();
this.balena =
balena !== null && balena !== void 0 ? balena : require('balena-sdk')
.fromSharedOptions();
this.stderr.pipe(this.bufferedStderr);
}
async _build() {
const files = ['Dockerfile', 'requirements.txt', 'src/preload.py'];
const name = 'Building Docker preloader image.';
this._progress(name, 0);
const tarStream = tarfs.pack(path.resolve(__dirname, '..'), {
entries: files,
});
const build = await this.docker.buildImage(tarStream, {
t: DOCKER_IMAGE_TAG,
});
await new Promise((resolve, reject) => {
this.docker.modem.followProgress(build, (error, output) => {
if (!error && output && output.length) {
error = output.pop().error;
}
if (error) {
reject(error);
}
else {
this._progress(name, 100);
resolve();
}
}, (event) => {
if (event.stream) {
const matches = event.stream.match(DOCKER_STEP_RE);
if (matches) {
this._progress(name, (parseInt(matches[1], 10) / (parseInt(matches[2], 10) + 1)) *
100);
}
this.stderr.write(event.stream);
}
});
});
}
async _fetchDeviceTypes() {
this.deviceTypes =
await this.balena.models.deviceType.getAll(deviceTypeQuery);
}
async _runWithSpinner(name, fn) {
this._startSpinner(name);
try {
return await fn();
}
finally {
this._stopSpinner(name);
}
}
_prepareErrorHandler() {
var _a;
(_a = this.container) === null || _a === void 0 ? void 0 : _a.wait().then((data) => {
if (data.StatusCode !== 0) {
const output = this.bufferedStderr.getData().toString('utf8').trim();
let error;
if (output.indexOf(GRAPHDRIVER_ERROR) !== -1 &&
output.indexOf(DOCKERD_USES_OVERLAY) !== -1) {
error = new this.balena.errors.BalenaError(OVERLAY_MODULE_MESSAGE);
}
else {
error = new Error(output);
error.code = data.StatusCode;
}
this.emit('error', error);
}
}).catch((error) => this.emit('error', error));
}
_runCommand(command, parameters) {
return new Promise((resolve, reject) => {
const cmd = JSON.stringify({ command, parameters }) + '\n';
this.stdout.once('error', reject);
this.stdout.once('data', (data) => {
let strData = data;
try {
strData = data.toString();
}
catch (_e) {
}
let response = {};
try {
response = JSON.parse(strData);
}
catch (error) {
response.statusCode = 1;
response.error = error;
}
if (response.statusCode === 0) {
resolve(response.result);
}
else {
const msg = [
`An error has occurred executing internal preload command '${command}':`,
cmd,
];
if (response.error) {
msg.push(`Status code: ${response.statusCode}`, `Error: ${response.error}`);
}
else {
msg.push(`Response: ${strData}`);
}
msg.push('');
reject(new Error(msg.join('\n')));
}
});
this.stdin.write(cmd);
});
}
_startSpinner(name) {
this.emit('spinner', { name, action: 'start' });
}
_stopSpinner(name) {
this.emit('spinner', { name, action: 'stop' });
}
_progress(name, percentage) {
this.emit('progress', { name, percentage });
}
_getStateVersion() {
if (this._supervisorLT7()) {
return 1;
}
else if (this._supervisorLT13()) {
return 2;
}
else {
return 3;
}
}
async _getStateWithRegistration(stateVersion) {
if (!this.appId) {
throw new Error(MISSING_APP_INFO_ERROR_MSG);
}
const uuid = this.balena.models.device.generateUniqueKey();
const deviceInfo = await this.balena.models.device.register(this.appId, uuid);
await this.balena.pine.patch({
resource: 'device',
id: deviceInfo.id,
body: {
is_pinned_on__release: this._getRelease().id,
},
});
const { body: state } = await this.balena.request.send({
headers: { 'User-Agent': SUPERVISOR_USER_AGENT },
baseUrl: this.balena.pine.API_URL,
url: `device/v${stateVersion}/${uuid}/state`,
});
this.state = state;
await this.balena.models.device.remove(uuid);
}
async _getStateFromTargetEndpoint(stateVersion) {
if (!this.appId) {
throw new Error(MISSING_APP_INFO_ERROR_MSG);
}
const release = this._getRelease();
const [{ uuid: appUuid }, state] = await Promise.all([
this.balena.models.application.get(this.appId, {
$select: 'uuid',
}),
this.balena.models.device.getSupervisorTargetStateForApp(this.appId, release.commit),
]);
if (stateVersion === 3) {
state.local = state[appUuid];
delete state[appUuid];
}
this.state = state;
}
async _getState() {
const stateVersion = this._getStateVersion();
if (stateVersion < 3) {
await this._getStateWithRegistration(stateVersion);
}
else {
await this._getStateFromTargetEndpoint(stateVersion);
}
}
async _getImageInfo() {
await this._runWithSpinner('Reading image information', async () => {
const info = (await this._runCommand('get_image_info', {}));
this.freeSpace = info.free_space;
this.preloadedBuilds = info.preloaded_builds;
this.supervisorVersion = info.supervisor_version;
this.balenaOSVersion = info.balena_os_version;
this.config = info.config;
});
}
_getCommit() {
return this.commit || this.application.should_be_running__release[0].commit;
}
_getRelease() {
const commit = this._getCommit();
const releases = this.application.owns__release;
if (commit === null && releases.length) {
return releases[0];
}
const release = _.find(releases, (r) => {
return r.commit.startsWith(commit);
});
if (!release) {
throw new this.balena.errors.BalenaReleaseNotFound(commit);
}
return release;
}
_getServicesFromApps(apps) {
var _a, _b;
const stateVersion = this._getStateVersion();
switch (stateVersion) {
case 1: {
const [appV1] = _.values(apps);
return [{ image: appV1.image }];
}
case 2: {
const [appV2] = _.values(apps);
return appV2.services;
}
case 3: {
const [appV3] = _.values(apps).filter((a) => a.id === this.appId);
const [release] = _.values((_a = appV3 === null || appV3 === void 0 ? void 0 : appV3.releases) !== null && _a !== void 0 ? _a : {});
return (_b = release === null || release === void 0 ? void 0 : release.services) !== null && _b !== void 0 ? _b : {};
}
}
}
_getImages() {
const images = this._getRelease().release_image.map((ri) => {
return _.merge({}, ri.image[0], {
is_stored_at__image_location: ri.image[0].is_stored_at__image_location.toLowerCase(),
});
});
const servicesImages = _.map(this._getServicesFromApps(this.state.local.apps), (service) => {
return service.image.toLowerCase();
});
_.each(images, (image) => {
image.is_stored_at__image_location = _.find(servicesImages, (serviceImage) => {
return serviceImage.startsWith(image.is_stored_at__image_location);
});
});
return images;
}
_getImagesToPreload() {
const preloaded = new Set(this.preloadedBuilds);
const toPreload = new Set(this._getImages());
for (const image of toPreload) {
if (preloaded.has(image.is_stored_at__image_location.split('@')[0])) {
toPreload.delete(image);
}
}
return Array.from(toPreload);
}
async registryRequest(url, registryToken, headers, responseFormat, followRedirect) {
if (typeof url === 'object') {
url = `https://${url.registryUrl}${url.layerUrl}`;
}
return await this.balena.request.send({
url,
headers: {
...headers,
...(registryToken != null && {
Authorization: `Bearer ${registryToken}`,
}),
},
responseFormat,
followRedirect,
sendToken: false,
refreshToken: false,
});
}
async _getLayerSize(registryToken, registryUrl, layerUrl) {
const headers = {
Range: 'bytes=-4',
};
let response = await this.registryRequest({ registryUrl, layerUrl }, registryToken, headers, 'blob', false);
if (response.statusCode === 206) {
}
else if ([301, 307].includes(response.statusCode)) {
const redirectUrl = response.headers.get('location');
if (redirectUrl == null) {
throw new Error('Response status code indicated a redirect but no redirect location was found in the response headers');
}
response = await this.registryRequest(redirectUrl, null, headers, 'blob', true);
}
else {
throw new Error('Unexpected status code from the registry: ' + response.statusCode);
}
const body = await response.body.arrayBuffer();
return Buffer.from(body).readUIntLE(0, 4);
}
_registryUrl(imageLocation) {
const slashIndex = imageLocation.search('/');
return `${imageLocation.substring(0, slashIndex)}`;
}
_imageManifestUrl(imageLocation) {
const slashIndex = imageLocation.search('/');
const atIndex = imageLocation.search('@');
return `/v2${imageLocation.substring(slashIndex, atIndex)}/manifests/${imageLocation.substring(atIndex + 1)}`;
}
_layerUrl(imageLocation, layerDigest) {
const slashIndex = imageLocation.search('/');
const atIndex = imageLocation.search('@');
return `/v2${imageLocation.substring(slashIndex, atIndex)}/blobs/${layerDigest}`;
}
async _getApplicationImagesManifests(imagesLocations, registryToken) {
return await limitedMap(imagesLocations, async (imageLocation) => {
const { body } = await this.registryRequest({
registryUrl: this._registryUrl(imageLocation),
layerUrl: this._imageManifestUrl(imageLocation),
}, registryToken, {}, 'json', true);
return { manifest: body, imageLocation };
}, { concurrency: CONCURRENT_REQUESTS_TO_REGISTRY });
}
async _getLayersSizes(manifests, registryToken) {
const digests = new Set();
const layersSizes = new Map();
const sizeRequests = [];
for (const manifest of manifests) {
for (const layer of manifest.manifest.layers) {
if (!digests.has(layer.digest)) {
digests.add(layer.digest);
sizeRequests.push({ imageLocation: manifest.imageLocation, layer });
}
}
}
await limitedMap(sizeRequests, async ({ imageLocation, layer }) => {
const size = await this._getLayerSize(registryToken, this._registryUrl(imageLocation), this._layerUrl(imageLocation, layer.digest));
layersSizes.set(layer.digest, { size, compressedSize: layer.size });
}, { concurrency: CONCURRENT_REQUESTS_TO_REGISTRY });
return layersSizes;
}
async _getApplicationSize() {
var _a;
const images = this._getImagesToPreload();
const imagesLocations = _.map(images, 'is_stored_at__image_location');
const registryToken = await this._getRegistryToken(imagesLocations);
const manifests = await this._getApplicationImagesManifests(imagesLocations, registryToken);
const layersSizes = await this._getLayersSizes(manifests, registryToken);
let extra = 0;
for (const { imageLocation, manifest } of manifests) {
const apiSize = (_a = _.find(images, {
is_stored_at__image_location: imageLocation,
})) === null || _a === void 0 ? void 0 : _a.image_size;
const size = _.sumBy(manifest.layers, (layer) => layersSizes.get(layer.digest).size);
if (apiSize != null && parseInt(apiSize, 10) > size) {
extra += parseInt(apiSize, 10) - size;
}
}
return _.sumBy([...layersSizes.values()], 'size') + extra;
}
async _getSize() {
const images = this._getImagesToPreload();
if (images.length === 1) {
return parseInt(images[0].image_size, 10);
}
return await this._getApplicationSize();
}
async _getRequiredAdditionalSpace() {
if (this.additionalSpace !== null) {
return this.additionalSpace;
}
const size = Math.round((await this._getSize()) * 1.4);
return Math.max(0, size - this.freeSpace);
}
_supervisorLT7() {
try {
return compareVersions(this.supervisorVersion, '7.0.0') === -1;
}
catch (e) {
return false;
}
}
_supervisorLT13() {
try {
return compareVersions(this.supervisorVersion, '13.0.0') === -1;
}
catch (e) {
return true;
}
}
async _getRegistryToken(images) {
const { body } = await this.balena.request.send({
baseUrl: this.balena.pine.API_URL,
url: '/auth/v1/token',
qs: {
service: this._registryUrl(images[0]),
scope: images.map((imageRepository) => `repository:${imageRepository.substr(imageRepository.search('/') + 1)}:pull`),
},
});
return body.token;
}
async _fetchApplication() {
const { appId } = this;
if (this.application || !appId) {
return;
}
await this._runWithSpinner(`Fetching application ${appId}`, async () => {
const releaseFilter = {
status: 'success',
};
if (this.commit === 'latest') {
const { should_be_running__release } = await this.balena.models.application.get(appId, {
$select: 'should_be_running__release',
});
releaseFilter.id =
should_be_running__release.__id;
}
else if (this.commit != null) {
releaseFilter.commit = { $startswith: this.commit };
}
const application = await this.balena.models.application.get(appId, getApplicationQuery(releaseFilter));
this.setApplication(application);
});
}
async _checkImage(image) {
const ok = await isReadWriteAccessibleFile(image);
if (!ok) {
console.warn('The image must be a read/write accessible file');
}
}
_pluralize(count, thing) {
return `${count} ${thing}${count !== 1 ? 's' : ''}`;
}
_deviceTypeArch(slug) {
var _a;
const deviceType = (_a = this.deviceTypes) === null || _a === void 0 ? void 0 : _a.find((dt) => {
return dt.slug === slug;
});
if (deviceType === undefined) {
throw new this.balena.errors.BalenaError(`No such deviceType: ${slug}`);
}
return deviceType.is_of__cpu_architecture[0].slug;
}
async prepare() {
await this._build();
await this._runWithSpinner('Checking that the image is a writable file', () => this._checkImage(this.image));
const port = await this._runWithSpinner('Finding a free tcp port', () => getPort());
this.dockerPort = port;
const container = await this._runWithSpinner('Creating preloader container', () => createContainer(this.docker, this.image, this.splashImage, this.dockerPort, this.proxy));
this.container = container;
await this._runWithSpinner('Starting preloader container', () => container.start());
for (const certificate of this.certificates) {
await this.container.putArchive(tarfs.pack(path.dirname(certificate), {
entries: [path.basename(certificate)],
}), {
path: '/usr/local/share/ca-certificates/',
noOverwriteDirNonDir: true,
});
}
this._prepareErrorHandler();
const stream = await this.container.attach({
stream: true,
stdout: true,
stderr: true,
stdin: true,
hijack: true,
});
this.stdin = stream;
this.docker.modem.demuxStream(stream, this.stdout, this.stderr);
await Promise.all([
this._getImageInfo(),
this._fetchDeviceTypes(),
this._fetchApplication(),
]);
}
async cleanup() {
await this._runWithSpinner('Cleaning up temporary files', async () => {
if (this.container) {
await Promise.all([this.kill(), this.container.wait()]);
await this.container.remove();
}
});
}
async kill() {
if (this.container) {
return this.container.kill().catch(() => undefined);
}
}
_ensureCanPreload() {
let msg;
if (this.application.owns__release.length === 0) {
msg = 'This application has no successful releases';
throw new this.balena.errors.BalenaError(msg);
}
if (this.dontCheckArch === false) {
const imageArch = this._deviceTypeArch(this.config.deviceType);
const applicationArch = this.application.is_for__device_type[0].is_of__cpu_architecture[0].slug;
if (!this.balena.models.os.isArchitectureCompatibleWith(imageArch, applicationArch)) {
msg = `Application architecture (${applicationArch}) and image architecture (${imageArch}) are not compatible.`;
throw new this.balena.errors.BalenaError(msg);
}
}
if (this._getImages().length > 1 && this._supervisorLT7()) {
msg = `Can't preload a multicontainer app on an image which supervisor version is < 7.0.0 (${this.supervisorVersion}).`;
throw new this.balena.errors.BalenaError(msg);
}
if (this._getImagesToPreload().length === 0) {
msg = 'Nothing new to preload.';
throw new this.balena.errors.BalenaError(msg);
}
}
_getAppData() {
if (this._supervisorLT7()) {
if (this.pinDevice === true) {
throw new this.balena.errors.BalenaError('Pinning releases only works with supervisor versions >= 7.0.0');
}
return _.map(this.state.local.apps, (value, appId) => {
return _.merge({}, _.omit(value, ['environment', 'image', 'serviceId']), { appId, env: value.environment, imageId: value.image });
});
}
else {
return _.merge(_.omit(this.state.local, 'name'), {
pinDevice: this.pinDevice,
});
}
}
_getSplashImagePath() {
try {
if (compareVersions(this.balenaOSVersion, '2.53.0') >= 0) {
return '/splash/balena-logo.png';
}
}
catch (err) {
}
return '/splash/resin-logo.png';
}
async preload() {
await this._getState();
this._ensureCanPreload();
const additionalBytes = await this._runWithSpinner('Estimating required additional space', () => this._getRequiredAdditionalSpace());
const images = _.map(this._getImagesToPreload(), 'is_stored_at__image_location');
await this._runWithSpinner('Resizing partitions and waiting for dockerd to start', () => this._runCommand('preload', {
app_data: this._getAppData(),
additional_bytes: additionalBytes,
splash_image_path: this._getSplashImagePath(),
}));
const registryToken = await this._getRegistryToken(images);
const opts = { authconfig: { registrytoken: registryToken } };
const innerDocker = new Docker({
host: os.platform() === 'win32' ? 'localhost' : '0.0.0.0',
port: this.dockerPort,
});
const innerDockerProgress = new dockerProgress.DockerProgress({
docker: innerDocker,
});
const pullingProgressName = `Pulling ${this._pluralize(images.length, 'image')}`;
const onProgressHandlers = innerDockerProgress.aggregateProgress(images.length, (e) => {
this._progress(pullingProgressName, e.percentage);
});
await Promise.all(images.map(async (image, index) => {
await innerDockerProgress.pull(image, onProgressHandlers[index], opts);
}));
this.stdin.write('\n');
await new Promise((resolve, reject) => {
this.stdout.once('error', reject);
this.stdout.once('data', resolve);
});
}
setApplication(application) {
this.appId = application.id;
this.application = application;
}
async setAppIdAndCommit(appIdOrSlug, commit) {
this.appId = appIdOrSlug;
this.commit = commit;
this.application = null;
await this._fetchApplication();
}
}
exports.Preloader = Preloader;
//# sourceMappingURL=preload.js.map