UNPKG

realm-object-server

Version:

Realm Object Server

716 lines 35.5 kB
"use strict"; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { return new (P || (P = Promise))(function (resolve, reject) { function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } function step(result) { result.done ? resolve(result.value) : new P(function (resolve) { resolve(result.value); }).then(fulfilled, rejected); } step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; Object.defineProperty(exports, "__esModule", { value: true }); const realm_sync_server_1 = require("realm-sync-server"); const stats_1 = require("../stats"); const KubernetesClient = require("kubernetes-client"); const diskusage = require("diskusage"); const events_1 = require("events"); const _1 = require("."); const getFolderSize = require("get-folder-size"); const util_1 = require("../shared/util"); const SyncService_1 = require("../services/SyncService"); const child_process_1 = require("child_process"); const path = require("path"); const tmp = require("tmp"); const fs = require("fs-extra"); const md5 = require("md5-file/promise"); const uuid = require("uuid"); class KubernetesSyncWorker extends events_1.EventEmitter { constructor(config) { super(); this.masterAnnotationKey = "sync.realm.io/master"; this.slaveAnnotationKey = "sync.realm.io/slave"; this.requestedOperationAnnotationKey = "sync.realm.io/requested-operation"; this.operationArgumentsAnnotationKey = "sync.realm.io/requested-operation-arguments"; this.currentResourceVersion = 0; this.config = config; this.config.listenAddress = "0.0.0.0"; this.config.listenPort = 0; this.disableSlave = config.disableSlave !== undefined ? config.disableSlave : false; this.disableDiskStats = config.disableDiskStats !== undefined ? config.disableDiskStats : false; } start(params) { return __awaiter(this, void 0, void 0, function* () { this.logger = params.logger; this.namespace = this.config.kubernetesConfig.namespace; this.logger.debug(`Found namespace in kubernetes config: ${this.namespace}`); this.coreApi = new KubernetesClient.Client1_10({ config: this.config.kubernetesConfig, version: "1.11" }).api.v1.namespace(this.namespace); this.configureDiskStats(params.statsSink); const statsdReceiver = params.statsdReceiver || new stats_1.StatsdReceiver({ logger: params.logger, }); this.statsdToStatsSink = new stats_1.StatsdToStatsSink({ logger: params.logger, statsSink: params.statsSink, }, statsdReceiver); this.statsdSocket = yield statsdReceiver.start(); const endpoints = yield this.registerCandidate(); yield this.reconcileEndpoints(endpoints); if (!params.disableWatcher) { const qs = { fieldSelector: `metadata.name=${this.config.endpointsName}` }; this.watcher = new _1.ResourceWatcher({ api: this.coreApi.endpoints, qs, logger: this.logger }); this.watcher.on("added", this.reconcileEndpoints.bind(this)); this.watcher.on("modified", this.reconcileEndpoints.bind(this)); this.watcher.on("deleted", (e) => { this.logger.warn("Endpoints record was deleted, shutting down"); this.shutdown().catch((error) => { this.logger.error("Failed to shutdown sync server", { error }); }); }); this.watcher.on("error", (error) => { this.logger.warn("Error from watcher, restarting", { error }); this.watcher.start(); }); this.watcher.start(endpoints.metadata.resourceVersion); } }); } shutdown(err) { return __awaiter(this, void 0, void 0, function* () { if (err) { this.logger.fatal("Shutting down sync server with error", err); } yield this.stopSyncServer().catch((error) => this.logger.warn("Could not stop sync server", { error })); if (this.statsdSocket) { yield new Promise((resolve) => this.statsdSocket.close(resolve)).catch((error) => this.logger.warn("Could not stop statsd listener socket", { error })); delete this.statsdSocket; } yield this.deregisterCandidate(); this.statsdToStatsSink.stop(); if (this.statsInterval) { clearInterval(this.statsInterval); delete this.statsInterval; } this.emit("shutdown", err); }); } reconcileEndpoints(endpoints) { return __awaiter(this, void 0, void 0, function* () { if (Number(endpoints.metadata.resourceVersion) < this.currentResourceVersion) { this.logger.debug("not processing old resourceVersion: " + `${endpoints.metadata.resourceVersion} < ${this.currentResourceVersion}`); return; } this.currentResourceVersion = Number(endpoints.metadata.resourceVersion); switch (this.role) { case "master": if (endpoints.metadata.annotations[this.masterAnnotationKey] !== this.config.syncWorkerId) { this.logger.info("Lost master lock, becoming spare"); this.role = "spare"; } break; case "slave": if (endpoints.metadata.annotations[this.masterAnnotationKey] === this.config.syncWorkerId) { this.logger.info("Slave now matches master annotation, promoting to master"); this.role = "master"; } else if (endpoints.metadata.annotations[this.slaveAnnotationKey] !== this.config.syncWorkerId) { this.logger.info("Lost slave lock, becoming spare"); this.role = "spare"; } break; default: if (!endpoints.metadata.annotations[this.masterAnnotationKey]) { return this.assumeRole("master"); } else if (endpoints.metadata.annotations[this.masterAnnotationKey] === this.config.syncWorkerId) { this.role = "master"; } else if (!endpoints.metadata.annotations[this.slaveAnnotationKey]) { return this.assumeRole("slave"); } else if (endpoints.metadata.annotations[this.slaveAnnotationKey] === this.config.syncWorkerId) { this.role = "slave"; } else if (this.role !== "spare") { this.role = "spare"; } break; } const requestedOperation = endpoints.metadata.annotations[this.requestedOperationAnnotationKey]; if (requestedOperation) { if (!this.currentOperation) { const operationArguments = endpoints.metadata.annotations[this.operationArgumentsAnnotationKey]; this.currentOperation = new Promise((resolve, reject) => __awaiter(this, void 0, void 0, function* () { try { switch (requestedOperation) { case "verify": yield this.stopSyncServer(); yield this.verifyFiles(operationArguments); break; case "delay": yield this.stopSyncServer(); this.logger.info("testing operation with delay..."); yield util_1.delay(operationArguments ? Number(operationArguments) : 60 * 1000); this.logger.info("delay complete!"); break; case "copyfile": yield this.copyAndUploadFile(JSON.parse(operationArguments)); break; case "quarantine": yield this.stopSyncServer(); yield this.quarantineFile(JSON.parse(operationArguments)); break; default: throw new Error(`Requested operation '${requestedOperation}' is not supported`); } } catch (err) { this.logger.error(`Operation '${requestedOperation}' failed: ${err.stack || err.message}`); } finally { delete this.currentOperation; const requestedOperationAnnotationPath = `/metadata/annotations/${this.requestedOperationAnnotationKey.replace("/", "~1")}`; const patchedEndpoints = [ { op: "remove", path: requestedOperationAnnotationPath, value: requestedOperation }, ]; if (operationArguments) { const operationArgumentsAnnotationPath = `/metadata/annotations/${this.operationArgumentsAnnotationKey.replace("/", "~1")}`; patchedEndpoints.push({ op: "remove", path: operationArgumentsAnnotationPath, value: operationArguments }); } yield this.patchEndpoints(patchedEndpoints).catch((err) => { this.logger.error(`Failed to remove requested-operation annotation: ${err.stack || err.message}`); }); } })); } else { this.logger.error("requested-operation annotation was removed while operation was running!"); } } else { const newSyncServerParams = this.generateSyncServerConfig(endpoints); const changed = JSON.stringify(this.currentSyncServerConfig) !== JSON.stringify(newSyncServerParams); if (changed) { this.logger.info("Confguration changed, restarting sync server"); yield this.stopSyncServer(); if (newSyncServerParams.pause === true) { this.logger.info("Server paused, skipping start"); } else { yield this.startSyncServer(newSyncServerParams); } } } }); } generateSyncServerConfig(endpoints) { const address = this.statsdSocket.address(); const config = SyncService_1.createSyncServerConfig({ syncServiceConfig: this.config, dataPath: this.config.dataPath, statsEndpoint: `127.0.0.1:${address.port}`, realmsEncryptionKey: this.config.realmsEncryptionKey, logger: this.logger, }); config.id = this.config.syncWorkerId; config.masterSlaveSharedSecret = "some-secret"; config.errorCallback = (name, message) => { const error = { name, message }; this.emit("error", error); this.shutdown(error).catch((error) => { this.logger.error("Failed to shutdown sync server", { error }); }); }; const annotations = endpoints.metadata.annotations || {}; if (annotations["sync.realm.io/enable-download-log-compaction"]) { config.enableDownloadLogCompaction = annotations["sync.realm.io/enable-download-log-compaction"] === "true"; } if (annotations["sync.realm.io/enable-log-compaction"]) { config.enableLogCompaction = annotations["sync.realm.io/enable-log-compaction"] === "true"; } if (annotations["sync.realm.io/max-download-size"]) { config.maxDownloadSize = Number(annotations["sync.realm.io/max-download-size"]); } if (annotations["sync.realm.io/log-level"]) { config.logLevel = annotations["sync.realm.io/log-level"]; } if (annotations["sync.realm.io/history-ttl"]) { config.historyTtl = Number(annotations["sync.realm.io/history-ttl"]); } if (annotations["sync.realm.io/history-compaction-interval"]) { config.historyCompactionInterval = Number(annotations["sync.realm.io/history-compaction-interval"]); } if (annotations["sync.realm.io/max-open-files"]) { config.maxOpenFiles = Number(annotations["sync.realm.io/max-open-files"]); } if (annotations["sync.realm.io/enable-realm-state-size-reporting"] === "true") { config.enableRealmStateSizeReporting = true; } else if (annotations["sync.realm.io/enable-realm-state-size-reporting"] === "false") { config.enableRealmStateSizeReporting = false; } if (annotations["sync.realm.io/pause"] === "true") { config.pause = true; } if (annotations["sync.realm.io/enable-download-boostrap-cache"] === "true") { config.enableDownloadBootstrapCache = true; } else if (annotations["sync.realm.io/enable-download-boostrap-cache"] === "false") { config.enableDownloadBootstrapCache = false; } if (annotations["sync.realm.io/disable-partial-sync-completer"] === "true") { config.disablePartialSyncCompleter = true; } else if (annotations["sync.realm.io/disable-partial-sync-completer"] === "false") { config.disablePartialSyncCompleter = false; } if (annotations["sync.realm.io/log-to-file"] === "true") { config.logToFile = true; } if (annotations["sync.realm.io/disable-state-realms"] === "true") { config.disableStateRealms = true; } if (annotations["sync.realm.io/num-aux-psync-threads"]) { config.numAuxPsyncThreads = Number(annotations["sync.realm.io/num-aux-psync-threads"]); } if (annotations["sync.realm.io/metrics-exclusions"]) { config.metricsExclusions = Number(annotations["sync.realm.io/metrics-exclusions"]); } if (annotations["sync.realm.io/history-compaction-ignore-clients"] === "true") { config.historyCompactionIgnoreClients = true; } const replicaCount = this.getReplicaCount(endpoints); switch (this.role) { case "master": if (replicaCount === 1) { config.operatingMode = realm_sync_server_1.RealmSyncServerOperatingMode.MasterWithNoSlave; } else { config.operatingMode = realm_sync_server_1.RealmSyncServerOperatingMode.MasterWithSynchronousSlave; config.slaveStatusCallback = (id, upToDate) => { this.logger.debug(`Slave status ${id} ${upToDate}`); this.syncWorkerSlaveStatus = { id, upToDate }; }; } break; case "slave": config.operatingMode = realm_sync_server_1.RealmSyncServerOperatingMode.Slave; config.masterAddress = this.config.serviceName; config.masterPort = 7800; break; case "spare": break; default: throw new Error("Unknown role"); } return config; } configureDiskStats(statsSink) { const diskUsage = statsSink.gauge({ name: "ros_sync_disk_usage_bytes", help: "Realm Object Server Sync Worker disk usage in bytes", labelNames: [], }); let diskSize, diskFree; if (!this.disableDiskStats) { diskSize = statsSink.gauge({ name: "ros_sync_disk_size_bytes", help: "Realm Object Server Sync Worker disk size in bytes", labelNames: [], }); diskFree = statsSink.gauge({ name: "ros_sync_disk_free_bytes", help: "Realm Object Server Sync Worker disk free space in bytes", labelNames: [], }); } const statsIntervalSeconds = process.env.DISK_STATS_INTERVAL ? Number(process.env.DISK_STATS_INTERVAL) : 60; this.statsInterval = setInterval(() => __awaiter(this, void 0, void 0, function* () { if (!this.disableDiskStats) { diskusage.check(this.config.dataPath, (error, result) => { if (error) { this.logger.warn("Error obtaining disk usage", { error }); } else { diskSize.set({}, result.total); diskUsage.set({}, result.total - result.free); diskFree.set({}, result.free); } }); } else { getFolderSize(this.config.dataPath, (error, size) => { if (error) { this.logger.warn("Error obtaining data directory size", { error }); } else { diskUsage.set({}, size); } }); } }), statsIntervalSeconds * 1000); } startSyncServer(config) { return __awaiter(this, void 0, void 0, function* () { if (!this.syncServerPromise) { const syncServer = new realm_sync_server_1.RealmSyncServer(config); this.currentSyncServerConfig = config; delete this.syncWorkerSlaveStatus; if (config.operatingMode !== undefined) { this.syncServerPromise = syncServer.start().then(() => __awaiter(this, void 0, void 0, function* () { this.startedAt = new Date(); if (this.role === "master") { yield this.reconcileSubsets([{ name: "sync", port: syncServer.address().port, protocol: "TCP", }]); } return syncServer; })); yield this.syncServerPromise; } else { this.logger.info("Starting as spare"); } } return this.syncServerPromise; }); } stopSyncServer() { return __awaiter(this, void 0, void 0, function* () { delete this.currentSyncServerConfig; if (!this.syncServerPromise) { return; } const syncServer = yield this.syncServerPromise.catch((err) => { }); if (!syncServer) { delete this.syncServerPromise; return; } try { delete this.syncServerPromise; if (this.role === "master" && this.syncWorkerSlaveStatus) { while (true) { if (this.syncWorkerSlaveStatus.upToDate) { break; } this.logger.warn("Waiting for slave to synchronize before shutting down..."); yield new Promise((resolve) => setTimeout(resolve, 1000)); } } yield syncServer.stop(); this.logger.info("Sync server has stopped."); const masterAnnotationPath = `/metadata/annotations/${this.masterAnnotationKey.replace("/", "~1")}`; const slaveAnnotationPath = `/metadata/annotations/${this.slaveAnnotationKey.replace("/", "~1")}`; switch (this.role) { case "master": if (this.syncWorkerSlaveStatus && this.syncWorkerSlaveStatus.upToDate) { this.logger.info(`delegating the slave '${this.syncWorkerSlaveStatus.id}' to become master`); yield this.patchEndpoints([ { op: "test", path: masterAnnotationPath, value: this.config.syncWorkerId }, { op: "replace", path: masterAnnotationPath, value: this.syncWorkerSlaveStatus.id }, { op: "remove", path: slaveAnnotationPath, value: this.syncWorkerSlaveStatus.id }, { op: "replace", path: "/subsets", value: [] }, ]); } delete this.syncWorkerSlaveStatus; break; case "slave": this.logger.info("Relinquishing slave lock"); yield this.patchEndpoints([ { op: "test", path: slaveAnnotationPath, value: this.config.syncWorkerId }, { op: "remove", path: slaveAnnotationPath }, ]).catch((err) => { this.logger.debug(`Error relinquishing slave lock: ${err.message}`); }); break; } } catch (err) { this.logger.warn(`Error stopping sync server: ${err.message}`); } }); } assumeRole(role) { return __awaiter(this, void 0, void 0, function* () { const otherRole = role === "master" ? "slave" : "master"; const endpoints = yield _1.retryApiRequest(() => this.coreApi.endpoints(this.config.endpointsName).get().then((r) => r.body)); const annotationKeyPrefix = "sync.realm.io"; const roleKey = `${annotationKeyPrefix}/${role}`; const otherRoleKey = `${annotationKeyPrefix}/${otherRole}`; const annotations = endpoints.metadata.annotations || {}; const rolePath = `/metadata/annotations/${roleKey.replace("/", "~1")}`; if (role === "slave" && this.disableSlave) { return; } const existingAnnotation = annotations[roleKey]; if (!existingAnnotation) { if (annotations[otherRoleKey] === this.config.syncWorkerId) { throw new Error(`Attempt to assume role while having another: ${otherRole}`); } try { const ops = [ { op: "test", path: "/metadata/resourceVersion", value: endpoints.metadata.resourceVersion }, { op: "add", path: rolePath, value: this.config.syncWorkerId }, ]; const response = yield this.patchEndpoints(ops); this.logger.info(`Acquired lock with response resourceVersion ${response.metadata.resourceVersion}`); return response; } catch (err) { if (err.code === 409 || err.code === 500) { this.logger.info(`Could not acquire endpoints ${role} lock: acquired by another worker`); } else { this.logger.error(`Failed to patch endpoints ${role} annotation`, err); } } } else { if (existingAnnotation === this.config.syncWorkerId) { this.logger.debug(`Existing ${role} annotation matches us.`); } else { this.logger.debug(`Existing ${role} annotation does not match us.`); } } }); } patchEndpoints(body) { return __awaiter(this, void 0, void 0, function* () { return _1.retryApiRequest(() => this.coreApi.endpoints(this.config.endpointsName).patch({ body, headers: { "content-type": "application/json-patch+json" }, }).then((r) => r.body)); }); } getReplicaCount(endpoints) { const annotations = endpoints.metadata.annotations; const candidates = {}; for (const key in annotations) { if (annotations[key]) { const matches = key.match(/^candidate\.sync\.realm\.io\/(.*)$/); if (matches) { const candidateName = matches[1]; candidates[candidateName] = JSON.parse(annotations[key]); } } } return Object.keys(candidates).length; } reconcileSubsets(ports) { return __awaiter(this, void 0, void 0, function* () { if (this.role !== "master") { throw new Error(`Cannot reconcile subsets as a ${this.role}`); } const subset = { addresses: [{ ip: this.config.podIp, nodeName: this.config.nodeName, targetRef: this.config.podRef, }], ports: ports || [], }; yield _1.retryApiRequest(() => this.coreApi.endpoints(this.config.endpointsName).patch({ body: { subsets: [] }, }).then((r) => r.body)); yield util_1.delay(5000); return _1.retryApiRequest(() => this.coreApi.endpoints(this.config.endpointsName).patch({ body: { subsets: [subset] }, }).then((r) => r.body)); }); } registerCandidate() { return __awaiter(this, void 0, void 0, function* () { const endpoints = yield _1.retryApiRequest(() => this.coreApi.endpoints(this.config.endpointsName).get().then((r) => r.body)); const annotations = endpoints.metadata.annotations || {}; const groupAnnotation = "sync.realm.io/group"; if (!annotations[groupAnnotation]) { throw new Error(`Annotation '${groupAnnotation}' not found on endnpoints resource`); } if (annotations[groupAnnotation] !== this.config.syncWorkerGroup) { throw new Error(`Annotation '${groupAnnotation}' value '${annotations[groupAnnotation]}' does not match our group: ` + `'${this.config.syncWorkerGroup}'`); } const annotationKey = `candidate.sync.realm.io/${this.config.syncWorkerId}`; const annotationData = JSON.stringify({ ref: this.config.podRef }); const path = `/metadata/annotations/${annotationKey.replace("/", "~1")}`; if (!annotations[annotationKey]) { return this.patchEndpoints([ { op: "add", path, value: annotationData }, ]); } else { return this.patchEndpoints([ { op: "replace", path, value: annotationData }, ]); } }); } deregisterCandidate() { return __awaiter(this, void 0, void 0, function* () { try { const annotationKey = `candidate.sync.realm.io/${this.config.syncWorkerId}`; const path = `/metadata/annotations/${annotationKey.replace("/", "~1")}`; return yield this.patchEndpoints([ { op: "remove", path }, ]); } catch (error) { this.logger.warn("Could not deregister ourselves as a candidate", { error }); } }); } verifyFiles(arg) { return __awaiter(this, void 0, void 0, function* () { const toolsPath = path.resolve(require.resolve("realm-sync-server"), `../../compiled/${process.platform}-${process.arch}`); const cmd = path.join(toolsPath, "realm-server-precheck"); const subcmd = path.join(toolsPath, "realm-precheck-server-file"); const args = ["--log-level", "debug", "--use-child-procs", "--child-proc-path", subcmd, "--perform-verification", "--perform-partial-sync", this.config.dataPath]; let encryptionKeyFilePath; if (this.config.realmsEncryptionKey) { encryptionKeyFilePath = yield new Promise((resolve, reject) => { tmp.file((err, path) => { if (err) { reject(err); } else { resolve(path); } }); }); yield fs.writeFile(encryptionKeyFilePath, this.config.realmsEncryptionKey); args.push("--encryption-key", encryptionKeyFilePath); } try { yield new Promise((resolve, reject) => { const childProcess = child_process_1.spawn(cmd, args, {}); const log = (data) => { const lines = data.toString().split(/\n/); for (const line of lines) { this.logger.info(line); } }; childProcess.on("error", (err) => { reject(err); }); childProcess.on("close", (code) => { if (code !== 0) { reject(new Error(`Verify process exited with non-zero result: ${code}`)); } else { resolve(); } }); childProcess.stdout.on("data", log); childProcess.stderr.on("data", log); }); } finally { if (encryptionKeyFilePath) { yield fs.remove(encryptionKeyFilePath); } } }); } copyAndUploadFile(args) { return __awaiter(this, void 0, void 0, function* () { if (args.pause) { yield this.stopSyncServer(); } const copyLocation = uuid.v4(); const files = (args.files || [args.file]).map(f => { return { file: f, fullPath: path.join(this.config.dataPath, f), copyPath: path.join(this.config.dataPath, copyLocation, f), }; }); if (files.length > 0 && !args.pause) { this.logger.warn("Multiple files are going to be copied while the server is running. This may lead to unpredictable state of the files."); } try { this.logger.info(`Uploading ${files.map(f => `'${f.file}'`).join(", ")}`); for (const file of files) { if (!(yield fs.pathExists(file.fullPath))) { throw new Error(`File at path '${file.fullPath}' doesn't exist.`); } yield this.copyFileSafely(file.fullPath, file.copyPath); } const archive = yield this.compressFolder(this.config.dataPath, copyLocation); if (this.config.fileUploadFunction) { yield this.config.fileUploadFunction(archive); yield fs.remove(archive); } else { this.logger.info(`Copied ${args.file} to '${archive}'. No upload function specified, so file has not been uploaded.`); } } catch (error) { this.logger.error(`Error uploading '${args.file}': ${error.stack}`, { error }); } finally { yield fs.remove(path.join(this.config.dataPath, copyLocation)); } }); } copyFileSafely(filePath, copyPath) { return __awaiter(this, void 0, void 0, function* () { let retries = 20; let originalMD5; let copyMD5; while (originalMD5 !== copyMD5 || copyMD5 === undefined) { try { if (yield fs.pathExists(copyPath)) { yield fs.remove(copyPath); } yield fs.copy(filePath, copyPath); originalMD5 = yield md5(filePath); copyMD5 = yield md5(copyPath); } catch (err) { this.logger.warn(`Failed to duplicate file: '${filePath}' to '${copyPath}': ${err.message}`); } if (retries-- === 0) { throw new Error(`Failed to copy file '${filePath}' after 20 retries, bailing...`); } } }); } compressFolder(rootFolder, directoryToCompress) { return __awaiter(this, void 0, void 0, function* () { const archiveName = `${directoryToCompress}.tar.gz`; const zipper = child_process_1.spawn("tar", ["-zcvf", archiveName, `${path.join(rootFolder, directoryToCompress)}/`], { stdio: "inherit" }); yield new Promise((resolve, reject) => { zipper.once("close", (code, signal) => { if (code) { reject(new Error(`gzip exited with non-zero exit code: ${code}`)); } else if (signal) { reject(new Error(`gzip crashed with signal ${signal}.`)); } else { resolve(); } }); }); return path.join(rootFolder, archiveName); }); } quarantineFile(args) { return __awaiter(this, void 0, void 0, function* () { const filePath = path.join(this.config.dataPath, args.file); const quarantinedPath = path.join(this.config.dataPath, "quarantined", args.file); try { const folder = quarantinedPath.substring(0, quarantinedPath.lastIndexOf(path.sep)); yield fs.mkdirp(folder); yield fs.rename(filePath, quarantinedPath); this.logger.info(`Quarantined ${args.file} to '${quarantinedPath}'.`); } catch (error) { this.logger.error(`Error quarantining '${args.file}': ${error.stack}`, { error }); } }); } } exports.KubernetesSyncWorker = KubernetesSyncWorker; //# sourceMappingURL=KubernetesSyncWorker.js.map