mnist-dataset/dist/index.js

✍️ Yann LeCun's MNIST handwritten digit dataset, made available to Node.js.

"use strict";function _typeof(obj){"@babel/helpers - typeof";if(typeof Symbol==="function"&&typeof Symbol.iterator==="symbol"){_typeof=function _typeof(obj){return typeof obj}}else{_typeof=function _typeof(obj){return obj&&typeof Symbol==="function"&&obj.constructor===Symbol&&obj!==Symbol.prototype?"symbol":typeof obj}}return _typeof(obj)}Object.defineProperty(exports,"__esModule",{value:true});exports.cache=exports.testLabelsUrl=exports.testImagesUrl=exports.trainingLabelsUrl=exports.trainingImagesUrl=exports.getTempPath=exports.gunzipUrl=void 0;var _fs=_interopRequireWildcard(require("fs"));var _http=_interopRequireDefault(require("http"));var _os=require("os");var _path=require("path");var _zlib=require("zlib");var _crypto=require("crypto");function _interopRequireDefault(obj){return obj&&obj.__esModule?obj:{default:obj}}function _getRequireWildcardCache(){if(typeof WeakMap!=="function")return null;var cache=new WeakMap;_getRequireWildcardCache=function _getRequireWildcardCache(){return cache};return cache}function _interopRequireWildcard(obj){if(obj&&obj.__esModule){return obj}if(obj===null||_typeof(obj)!=="object"&&typeof obj!=="function"){return{default:obj}}var cache=_getRequireWildcardCache();if(cache&&cache.has(obj)){return cache.get(obj)}var newObj={};var hasPropertyDescriptor=Object.defineProperty&&Object.getOwnPropertyDescriptor;for(var key in obj){if(Object.prototype.hasOwnProperty.call(obj,key)){var desc=hasPropertyDescriptor?Object.getOwnPropertyDescriptor(obj,key):null;if(desc&&(desc.get||desc.set)){Object.defineProperty(newObj,key,desc)}else{newObj[key]=obj[key]}}}newObj["default"]=obj;if(cache){cache.set(obj,newObj)}return newObj}function _slicedToArray(arr,i){return _arrayWithHoles(arr)||_iterableToArrayLimit(arr,i)||_nonIterableRest()}function _nonIterableRest(){throw new TypeError("Invalid attempt to destructure non-iterable instance")}function _iterableToArrayLimit(arr,i){if(!(Symbol.iterator in Object(arr)||Object.prototype.toString.call(arr)==="[object Arguments]")){return}var _arr=[];var _n=true;var _d=false;var _e=undefined;try{for(var _i=arr[Symbol.iterator](),_s;!(_n=(_s=_i.next()).done);_n=true){_arr.push(_s.value);if(i&&_arr.length===i)break}}catch(err){_d=true;_e=err}finally{try{if(!_n&&_i["return"]!=null)_i["return"]()}finally{if(_d)throw _e}}return _arr}function _arrayWithHoles(arr){if(Array.isArray(arr))return arr}function _toConsumableArray(arr){return _arrayWithoutHoles(arr)||_iterableToArray(arr)||_nonIterableSpread()}function _nonIterableSpread(){throw new TypeError("Invalid attempt to spread non-iterable instance")}function _iterableToArray(iter){if(Symbol.iterator in Object(iter)||Object.prototype.toString.call(iter)==="[object Arguments]")return Array.from(iter)}function _arrayWithoutHoles(arr){if(Array.isArray(arr)){for(var i=0,arr2=new Array(arr.length);i<arr.length;i++){arr2[i]=arr[i]}return arr2}}var mnistTmpDir="".concat((0,_os.tmpdir)()).concat(_path.sep,"mnist-dataset");var gunzipUrl=function gunzipUrl(url){return Promise.resolve([]).then(function(buf){return new Promise(function(resolve,reject){return _http["default"].get(url,function(res){return res.pipe((0,_zlib.createGunzip)()).on("data",function(data){return buf.push(data)}).on("end",function(){return resolve(buf)}).on("error",reject)}).on("error",reject)})}).then(function(arr){var _ref;return(_ref=[]).concat.apply(_ref,_toConsumableArray(arr))}).then(function(arr){return Buffer.concat(arr)})};exports.gunzipUrl=gunzipUrl;var getTempPath=function getTempPath(key){if(!(0,_fs.existsSync)(mnistTmpDir))(0,_fs.mkdirSync)(mnistTmpDir);if(typeof key==="string"){return"".concat(mnistTmpDir).concat(_path.sep).concat((0,_crypto.createHash)("md5").update(key).digest("hex"))}throw new Error("Expected string key, encountered ".concat(key,"."))};exports.getTempPath=getTempPath;var magicNumbers=Object.freeze([2051,2049]);var _magicNumbers=_slicedToArray(magicNumbers,2),imagesMagicNumber=_magicNumbers[0],labelsMagicNumber=_magicNumbers[1];var images=function images(buf){return Promise.resolve().then(function(){var _ref2=[buf.readInt32BE(0),buf.readInt32BE(4),buf.readInt32BE(8),buf.readInt32BE(12)],magicNumber=_ref2[0],numberOfImages=_ref2[1],numberOfRows=_ref2[2],numberOfColumns=_ref2[3];var _buf$slice$toJSON=buf.slice(16).toJSON(),data=_buf$slice$toJSON.data;var pixels=numberOfRows*numberOfColumns;return _toConsumableArray(Array(numberOfImages)).map(function(_,i){return data.slice(i*pixels,(i+1)*pixels)})})};var labels=function labels(buf){return Promise.resolve().then(function(){var _ref3=[buf.readInt32BE(0),buf.readInt32BE(4)],magicNumber=_ref3[0],numberOfLabels=_ref3[1];var _buf$slice$toJSON2=buf.slice(8).toJSON(),data=_buf$slice$toJSON2.data;return _toConsumableArray(Array(numberOfLabels)).map(function(_,i){return data.slice(i,i+1)})})};var parse=function parse(buf){return Promise.resolve().then(function(){var magicNumber=buf.readInt32BE(0);if(magicNumber===imagesMagicNumber){return images(buf)}else if(magicNumber===labelsMagicNumber){return labels(buf)}return Promise.reject(new Error("Expected one of magic numbers ".concat(magicNumbers,", encountered ").concat(magicNumber,".")))})};var maybeDownload=function maybeDownload(url){return Promise.resolve().then(function(){var path=getTempPath(url);if(!_fs["default"].existsSync(path)){return gunzipUrl(url).then(function(data){return _fs["default"].promises.writeFile(path,data)}).then(function(){return path})}return path})};var trainingImagesUrl="http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz";exports.trainingImagesUrl=trainingImagesUrl;var trainingLabelsUrl="http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz";exports.trainingLabelsUrl=trainingLabelsUrl;var testImagesUrl="http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz";exports.testImagesUrl=testImagesUrl;var testLabelsUrl="http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz";exports.testLabelsUrl=testLabelsUrl;var cache=function cache(url){return maybeDownload(url).then(function(path){return _fs["default"].promises.readFile(path)}).then(parse)};exports.cache=cache;