geoip-lite2/utils-minified/updatedb.js

A light weight native JavaScript implementation of GeoIP API from MaxMind. Improved and faster version by Sefinek.

const{name:e,version:t}=require('../package.json'),n=`Mozilla/5.0 (compatible; ${e}/${t}; +https://github.com/sefinek/geoip-lite2)`,o=require('fs'),r=require('http'),s=require('https'),i=require('path'),c=require('zlib'),a=require('readline'),l=require('async'),{decodeStream:u}=require('iconv-lite'),d=require('rimraf').sync,p=require('adm-zip'),f=require('../lib/utils.js'),{Address6:h,Address4:g}=require('ip-address'),w=process.argv.slice(2);let m=w.find((e=>null!==e.match(/^license_key=[a-zA-Z0-9]+/)));void 0===m&&void 0!==process.env.LICENSE_KEY&&(m=`license_key=${process.env.LICENSE_KEY}`);let y=w.find((e=>null!==e.match(/^geoDataDir=[\w./]+/)));void 0===y&&void 0!==process.env.GEODATADIR&&(y=`geoDataDir=${process.env.GEODATADIR}`);let I=i.resolve(__dirname,'..','geoip-data');void 0!==y&&(I=i.resolve(process.cwd(),y.split('=')[1]),o.existsSync(I)||(console.log('ERROR: Directory doesn\'t exist: '+I),process.exit(1)));const v=process.env.GEOTMPDIR||i.resolve(__dirname,'..','tmp'),S={},E={NaN:-1},x=[{type:'country',url:`https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&suffix=zip&${m}`,checksum:`https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&suffix=zip.sha256&${m}`,fileName:'GeoLite2-Country-CSV.zip',src:['GeoLite2-Country-Locations-en.csv','GeoLite2-Country-Blocks-IPv4.csv','GeoLite2-Country-Blocks-IPv6.csv'],dest:['','geoip-country.dat','geoip-country6.dat']},{type:'city',url:`https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City-CSV&suffix=zip&${m}`,checksum:`https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City-CSV&suffix=zip.sha256&${m}`,fileName:'GeoLite2-City-CSV.zip',src:['GeoLite2-City-Locations-en.csv','GeoLite2-City-Blocks-IPv4.csv','GeoLite2-City-Blocks-IPv6.csv'],dest:['geoip-city-names.dat','geoip-city.dat','geoip-city6.dat']}];function k(e){const t=i.dirname(e);o.existsSync(t)||o.mkdirSync(t)}const D=/^\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*(?:,\s*(?:'[^'\\]*(?:\\[\S\s][^'\\]*)*'|"[^"\\]*(?:\\[\S\s][^"\\]*)*"|[^,'"\s\\]*(?:\s+[^,'"\s\\]+)*)\s*)*$/,C=/(?!\s*$)\s*(?:'([^'\\]*(?:\\[\S\s][^'\\]*)*)'|"([^"\\]*(?:\\[\S\s][^"\\]*)*)"|([^,'"\s\\]*(?:\s+[^,'"\s\\]+)*))\s*(?:,|$)/g;function R(e){if(!D.test(e)&&(e=function(e){let t=0,n=-1;for(e=e.replace(/""/,'\\"').replace(/'/g,'\\\'');t<e.length&&n<e.length;)t=n,n=e.indexOf(',',t+1),n<0&&(n=e.length),e.indexOf('\'',t||0)>-1&&e.indexOf('\'',t)<n&&'"'!=e[t+1]&&'"'!=e[n-1]&&(n=(e=e.substr(0,t+1)+'"'+e.substr(t+1,n-t-1)+'"'+e.substr(n,e.length-n)).indexOf(',',n+1),n<0&&(n=e.length));return e}(e),!D.test(e)))return null;const t=[];return e.replace(C,((e,n,o,r)=>(void 0!==n?t.push(n.replace(/\\'/g,'\'')):void 0!==o?t.push(o.replace(/\\"/g,'"').replace(/\\'/g,'\'')):void 0!==r&&t.push(r),''))),/,\s*$/.test(e)&&t.push(''),t}function B(e){const t=new URL(e),o={protocol:t.protocol,host:t.host,path:t.pathname+t.search,headers:{'User-Agent':n}};if(process.env.http_proxy||process.env.https_proxy)try{const e=require('https-proxy-agent');o.agent=new e(process.env.http_proxy||process.env.https_proxy)}catch(e){console.error(`Install https-proxy-agent to use an HTTP/HTTPS proxy. ${e.message}`),process.exit(-1)}return o}function _(e,t){if(-1!==w.indexOf('force'))return t(null,e);const n=e.checksum;if(void 0===n)return t(null,e);o.readFile(i.join(I,`${e.type}.checksum`),{encoding:'utf8'},((o,i)=>{!o&&i&&i.length&&(e.checkValue=i),console.log('Checking',e.fileName);var c=s.get(B(n),(function n(o){const i=o.statusCode;if([301,302,303,307,308].includes(i))return s.get(B(o.headers.location),n);200!==i&&(console.error(o.data),console.error('ERROR: HTTP Request Failed [%d %s]',i,r.STATUS_CODES[i]),c.end(),process.exit(1));let a='';o.on('data',(e=>{a+=e})),o.on('end',(()=>{a&&a.length?a===e.checkValue?(console.log(`Database "${e.type}" is up to date`),e.skip=!0):(console.log(`Database "${e.type}" has new data`),e.checkValue=a):(console.error(`ERROR: Could not retrieve checksum for ${e.type}. Aborting.`),console.error('Run with "force" to update without checksum'),c.end(),process.exit(1)),t(null,e)}))}))}))}function A(e,t){if(e.skip)return t(null,null,null,e);const n=e.url;let a=e.fileName;const l='.gz'===i.extname(a);l&&(a=a.replace('.gz',''));const u=i.join(v,a);if(o.existsSync(u))return t(null,u,a,e);console.log('Fetching',a),k(u);var d=s.get(B(n),(function n(i){const p=i.statusCode;if([301,302,303,307,308].includes(p))return s.get(B(i.headers.location),n);let f;200!==p&&(console.error('ERROR: HTTP Request Failed [%d %s]',p,r.STATUS_CODES[p]),d.end(),process.exit(1));const h=o.createWriteStream(u);f=l?i.pipe(c.createGunzip()).pipe(h):i.pipe(h),f.on('close',(()=>{console.log(' DONE'),t(null,u,a,e)}))}));process.stdout.write(`Retrieving ${a}...`)}function O(e,t,n,r){if(n.skip)return r(null,n);'.zip'!==i.extname(t)||(process.stdout.write('Extracting '+t+'...'),new p(e).getEntries().forEach((e=>{if(e.isDirectory)return;const t=e.entryName.split('/'),n=t[t.length-1],r=i.join(v,n);o.writeFileSync(r,e.getData())})),console.log(' DONE')),r(null,n)}async function b(e,t){var n,r;let s=0;function c(e){const t=R(e);if(!t||t.length<6)return console.warn('weird line: %s::',e);let o,i,c;s++;const a=S[t[1]];let l,u,d;if(a){if(t[0].match(/:/)){for(u=34,c=new h(t[0]),o=f.aton6(c.startAddress().correctForm()),i=f.aton6(c.endAddress().correctForm()),l=Buffer.alloc(u),d=0;d<o.length;d++)l.writeUInt32BE(o[d],4*d);for(d=0;d<i.length;d++)l.writeUInt32BE(i[d],16+4*d)}else u=10,c=new g(t[0]),o=parseInt(c.startAddress().bigInt(),10),i=parseInt(c.endAddress().bigInt(),10),l=Buffer.alloc(u),l.fill(0),l.writeUInt32BE(o,0),l.writeUInt32BE(i,4);return l.write(a,u-2),Date.now()-n>5e3&&(n=Date.now(),process.stdout.write(`\nStill working (${s})...`)),r._writableState.needDrain?new Promise((e=>{r.write(l,e)})):r.write(l)}}const l=i.join(I,t),u=i.join(v,e);d(l),k(l),process.stdout.write('\nProcessing data (may take a moment)...'),n=Date.now(),r=o.createWriteStream(l);const p=a.createInterface({input:o.createReadStream(u),crlfDelay:1/0});let w=0;for await(const e of p)w++,1!==w&&await c(e);r.close(),console.log(' DONE')}async function $(e,t){var n,r;let s=0;async function c(e){if(e.match(/^Copyright/)||!e.match(/\d/))return;const t=R(e);if(!t)return console.warn('Weird line: %s::',e);let o,i,c,a,l,u,d,p,w,m;if(s++,t[0].match(/:/)){let e=0;for(u=48,c=new h(t[0]),o=f.aton6(c.startAddress().correctForm()),i=f.aton6(c.endAddress().correctForm()),a=parseInt(t[1],10),a=E[a],l=Buffer.alloc(u),l.fill(0),m=0;m<o.length;m++)l.writeUInt32BE(o[m],e),e+=4;for(m=0;m<i.length;m++)l.writeUInt32BE(i[m],e),e+=4;l.writeUInt32BE(a>>>0,32),d=Math.round(1e4*parseFloat(t[7])),p=Math.round(1e4*parseFloat(t[8])),w=parseInt(t[9],10),l.writeInt32BE(d,36),l.writeInt32BE(p,40),l.writeInt32BE(w,44)}else u=24,c=new g(t[0]),o=parseInt(c.startAddress().bigInt(),10),i=parseInt(c.endAddress().bigInt(),10),a=parseInt(t[1],10),a=E[a],l=Buffer.alloc(u),l.fill(0),l.writeUInt32BE(o>>>0,0),l.writeUInt32BE(i>>>0,4),l.writeUInt32BE(a>>>0,8),d=Math.round(1e4*parseFloat(t[7])),p=Math.round(1e4*parseFloat(t[8])),w=parseInt(t[9],10),l.writeInt32BE(d,12),l.writeInt32BE(p,16),l.writeInt32BE(w,20);return Date.now()-n>5e3&&(n=Date.now(),process.stdout.write('\nStill working ('+s+')...')),r._writableState.needDrain?new Promise((e=>{r.write(l,e)})):r.write(l)}const l=i.join(I,t),u=i.join(v,e);d(l),process.stdout.write('\nProcessing data (may take a moment)...'),n=Date.now(),r=o.createWriteStream(l);const p=a.createInterface({input:o.createReadStream(u),crlfDelay:1/0});let w=0;for await(const e of p)w++,1!==w&&await c(e);r.close()}function q(e,t){if(e.skip)return t(null,e);const n=e.type,r=e.src,s=e.dest;'country'===n?Array.isArray(r)?function(e,t){const n=i.join(v,e);process.stdout.write('Processing lookup data (may take a moment)...');const r=a.createInterface({input:o.createReadStream(n).pipe(u('latin1')),output:process.stdout,terminal:!1});let s=0;r.on('line',(e=>{s>0&&function(e){const t=R(e);!t||t.length<6?console.log('Weird line: %s::',e):S[t[0]]=t[4]}(e),s++})),r.on('close',(()=>{console.log(' DONE'),t()}))}(r[0],(()=>{b(r[1],s[1]).then((()=>b(r[2],s[2]))).then((()=>{t(null,e)}))})):b(r,s):'city'===n&&function(e,t,n){let r=null,s=0;const c=i.join(I,t),l=i.join(v,e);d(c);var p=o.openSync(c,'w');const f=a.createInterface({input:o.createReadStream(l).pipe(u('utf-8')),output:process.stdout,terminal:!1});let h=0;f.on('line',(e=>{h>0&&function(e){if(e.match(/^Copyright/)||!e.match(/\d/))return;const t=Buffer.alloc(88),n=R(e);if(!n)return void console.warn('Weird line: %s::',e);r=parseInt(n[0]),E[r]=s;const i=n[4],c=n[6],a=n[10],l=parseInt(n[11]),u=n[12],d=n[13];t.fill(0),t.write(i,0),t.write(c,2),l&&t.writeInt32BE(l,5),t.write(d,9),t.write(u,10),t.write(a,42),o.writeSync(p,t,0,t.length,null),s++}(e),h++})),f.on('close',n)}(r[0],s[0],(()=>{$(r[1],s[1]).then((()=>(console.log('\nCity data processed'),$(r[2],s[2])))).then((()=>{console.log(' DONE'),t(null,e)}))}))}function L(e,t){if(e.skip||!e.checkValue)return t();o.writeFile(i.join(I,e.type+'.checksum'),e.checkValue,'utf8',(n=>{n&&console.log('Failed to Update checksums! Database:',e.type),t()}))}m||(console.error('ERROR: Missing license_key'),process.exit(1)),d(v),k(v),l.eachSeries(x,((e,t)=>{l.seq(_,A,O,q,L)(e,t)}),(e=>{e?(console.error('Failed to update databases from MaxMind!',e),process.exit(1)):(console.log('Successfully updated databases from MaxMind'),-1!==w.indexOf('debug')||d(v),process.exit(0))}));