ip-location-api
Version:
Fast location lookup from IP address
624 lines (573 loc) • 18.6 kB
JavaScript
const fs = require('fs/promises')
const fsSync = require('fs')
const path = require('path')
const { exec, execSync } = require('child_process')
const { countries, continents } = require('countries-list')
const { CronJob } = require('cron')
const { setting, setSetting, getSettingCmd, consoleLog, consoleWarn } = require('./setting.cjs')
const { num37ToStr, getSmallMemoryFile, getZeroFill, aton6Start, aton4 } = require('./utils.cjs')
const { update as updataDbAsync } = require('./db.cjs')
const v4db = setting.v4
const v6db = setting.v6
const locFieldHash = setting.locFieldHash
const mainFieldHash = setting.mainFieldHash
/**
* @typedef {Object} LookupResult
* @property {number} [latitude]
* @property {number} [longitude]
* @property {string} [postcode]
* @property {string} [area]
* @property {string} [country]
* @property {boolean} [eu]
* @property {string} [region1]
* @property {string} [region1_name]
* @property {string} [region2]
* @property {string} [region2_name]
* @property {number} [metro]
* @property {string} [timezone]
* @property {string} [city]
* @property {string} country_name
* @property {string} country_native
* @property {string} continent
* @property {string} continent_name
* @property {string} capital
* @property {number[]} phone
* @property {string[]} currency
* @property {string[]} languages
*/
/**
* lookup ip address
* @param {string} ip - ipv4 or ipv6 formatted address
* @return {LookupResult | Promise<LookupResult | null> | null} location information
*/
const lookup = (ip) => {
var isIpv6
if(ip.includes(':')){
ip = aton6Start(ip)
isIpv6 = ip.constructor === BigInt
} else {
ip = aton4(ip)
isIpv6 = false
}
const db = isIpv6 ? v6db : v4db
if(!(ip >= db.firstIp)) return null
const list = db.startIps
var fline = 0, cline = db.lastLine, line
for(;;){
line = fline + cline >> 1
if(ip < list[line]){
if(cline - fline < 2) return null
cline = line - 1
} else {
if(fline === line) {
if(cline !== line && ip >= list[cline]) {
line = cline
}
break
}
fline = line
}
}
if(setting.smallMemory){
return lineToFile(line, db).then(buffer => {
var endIp = isIpv6 ? buffer.readBigUInt64LE(0) : buffer.readUInt32LE(0)
if(ip > endIp) return null
if(setting.isCountry){
return setCountryInfo({
country: buffer.toString('latin1', isIpv6 ? 8 : 4, isIpv6 ? 10 : 6)
})
}
return setCityRecord(buffer, {}, isIpv6 ? 8 : 4)
})
}
if(ip > db.endIps[line]) return null
if(setting.isCountry){
return setCountryInfo({
country: db.mainBuffer.toString('latin1', line * db.recordSize, line * db.recordSize + 2)
})
}
return setCityRecord(db.mainBuffer, {}, line * db.recordSize)
}
/**
* lookup ip address
* @param {number | BigInt} ip - ipv4 or ipv6 numeric address
* @return {LookupResult | Promise<LookupResult | null> | null} location information
*/
const lookupNumber = (ip) => {
var isIpv6 = ip.constructor === BigInt
if(isIpv6 && ip < BigInt(4294967296)) {
ip = Number(ip)
isIpv6 = false
}
const db = isIpv6 ? v6db : v4db
if(!(ip >= db.firstIp)) return null
const list = db.startIps
var fline = 0, cline = db.lastLine, line
for(;;){
line = fline + cline >> 1
if(ip < list[line]){
if(cline - fline < 2) return null
cline = line - 1
} else {
if(fline === line) {
if(cline !== line && ip >= list[cline]) {
line = cline
}
break
}
fline = line
}
}
if(setting.smallMemory){
return lineToFile(line, db).then(buffer => {
var endIp = isIpv6 ? buffer.readBigUInt64LE(0) : buffer.readUInt32LE(0)
if(ip > endIp) return null
if(setting.isCountry){
return setCountryInfo({
country: buffer.toString('latin1', isIpv6 ? 8 : 4, isIpv6 ? 10 : 6)
})
}
return setCityRecord(buffer, {}, isIpv6 ? 8 : 4)
})
}
if(ip > db.endIps[line]) return null
if(setting.isCountry){
return setCountryInfo({
country: db.mainBuffer.toString('latin1', line * db.recordSize, line * db.recordSize + 2)
})
}
return setCityRecord(db.mainBuffer, {}, line * db.recordSize)
}
/**
* lookup ip address
* @param {string | number | BigInt} ip - ipv4 or ipv6 formatted address or numeric address
* @return {LookupResult | Promise<LookupResult | null> | null} location information
*/
const lookupAny = (ip) => {
if(typeof ip === 'string'){
if(isNaN(ip)){
return lookup(ip)
} else if(ip > 4294967295){
return lookupNumber(BigInt(ip))
}
return lookupNumber(Number(ip))
}
if(ip.constructor === BigInt || ip.constructor === Number){
return lookupNumber(ip)
}
return null
}
/**
* setup database without reload
* @param {object} [_setting]
* @return {void}
*/
const setupWithoutReload = setSetting
/**
* clear in-memory database
* @type {function}
* @return {void}
*/
const clear = () => {
v4db.startIps = v6db.startIps = v4db.endIps = v6db.endIps = v4db.mainBuffer = v6db.mainBuffer = null
Region1NameJson = Region2NameJson = TimezoneJson = LocBuffer = CityNameBuffer = EuJson = null
}
var Region1NameJson, Region2NameJson, TimezoneJson, LocBuffer, CityNameBuffer, AreaJson, EuJson
var updateJob
/**
* reload in-memory database
* @type {function}
* @param {object} [_setting] - if you need to update the database with different setting
* @param {boolean} [sync] - sync mode
* @param {boolean} [_runningUpdate] - if it's running update [internal use]
* @return {Promise|void}
*/
const reload = async (_setting, sync, _runningUpdate) => {
var curSetting = setting
if(_setting){
var oldSetting = Object.assign({}, setting)
setSetting(_setting)
curSetting = Object.assign({}, setting)
Object.assign(setting, oldSetting)
}
const dataDir = curSetting.fieldDir
const v4 = v4db, v6 = v6db
var dataFiles = {
v41: path.join(dataDir, '4-1.dat'),
v42: path.join(dataDir, '4-2.dat'),
v43: path.join(dataDir, '4-3.dat'),
v61: path.join(dataDir, '6-1.dat'),
v62: path.join(dataDir, '6-2.dat'),
v63: path.join(dataDir, '6-3.dat'),
cityLocation: path.join(dataDir, 'location.dat'),
cityName: path.join(dataDir, 'name.dat'),
citySub: path.join(dataDir, 'sub.json')
}
var locBuffer, cityNameBuffer, subBuffer
var buffer41, buffer42, buffer43, buffer61, buffer62, buffer63
if(sync){
if(!fsSync.existsSync(dataFiles.v41)){
consoleLog('Database creating ...')
updateDb(_setting && curSetting, true, true)
consoleLog('Database created')
}
buffer41 = fsSync.readFileSync(dataFiles.v41)
buffer61 = fsSync.readFileSync(dataFiles.v61)
if(!curSetting.smallMemory){
buffer42 = fsSync.readFileSync(dataFiles.v42)
buffer43 = fsSync.readFileSync(dataFiles.v43)
buffer62 = fsSync.readFileSync(dataFiles.v62)
buffer63 = fsSync.readFileSync(dataFiles.v63)
}
if(curSetting.locFile){
locBuffer = fsSync.readFileSync(dataFiles.cityLocation)
if(locFieldHash.city){
cityNameBuffer = fsSync.readFileSync(dataFiles.cityName)
}
if(locFieldHash.region1_name || locFieldHash.region2_name || locFieldHash.timezone || mainFieldHash.area || locFieldHash.eu){
subBuffer = fsSync.readFileSync(dataFiles.citySub)
}
}
} else {
if(!fsSync.existsSync(dataFiles.v41)){
consoleLog('Database creating ...')
await updateDb(_setting && curSetting, true)
consoleLog('Database created')
}
var prs = [
fs.readFile(dataFiles.v41).then(data => buffer41 = data),
fs.readFile(dataFiles.v61).then(data => buffer61 = data),
]
if(!curSetting.smallMemory){
prs.push(
fs.readFile(dataFiles.v42).then(data => buffer42 = data),
fs.readFile(dataFiles.v43).then(data => buffer43 = data),
fs.readFile(dataFiles.v62).then(data => buffer62 = data),
fs.readFile(dataFiles.v63).then(data => buffer63 = data)
)
}
if(curSetting.locFile){
prs.push(fs.readFile(dataFiles.cityLocation).then(data => locBuffer = data))
if(locFieldHash.city){
prs.push(fs.readFile(dataFiles.cityName).then(data => cityNameBuffer = data))
}
if(locFieldHash.region1_name || locFieldHash.region2_name || locFieldHash.timezone || mainFieldHash.area || locFieldHash.eu){
prs.push(fs.readFile(dataFiles.citySub).then(data => subBuffer = data))
}
}
await Promise.all(prs)
}
if(_setting){
Object.assign(setting, curSetting)
}
v4.startIps = new Uint32Array(buffer41.buffer, 0, buffer41.byteLength >> 2)
v6.startIps = new BigUint64Array(buffer61.buffer, 0, buffer61.byteLength >> 3)
if(!curSetting.smallMemory){
v4.endIps = new Uint32Array(buffer42.buffer, 0, buffer42.byteLength >> 2)
v4.mainBuffer = buffer43
v6.endIps = new BigUint64Array(buffer62.buffer, 0, buffer62.byteLength >> 3)
v6.mainBuffer = buffer63
}
v4.lastLine = v4.startIps.length - 1
v6.lastLine = v6.startIps.length - 1
v4.firstIp = v4.startIps[0]
v6.firstIp = v6.startIps[0]
if(curSetting.isCity){
LocBuffer = locBuffer
CityNameBuffer = cityNameBuffer
if(subBuffer){
var tmpJson = JSON.parse(subBuffer)
if(locFieldHash.region1_name) Region1NameJson = tmpJson.region1_name
if(locFieldHash.region2_name) Region2NameJson = tmpJson.region2_name
if(locFieldHash.timezone) TimezoneJson = tmpJson.timezone
if(mainFieldHash.area) AreaJson = tmpJson.area
if(locFieldHash.eu) EuJson = tmpJson.eu
}
}
if(setting.smallMemory && _runningUpdate){
const rimraf = (dir) => {
if(fs.rm){
return fs.rm(dir, {recursive: true, force: true, maxRetries: 3})
}
return fs.rmdir(dir, {recursive: true, maxRetries: 3})
}
fsSync.cpSync(path.join(setting.fieldDir, 'v4-tmp'), path.join(setting.fieldDir, 'v4'), {recursive: true, force: true})
fsSync.cpSync(path.join(setting.fieldDir, 'v6-tmp'), path.join(setting.fieldDir, 'v6'), {recursive: true, force: true})
rimraf(path.join(setting.fieldDir, 'v4-tmp')).catch(consoleWarn)
rimraf(path.join(setting.fieldDir, 'v6-tmp')).catch(consoleWarn)
}
if(!updateJob && setting.autoUpdate){
updateJob = new CronJob(setting.autoUpdate, () => {
updateDb().finally(() => {})
}, null, true, 'UTC')
} else if(updateJob && !setting.autoUpdate){
updateJob.stop()
updateJob = null
}
}
const watchHash = {}
/**
* Watch database directory.
* When database file is updated, it reload the database automatically
* This causes error if you use ILA_SMALL_MEMORY=true
* @type {function}
* @param {string} [name] - name of watch. If you want to watch multiple directories, you can set different name for each directory
*/
const watchDb = (name = 'ILA') => {
var watchId = null
watchHash[name] = fsSync.watch(setting.fieldDir, (eventType, filename) => {
if(!filename.endsWith('.dat')) return;
if(fsSync.existsSync(path.join(setting.fieldDir, filename))) {
if(watchId) clearTimeout(watchId)
watchId = setTimeout(reload, 30 * 1000)
}
})
}
/**
* Stop watching database directory
* @type {function}
* @param {string} [name]
*/
const stopWatchDb = (name = 'ILA') => {
if(watchHash[name]){
watchHash[name].close()
delete watchHash[name]
}
}
/**
* Update database and auto reload database
* @type {function}
* @param {object} [_setting] - if you need to update the database with different setting
* @param {boolean} [noReload] - if you don't want to reload the database after update
* @param {boolean} [sync] - if you want to update the database in sync mode
* @return {Promise<boolean>} - true if database is updated, false if no need to update
*/
const updateDb = (_setting, noReload, sync) => {
var arg, runningUpdate = false
if(_setting){
var oldSetting = Object.assign({}, setting)
setSetting(_setting)
arg = getSettingCmd()
Object.assign(setting, oldSetting)
} else {
arg = getSettingCmd()
}
if(!_setting && !sync && !setting.smallMemory){
var sameDbSetting = setting.sameDbSetting
if(!sameDbSetting) setting.sameDbSetting = true
return updataDbAsync().then((r) => {
setting.sameDbSetting = sameDbSetting
if(r === true){
if(!noReload) reload()
}
return true
}).catch(e => {
setting.sameDbSetting = sameDbSetting
throw e
})
}
var scriptPath = path.resolve(_setting ? _setting.apiDir : setting.apiDir, 'script', 'updatedb.mjs')
if(scriptPath.includes(' ')) scriptPath = '"' + scriptPath + '"'
var cmd = 'node ' + scriptPath
if(!_setting){
arg += ' ILA_SAME_DB_SETTING=true'
}
if(_setting && _setting.smallmemory || !_setting && setting.smallMemory){
runningUpdate = true
arg += ' ILA_RUNNING_UPDATE=true'
}
if(arg){
cmd += ' ' + arg
}
if(sync){
try{
var stdout = execSync(cmd)
if(stdout.includes('NO NEED TO UPDATE')){
return true
}
if(stdout.includes('SUCCESS TO UPDATE')){
if(!noReload){
reload(_setting, sync)
}
return true
}
return false
}catch(e){
consoleWarn(e)
return false
}
}
return new Promise((resolve, reject) => {
exec(cmd, (err, stdout, stderr) => {
if(err) {
consoleWarn(err)
}
if(stderr) {
consoleWarn(stderr)
}
if(stdout) {
consoleLog(stdout)
}
if(err) {
reject(err)
} else if(stdout.includes('ERROR TO UPDATE')){
reject(new Error('ERROR TO UPDATE'))
} else if(stdout.includes('NO NEED TO UPDATE')){
resolve(false)
} else if(stdout.includes('SUCCESS TO UPDATE')){
if(noReload){
resolve(true)
} else {
reload(_setting, false, runningUpdate).then(() => {
resolve(true)
}).catch(reject)
}
} else {
consoleLog('UNKNOWN ERROR')
reject(new Error('UNKNOWN ERROR'))
}
})
})
}
/*
-- Remain this code for better performance check
const lineToFile = (line, db) => {
const [ dir, file, offset ] = getSmallMemoryFile(line, db)
return new Promise((resolve, reject) => {
fs.open(path.join(dir, file), 'r').then(fd => {
const buffer = Buffer.alloc(db.recordSize)
fd.read(buffer, 0, db.recordSize, offset).then(() => {
fd.close().catch(reject)
resolve(buffer)
}).catch(reject)
}).catch(reject)
})
}
*/
const lineToFile = async (line, db) => {
const [ dir, file, offset ] = getSmallMemoryFile(line, db)
const fd = await fs.open(path.join(setting.fieldDir, dir, file), 'r')
const buffer = Buffer.alloc(db.recordSize)
await fd.read(buffer, 0, db.recordSize, offset)
fd.close().catch(consoleWarn)
return buffer
}
/**
* Set city record
* @param {any} buffer
* @param {LookupResult} geodata
* @param {number} offset
* @return {LookupResult}
*/
const setCityRecord = (buffer, geodata, offset) => {
var locId
if(setting.locFile){
locId = buffer.readUInt32LE(offset)
offset += 4
}
if(mainFieldHash.latitude){
geodata.latitude = buffer.readInt32LE(offset) / 10000
offset += 4
}
if(mainFieldHash.longitude){
geodata.longitude = buffer.readInt32LE(offset) / 10000
offset += 4
}
if(mainFieldHash.postcode){
var postcode2 = buffer.readUInt32LE(offset)
var postcode1 = buffer.readInt8(offset + 4)
if (postcode2) {
var postcode, tmp
if(postcode1 < -9){
tmp = (-postcode1).toString()
postcode = postcode2.toString(36)
postcode = getZeroFill(postcode.slice(0, -tmp[1]), tmp[0]-0) + '-' + getZeroFill(postcode.slice(-tmp[1]), tmp[1]-0)
} else if(postcode1 < 0){
postcode = getZeroFill(postcode2.toString(36), -postcode1)
} else if(postcode1 < 10){
postcode = getZeroFill(postcode2.toString(10), postcode1)
} else if(postcode1 < 72){
postcode1 = String(postcode1)
postcode = getZeroFill(postcode2.toString(10), (postcode1[0]-0) + (postcode1[1]-0))
postcode = postcode.slice(0, postcode1[0]-0) + '-' + postcode.slice(postcode1[0]-0)
} else {
postcode = postcode1.toString(36).slice(1) + postcode2.toString(36)
}
geodata.postcode = postcode.toUpperCase()
}
offset += 5
}
if(mainFieldHash.area){
geodata.area = AreaJson[buffer.readUInt8(offset)]
offset += 1
}
if(locId){
var locOffset = (locId-1) * setting.locRecordSize
if(locFieldHash.country){
geodata.country = LocBuffer.toString('utf8', locOffset, locOffset += 2)
if(locFieldHash.eu){
geodata.eu = EuJson[geodata.country]
}
}
if(locFieldHash.region1){
var region1 = LocBuffer.readUInt16LE(locOffset)
locOffset += 2
if(region1 > 0) geodata.region1 = num37ToStr(region1)
}
if(locFieldHash.region1_name){
var region1_name = LocBuffer.readUInt16LE(locOffset)
locOffset += 2
if(region1_name > 0) geodata.region1_name = Region1NameJson[region1_name]
}
if(locFieldHash.region2){
var region2 = LocBuffer.readUInt16LE(locOffset)
locOffset += 2
if(region2 > 0) geodata.region2 = num37ToStr(region2)
}
if(locFieldHash.region2_name){
var region2_name = LocBuffer.readUInt16LE(locOffset)
locOffset += 2
if(region2_name > 0) geodata.region2_name = Region2NameJson[region2_name]
}
if(locFieldHash.metro){
var metro = LocBuffer.readUInt16LE(locOffset)
locOffset += 2
if(metro > 0) geodata.metro = metro
}
if(locFieldHash.timezone){
var timezone = LocBuffer.readUInt16LE(locOffset)
locOffset += 2
if(timezone > 0) geodata.timezone = TimezoneJson[timezone]
}
if(locFieldHash.city){
var city = LocBuffer.readUInt32LE(locOffset)
locOffset += 4
if(city > 0){
var start = city >>> 8
geodata.city = CityNameBuffer.toString('utf8', start, start + (city & 255))
}
}
}
return setCountryInfo(geodata)
}
/**
* Set country information
* @param {LookupResult} geodata
* @return {LookupResult}
*/
const setCountryInfo = (geodata) => {
if(setting.addCountryInfo){
var h = countries[geodata.country]
geodata.country_name = h.name
geodata.country_native = h.native
geodata.continent = h.continent
geodata.continent_name = continents[h.continent]
geodata.capital = h.capital
geodata.phone = h.phone
geodata.currency = h.currency
geodata.languages = h.languages
}
return geodata
}
reload(undefined, true)
module.exports={lookup:lookup,lookupNumber:lookupNumber,lookupAny:lookupAny,setupWithoutReload:setupWithoutReload,clear:clear,reload:reload,watchDb:watchDb,stopWatchDb:stopWatchDb,updateDb:updateDb}