redis-collections
Version:
Collection based views for Redis
272 lines (257 loc) • 10.3 kB
JavaScript
'use strict'
const chai = require('chai')
const expect = chai.expect
const redis = require('redis')
const {Store} = require("..")
// const createSections=(list,depth)=>{
// const sections = []
// if(list.length>0) {
// let section = []
// sections.push(section)
// section.push(list[0])
// for (let i = 1; i < list.length; i++) {
// const prevItem = list[i - 1]
// const item = list[i]
// let isDifferent = prevItem.length !== item.length
// for(let d=0;!isDifferent&&d<depth;d++) isDifferent=item[d] !== prevItem[d]
// if (isDifferent && sections.length > 0)
// sections.push(section=[])
// section.push(item)
// }
// }
// if(sections.length>0)
// return sections
// else
// return list
// }
// const createSubSections=(list,d)=>{
// if(list.length==0) return list
// let maxDepth=list[list.length-1].length-1
// for(let d=0;d<maxDepth;d++) {
// const sections = createSections(list)
// }
// return sections
// }
function keyTypesToSortedKeyParts(allKeyTypes) {
const keyParts=[]
allKeyTypes.forEach(keyType => {
const [key,type]=keyType
const parts=key.split(':')///[:]+/)
parts.push(type)
keyParts.push(parts)
})
sortKeyParts(keyParts)
return keyParts
}
function sortKeyParts(list) {
list.sort((a,b)=>{
if(a.length!==b.length) return a.length-b.length
for(let i=0;i<a.length;i++) if(a[i]!==b[i]) return a[i].localeCompare(b[i])
return 0
})
}
function replaceNumericParts(keyParts) {
const template = []
for(let p=0;p<keyParts.length;p++) {
const part=keyParts[p]
const replace=/[\d@]/.test(part)
const add=replace?'*':part
template.push(add)
}
return template
}
function createTemplateKey(keyParts, templatesCreated) {
const template=replaceNumericParts(keyParts)
let templateKey=template.join(':')
let alreadyHaveItsTemplate=templatesCreated[templateKey]
for(let p=0;!alreadyHaveItsTemplate&&p<template.length;p++) {
const rememberPart=template[p]
template[p]='*'
const generalizedTemplateKey = template.join(":")
template[p]=rememberPart
if(templatesCreated[generalizedTemplateKey]) {
templateKey=generalizedTemplateKey
alreadyHaveItsTemplate=true
}
}
const key=keyParts.join(":")
if(!alreadyHaveItsTemplate) {
templatesCreated[templateKey]=[key]
}
else templatesCreated[templateKey].push(key)//=templatesCreated[templateKey]+1
return templateKey
}
function processIds(keyParts, templateKey, idInTemplates, idTemplateToIds) {
// store id -> idTemplate
const template=templateKey.split(':')
// let idCount=0
for(let p=0;p<keyParts.length;p++) {
if(template[p]!=='*') continue
const idPart=keyParts[p]
let idTemplateCounter=idInTemplates[idPart]
if(!idTemplateCounter) idTemplateCounter=idInTemplates[idPart]={}
const idTemplateParts=templateKey.split(':')
idTemplateParts[p]='{}'
const idTemplateKey=idTemplateParts.join(':')
idTemplateCounter[idTemplateKey]=(idTemplateCounter[idTemplateKey]||0)+1
// idCount++
let idSet=idTemplateToIds[idTemplateKey]
if(!idSet) idSet=idTemplateToIds[idTemplateKey]={}
idSet[idPart]=(idSet[idPart]||0)+1
}
}
function getSimpleIdTemplates(idTemplateToIds) {
const simpleIdTemplates={}
for(const k in idTemplateToIds) {
if(/\*/.test(k)) continue
const idSet=idTemplateToIds[k]
simpleIdTemplates[k]=idSet
}
return simpleIdTemplates
}
function getBiggestSimpleIdTemplates(simpleIdTemplates) {
const biggests=[]
for(const k in simpleIdTemplates) {
biggests.push([Object.keys(simpleIdTemplates[k]).length,k])
}
biggests.sort((a,b)=>{
const big=b[0]-a[0]
if(big!==0) return big
return a[1].localeCompare(b[1])
})
return biggests
}
function camelize(str) {
return str.replace(/(?:^\w|[A-Z]|\b\w|\s+)/g, function(match, index) {
if (+match === 0) return ""; // or if (/\s+/.test(match)) for white spaces
return index == 0 ? match.toLowerCase() : match.toUpperCase();
});
}
function getNickName(key,usedUpNames) {
key.replace()
const parts = key.split(/[:.]/)
let firstName=null
for(let i=1;i<parts.length;i++) {
firstName = parts.slice(0,i).join(" ").split(/\W/).join(' ').trim().toLowerCase()
firstName = camelize(firstName)//.split(' ').join(" "))
if(!usedUpNames[firstName]) break
}
if(usedUpNames[firstName]) {
firstName+="2"
}
usedUpNames[firstName]=1
return firstName
}
function printTemplateList(list,examples) {
const lines=[]
lines.push("const store = {")
for(const key of list) {
const words=key.split(/\W/)
words.pop()
const collName = camelize(words.join(" ").trim())
const example=examples[key][0]
const exampleParts=example.split(":")
// let idCount=0
const parts=key.split(":").map((part,i)=>part==='*'?"${"+exampleParts[i]+"}":part)
const type=parts.pop()
const idCount=(key.match(/\*/g)||[]).length
const pre=(idCount==0?"":idCount==1?"IdTo":idCount==2?"IdPairTo":"Id"+idCount+"To")
const post=type==="hash"?"Map":type==="set"?"Set":type==="zset"?"SortedSet":type==="string"?"Value":"Object"
const collType="Redis"+pre+post
const line=collName+": new "+collType+"(\""+parts.join(":")+"\"),"
lines.push(" "+line)
}
lines.push("}")
console.log(lines.join("\n"))
}
describe('scan:', function () {
this.timeout(0)
it('scan', async() => {
// console.log(getNickName('-api-notification:test:*',{}))
// return
const config = {
"host": "79.172.210.37",
"port": 6379,
"db": 11,
"password": "TuRMAUxvaR7)pWebwcr2b6Q3}EVCp7cqnBH+xhTPYpmsRs8Dpk3Tdys8H4CW@tgoyLmh6DMoF)LXRjUPy=cvb36DEgJNtcqiQBMAR96}VxbcXEgikDJuVZftVMC79p8AbaoEZGtnxpcXH=*kuEQuN6"
}
const client = redis.createClient(config)
const store = new Store(client)
const allKeyTypes=[]
await store.promiseTypeScan(10000, (keyTypes) => {
allKeyTypes.push(...keyTypes)
})
const allKeyParts=keyTypesToSortedKeyParts(allKeyTypes)
// console.log("map=",JSON.stringify(map,null,4))
// if(allKeyParts.length>0) {
const templateToKeys={}
const idToIdTemplates={}
const idTemplateToIds={}
let lastTemplate=null
for (let i = 0; i < allKeyParts.length; i++) {
const keyParts = allKeyParts[i]
const templateKey = createTemplateKey(keyParts, templateToKeys)
processIds(keyParts,templateKey,idToIdTemplates,idTemplateToIds)
}
// console.log("templatesCreated=",templateToKeys)
let sum=0
for(const k in templateToKeys) {
sum+=templateToKeys[k].length
}
console.log("list vs sum",allKeyParts.length,sum)
const templateList=Object.keys(templateToKeys)
templateList.sort()
// console.log("templateList=",templateList)
printTemplateList(templateList,templateToKeys)
if(false) {
console.log("idToIdTemplates=", idToIdTemplates)
console.log("idTemplateToIds=", idTemplateToIds)
// simpleIdTemplates
const simpleIdTemplates = getSimpleIdTemplates(idTemplateToIds)
console.log("simpleIdTemplates=", simpleIdTemplates)
const biggestSimpleIdTemplates = getBiggestSimpleIdTemplates(simpleIdTemplates)
console.log("biggestSimpleIdTemplates=", biggestSimpleIdTemplates)
const renamed = {}
const usedUpNames = {}
for (let which = 0; /*which<2&&*/which < biggestSimpleIdTemplates.length; which++) {
const biggestSimple = biggestSimpleIdTemplates[which][1]
const nickName = getNickName(biggestSimple, usedUpNames) + "Id"
const idsOfBiggestSimple = idTemplateToIds[biggestSimple]
// console.log("big:", idsOfBiggestSimple)
const templatesToRename = {}
for (const id in idsOfBiggestSimple) {
const templates = idToIdTemplates[id]
for (const k in templates) templatesToRename[k] = 1
}
// console.log("templatesToRename=", templatesToRename)
for (const idTemplate in templatesToRename) {
if (renamed[idTemplate]) continue
renamed[idTemplate] = nickName
// const newIdTemplate = idTemplate.replace("{}", "${" + nickName + "}")
// console.log("newIdTemplate=", newIdTemplate)
}
}
console.log("renamed=", renamed)
}
// const pairs=[]
// for(const k in idToIdTemplates) {
// pairs.push([Object.keys(idToIdTemplates[k]).length,k,idToIdTemplates[k]])
// }
// pairs.sort((a,b)=>{
// const less=a[0]-b[0]
// if(less) return less
// return a[1].localeCompare(b[1])
// })
// console.log("pairs=",pairs)
//
// // remove multi:
// for(const pair of pairs) {
// const map=pair[2]
// for(const k in map) {
// if(map[k]>1) delete map[k]
// }
// }
// console.log("pairs2=",pairs)
// }
})
})