umc-managed-store/verify/store.js

extend umc store with structure-ensure and auto-clear to make store safety

'use strict'

const bytes = require('bytes');
const cache = {};

const STORE_CACHE_KEY = 'fyadmin:all-in-one-store';//注意该名称不能更改!!!!!!!!!!!!!!, 否则会导致用户缓存数据丢失, 要重新登录
const CLEAR_THRESHOLD_SIZE = bytes('10mb') || 10*1024*1024;

const PORTAL_FEED_TYPES = 'feed,activity,work'.split(',');
const PORTAL_MEM_TYPES = 'friend,followed,follower'.split(',');
const PORTAL_TYPES = PORTAL_FEED_TYPES.concat(PORTAL_MEM_TYPES).concat(('team,activityOrder'.split(',')));
const REC_TYPES = 'member,activity,work,team'.split(',');
const PULL_TYPES = 'feed,liked,at,reply'.split(',');

const RL_PERIODS = 'forever,month,day'.split(',');
const RL_MEMBERS = RL_PERIODS.map(key=>'member-'+key);
global.WORK_CATEGORY = ['相当']
const RL_WORKS = RL_PERIODS.map(key=>'work-'+key).concat(global.WORK_CATEGORY.map(cat=>'work-forever-'+cat));
const RL_NAMES = RL_MEMBERS.concat(RL_WORKS);

const SEARCH_FEED_TYPES = 'feed,activity,work,thread'.split(',');
const SEARCH_MEM_TYPES = 'member,friend,followed,follower'.split(',');
const SEARCH_TYPES = SEARCH_FEED_TYPES.concat(SEARCH_MEM_TYPES, 'team');

const STATIC_ROOMS = {
  '#at': {id: '#at', type: 'static'}, 
  '#reply': {id: '#reply', type: 'static'}, 
  '#liked': {id: '#liked', type: 'static'}, 
}

const umc = require('umc');
let now = Date.now();
let initialState = ensureStore({});
console.log('initialState elapsed ', Date.now()-now)
const store = umc(initialState);
const DEFAULT_SET_OPTIONS = {ensure: true, save: true};

const oldSetState = store.setState;
store.setState = function(state, options){
  options || (options = {});
  options = Object.assign({}, DEFAULT_SET_OPTIONS, options);

  //确保数据结构安全
  if(options.ensure){
    state = Object.assign({}, store.state, state);
    state = ensureStore(state);
  }

  oldSetState.call(store, state);

  //save to the cache
  if(options.save){
    if(saveStore) saveStore(store.state);//should use store.state, for state maybe incomplete
  }
}

//更改store的结构后, 请同步在store.readme.md文件中修改
//!!!! 这个方法要尽可能的安全, 防止因为clear/unload等其他因素导致store.state被破坏, 程序直接gg
//!!!! 所谓安全, 就是个个key都要确保存在, 且其值类型正确(如array不能为null等); 不存在的实体数据的引用要及时删除
function ensureStore(state){
  if(!state) state = {};
  //以下定义中, 如无特殊说明, rows都应该是是id的集合

  //当前登陆者
  if(!state.auth) state.auth = {isLogin: false, token: '', id: ''};

  //行数据集, 基本上上所有的行数据都在这里, 其他地方都是对它们的引用. 结构基本为: {id: row}
  if(!state.members) state.members = {};//所有的会员
  if(!state.feeds) state.feeds = {};//所有的feed, 包括作品/活动/帖子/评论
  if(!state.teams) state.teams = {};//所有的小组
  if(!state.activityOrders) state.activityOrders = {};//活动报名的订单信息, {orderId: order}
  const members = state.members, feeds = state.feeds, teams = state.teams, activityOrders = state.activityOrders;

  //各feed的关系数据. 示例结构如下. 
  if(!state.feedRelations){
    state.feedRelations = {
      '-1': {//示例数据.
        reply: {rows: []},
        hotReply: {rows: []},
        liked: {rows: []},
        activityOrder: {rows: []},
      }
    }
  }else{
    for(let kk in state.feedRelations){
      let vv = state.feedRelations[kk];
      
      if(!vv.reply) vv.reply = {rows: []};
      if(!vv.reply.rows) vv.reply.rows = [];
      vv.reply.rows = vv.reply.rows.filter(rowId=>!!feeds[rowId])

      if(!vv.hotReply) vv.hotReply = {rows: []};
      if(!vv.hotReply.rows) vv.hotReply.rows = [];
      vv.hotReply.rows = vv.hotReply.rows.filter(rowId=>!!feeds[rowId])

      if(!vv.liked) vv.liked = {rows: []};
      if(!vv.liked.rows) vv.liked.rows = [];
      vv.liked.rows = vv.liked.rows.filter(rowId=>!!members[rowId])

      if(!vv.activityOrder) vv.activityOrder = {rows: []};
      if(!vv.activityOrder.rows) vv.activityOrder.rows = [];
      vv.activityOrder.rows = vv.activityOrder.rows.filter(rowId=>!!activityOrders[rowId])
    }
  }

  //team的相关数据, 
  if(!state.teamRelations){
    state.teamRelations = {
      '-1': {
        thread: {rows: []},
        member: {rows: []},
      }
    }
  }else{
    for(let kk in state.teamRelations){
      let vv = state.teamRelations[kk];
      if(!vv.thread) vv.thread = {rows: []};
      if(!vv.thread.rows) vv.thread.rows = [];
      vv.thread.rows = vv.thread.rows.filter(rowId=>!!feeds[rowId])

      if(!vv.member) vv.member = {rows: []};
      if(!vv.member.rows) vv.member.rows = [];
      vv.member.rows = vv.member.rows.filter(rowId=>!!members[rowId])
    }
  }

  //会员自身额外的一些信息
  if(!state.portal) state.portal = {};
  const portal = state.portal;
  PORTAL_TYPES.forEach(type=>{
    if(!portal[type]) portal[type] = {rows: [], count: 0};
    if(!portal[type].count) portal[type].count = 0;
    if(!portal[type].rows) portal[type].rows = [];

    let removed = 0;
    let rows = portal[type].rows.filter(rowId=>{
      let exists = true;
      if(PORTAL_FEED_TYPES.indexOf(type)>=0) exists = !!feeds[rowId];
      else if(PORTAL_MEM_TYPES.indexOf(type)>=0) exists = !!members[rowId];
      else if(type==='team') exists = !!teams[rowId];
      
      if(!exists) removed++;
      return exists;
    })
    portal[type].rows = rows;
    portal[type].count = Math.max(portal[type].count-removed, 0);
  })

  //推荐内容
  if(!state.recommend) state.recommend = {};
  const recommend = state.recommend;
  REC_TYPES.forEach(type=>{
    if(!recommend[type]) recommend[type] = {rows: []};
    if(!recommend[type].rows) recommend[type].rows = [];
    recommend[type].rows = recommend[type].rows.filter(rowId=>{
      if(type==='work' || type==='activity') return !!feeds[rowId];
      else if(type==='member') return !!members[rowId];
      else if(type==='team') return !!teams[rowId];
      return true;
    })
  })

  //定时拉取的消息: 关注/点赞/@/评论
  if(!state.pull) state.pull = {};
  const pull = state.pull;
  PULL_TYPES.forEach(type=>{
    if(!pull[type]) pull[type] = {rows: [], newlyCount: 0};
    if(!pull[type].newlyCount) pull[type].newlyCount = 0;
    if(!pull[type].rows) pull[type].rows = [];
    pull[type].rows = pull[type].rows.filter(rowId=>!!feeds[rowId]);
  })

  //排行榜
  if(!state.ranklist) state.ranklist = {};
  const ranklist = state.ranklist;
  RL_NAMES.forEach(name=>{
    if(!ranklist[name]) ranklist[name] = {rows: [], };
    if(!ranklist[name].rows) ranklist[name].rows = [];
    ranklist[name].rows = ranklist[name].rows.filter(rowId=>{
      if(RL_MEMBERS.indexOf(name)>=0) return !!members[rowId];
      else if(RL_WORKS.indexOf(name)>=0) return !!feeds[rowId];
      return true;
    })
  })

  //搜索结果
  if(!state.search) state.search = {};
  const search = state.search;
  SEARCH_TYPES.forEach(type=>{
    if(!search[type]) search[type] = {rows: [], term: ''};
    if(!search[type].rows) search[type].rows = [];
    search[type].rows = search[type].rows.filter(rowId=>{
      if(SEARCH_FEED_TYPES.indexOf(type)>=0) return !!feeds[rowId];
      else if(SEARCH_MEM_TYPES.indexOf(type)>=0) return !!members[rowId];
      else if(type==='team') return !!teams[rowId];
      return true;
    })
  })

  //所有的房间, exchange里用的, room: {id: string, type: enum(static|team|member|group), 
  // desc: string, updatedAt: timestamp, unreadCount: integer}
  if(!state.rooms){
    state.rooms = STATIC_ROOMS;
  }
  if(!state.chats) state.chats = {};//所有含聊天房间的会话信息, 结构: {roomId: [messages]}. 消息都存储在sqlite里
  if(!state.exchange){
    state.exchange = {
      rooms: Object.keys(STATIC_ROOMS),
      unreadCount: 0, //totally unread message count for all rooms
      open: '',//打开的room id
    }
  }

  //自动更新
  if(!state.update){
    state.update = {
      version: '',//当前native code的版本
      local: {//本地包
        isFirstRun: false,//当前更新是否第一次运行
        description: '',//当前更新的描述
        binaryVersion: '',//app安装时的版本号
        labelVersion: '',//codepush update label
        lastUpdateSize: '',//上次下载的更新包大小
      },
      remote: {//远程更新包
        status: '',//found/ignored/downloading/background-downloading/downloaded/restart-later
        description: '',
        binaryVersion: '',
        labelVersion: '',
        progress: 0,//下载进度, 0-100
        size: '',//更新包总大小
      },
    }
  }

  if(!state.ui) state.ui = {selectedMainBar: 'home'};

  return state;
}

function loadFromCache(){
  // return cache.get(STORE_CACHE_KEY).then(state=>{
  //   if(!state) return;
  //   if(typeof state !== 'object') return;

  //   //初始化部分数据
  //   const {auth, ui, exchange, rooms} = state;

  //   if(auth) auth.isLogin = false;

  //   if(ui) ui.selectedMainBar = 'home';

  //   //修正exchange的数据
  //   if(exchange){
  //     let unreadCount = 0;
  //     if(rooms && Array.isArray(exchange.rooms)){
  //       exchange.rooms.forEach(roomId=>{
  //         let room = rooms[roomId];
  //         if(!room) return;
  //         unreadCount += (room.unreadCount || 0);
  //       })
  //     }
      
  //     exchange.unreadCount = unreadCount;
  //     exchange.open = '';
  //   }
    
  //   store.setState(state);
  // })
}
store.load = loadFromCache;

let lastAutoClear = 0;
const AUTO_CLEAR_INTERVAL = 5*60*1000;//5mins
function saveStore(state){
  if(!state) return;
  let saved = {};
  for(let kk in state){
    if(kk==='chats') continue;
    saved[kk] = state[kk];
  }
  return cache.set(STORE_CACHE_KEY, saved, {returnSize: true}).then(result=>{
    if(!result) return;
    state.size = bytes(result.size);//这里直接设置size没有问题(不需要setState, 影响性能)
    // console.debug('size', state.size);

    if(result.size>=CLEAR_THRESHOLD_SIZE){
      let now = Date.now();
      if(now-lastAutoClear<AUTO_CLEAR_INTERVAL) return;
      lastAutoClear = now;
      return store.clear();
    }
  })
}

//一般是登出时调用
//清除登陆者特有的数据, 一般只要清除引用数据即可, 不需要清除实体数据(实体数据可以通过clearStore来清理)
function unloadStore(){
  // const newState = {...store.state};
  // const {pull, portal, teams, members, feeds, exchange, ui, feedRelations} = newState;

  // ui.selectedMainBar = 'home';

  // PULL_TYPES.forEach(key=>{
  //   pull[key] = {rows: [], newlyCount: 0}
  // })

  // PORTAL_TYPES.forEach(key=>{
  //   portal[key] = {rows: [], count: 0, }
  // })

  // exchange.unreadCount = 0;
  // newState.rooms = {...STATIC_ROOMS};
  // exchange.rooms = Object.keys(STATIC_ROOMS);
  // exchange.open = '';

  // for(let memberId in members){
  //   members[memberId] = {...members[memberId], followed: false};
  // }
  // for(let teamId in teams){
  //   teams[teamId] = {...teams[teamId], joined: false};
  // }
  // for(let feedId in feeds){
  //   feeds[feedId] = {...feeds[feedId], liked: false, activityOrderStatus: ''};
  // }
  // for(let feedId in feedRelations){
  //   feedRelations[feedId].activityOrder = {rows: []};//非登录用户既看不到自己参与的活动, 也看不到活动的报名情况
  // }
  // newState.activityOrders = {};//非登录用户既看不到自己参与的活动, 也看不到活动的报名情况

  // store.setState(newState);
}
store.unload = unloadStore;

//清理store, 将无用的数据移除
//options.truncate: 对引用进行截断, 取前多少行id, 值为number|true. 默认行数为60. 
//  这个参数是为了避免循环引用导致无法释放(如reply与feed, activity与order之间)
function clearStore(options){
  options || (options = {});
  if(options.truncate===true) options.truncate = 60;
  options.truncate = ~~(+options.truncate);
  if(options.truncate<=0) options.truncate = undefined;

  const newState = Object.assign({}, store.state);
  const feeds = newState.feeds, teams = newState.teams, members = newState.members, feedRelations = newState.feedRelations, 
    teamRelations = newState.teamRelations, activityOrders = newState.activityOrders, rooms = newState.rooms, 
    auth = newState.auth, portal = newState.portal, pull = newState.pull, recommend = newState.recommend,
    search = newState.search, ranklist = newState.ranklist;

  //先计算各个实体的引用情况
  const feedHasRef = {}, memberHasRef = {}, teamHasRef = {}, activityOrderHasRef = {};

  if(auth.id) memberHasRef[auth.id] = true;

  for(let feedId in feeds){
    let feed = feeds[feedId];
    if(feed.feedId) feedHasRef[feed.feedId] = true;
    if(feed.memberId) memberHasRef[feed.memberId] = true;
  }

  for(let kk in activityOrders){
    let order = activityOrders[kk];
    if(order.activityId) feedHasRef[order.activityId] = true;
    if(order.memberId) memberHasRef[order.memberId] = true;
  }

  for(let kk in feedRelations){
    let vv = feedRelations[kk];
    vv.reply.rows.slice(0, options.truncate).forEach(rowId=>(feedHasRef[rowId] = true));
    vv.hotReply.rows.slice(0, options.truncate).forEach(rowId=>(feedHasRef[rowId] = true))
    vv.liked.rows.slice(0, options.truncate).forEach(rowId=>(memberHasRef[rowId] = true))
    vv.activityOrder.rows.slice(0, options.truncate).forEach(rowId=>(activityOrderHasRef[rowId] = true))
  }
 
  for(let kk in teamRelations){
    let vv = teamRelations[kk];
    vv.thread.rows.slice(0, options.truncate).forEach(rowId=>(feedHasRef[rowId] = true));
    vv.member.rows.slice(0, options.truncate).forEach(rowId=>(memberHasRef[rowId] = true));
  }

  PORTAL_TYPES.forEach(type=>{
    portal[type].rows.slice(0, options.truncate).forEach(rowId=>{
      if(PORTAL_FEED_TYPES.indexOf(type)>=0) feedHasRef[rowId] = true; 
      else if(PORTAL_MEM_TYPES.indexOf(type)>=0) memberHasRef[rowId] = true;
      else if(type==='team') teamHasRef[rowId] = true;
      else if(type==='activityOrder') activityOrderHasRef[rowId] = true;
    })
  })

  REC_TYPES.forEach(type=>{
    recommend[type].rows.slice(0, options.truncate).forEach(rowId=>{
      if(type==='work' || type==='activity') feedHasRef[rowId] = true; 
      else if(type==='member') memberHasRef[rowId] = true;
      else if(type==='team') teamHasRef[rowId] = true;
    })
  })

  PULL_TYPES.forEach(type=>{
    pull[type].rows.slice(0, options.truncate).forEach(rowId=>(feedHasRef[rowId] = true));
  })

  RL_NAMES.forEach(name=>{
    ranklist[name].rows.slice(0, options.truncate).forEach(rowId=>{
      if(RL_MEMBERS.indexOf(name)>=0) memberHasRef[rowId] = true;
      else if(RL_WORKS.indexOf(name)>=0) feedHasRef[rowId] = true; 
    })
  })

  SEARCH_TYPES.forEach(type=>{
    search[type].rows.slice(0, options.truncate).forEach(rowId=>{
      if(SEARCH_FEED_TYPES.indexOf(type)>=0) feedHasRef[rowId] = true; 
      else if(SEARCH_MEM_TYPES.indexOf(type)>=0) memberHasRef[rowId] = true;
      else if(type==='team') teamHasRef[rowId] = true;
    })
  })

  for(let kk in rooms){
    let room = rooms[kk];
    if(room.type==='member') memberHasRef[room.id] = true;
    else if(room.type==='team') teamHasRef[room.id] = true;
  }

  //清理各实体数据及其对应的关系数据
  Object.keys(feeds).forEach(feedId=>{
    if(!feedHasRef[feedId]){
      delete feeds[feedId];
      delete feedRelations[feedId];
    }
  })
  Object.keys(members).forEach(memberId=>{
    if(!memberHasRef[memberId]) delete members[memberId];
  })
  Object.keys(teams).forEach(teamId=>{
    if(!teamHasRef[teamId]){
      delete teams[teamId];
      delete teamRelations[teamId];
    }
  })
  Object.keys(activityOrders).forEach(orderId=>{
    if(!activityOrderHasRef[orderId]) delete activityOrders[orderId];
  })

  store.setState(newState);
}
store.clear = clearStore;

module.exports = store;