owhat-job
Version:
owhat红包清理
1,218 lines (1,148 loc) • 50.1 kB
JavaScript
//console.log = function() {};
import V from 'gcl';
import pro from 'myprocesscontroller';
import OSS from 'ali-oss';
import { bulkArticle, delArticleByID } from './util/eshelper';
pro.start(async function() {
const conf = V.AppSettings(this.Config, "add_file");
if (!conf) throw new Error('add_static配置获取失败!');
const count = parseInt(conf.count || 1000);
const max = parseInt(conf.max || 1);
const threadcount = conf.threadcount || 1;
console.log('数据库初始化完成', threadcount, max);
const that = this;
return new pro.CACProcess(new class extends pro.ADataWorker {
constructor() {
super('file', threadcount, max);
}
async getData(data) {
console.log('file-schedule获取数据');
let ret = null;
try {
ret = await that.Ni.excute('add.redis.getstaticmq', {
count: count
});
console.log('getschedulemq', ret.hasData() && ret.last()[0]);
if (ret.hasData() && ret.last()[0].length && !ret.last()[0][0].affected) {
let result = [];
const map = {};
ret.last()[0].map(v => {
try {
let value = V.json(v.value);
result[result.length] = { type: value.Type, params: value.Params };
} catch (ee) {
that.Log.release('add.redis.getstaticmq-单条 -' + ee.message + ',json=' + V.toJsonString(v));
console.log('add.redis.getstaticmq-单条 -', ee.stack, V.toJsonString(v));
}
});
console.log('add.redis.getstaticmq', `获取${ret.last()[0].length}条数据`, result);
return result;
} else if (!ret.hasData()) {
console.log('没有返回数据!', ret.last());
} else {
console.log('奇怪错误 未返回返回数据 也没有此次请求的num');
}
} catch (e) {
that.Log.release('add.redis.getschedulemsmq -' + e.message + ',json=' + V.toJsonString(ret.last()));
console.log('add.redis.getschedulemsmq -', e.stack, ret.last());
}
if (ret) {
ret.clear();
}
return null;
}
async custom(ret, data) {
if (!data.array) data.array = 1;
else data.array++;
if (!data.Log) {
data.Log = that.Log;
data.Ni = that.Middler.getObjectByAppName('Ni', 'templatemanager');
data.Config = that.Config;
}
let res = null;
let name;
let result;
let params = ret.params;
let _this = this;
try {
switch (ret.type.toLowerCase()) {
case 'index.banner':
await this.buildIndexBanner(data.Ni);
break;
case 'index.author':
await this.buildIndexAuthor(data.Ni);
break;
case 'authors':
await this.buildAuthors(data.Ni);
break;
case 'author':
await this.buildAuthor(data.Ni, params);
break;
case 'index.article':
await this.buildIndexArticle(data.Ni);
break;
case 'index.articleexcel':
await this.buildIndexArticleExcel(data.Ni);
break;
case 'index.articlehot':
await this.buildIndexArticleHot(data.Ni);
break;
case 'article':
await this.buildArticle(data.Ni, params);
break;
case 'index.excel':
await this.buildIndexExcel(data.Ni);
break;
case 'article.comment':
await this.buildArticleComment(data.Ni, params); //{commentCount:222,comments:[{}]};
break;
case 'comment.report':
await this.buildCommentReport(data.Ni, params);
break;
case 'index.topic':
await this.buildIndexTopic(data.Ni);
break;
case 'topics':
await this.buildTopics(data.Ni);
break;
case 'topic':
await this.buildTopic(data.Ni, params);
break;
case 'index.finance':
await this.buildIndexFinance(data.Ni);
break;
case 'index.category':
await this.buildIndexCategory(data.Ni);
break;
case 'index.motto':
await this.buildIndexMotto(data.Ni);
break;
case 'index.excels':
await this.buildExcels(data.Ni);
break;
case 'index.partner':
await this.buildIndexPartner(data.Ni);
break;
default:
console.log('不支持的命令', V.toJsonString(ret));
break;
}
console.log(ret.type + '完成', ret);
} catch (e) {
console.log("单条处理失败" + e.message, V.toJsonString(ret));
that.Log.release("单条处理失败" + e.message + ' ' + V.toJsonString(ret));
} finally {}
if (res) {
res.clear();
}
return false;
}
async rollback(v, data) {
console.log('rollback 抛弃数据', v);
that.Log.release("单条处理放弃" + ' ' + V.toJsonString(v));
return false;
}
async dispose(data) {
try {
await super.dispose(data);
} catch (e) {
console.log('dispose', e.message);
} finally {
//todo 关闭特有资源
delete data.Log;
delete data.Config;
that.Middler.setObjectByAppName('Ni', "MT", data.Ni);
delete data.Ni;
}
return false;
}
async refreshOSS(name, result) {
//上传
var oss = conf.oss;
let client = new OSS({
region: oss.region,
bucket: oss.bucket,
accessKeyId: oss.key,
accessKeySecret: oss.secret,
secure: true,
});
console.log('oss待上传--' + oss.path + name, (result && result.length ? result.length : result.ID));
await client.put(oss.path + name, Buffer.from(typeof(result) === 'string' ? result : V.toJsonString(result)));
console.log('oss已上传--' + oss.path + name);
return false;
}
async delOSS(name) {
//上传
var oss = conf.oss;
let client = new OSS({
region: oss.region,
bucket: oss.bucket,
accessKeyId: oss.key,
accessKeySecret: oss.secret,
secure: true,
});
await client.deleteMulti(name.split(','), {
quiet: true
});
return false;
}
async buildIndexBanner(ni) {
var res = await ni.excute('r_emb', '~.l_banner.select', {
$delFlag: 3,
$startTime: '<=now()',
$endTime: '>=now()',
_order: 'weight desc,ID desc',
_page: '0,6',
ID: true,
Name: true,
url: true,
img: true
});
await this.refreshOSS(conf.indexbanner, `var Banners = ` + V.toJsonString(res.hasData() ? res.last()[0] : []) + ';');
res.clear();
}
async buildIndexCategory(ni) {
var res = await ni.excute('r_emb', '~.l_category.select', {
$delFlag: 3,
_order: 'ParentID asc,weight asc',
ID: true,
Name: true,
ParentID: true,
SonCates: true
});
var CateIdic = {};
var Cates = {};
var CateParent = {};
res.hasData() && res.last()[0].map(v => {
CateIdic[v.ID] = v.Name;
if (v.ParentID) {
var ParentName = CateIdic[v.ParentID];
if (ParentName) {
Cates[ParentName].Children[v.Name] = { ID: v.ID };
CateParent[v.ID] = ParentName;
}
} else {
Cates[v.Name] = { ID: v.ID };
!!v.SonCates && (Cates[v.Name].Children = {});
}
return null;
});
await this.refreshOSS(conf.indexcategory, `var Cates=${V.toJsonString(Cates)};var CateIdic=${V.toJsonString(CateIdic)};var CateParent=${V.toJsonString(CateParent)};`);
res.clear();
}
async buildIndexFinance(ni) {
var res = await ni.excute('r_emb', '~.l_financeinfo.select', {
$delFlag: 3,
$createTime: '>=date_add(now(),interval -30 day)',
_page: '0,3',
_order: 'ID desc',
ID: true,
intro: true,
company: true
});
await this.refreshOSS(conf.indexfinance, `var Finances=${V.toJsonString(res.hasData() ? res.last()[0]:[]) };`);
res.clear();
}
async buildIndexMotto(ni) {
var res = await ni.excute('r_emb', '~.l_motto.select', {
$delFlag: 3,
$topStartTime: '<=now()',
$endStartTime: '>=now()',
_order: 'ID desc',
_page: '0,1',
ID: true,
headerImage: true,
intro: true,
name: true
});
if (!res.hasData()) {
res = await ni.excute('r_emb', '~.l_motto.select', {
$delFlag: 3,
_order: 'uuid() asc',
_page: '0,1',
ID: true,
headerImage: true,
intro: true,
name: true
});
}
await this.refreshOSS(conf.indexmotto, `var Mottos=${V.toJsonString(res.hasData() ? res.last()[0]:[]) };`);
res.clear();
}
async buildIndexPartner(ni) {
var res = await ni.excute('r_emb', '~.l_partner.select', {
$delFlag: 3,
_order: 'ID desc',
ID: true,
Name: true,
URL: true,
Type: true
});
var cooperater = [],
joiner = [];
res.hasData() && res.last().map(v => !v.Type ? cooperater.push(v) : joiner.push(v));
await this.refreshOSS(conf.indexpartner, `var Partners=${V.toJsonString({
cooperater,joiner
}) };`);
res.clear();
}
async buildIndexExcel(ni) {
var res = await ni.excute('r_emb', '~.l_article.select', {
$delFlag: 3,
$isExcel: 1,
_order: 'createTime desc',
_page: '0,6',
ID: true,
Name: true,
intro: true,
createrName: true
});
await this.refreshOSS(conf.indexexcel, `var Excels=${V.toJsonString(res.hasData() ? res.last()[0]:[]) };`);
res.clear();
}
async buildIndexTopic(ni) {
var res = await ni.excute('r_emb', '~.l_topic.select', {
$delFlag: '~=3,4',
_order: 'createTime desc',
_page: '0,6',
ID: true,
Name: true,
categoryID: true,
delFlag: true,
authorID: true,
intro: true
});
var ending = res.hasData() ? res.last()[0] : [];
res = await ni.excute('r_emb', '~.l_topic.select', {
$delFlag: '5',
_order: 'createTime desc',
_page: '0,6',
ID: true,
Name: true,
categoryID: true,
delFlag: true,
authorID: true,
intro: true
});
var ended = res.hasData() ? res.last()[0] : [];
var author = {};
ended.map(v => {
if (author[v.authorID]) author[v.authorID].push(v);
else author[v.authorID] = [v];
});
res = await ni.excute('r_emb', '~.l_author.select', {
$ID: '~="' + Object.keys(author).join(',') + '"',
ID: true,
Name: true
});
if (res.hasData())
res.last()[0].map(v => author[v.ID].map(v2 => v2.author = v.Name));
await this.refreshOSS(conf.indextopic, `var Topics=${V.toJsonString({ending,ended})}`);
res.clear();
}
async buildIndexAuthor(ni) {
var res = await ni.excute('r_emb', '~.l_author.select', {
$delFlag: 3,
$topStartTime: '<=now()',
$topEndTime: '>=now()',
_order: 'createTime desc',
_page: '0,3',
ID: true,
image: true,
intro: true,
Name: true,
company: true,
position: true,
});
var authors = res.hasData() ? res.last()[0] : [];
if (authors.length < 3) {
res = await ni.excute('r_emb', '~.l_author.select', {
$delFlag: 3 - authors.length,
_order: 'lastPublisTime desc',
_page: '0,3',
ID: true,
image: true,
intro: true,
Name: true,
company: true,
position: true,
passArticleCount: true,
passTopicCount: true
});
res.hasData() &&
authors.concat(res.last()[0]);
}
await this.refreshOSS(conf.indexauthor, `var Authors=${V.toJsonString(authors) };`);
res.clear();
}
async buildArticle(ni, ID) {
var res = await ni.excute('r_emb', '~.l_category.select', {
$delFlag: 3,
_order: 'ParentID asc,weight asc',
ID: true,
Name: true,
ParentID: true
});
var CateParent = {},
CateIdic = {};
res.hasData() && res.last()[0].map(v => {
if (v.ParentID) {
CateParent[v.ID] = v.ParentID;
}
CateIdic[v.ID] = v.Name;
return null;
});
res.clear();
res = await ni.excute('r_emb', '~.l_article.select', {
$ID: ID,
$delFlag: 3,
ID: true,
Name: true,
categoryID: true,
authorID: true,
coverImage: true,
intro: true,
content: true,
keyWords: true,
canComment: true,
isTopic: true,
isTop: true,
isHomeTop: true,
isExcel: true,
createrName: true,
createTime: true,
source: true,
sourceurl: true,
});
var article = res.hasData() ? res.single() : {};
if (res.hasData()) {
if (!article.source.length) {
res = await ni.excute('r_emb', '~.l_author.select', {
$ID: article.authorID,
Name: true
});
res.hasData() && (article.author = res.single().Name);
article.author = article.source;
}
try {
var vals = [article].map(v => {
return {
id: v.ID,
name: V.decodeEmoji(v.Name || ''),
authorID: v.authorID,
authorName: V.decodeEmoji(v.author || ''),
coverImage: v.coverImage,
topicID: v.topicID || null,
firstCategory: {
categoryID: CateParent[v.categoryID],
categoryName: CateIdic[CateParent[v.categoryID]]
},
secondCategory: {
categoryID: v.categoryID,
categoryName: CateIdic[v.categoryID]
},
keyWords: V.decodeEmoji(v.keyWords || '').split(','),
intro: V.decodeEmoji(v.intro),
content: V.decodeEmoji(v.content).replace(/<[^>]*>/g, ''),
source: v.source,
sourceurl: v.sourceurl,
isTopic: v.isTopic,
isTop: v.isTop,
isHomeTop: v.isHomeTop,
isExcel: v.isExcel,
canComment: v.canComment,
allowComment: 1,
commentCount: 0,
viewCount: v.viewCount,
createTime: v.createTime
};
});
await bulkArticle(vals);
} catch (e) {
console.log('buildArticle error', e.message);
}
} else {
await delArticleByID(ID);
console.log('del ES success', ID);
}
await this.refreshOSS(conf.article + '/' + ID + '/body.js', `var Article=${V.toJsonString(article) };`);
res.clear();
}
async buildAuthor(ni, ID) {
var res = await ni.excute('r_emb', '~.l_author.select', {
$ID: ID,
$delFlag: 3,
Name: true,
categoryIDs: true,
ID: true,
image: true,
intro: true,
position: true,
company: true,
passArticleCount: true,
passTopicCount: true,
commentCount: true,
collectCount: true,
});
var author = res.hasData() ? res.single() : {};
var articles = [],
topics = [];
if (res.hasData()) {
res = await ni.excute('r_emb', '~.l_article.select', {
$authorID: ID,
$delFlag: 3,
_order: 'createTime desc',
ID: true,
Name: true,
categoryID: true,
authorID: true,
coverImage: true,
intro: true,
content: true,
keyWords: true,
canComment: true,
isTopic: true,
isTop: true,
isHomeTop: true,
isExcel: true,
createrName: true,
createTime: true,
source: true,
sourceurl: true,
});
res.hasData() && (articles = res.last()[0]);
res = await ni.excute('r_emb', '~.l_topic.select', {
$authorID: ID,
$delFlag: '~=3,4,5',
_order: 'createTime desc',
ID: true,
Name: true,
categoryID: true,
intro: true,
createTime: true,
delFlag: true
});
res.hasData() && (topics = res.last()[0]);
}
await this.refreshOSS(conf.author + '/' + ID + '/author.js', `var Author =${V.toJsonString(author) };`);
await this.refreshOSS(conf.author + '/' + ID + '/author.json', `${V.toJsonString(author) }`);
await this.refreshOSS(conf.author + '/' + ID + '/articles.js', `var Articles =${V.toJsonString(articles) };`);
await this.refreshOSS(conf.author + '/' + ID + '/topics.js', `var Topics =${V.toJsonString(topics) };`);
res.clear();
}
async buildTopic(ni, ID) {
var res = await ni.excute('r_emb', '~.l_topic.select', {
$ID: ID,
$delFlag: '~=3,4,5',
ID: true,
Name: true,
categoryID: true,
authorID: true,
articleID: true,
intro: true,
createTime: true,
delFlag: true
});
var topic = res.hasData() ? res.single() : {};
await this.refreshOSS(conf.topic + '/' + ID + '.js', `var Topic =${V.toJsonString(topic) };`);
res.clear();
}
async buildAuthors(ni, ID) {
var res = await ni.excute('r_emb', '~.l_author.select', {
$delFlag: '~=3,4,5',
_order: 'PY asc',
ID: true,
image: true,
intro: true,
Name: true,
company: true,
position: true,
});
var authors = res.hasData() ? res.last()[0] : [];
var i = 1;
while (authors.length) {
var data = authors.splice(0, 10);
await this.refreshOSS(conf.authors + '/' + i++ + '.json', `${V.toJsonString(data) }`);
}
//主动删除最后+1页 防止因为文件大批量下架而导致静态文件未重新处理干净
await this.delOSS(conf.authors + '/' + i + '.json');
res.clear();
}
async buildExcels(ni) {
var res = await ni.excute('r_emb', '~.l_article.select', {
$isExcel: 1,
$delFlag: 3,
_order: 'createTime desc',
ID: true,
Name: true,
categoryID: true,
authorID: true,
coverImage: true,
intro: true,
content: true,
keyWords: true,
canComment: true,
isTopic: true,
isTop: true,
isHomeTop: true,
isExcel: true,
createrName: true,
createTime: true,
source: true,
sourceurl: true,
});
var articles = res.hasData() ? res.last()[0] : [];
var i = 1;
while (articles.length) {
var data = articles.splice(0, 10);
await this.refreshOSS(conf.excels + '/' + i++ + '.json', `${V.toJsonString(data) }`);
}
//主动删除最后+1页 防止因为文件大批量下架而导致静态文件未重新处理干净
await this.delOSS(conf.excels + '/' + i + '.json');
res.clear();
}
async buildTopics(ni) {
var res = await ni.excute('r_emb', '~.l_topic.select', {
$delFlag: '~=3,4',
_order: 'createTime desc',
ID: true,
Name: true,
categoryID: true,
delFlag: true,
authorID: true,
articleID: true,
});
var ending = res.hasData() ? res.last()[0] : [];
var i = 1;
while (ending.length) {
var data = ending.splice(0, 10);
await this.refreshOSS(conf.topics + '/ending/' + i++ + '.json', `${V.toJsonString(data) }`);
}
//主动删除最后+1页 防止因为文件大批量下架而导致静态文件未重新处理干净
await this.delOSS(conf.topics + '/ending/' + i + '.json');
res.clear();
res = await ni.excute('r_emb', '~.l_topic.select', {
$delFlag: '5',
_order: 'createTime desc',
ID: true,
Name: true,
categoryID: true,
delFlag: true,
authorID: true,
articleID: true,
});
var ended = res.hasData() ? res.last()[0] : [];
var author = {};
ended.map(v => {
if (author[v.authorID]) author[v.authorID].push(v);
else author[v.authorID] = [v];
});
res = await ni.excute('r_emb', '~.l_author.select', {
$ID: '~="' + Object.keys(author).join(',') + '"',
ID: true,
Name: true
});
if (res.hasData())
res.last()[0].map(v => author[v.ID].map(v2 => v2.author = v.Name));
i = 1;
while (ended.length) {
var data = ended.splice(0, 10);
await this.refreshOSS(conf.topics + '/ended/' + i++ + '.json', `${V.toJsonString(data) }`);
}
//主动删除最后+1页 防止因为文件大批量下架而导致静态文件未重新处理干净
await this.delOSS(conf.topics + '/ended/' + i + '.json');
res.clear();
}
async buildIndexArticle(ni) {
var res = await ni.excute('r_emb', '~.l_category.select', {
$delFlag: 3,
$ParentID: '!=""',
_order: 'ParentID asc,weight asc',
ID: true,
Name: true,
ParentID: true
});
var CateParent = {};
res.hasData() && res.last()[0].map(v => {
if (v.ParentID) {
CateParent[v.ID] = v.ParentID;
}
return null;
});
res.clear();
//准备好父类的all 和 all
//文章量大了会有问题 先算置顶的 再算全部 再1000行 1000行处理
//todo Cates设置全部和二级全部：一级ID
res = await ni.excute('r_emb', '~.l_article.select', {
'||': V.toJsonString({
$isTop: 1,
$isHomeTop: 1
}),
$delFlag: 3,
_order: 'topStartTime desc,createTime desc',
ID: true,
Name: true,
categoryID: true,
authorID: true,
coverImage: true,
intro: true,
//content: true,
keyWords: true,
canComment: true,
isTopic: true,
isTop: true,
isHomeTop: true,
isExcel: true,
topicID: true,
createrName: true,
createTime: true,
source: true,
sourceurl: true,
topStartTime: true,
topEndTime: true
});
var articles = res.hasData() ? res.last()[0] : [];
var cates = { all: [] },
tops = {},
that = this;
var catelength = { all: 1 };
await V.each(articles, async function(v) {
var now = +new Date,
startTime = v.topStartTime ? Date.parse(v.topStartTime) : null;
var endTime = v.topEndTime ? Date.parse(v.topEndTime) : null;
if ((startTime == null || startTime >= now) && (endTime == null || endTime <= now)) {
tops[v.ID] = true;
v.isTop = true;
//置顶处理
if (v.isHomeTop) {
cates['all'].push(v);
if (cates['all'].length >= 10) {
var jsons = cates['all'];
cates['all'] = [];
await that.refreshOSS(conf.indexarticle + '/all/' + catelength['all']++ + '.json', `${V.toJsonString(jsons) }`);
}
}
if (v.isTop) {
var parentID = CateParent[v.categoryID];
if (parentID) {
if (cates[parentID]) {
cates[parentID].push(v);
if (cates[parentID].length >= 10) {
var jsons = cates[parentID];
delete cates[parentID];
await that.refreshOSS(conf.indexarticle + '/' + parentID + '/' + catelength[parentID]++ + '.json', `${V.toJsonString(jsons) }`);
}
} else {
if (!catelength[parentID]) catelength[parentID] = 1;
cates[parentID] = [v];
}
}
if (cates[v.categoryID]) {
cates[v.categoryID].push(v);
if (cates[v.categoryID].length >= 10) {
var jsons = cates[v.categoryID];
delete cates[v.categoryID];
await that.refreshOSS(conf.indexarticle + '/' + v.categoryID + '/' + catelength[v.categoryID]++ + '.json', `${V.toJsonString(jsons) }`);
}
} else {
if (!catelength[v.categoryID]) catelength[v.categoryID] = 1;
cates[v.categoryID] = [v];
}
}
}
return false;
}, 2);
res.clear();
var isRun = true,
w = 0;
await V.whileC(() => (isRun) ? 1 : null, async function() {
res = await ni.excute('r_emb', '~.l_article.select', {
$delFlag: 3,
_order: 'createTime desc',
_page: `${w++*1000},1000`,
ID: true,
Name: true,
categoryID: true,
authorID: true,
coverImage: true,
intro: true,
content: true,
keyWords: true,
canComment: true,
isTopic: true,
isExcel: true,
createrName: true,
createTime: true,
source: true,
sourceurl: true,
topStartTime: true,
topEndTime: true
});
isRun = res.hasData();
if (isRun) {
var articles = res.last()[0];
res.clear();
await V.each(articles, async function(v) {
if (tops[v.ID]) return false;
v.isTop = false;
cates['all'].push(v);
if (cates['all'].length >= 10) {
var jsons = cates['all'];
cates['all'] = [];
await that.refreshOSS(conf.indexarticle + '/all/' + catelength['all']++ + '.json', `${V.toJsonString(jsons) }`);
}
var parentID = CateParent[v.categoryID];
if (parentID) {
if (cates[parentID]) {
cates[parentID].push(v);
if (cates[parentID].length >= 10) {
var jsons = cates[parentID];
delete cates[parentID];
await that.refreshOSS(conf.indexarticle + '/' + parentID + '/' + catelength[parentID]++ + '.json', `${V.toJsonString(jsons) }`);
}
} else {
if (!catelength[parentID]) catelength[parentID] = 1;
cates[parentID] = [v];
}
}
if (cates[v.categoryID]) {
cates[v.categoryID].push(v);
if (cates[v.categoryID].length >= 10) {
var jsons = cates[v.categoryID];
delete cates[v.categoryID];
await that.refreshOSS(conf.indexarticle + '/' + v.categoryID + '/' + catelength[v.categoryID]++ + '.json', `${V.toJsonString(jsons) }`);
}
} else {
if (!catelength[v.categoryID]) catelength[v.categoryID] = 1;
cates[v.categoryID] = [v];
}
return false;
}, 2);
}
return false;
}, true);
//剩余数据处理
await V.forC(cates, async function(k, v) {
if (v.length)
await that.refreshOSS(conf.indexarticle + '/' + k + '/' + catelength[k]++ + '.json', `${V.toJsonString(v) }`);
await that.delOSS(conf.indexarticle + '/' + k + '/' + catelength[k] + '.json');
return false;
}, 2);
//主动删除最后+1页 防止因为文件大批量下架而导致静态文件未重新处理干净
res.clear();
}
async buildIndexArticleExcel(ni) {
var res = await ni.excute('r_emb', '~.l_category.select', {
$delFlag: 3,
$ParentID: '!=""',
_order: 'ParentID asc,weight asc',
ID: true,
Name: true,
ParentID: true
});
var CateParent = {};
res.hasData() && res.last()[0].map(v => {
if (v.ParentID) {
CateParent[v.ID] = v.ParentID;
}
return null;
});
res.clear();
//准备好父类的all 和 all
//文章量大了会有问题 先算置顶的 再算全部 再1000行 1000行处理
//todo Cates设置全部和二级全部：一级ID
res = await ni.excute('r_emb', '~.l_article.select', {
'||': V.toJsonString({
$isTop: 1,
$isHomeTop: 1
}),
$isExcel: 1,
$delFlag: 3,
_order: 'createTime desc',
ID: true,
Name: true,
categoryID: true,
authorID: true,
coverImage: true,
intro: true,
content: true,
keyWords: true,
canComment: true,
isTopic: true,
isExcel: true,
createrName: true,
createTime: true,
source: true,
sourceurl: true,
topStartTime: true,
topEndTime: true
});
var articles = res.hasData() ? res.last()[0] : [];
var cates = { all: [] },
tops = {},
catelength = { all: 1 };
await V.each(articles, async function(v) {
var now = +new Date,
startTime = v.topStartTime ? Date.parse(v.topStartTime) : null;
var endTime = v.topEndTime ? Date.parse(v.topEndTime) : null;
if ((startTime == null || startTime >= now) && (endTime == null || endTime <= now)) {
tops[v.ID] = true;
v.isTop = true;
//置顶处理
if (v.isHomeTop) {
cates['all'].push(v);
if (cates['all'].length >= 10) {
var jsons = cates['all'];
cates['all'] = [];
await that.refreshOSS(conf.indexarticleexcel + '/all/' + catelength['all']++ + '.json', `${V.toJsonString(jsons) }`);
}
}
if (v.isTop) {
var parentID = CateParent[v.categoryID];
if (parentID) {
if (cates[parentID]) {
cates[parentID].push(v);
if (cates[parentID].length >= 10) {
var jsons = cates[parentID];
delete cates[parentID];
await that.refreshOSS(conf.indexarticleexcel + '/' + parentID + '/' + catelength[parentID]++ + '.json', `${V.toJsonString(jsons) }`);
}
} else {
if (!catelength[parentID]) catelength[parentID] = 1;
cates[parentID] = [v];
}
}
if (cates[v.categoryID]) {
cates[v.categoryID].push(v);
if (cates[v.categoryID].length >= 10) {
var jsons = cates[v.categoryID];
delete cates[v.categoryID];
await that.refreshOSS(conf.indexarticleexcel + '/' + v.categoryID + '/' + catelength[v.categoryID]++ + '.json', `${V.toJsonString(jsons) }`);
}
} else {
if (!catelength[v.categoryID]) catelength[v.categoryID] = 1;
cates[v.categoryID] = [v];
}
}
}
return false;
}, 2);
res.clear();
var isRun = true,
w = 0;
await V.whileC(() => (isRun) ? 1 : null, async function() {
res = await ni.excute('r_emb', '~.l_article.select', {
$delFlag: 3,
$isExcel: 1,
_order: 'createTime desc',
_page: `${w++*1000},1000`,
ID: true,
Name: true,
categoryID: true,
authorID: true,
coverImage: true,
intro: true,
content: true,
keyWords: true,
canComment: true,
isTopic: true,
isExcel: true,
createrName: true,
createTime: true,
source: true,
sourceurl: true,
topStartTime: true,
topEndTime: true
});
isRun = res.hasData();
if (isRun) {
var articles = res.last()[0];
res.clear();
await V.each(articles, async function(v) {
if (tops[v.ID]) return false;
v.isTop = false;
cates['all'].push(v);
if (cates['all'].length >= 10) {
var jsons = cates['all'];
cates['all'] = [];
await that.refreshOSS(conf.indexarticleexcel + '/all/' + catelength['all']++ + '.json', `${V.toJsonString(jsons) }`);
}
var parentID = CateParent[v.categoryID];
if (parentID) {
if (cates[parentID]) {
cates[parentID].push(v);
if (cates[parentID].length >= 10) {
var jsons = cates[parentID];
delete cates[parentID];
await that.refreshOSS(conf.indexarticleexcel + '/' + parentID + '/' + catelength[parentID]++ + '.json', `${V.toJsonString(jsons) }`);
}
} else {
if (!catelength[parentID]) catelength[parentID] = 1;
cates[parentID] = [v];
}
}
if (cates[v.categoryID]) {
cates[v.categoryID].push(v);
if (cates[v.categoryID].length >= 10) {
var jsons = cates[v.categoryID];
delete cates[v.categoryID];
await that.refreshOSS(conf.indexarticleexcel + '/' + v.categoryID + '/' + catelength[v.categoryID]++ + '.json', `${V.toJsonString(jsons) }`);
}
} else {
if (!catelength[v.categoryID]) catelength[v.categoryID] = 1;
cates[v.categoryID] = [v];
}
return false;
}, 2);
}
return false;
}, true);
//剩余数据处理
await V.forC(cates, async function(k, v) {
if (v.length)
await that.refreshOSS(conf.indexarticleexcel + '/' + k + '/' + catelength[k]++ + '.json', `${V.toJsonString(v) }`);
await that.delOSS(conf.indexarticleexcel + '/' + k + '/' + catelength[k] + '.json');
return false;
}, 2);
//主动删除最后+1页 防止因为文件大批量下架而导致静态文件未重新处理干净
res.clear();
}
async buildIndexArticleHot(ni) {
var res = await ni.excute('r_emb', '~.l_category.select', {
$delFlag: 3,
$ParentID: '!=""',
_order: 'ParentID asc,weight asc',
ID: true,
Name: true,
ParentID: true
});
var CateParent = {};
res.hasData() && res.last()[0].map(v => {
if (v.ParentID) {
CateParent[v.ID] = v.ParentID;
}
return null;
});
res.clear();
//准备好父类的all 和 all
//文章量大了会有问题 先算置顶的 再算全部 再1000行 1000行处理
//todo Cates设置全部和二级全部：一级ID
var cates = { all: [] },
tops = {},
catelength = { all: 1 };
res.clear();
var isRun = true,
w = 0;
await V.whileC(() => (isRun) ? 1 : null, async function() {
res = await ni.excute('r_emb', '~.l_article.select', {
$delFlag: 3,
_order: 'commentCount desc,viewCount desc',
_page: `${w++*1000},1000`,
ID: true,
Name: true,
categoryID: true,
authorID: true,
coverImage: true,
intro: true,
content: true,
keyWords: true,
canComment: true,
isTopic: true,
isExcel: true,
createrName: true,
createTime: true,
source: true,
sourceurl: true,
topStartTime: true,
topEndTime: true
});
isRun = res.hasData();
if (isRun) {
var articles = res.last()[0];
res.clear();
await V.each(articles, async function(v) {
v.isTop = false;
cates['all'].push(v);
if (cates['all'].length >= 10) {
var jsons = cates['all'];
cates['all'] = [];
await that.refreshOSS(conf.indexarticlehot + '/all/' + catelength['all']++ + '.json', `${V.toJsonString(jsons) }`);
}
var parentID = CateParent[v.categoryID];
if (parentID) {
if (cates[parentID]) {
cates[parentID].push(v);
if (cates[parentID].length >= 10) {
var jsons = cates[parentID];
delete cates[parentID];
await that.refreshOSS(conf.indexarticlehot + '/' + parentID + '/' + catelength[parentID]++ + '.json', `${V.toJsonString(jsons) }`);
}
} else {
if (!catelength[parentID]) catelength[parentID] = 1;
cates[parentID] = [v];
}
}
if (cates[v.categoryID]) {
cates[v.categoryID].push(v);
if (cates[v.categoryID].length >= 10) {
var jsons = cates[v.categoryID];
delete cates[v.categoryID];
await that.refreshOSS(conf.indexarticlehot + '/' + v.categoryID + '/' + catelength[v.categoryID]++ + '.json', `${V.toJsonString(jsons) }`);
}
} else {
if (!catelength[v.categoryID]) catelength[v.categoryID] = 1;
cates[v.categoryID] = [v];
}
return false;
}, 2);
}
return false;
}, true);
//剩余数据处理
await V.forC(cates, async function(k, v) {
if (v.length)
await that.refreshOSS(conf.indexarticlehot + '/' + k + '/' + catelength[k]++ + '.json', `${V.toJsonString(v) }`);
await that.delOSS(conf.indexarticlehot + '/' + k + '/' + catelength[k] + '.json');
return false;
}, 2);
//主动删除最后+1页 防止因为文件大批量下架而导致静态文件未重新处理干净
res.clear();
}
async buildCommentReport(ni, params) {
var res = await ni.excute('template9', `hgets from ${params}<ID>`, {});
if ('comment_last'.eq(params))
await this.refreshOSS(conf.commentreport + '/' + params + '.js', `var CommentReport =${V.toJsonString(res.hasData()?res.single():{}) }`);
await this.refreshOSS(conf.commentreport + '/' + params + '.json', `${V.toJsonString(res.hasData()?res.single():{}) }`);
if (res.last()[0].length > 1) console.error('buildCommentReport 数据结构可能需要数组转对象!');
//主动删除最后+1页 防止因为文件大批量下架而导致静态文件未重新处理干净
res.clear();
}
async buildArticleComment(ni, params) {
var res = await ni.excute('r_emb', '~.l_article.select', {
$ID: params,
$delFlag: 3,
commentCount: true,
});
var commentCount = res.hasData() ? res.single().commentCount : 0;
res.clear();
res = await ni.excute('r_emb', '~.l_comment.select', {
$articleID: params,
$parentCommentID: 0,
_order: 'ID desc',
_page: '0,10',
ID: true,
createrName: true,
createrImage: true,
createTime: true,
comments: true,
comment: true
});
await this.refreshOSS(conf.articlecomment + '/' + params + '/comments.js', `var Comments = ${V.toJsonString({
commentCount,
comments:res.hasData()?res.last()[0]:[]
}) }`);
//主动删除最后+1页 防止因为文件大批量下架而导致静态文件未重新处理干净
res.clear();
}
}, this.Log, threadcount, 10000);
});