UNPKG

owhat-job

Version:

owhat红包清理

1,205 lines (1,085 loc) 38 kB
'use strict' require('array.prototype.flatmap').shim(); // const config = require("config"); // console.log(config); // const esConfig = config.get('EsConnect.dbConfig'); const { Client } = require('@elastic/elasticsearch') const client = new Client({ node: "http://172.17.102.128:9200" //esConfig.url }) const articleIndexName = "law-article", topicIndexName = "law-topic"; /******* * 文章映射 * **** */ const articleMapping = { properties: { id: { type: 'keyword', }, name: { type: 'text', analyzer: 'ik_max_word', "search_analyzer": "ik_smart", "fields": { "full_pinyin": { //全拼分词 "type": "completion", "analyzer": "full_pinyin_analyzer", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "prefix_pinyin": { //拼音首字母分词 "type": "completion", "analyzer": "prefix_pinyin_analyzer", "search_analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "standardText": { //单字分词 "type": "completion", "analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 } } }, authorID: { type: 'keyword', }, authorName: { type: 'keyword', }, coverImage: { type: 'keyword', }, topicID: { type: 'keyword' }, firstCategory: { type: 'nested', properties: { categoryID: { type: 'integer' }, categoryName: { type: 'text', analyzer: 'ik_max_word', "search_analyzer": "ik_smart", "fields": { "full_pinyin": { //全拼分词 "type": "completion", "analyzer": "full_pinyin_analyzer", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "prefix_pinyin": { //拼音首字母分词 "type": "completion", "analyzer": "prefix_pinyin_analyzer", "search_analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "standardText": { //单字分词 "type": "completion", "analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 } } } } }, secondCategory: { type: 'nested', properties: { categoryID: { type: 'integer' }, categoryName: { type: 'text', analyzer: 'ik_max_word', "search_analyzer": "ik_smart", "fields": { "full_pinyin": { //全拼分词 "type": "completion", "analyzer": "full_pinyin_analyzer", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "prefix_pinyin": { //拼音首字母分词 "type": "completion", "analyzer": "prefix_pinyin_analyzer", "search_analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "standardText": { //单字分词 "type": "completion", "analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 } } } } }, intro: { type: 'text', analyzer: 'ik_max_word', "search_analyzer": "ik_smart", "fields": { "full_pinyin": { //全拼分词 "type": "completion", "analyzer": "full_pinyin_analyzer", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "prefix_pinyin": { //拼音首字母分词 "type": "completion", "analyzer": "prefix_pinyin_analyzer", "search_analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "standardText": { //单字分词 "type": "completion", "analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 } } }, content: { type: 'text', analyzer: 'ik_max_word', "search_analyzer": "ik_smart", "fields": { "full_pinyin": { //全拼分词 "type": "completion", "analyzer": "full_pinyin_analyzer", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "prefix_pinyin": { //拼音首字母分词 "type": "completion", "analyzer": "prefix_pinyin_analyzer", "search_analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "standardText": { //单字分词 "type": "completion", "analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 } } }, keyWords: { type: 'keyword' // , // "fields": { // "keyword": { // "type": "keyword", // "ignore_above": 2048 // } // } }, source: { type: 'keyword' }, sourceurl: { type: 'keyword' }, isTopic: { type: 'integer' }, isTop: { type: 'integer' }, isHomeTop: { type: 'integer' }, isExcel: { type: 'integer' }, canComment: { type: 'integer' }, allowComment: { type: 'integer' }, commentCount: { type: 'integer' }, viewCount: { type: 'integer' }, createTime: { type: 'date' // , // format: "YYYY-MM-DD HH:mm:ss" } // , // "full_pinyin": { //全拼分词 // "type": "completion", // "analyzer": "full_pinyin_analyzer", // // "payloads": true, // "preserve_separators": false, // "preserve_position_increments": true, // "max_input_length": 50 // }, // "prefix_pinyin": { //拼音首字母分词 // "type": "completion", // "analyzer": "prefix_pinyin_analyzer", // "search_analyzer": "standard", // // "payloads": true, // "preserve_separators": false, // "preserve_position_increments": true, // "max_input_length": 50 // }, // "suggestText": { //单字分词 // "type": "completion", // "analyzer": "standard", // // "payloads": true, // "preserve_separators": false, // "preserve_position_increments": true, // "max_input_length": 50 // } } }; const setting = { "analysis": { "filter": { // "_pattern": { // "type": "pattern_capture", // "preserve_original": "true", // "patterns": ["([0-9])", "([a-z])"] // }, "full_pinyin": { "keep_first_letter": "false", "keep_none_chinese_in_first_letter": "false", "type": "pinyin", "keep_original": "false", "keep_full_pinyin": "true" }, "prefix_pinyin": { "keep_first_letter": "true", "none_chinese_pinyin_tokenize": "false", "type": "pinyin", "keep_original": "false", "keep_full_pinyin": "false" }, "ik_pinyin": { "type": "pinyin", "keep_separate_first_letter": true, "keep_full_pinyin": true, "keep_original": true, "limit_first_letter_length": 16, "lowercase": true, "remove_duplicated_term": true } }, "analyzer": { "full_pinyin_analyzer": { "filter": ["lowercase", "full_pinyin"], "tokenizer": "standard" }, "prefix_pinyin_analyzer": { "filter": ["lowercase", "prefix_pinyin"], "tokenizer": "standard" }, "ik_smart_pinyin": { "type": "custom", "tokenizer": "ik_smart", "filter": ["ik_pinyin", "word_delimiter"] }, "ik_max_word_pinyin": { "type": "custom", "tokenizer": "ik_max_word", "filter": ["ik_pinyin", "word_delimiter"] } } } } /******* * 课题映射 * **** */ const topicMapping = { properties: { id: { type: 'keyword', }, name: { type: 'text', analyzer: 'ik_max_word', "search_analyzer": "ik_smart", "fields": { "full_pinyin": { //全拼分词 "type": "completion", "analyzer": "full_pinyin_analyzer", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "prefix_pinyin": { //拼音首字母分词 "type": "completion", "analyzer": "prefix_pinyin_analyzer", "search_analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "standardText": { //单字分词 "type": "completion", "analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 } } }, firstCategory: { type: 'nested', properties: { categoryID: { type: 'integer' }, categoryName: { type: 'text', analyzer: 'ik_max_word', "search_analyzer": "ik_smart", "fields": { "full_pinyin": { //全拼分词 "type": "completion", "analyzer": "full_pinyin_analyzer", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "prefix_pinyin": { //拼音首字母分词 "type": "completion", "analyzer": "prefix_pinyin_analyzer", "search_analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "standardText": { //单字分词 "type": "completion", "analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 } } } } }, secondCategory: { type: 'nested', properties: { categoryID: { type: 'integer' }, categoryName: { type: 'text', analyzer: 'ik_max_word', "search_analyzer": "ik_smart", "fields": { "full_pinyin": { //全拼分词 "type": "completion", "analyzer": "full_pinyin_analyzer", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "prefix_pinyin": { //拼音首字母分词 "type": "completion", "analyzer": "prefix_pinyin_analyzer", "search_analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "standardText": { //单字分词 "type": "completion", "analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 } } } } }, intro: { type: 'text', analyzer: 'ik_max_word', "search_analyzer": "ik_smart", "fields": { "full_pinyin": { //全拼分词 "type": "completion", "analyzer": "full_pinyin_analyzer", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "prefix_pinyin": { //拼音首字母分词 "type": "completion", "analyzer": "prefix_pinyin_analyzer", "search_analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 }, "standardText": { //单字分词 "type": "completion", "analyzer": "standard", // "payloads": true, "preserve_separators": false, "preserve_position_increments": true, "max_input_length": 50 } } }, authorID: { type: 'keyword' }, articleID: { type: 'keyword' }, creater: { type: 'keyword' }, // creater: { // type: 'text', // analyzer: 'ik_max_word', // "search_analyzer": "ik_smart", // fields: { // keyword: { // type: "keyword", // ignore_above: 256 // } // } // }, createTime: { type: 'date' }, delFlag: { type: 'integer' } } } /******* * 检查数据的索引是否准确 * **** */ let indices = function indices() { return esClient.cat.indices({ v: true }) .then(console.log) .catch(err => console.error(`Error connecting to the es client: ${err}`)); }; let existsByID = async(index, id) => { const { body } = await client.exists({ index: index, id: id }) console.log(body) // true return body; }; /******* * 判断索引是否存在 * **** */ //index string | string[] let exists = async(index) => { const { body } = await client.indices.exists({ index: index }); return body; } /******* * 创建索引(同时mapping) * **** */ let createIndex = async(indexName, mappings = {}, settings = {}) => { const { body } = await client.indices.create({ index: indexName, body: { mappings: mappings, settings: settings } }, { ignore: [400] }); return body; } // createIndex(articleIndexName, articleMapping, setting).then(item => { // console.log(item); // }); // createIndex(topicIndexName, topicMapping, setting).then(item => { // console.log(item); // }); // exists(["tweets"]).then(item => { // console.log(item); // }) // existsByID("tweets", 1).catch(console.log).then(v => { // console.log("vvvvvvvvvvvv" + v); // }); /******* * 批量操作 * **** */ let bulk_law = async function(indexName, data) { const body = data.flatMap(doc => [{ index: { _index: indexName, _id: doc.id } }, doc]); // console.log(body); const { body: bulkResponse } = await client.bulk({ refresh: true, body }) if (bulkResponse.errors) { console.log(bulkResponse) process.exit(1) } } /******* * 批量添加文章 * **** */ exports.bulkArticle = async function(data) { // const dataset = [{ // id: '1ecab369caee11e9a66300163e258174', // name: '张三', // authorID: '2962b37acaee11e9a66300163e308174', // authorName: '中国人', // coverImage: 'http://localhost:5251/80909c3a9cb4c0b3fdfbd7785f2d008f0732f90859da1d8d4b84445125fab616.jpg', // topicID: 'd5dd6691cbf911e9a66300163e308174', // firstCategory: { // categoryID: 1, // categoryName: '互联网金融' // }, // secondCategory: { // categoryID: 2, // categoryName: '银行' // }, // keyWords: ['互联网'], // intro: 'dddddddddddddddd', // content: '祖国统一祖国统一祖国统一祖国统一祖国统一祖国统一祖国统一祖国统一祖国统一祖国统一祖国统一祖国统一', // source: '', // sourceurl: '', // isTopic: 1, // isTop: 0, // isHomeTop: 0, // isExcel: 1, // canComment: 1, // allowComment: 1, // commentCount: 10, // viewCount: 1000, // createTime: new Date() // }, { // id: '873300fcce3b22e9a66300163e308174', // name: '第5个话题', // authorID: '2962b37acaee11e9a66300163e308174', // authorName: '黎明', // coverImage: 'http://localhost:5251/998146b732a0a47b68ff38322901be645b6c4f1da0c0c0c1d0647496eb677ef0.jpg', // firstCategory: { // categoryID: 1, // categoryName: '互联网金融' // }, // secondCategory: { // categoryID: 2, // categoryName: '银行' // }, // intro: 'sd的撒发达阿迪斯发啊中国', // content: '地方你il几给奥', // keyWords: ['金融'], // source: '', // sourceurl: '', // isTopic: 0, // isTop: 1, // isHomeTop: 0, // isExcel: 1, // canComment: 1, // allowComment: 1, // commentCount: 55, // viewCount: 22, // createTime: new Date() // }, { // id: 'd8216558ce2311e9a66300163e308174', // name: '东方之珠-香港', // authorID: '2962b37acaee11e9a66300163e308174', // authorName: '中国人', // coverImage: 'http://localhost:5251/fc2cb6078e42d390f548d85e88d0f6f69acaa30121e73435779c0e876df361f7.jpg', // firstCategory: { // categoryID: 12, // categoryName: '电子商务与社交' // }, // secondCategory: { // categoryID: 13, // categoryName: '购物' // }, // intro: 'aaaaaaaaaaaaaaa', // content: '这是香港的内容', // keyWords: ['其他'], // source: '', // sourceurl: '', // isTopic: 0, // isTop: 0, // isHomeTop: 0, // isExcel: 1, // canComment: 1, // allowComment: 1, // commentCount: 900, // viewCount: 5000, // createTime: new Date() // }]; // var data = [{ // id: '16b750f3f7ac48ec810b9a7cf6bb4832', // name: '222333', // authorID: 'bd34c1dec9ab11e9b332000c29187cba', // authorName: undefined, // coverImage: 'https://data.elecadd.com/web/temp/2019-10-09/282697bba98d744356f220631ec0ceb3.jpeg', // topicID: null, // firstCategory: { // categoryID: 13, // categoryName: '国际时局' // }, // secondCategory: { // categoryID: 14, // categoryName: '经济' // }, // keyWords: ['测试'], // intro: '333222', // content: '<p>333222</p >', // source: '哈哈', // sourceurl: 'https://www.baidu.com', // isTopic: 0, // isTop: 0, // isHomeTop: 0, // isExcel: 0, // canComment: 1, // allowComment: 1, // commentCount: 0, // viewCount: undefined, // createTime: '2019-10-09T07:48:37.000Z' // }]; await bulk_law(articleIndexName, data); }; // exports.bulkArticle(); /******* * 批量添加课题 * **** */ exports.bulkTopic = async function(data) { const dataset = [{ id: '314124cdcbfa11e9a66300163e308174', name: '测试进行中课题', firstCategory: { categoryID: 1, categoryName: '互联网金融' }, secondCategory: { categoryID: 2, categoryName: '银行' }, intro: 'dddddddddddddddd', creater: '0ced5fefcbfa11e9a66300163e308174', createTime: new Date('2019/9/16 15:03:30'), delFlag: 3 }, { id: 'd5dd6691cbf911e9a66300163e308174', name: '课题1', firstCategory: { categoryID: 12, categoryName: '电子商务与社交' }, secondCategory: { categoryID: 13, categoryName: '购物' }, authorID: '6cbb1c41cbf611e9a66300163e308174', articleID: '1ecab369caee11e9a66300163e308174', intro: '课题1介绍', creater: '0ced5fefcbfa11e9a66300163e308174', createTime: new Date('2019/9/16 15:03:30'), delFlag: 3 }]; await bulk_law(topicIndexName, dataset); } // exports.bulkTopic(); /******* * 获取文章列表 * **** */ exports.getArticles = async(searchStr, pageSize = 10, pageIndex = 1, searchAfterPros = []) => { // const deepCount = 10000; // const deepPage = deepCount % pageSize == 0 ? deepCount / pageSize : deepCount / pageSize + 1; // //true 深分页 false 浅分页 // const sType = pageIndex > deepPage; //pageIndex * pageSize > 10; const itemIndex = (pageIndex - 1) * pageSize; let query = {}; if (!searchStr || searchStr === "") { query["match_all"] = {}; } else { query["bool"] = { should: [{ nested: { path: [ "firstCategory" ], query: { bool: { must: [{ match: { "firstCategory.categoryName": searchStr } }] } } } }, { nested: { path: [ "secondCategory" ], query: { bool: { must: [{ match: { "secondCategory.categoryName": searchStr } }] } } } }, { multi_match: { query: searchStr, type: "best_fields", //完全匹配的文档占的评分比较高,则需要使用best_fields fields: ["name", "name.full_pinyin", "name.prefix_pinyin", "name.standardText", "content", "content.full_pinyin", "content.prefix_pinyin", "content.standardText" ], tie_breaker: 0.3 } }] } } const { body } = await client.search({ index: articleIndexName, body: { query: query, from: itemIndex, size: pageSize, highlight: { number_of_fragments: 1, //返回结果最多可以包含几段不连续的文字 fragment_size: 10, //一段fragment 包含多少个字符 // no_match_size: 2, //即使字段中没有关键字命中,也可以返回一段文字,该参数表示从0-2索引字符返回 fields: { "content": { // number_of_fragments: 0, //返回结果最多可以包含几段不连续的文字 // fragment_size: 10, //一段fragment 包含多少个字符 "pre_tags": ['<i class="ic">'], "post_tags": ['</i>'], "no_match_size": 1 }, "name": { "pre_tags": ["<em>"], "post_tags": ["</em>"], "no_match_size": 1 } } }, sort: [{ "isHomeTop": "desc" }, { "isTop": "desc" }, { "createTime": "desc" } ] } }); const hits = body.hits; // console.log(hits) const total = hits.total.value; // console.log(hits.hits); let items = hits.hits.map(item => { let obj = Object.assign({}, item._source); obj["ID"] = obj.id; obj["highlight"] = item.highlight; delete obj.id; return obj; }); let returnObj = { "total": total, "items": items, "searchAfterPros": [] } console.log(returnObj); return returnObj; } // exports.getArticles("222333", 10, 1) /******* * 根据文章ID删除文章 * **** */ exports.delArticleByID = async(articleID) => { var res = await client.delete({ id: articleID, index: articleIndexName, refresh: true }).catch(error => { console.log(error.meta.body); }); if (res) { console.log("======文章搜索引擎下架成功======" + res); } } // exports.delArticleByID('16b750f3f7ac48ec810b9a7cf6bb4832') //获取课题列表 exports.getTopics = async(searchStr, pageSize = 10, pageIndex = 1, searchAfterPros = []) => { // const deepCount = 10000; // const deepPage = deepCount % pageSize == 0 ? deepCount / pageSize : deepCount / pageSize + 1; // //true 深分页 false 浅分页 // const sType = pageIndex > deepPage; //pageIndex * pageSize > 10; const itemIndex = (pageIndex - 1) * pageSize; let query = {}; if (!searchStr || searchStr === "") { query["match_all"] = {}; } else { query["bool"] = { should: [{ nested: { path: [ "firstCategory" ], query: { bool: { must: [{ match: { "firstCategory.categoryName": searchStr } }] } } } }, { nested: { path: [ "secondCategory" ], query: { bool: { must: [{ match: { "secondCategory.categoryName": searchStr } }] } } } }, { multi_match: { query: searchStr, type: "best_fields", //完全匹配的文档占的评分比较高,则需要使用best_fields fields: ["name", "name.full_pinyin", "name.prefix_pinyin", "name.standardText", "intro", "intro.full_pinyin", "intro.prefix_pinyin", "intro.standardText" ], tie_breaker: 0.3 } }] } // query["multi_match"] = { // query: searchStr, // type: "best_fields", //完全匹配的文档占的评分比较高,则需要使用best_fields // fields: ["name", "content"], // tie_breaker: 0.3 // }; } const { body } = await client.search({ index: topicIndexName, body: { query: query, from: itemIndex, size: pageSize, highlight: { fields: { "intro": { // number_of_fragments: 0, //返回结果最多可以包含几段不连续的文字 // fragment_size: 10, //一段fragment 包含多少个字符 "pre_tags": ["<em>"], "post_tags": ["</em>"], }, "name": { "pre_tags": ["em"], "post_tags": ["</em>"] } } }, sort: [{ "createTime": "desc" }] } }); const hits = body.hits; // console.log(hits) const total = hits.total.value; // console.log(hits.hits); let items = hits.hits.map(item => { let obj = Object.assign({}, item._source); obj["ID"] = obj.id; obj["Name"] = obj.name; obj["highlight"] = item.highlight; delete obj.id; delete obj.name; return obj; }); let returnObj = { "total": total, "items": items, "searchAfterPros": [] } console.log(returnObj); return returnObj; } // exports.getTopics("课题1"); //“文章”搜索建议 exports.suggestArticle = async(searchStr) => { const { body } = await client.search({ index: articleIndexName, body: { // query: { // multi_match: { // query: searchStr, // type: "best_fields", //完全匹配的文档占的评分比较高,则需要使用best_fields // fields: ["name", "content"], // tie_breaker: 0.3 // } // }, suggest: { text: searchStr, // "suggest-keyWords": { // // text: searchStr, // term: { // // "suggest_mode": "missing", // field: 'keyWords.keyword' // } // }, "suggest-content": { // text: searchStr, term: { "suggest_mode": "always", field: 'content', analyzer: 'ik_smart' } } } } }).catch(error => { console.log(error.meta.body); }); console.log(body.suggest); } // exports.suggestArticle("心"); //相似文章 exports.likeArticles = async(keyWords = [], size = 10) => { const { body } = await client.search({ index: articleIndexName, body: { query: { bool: { must: [{ terms: { "keyWords": keyWords } }] } }, from: 0, size: size, sort: [{ "createTime": "desc" }] } }); const hits = body.hits; // console.log(hits) const total = hits.total.value; // console.log(hits.hits); let items = hits.hits.map(item => { let obj = Object.assign({}, item._source); obj["ID"] = obj.id; obj["highlight"] = item.highlight; delete obj.id; return obj; }); let returnObj = { "total": total, "items": items, "searchAfterPros": [] } console.log(returnObj); return returnObj; } // exports.likeArticles(['其他']);