w-dwdata-ftp
Version:
A downloader for ftp data.
642 lines (517 loc) • 19.7 kB
HTML
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>WDwdataFtp.mjs - Documentation</title>
<script src="scripts/prettify/prettify.js"></script>
<script src="scripts/prettify/lang-css.js"></script>
<!--[if lt IE 9]>
<script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
<link type="text/css" rel="stylesheet" href="styles/prettify.css">
<link type="text/css" rel="stylesheet" href="styles/jsdoc.css">
<script src="scripts/nav.js" defer></script>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
</head>
<body>
<input type="checkbox" id="nav-trigger" class="nav-trigger" />
<label for="nav-trigger" class="navicon-button x">
<div class="navicon"></div>
</label>
<label for="nav-trigger" class="overlay"></label>
<nav >
<h2><a href="index.html">Home</a></h2><h3>Global</h3><ul><li><a href="global.html#WDwdataFtp">WDwdataFtp</a></li></ul>
</nav>
<div id="main">
<h1 class="page-title">WDwdataFtp.mjs</h1>
<section>
<article>
<pre class="prettyprint source linenums"><code>import fs from 'fs'
import get from 'lodash-es/get.js'
import size from 'lodash-es/size.js'
import each from 'lodash-es/each.js'
import map from 'lodash-es/map.js'
import cloneDeep from 'lodash-es/cloneDeep.js'
import isbol from 'wsemi/src/isbol.mjs'
import isestr from 'wsemi/src/isestr.mjs'
import isp0int from 'wsemi/src/isp0int.mjs'
import isfun from 'wsemi/src/isfun.mjs'
import ispm from 'wsemi/src/ispm.mjs'
import cdbl from 'wsemi/src/cdbl.mjs'
import pmSeries from 'wsemi/src/pmSeries.mjs'
import getErrorMessage from 'wsemi/src/getErrorMessage.mjs'
import fsIsFolder from 'wsemi/src/fsIsFolder.mjs'
import fsCopyFile from 'wsemi/src/fsCopyFile.mjs'
import fsCleanFolder from 'wsemi/src/fsCleanFolder.mjs'
import fsCreateFolder from 'wsemi/src/fsCreateFolder.mjs'
import fsDeleteFolder from 'wsemi/src/fsDeleteFolder.mjs'
import fsSyncFolder from 'wsemi/src/fsSyncFolder.mjs'
import fsTreeFolderWithHash from 'wsemi/src/fsTreeFolderWithHash.mjs'
import fsGetFileBasicHash from 'wsemi/src/fsGetFileBasicHash.mjs'
import WDwdataBuilder from 'w-dwdata-builder/src/WDwdataBuilder.mjs'
import downloadFiles from './downloadFiles.mjs'
/**
* 基於檔案之下載FTP數據與任務建構器
*
* 執行階段最新hash數據放置於fdDwAttime，前次hash數據會於結束前自動備份至fdDwCurrent
*
* 執行階段最新數據放置於fdDwStorageTemp，前次數據放置於fdDwStorage，於結束前會將fdDwStorage清空，將fdDwStorageTemp複製至fdDwStorage
*
* @param {String} st 輸入設定FTP連線資訊物件
* @param {String} [st.transportation='FTP'] 輸入傳輸協定字串，可選'FTP'、'SFTP'，預設'FTP'
* @param {String} [st.hostname=''] 輸入hostname字串，預設''
* @param {Integer} [st.port=21|22] 輸入port正整數，當transportation='FTP'預設21，當transportation='SFTP'預設22
* @param {String} [st.username=''] 輸入帳號字串，預設''
* @param {String} [st.password=''] 輸入密碼字串，預設''
* @param {String} [st.fdIni='./'] 輸入同步資料夾字串，預設'./'
* @param {Object} [opt={}] 輸入設定物件，預設{}
* @param {Boolean} [opt.useExpandOnOldFiles=false] 輸入來源檔案是否僅為增量檔案布林值，預設false
* @param {Boolean} [opt.useSimulateFiles=false] 輸入是否使用模擬取得FTP數據布林值，預設false
* @param {String} [opt.fdTagRemove='./_tagRemove'] 輸入暫存標記為刪除數據資料夾字串，預設'./_tagRemove'
* @param {String} [opt.fdDwStorageTemp='./_dwStorageTemp'] 輸入最新下載檔案存放資料夾字串，預設'./_dwStorageTemp'
* @param {String} [opt.fdDwStorage='./_dwStorage'] 輸入合併儲存檔案資料夾字串，預設'./_dwStorage'
* @param {String} [opt.fdDwAttime='./_dwAttime'] 輸入當前下載供比對hash用之數據資料夾字串，預設'./_dwAttime'
* @param {String} [opt.fdDwCurrent='./_dwCurrent'] 輸入已下載供比對hash用之數據資料夾字串，預設'./_dwCurrent'
* @param {String} [opt.fdResult=`./_result`] 輸入已下載數據所連動生成數據資料夾字串，預設`./_result`
* @param {String} [opt.fdTaskCpActualSrc='./_taskCpActualSrc'] 輸入任務狀態之來源端完整資料夾字串，預設'./_taskCpActualSrc'
* @param {String} [opt.fdTaskCpSrc='./_taskCpSrc'] 輸入任務狀態之來源端資料夾字串，預設'./_taskCpSrc'
* @param {String} [opt.fdLog='./_logs'] 輸入儲存log資料夾字串，預設'./_logs'
* @param {Function} [opt.funDownload=null] 輸入取得最新下載檔案hash之函數，回傳資料陣列，預設null
* @param {Function} [opt.funGetCurrent=null] 輸入取得已下載檔案hash之函數，回傳資料陣列，預設null
* @param {Function} [opt.funAdd=null] 輸入當有新資料時，需要連動處理之函數，預設null
* @param {Function} [opt.funModify=null] 輸入當有資料需更新時，需要連動處理之函數，預設null
* @param {Function} [opt.funRemove=null] 輸入當有資料需刪除時，需要連動處理之函數，預設null
* @param {Function} [opt.funAfterStart=null] 輸入偵測程序剛開始啟動時，需要處理之函數，預設null
* @param {Function} [opt.funBeforeEnd=null] 輸入偵測程序要結束前，需要處理之函數，預設null
* @param {Number} [opt.timeToleranceRemove=0] 輸入刪除任務之防抖時長，單位ms，預設0，代表不使用
* @returns {Object} 回傳事件物件，可呼叫函數on監聽change事件
* @example
*
* import w from 'wsemi'
* import WDwdataFtp from './src/WDwdataFtp.mjs'
*
* let st = {
* 'hostname': '{hostname}',
* 'port': 21,
* 'username': '{username}',
* 'password': '{password}',
* 'fdIni': './'
* }
* // console.log('st', st)
*
* //fdTagRemove
* let fdTagRemove = `./_tagRemove`
* w.fsCleanFolder(fdTagRemove)
*
* //fdDwStorageTemp
* let fdDwStorageTemp = `./_dwStorageTemp`
* w.fsCleanFolder(fdDwStorageTemp)
*
* //fdDwStorage
* let fdDwStorage = `./_dwStorage`
* w.fsCleanFolder(fdDwStorage)
*
* //fdDwAttime
* let fdDwAttime = `./_dwAttime`
* w.fsCleanFolder(fdDwAttime)
*
* //fdDwCurrent
* let fdDwCurrent = `./_dwCurrent`
* w.fsCleanFolder(fdDwCurrent)
*
* //fdResult
* let fdResult = `./_result`
* w.fsCleanFolder(fdResult)
*
* //fdTaskCpActualSrc
* let fdTaskCpActualSrc = `./_taskCpActualSrc`
* w.fsCleanFolder(fdTaskCpActualSrc)
*
* //fdTaskCpSrc
* let fdTaskCpSrc = `./_taskCpSrc`
* w.fsCleanFolder(fdTaskCpSrc)
*
* let opt = {
* useExpandOnOldFiles: false, //true, false
* fdTagRemove,
* fdDwStorageTemp,
* fdDwStorage,
* fdDwAttime,
* fdDwCurrent,
* fdResult,
* fdTaskCpActualSrc,
* fdTaskCpSrc,
* // fdLog,
* // funDownload,
* // funGetCurrent,
* // funRemove,
* // funAdd,
* // funModify,
* }
* let ev = await WDwdataFtp(st, opt)
* .catch((err) => {
* console.log(err)
* })
* ev.on('change', (msg) => {
* delete msg.type
* console.log('change', msg)
* })
* // change { event: 'start', msg: 'running...' }
* // change { event: 'proc-callfun-afterStart', msg: 'start...' }
* // change { event: 'proc-callfun-afterStart', msg: 'done' }
* // change { event: 'proc-callfun-download', msg: 'start...' }
* // change { event: 'proc-callfun-download', num: 2, msg: 'done' }
* // change { event: 'proc-callfun-getCurrent', msg: 'start...' }
* // change { event: 'proc-callfun-getCurrent', num: 0, msg: 'done' }
* // change { event: 'proc-compare', msg: 'start...' }
* // change { event: 'proc-compare', numRemove: 0, numAdd: 2, numModify: 0, numSame: 0, msg: 'done' }
* // change { event: 'proc-add-callfun-add', id: 'test1.txt', msg: 'start...' }
* // change { event: 'proc-add-callfun-add', id: 'test1.txt', msg: 'done' }
* // change { event: 'proc-add-callfun-add', id: 'test2.txt', msg: 'start...' }
* // change { event: 'proc-add-callfun-add', id: 'test2.txt', msg: 'done' }
* // ...
*
*/
let WDwdataFtp = async(st, opt = {}) => {
//useExpandOnOldFiles
let useExpandOnOldFiles = get(opt, 'useExpandOnOldFiles')
if (!isbol(useExpandOnOldFiles)) {
useExpandOnOldFiles = false
}
//useSimulateFiles, 檔案得預先給予至fdDwStorageTemp
let useSimulateFiles = get(opt, 'useSimulateFiles')
if (!isbol(useSimulateFiles)) {
useSimulateFiles = false
}
//fdTagRemove
let fdTagRemove = get(opt, 'fdTagRemove')
if (!isestr(fdTagRemove)) {
fdTagRemove = `./_tagRemove`
}
//fdDwStorageTemp, 最新下載檔案存放資料夾
let fdDwStorageTemp = get(opt, 'fdDwStorageTemp')
if (!isestr(fdDwStorageTemp)) {
fdDwStorageTemp = `./_dwStorageTemp`
}
if (!fsIsFolder(fdDwStorageTemp)) {
fsCreateFolder(fdDwStorageTemp)
}
//fdDwStorage, 合併儲存檔案資料夾
let fdDwStorage = get(opt, 'fdDwStorage')
if (!isestr(fdDwStorage)) {
fdDwStorage = `./_dwStorage`
}
if (!fsIsFolder(fdDwStorage)) {
fsCreateFolder(fdDwStorage)
}
//fdDwAttime
let fdDwAttime = get(opt, 'fdDwAttime')
if (!isestr(fdDwAttime)) {
fdDwAttime = `./_dwAttime`
}
if (!fsIsFolder(fdDwAttime)) {
fsCreateFolder(fdDwAttime)
}
//fdDwCurrent
let fdDwCurrent = get(opt, 'fdDwCurrent')
if (!isestr(fdDwCurrent)) {
fdDwCurrent = `./_dwCurrent`
}
if (!fsIsFolder(fdDwCurrent)) {
fsCreateFolder(fdDwCurrent)
}
//fdResult
let fdResult = get(opt, 'fdResult')
if (!isestr(fdResult)) {
fdResult = `./_result`
}
if (!fsIsFolder(fdResult)) {
fsCreateFolder(fdResult)
}
//fdTaskCpActualSrc
let fdTaskCpActualSrc = get(opt, 'fdTaskCpActualSrc')
if (!isestr(fdTaskCpActualSrc)) {
fdTaskCpActualSrc = `./_taskCpActualSrc`
}
if (!fsIsFolder(fdTaskCpActualSrc)) {
fsCreateFolder(fdTaskCpActualSrc)
}
//fdTaskCpSrc
let fdTaskCpSrc = get(opt, 'fdTaskCpSrc')
if (!isestr(fdTaskCpSrc)) {
fdTaskCpSrc = './_taskCpSrc'
}
if (!fsIsFolder(fdTaskCpSrc)) {
fsCreateFolder(fdTaskCpSrc)
}
//fdLog
let fdLog = get(opt, 'fdLog')
if (!isestr(fdLog)) {
fdLog = './_logs'
}
if (!fsIsFolder(fdLog)) {
fsCreateFolder(fdLog)
}
//funDownload
let funDownload = get(opt, 'funDownload')
//funGetCurrent
let funGetCurrent = get(opt, 'funGetCurrent')
//funAdd
let funAdd = get(opt, 'funAdd')
//funModify
let funModify = get(opt, 'funModify')
//funRemove
let funRemove = get(opt, 'funRemove')
//funAfterStartCall
let funAfterStartCall = get(opt, 'funAfterStart')
//funBeforeEndCall
let funBeforeEndCall = get(opt, 'funBeforeEnd')
//timeToleranceRemove
let timeToleranceRemove = get(opt, 'timeToleranceRemove')
if (!isp0int(timeToleranceRemove)) {
timeToleranceRemove = 0
}
timeToleranceRemove = cdbl(timeToleranceRemove)
//getFilesHash
let getFilesHash = async(vfps) => {
//ltdtHash
let ltdtHash = await pmSeries(vfps, async(v) => {
let id = v.name //用檔名做id
let hash = await fsGetFileBasicHash(v.path, { type: 'md5' })
return {
id,
hash,
}
})
return ltdtHash
}
//treeFolderAndGetFilesHash
let treeFolderAndGetFilesHash = async (fd) => {
//vfps
let vfps = await fsTreeFolderWithHash(fd, 1, { type: 'md5', forFile: true, forFolder: false })
//ltdtHash
let ltdtHash = map(vfps, (v) => {
let id = v.name //用檔名做id
let hash = v.hash
return {
id,
hash,
}
})
return ltdtHash
}
//cvLtdtToKp
let cvLtdtToKp = (ltdt) => {
let kp = {}
each(ltdt, (v) => {
kp[v.id] = v.hash
})
return kp
}
//mergeLtdt
let mergeLtdt = (ltdtNew, ltdtOld) => {
let kpNew = cvLtdtToKp(ltdtNew)
let kpOld = cvLtdtToKp(ltdtOld)
let kp = cloneDeep(kpOld)
each(kpNew, (hash, id) => {
kp[id] = hash
})
let ltdt = []
each(kp, (hash, id) => {
ltdt.push({
id,
hash,
})
})
return ltdt
}
//vfpsDw
let vfpsDw = []
//funDownloadDef
let funDownloadDef = async() => {
//vfpsDw, 為下載所得新增檔案清單, 檔案放置於fdDwStorageTemp內
vfpsDw = await downloadFiles(st, fdDwStorageTemp, {
useExpandOnOldFiles,
useSimulateFiles,
})
// console.log('vfpsDw', vfpsDw[0], size(vfpsDw))
//check, 無檔案須報錯, 底層會中止與計算hash與計算檔案hash差異
if (size(vfpsDw) === 0) {
throw new Error(`no files`)
}
//ltdtHashNewTemp, 計算新增檔案hash值
let ltdtHashNewTemp = await getFilesHash(vfpsDw)
//ltdtHashNew
let ltdtHashNew = []
if (useExpandOnOldFiles) {
//ltdtHashOld, 數據來源為fdDwStorage, 為舊hash清單
let ltdtHashOld = await treeFolderAndGetFilesHash(fdDwStorage)
//最新合併後檔案hash值清單
ltdtHashNew = mergeLtdt(ltdtHashNewTemp, ltdtHashOld)
}
else {
//當前下載檔案為全部檔案, 各檔案計算hash值皆須為新hash
ltdtHashNew = ltdtHashNewTemp
}
// console.log('ltdtHashNew', ltdtHashNew)
//清空fdDwAttime
fsCleanFolder(fdDwAttime)
//儲存新hash檔案至fdDwAttime
each(ltdtHashNew, (v) => {
//fp
let fp = `${fdDwAttime}/${v.id}.json` //v.id雖為檔名但視為id使用, fdDwAttime與fdDwCurrent內檔案皆為對應hash檔案, 副檔名為.json
//writeFileSync
fs.writeFileSync(fp, JSON.stringify(v), 'utf8')
})
return ltdtHashNew
}
if (!isfun(funDownload)) {
funDownload = funDownloadDef
}
//funGetCurrentDef
let funGetCurrentDef = async() => {
//ltdtHashOld, 數據來源為fdDwStorage, 為舊hash清單
let ltdtHashOld = await treeFolderAndGetFilesHash(fdDwStorage)
return ltdtHashOld
}
if (!isfun(funGetCurrent)) {
funGetCurrent = funGetCurrentDef
}
//funRemoveDef
let funRemoveDef = async(v) => {
let fd = `${fdResult}/${v.id}`
if (fsIsFolder(fd)) {
let r = fsDeleteFolder(fd)
if (r.error) {
throw new Error(r.error)
}
}
}
if (!isfun(funRemove)) {
funRemove = funRemoveDef
}
//funAddDef
let funAddDef = async(v) => {
let fd = `${fdResult}/${v.id}`
if (!fsIsFolder(fd)) {
fsCreateFolder(fd)
}
fsCleanFolder(fd)
let fpStorage = `${fdDwStorageTemp}/${v.id}` //fdDwStorageTemp為新下載檔案, 使用v.id為實際檔案名稱
let fpResult = `${fd}/${v.id}`
let r = fsCopyFile(fpStorage, fpResult)
if (r.error) {
throw new Error(r.error)
}
}
if (!isfun(funAdd)) {
funAdd = funAddDef
}
//funModifyDef
let funModifyDef = async(v) => {
let fd = `${fdResult}/${v.id}`
if (!fsIsFolder(fd)) {
fsCreateFolder(fd)
}
fsCleanFolder(fd)
let fpStorage = `${fdDwStorageTemp}/${v.id}` //fdDwStorageTemp為新下載檔案, 使用v.id為實際檔案名稱
let fpResult = `${fd}/${v.id}`
let r = fsCopyFile(fpStorage, fpResult)
if (r.error) {
throw new Error(r.error)
}
}
if (!isfun(funModify)) {
funModify = funModifyDef
}
//funBeforeEndNec
let funBeforeEndNec = async() => {
try {
srlog.info({ event: 'move-files-to-storage', msg: 'start...' })
//check, 無檔案須報錯, 底層會中止與計算hash與計算檔案hash差異
if (size(vfpsDw) === 0) {
throw new Error(`no files`)
}
//useExpandOnOldFiles
if (useExpandOnOldFiles) {
//增量模式, 僅將新下載檔案儲存至fdDwStorage
//複製fdDwStorageTemp內新下載檔案至合併儲存資料夾fdDwStorage
each(vfpsDw, (v) => {
//fsCopyFile
let fpSrc = v.path
let fpTar = `${fdDwStorage}/${v.name}`
let r = fsCopyFile(fpSrc, fpTar)
//check
if (r.error) {
throw new Error(r.error)
}
})
}
else {
//全量模式, 將新下載檔案視為全部檔案並儲存至fdDwStorage
//fsSyncFolder, 將fdDwStorageTemp完全同步至fdDwStorage
await fsSyncFolder(fdDwStorageTemp, fdDwStorage)
}
//清空fdDwStorageTemp
fsCleanFolder(fdDwStorageTemp)
srlog.info({ event: 'move-files-to-storage', msg: 'done' })
}
catch (err) {
console.log(err)
srlog.error({ event: 'move-files-to-storage', msg: getErrorMessage(err) })
}
}
let funAfterStart = async() => {
if (isfun(funAfterStartCall)) {
let r = funAfterStartCall()
if (ispm(r)) {
r = await r
}
}
//無funAfterStartNec
}
let funBeforeEnd = async() => {
await funBeforeEndNec()
if (isfun(funBeforeEndCall)) {
let r = funBeforeEndCall()
if (ispm(r)) {
r = await r
}
}
}
let optBdr = {
fdTagRemove,
fdDwAttime,
fdDwCurrent,
fdResult,
fdTaskCpActualSrc,
fdTaskCpSrc,
fdLog,
funDownload,
funGetCurrent,
funRemove,
funAdd,
funModify,
funAfterStart,
funBeforeEnd,
timeToleranceRemove,
}
let ev = await WDwdataBuilder(optBdr)
//srlog
let srlog = ev.srlog
return ev
}
export default WDwdataFtp
</code></pre>
</article>
</section>
</div>
<br class="clear">
<footer>
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 4.0.4</a> on Tue Sep 23 2025 22:03:35 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
</footer>
<script>prettyPrint();</script>
<script src="scripts/polyfill.js"></script>
<script src="scripts/linenumber.js"></script>
</body>
</html>