@letsscrapedata/scraper
Version:
Web scraper that scraping web pages by LetsScrapeData XML template
1 lines • 137 kB
JavaScript
"use strict";var __create=Object.create,__defProp=Object.defineProperty,__getOwnPropDesc=Object.getOwnPropertyDescriptor,__getOwnPropNames=Object.getOwnPropertyNames,__getProtoOf=Object.getPrototypeOf,__hasOwnProp=Object.prototype.hasOwnProperty,__export=(t,e)=>{for(var r in e)__defProp(t,r,{get:e[r],enumerable:!0})},__copyProps=(t,e,r,a)=>{if(e&&"object"==typeof e||"function"==typeof e)for(let s of __getOwnPropNames(e))__hasOwnProp.call(t,s)||s===r||__defProp(t,s,{get:()=>e[s],enumerable:!(a=__getOwnPropDesc(e,s))||a.enumerable});return t},__toESM=(t,e,r)=>(r=null!=t?__create(__getProtoOf(t)):{},__copyProps(!e&&t&&t.__esModule?r:__defProp(r,"default",{value:t,enumerable:!0}),t)),__toCommonJS=t=>__copyProps(__defProp({},"__esModule",{value:!0}),t),src_exports={};__export(src_exports,{TaskParser:()=>TaskParser,TemplateManagerInScraper:()=>TemplateManagerInScraper,performOneTask:()=>performOneTask,scraper:()=>scraper,setScraperLogFun:()=>setScraperLogFun,updateScraperConfig:()=>updateScraperConfig}),module.exports=__toCommonJS(src_exports);var import_controller=require("@letsscrapedata/controller"),import_proxy=require("@letsscrapedata/proxy"),import_utils=require("@letsscrapedata/utils"),pkgLog=import_utils.log;function setScraperLogFun(t){return"function"==typeof t&&(pkgLog=t,(0,import_controller.setControllerLogFun)(pkgLog),(0,import_proxy.setProxyLogFun)(pkgLog),!0)}async function logdbg(...t){await pkgLog(import_utils.LogLevel.DBG,...t)}async function loginfo(...t){await pkgLog(import_utils.LogLevel.INF,...t)}async function logwarn(...t){await pkgLog(import_utils.LogLevel.WRN,...t)}async function logerr(...t){await pkgLog(import_utils.LogLevel.ERR,...t)}(0,import_controller.setControllerLogFun)(pkgLog),(0,import_proxy.setProxyLogFun)(pkgLog),(0,import_utils.setLogLevel)(import_utils.LogLevel.INF);var import_xml2js_lsd=__toESM(require("xml2js-lsd"),1),myfunCfg={type:void 0,arg2:"",arg3:"",arg4:"",arg5:""},defaultSplitOfParases="[\r\n\t,]+",defaultSeperatorOfParas="::",newLineChar="\n",defaultElementCfg={template:{tid:void 0,version:void 0,commited:!1},attrs:{},attr:{name:void 0,value:void 0},paras:{},para:{paraname:void 0,name:"",desc:"",uitype:"",defval:"",listid:"0",listparas:"",min:"",max:"",pattern:"",alert:""},element:{loc:void 0,source:"default",absolute:!1},iframe:{srcprefix:"",loc:""},templstr:{templ:void 0},elecontent_attr:{type:void 0,attrname:void 0,attrname2:"",attrname3:"",attrname4:"",trim:!1,pattern:"",boolattr:!1,multieles:!0,join:newLineChar,line:!1,absolute:!1},elecontent_imgbase64:{type:void 0},elecontent_innerhtml:{type:void 0},elecontent_length:{type:void 0},elecontent_outerhtml:{type:void 0},elecontent_text:{type:void 0,trim:!0,multieles:!0,join:newLineChar,line:!1,top:!1},decodefontsvg:{outtype:"text"},decodefontttf:{fflocs:"",intype:"outerhtml",outtype:"text"},ocrcfg_api:{type:void 0,nlchars:"",seperator:!1,lang:"zh"},ocrcfg_tesseract:{type:void 0,nlchars:"",seperator:!1,lang:"zh"},actions:{},action_break:{type:void 0,id:"0"},action_captcha:{type:void 0,try:"1",errname:"captchafailed",id:"0"},action_click:{type:void 0,wait:"-1",gen:!0,cap:!0,clicktype:"click",try:"1",errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!0,captcha:!0,navigate:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",downloadpath:"",filename:"",pathvarname:"",id:"0"},action_continue:{type:void 0,id:"0"},action_misc:{type:void 0,id:"0"},misc_delsyncdbdata:{type:void 0,tids:"",retention:"2592000",minretention:"86400"},misc_extractdata:{type:void 0,name:void 0,method:"put",paras:!0,execdata:!0,subtasks:!0,credits:!0},misc_getstatedata:{type:void 0,cookies:"new",localstorage:"new",headers:"none",userdata:"none"},action_exit:{type:void 0,errname:void 0,id:"0"},action_extract:{type:void 0,tabname:void 0,id:"0"},column_element:{type:void 0,colname:void 0,nickname:"",completed:!1,setvar:!1,datatype:"string",errname:"ignore",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_ocr:{type:void 0,colname:void 0,nickname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_subtask:{type:void 0,colname:void 0,nickname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},column_templstr:{type:void 0,colname:void 0,nickname:void 0,completed:!1,setvar:!1,datatype:"string",valerrname:"ignore",pattern:"",flags:"",id:"0",discarded:!1},action_goto:{type:void 0,url:void 0,reuse:!1,wait:"-1",encodeuri:!1,gen:!0,cap:!0,datapage:"current",popupsubtask:!1,login:!0,captcha:!0,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",headers:"",referer:"",id:"0"},action_hover:{type:void 0,try:"1",gen:!1,cap:!1,wait:"-1",errname:"cfginvalid",id:"0"},action_ifelse:{type:void 0,id:"0"},condition_element:{type:void 0},condition_else:{type:void 0},condition_templstr:{type:void 0},action_input:{type:void 0,content:void 0,enter:!1,replace:!1,gen:!1,cap:!1,try:"1",wait:"-1",errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_intercept_clear:{type:void 0,subtype:void 0},action_intercept_set:{type:void 0,subtype:void 0},request_abort:{type:void 0,method:"all",url:"",resourcetype:"image",postdata:""},response_cache:{type:void 0,method:"all",url:"",resourcetype:"xhr",postdata:"",minsize:"0",maxsize:"0",contenttype:"text",responses:!0,tabname:"",requestheaders:!1,responseheaders:!1},response_save:{type:void 0,method:"all",url:"",resourcetype:"image",postdata:"",minsize:"0",maxsize:"0",basedir:"response",pathtype:"path_name",hashmethod:"MD5",hostnameparts:"0",extname:"",validextnames:"",params:"",encoding:"",tabname:""},action_loopdowhile_element:{type:void 0,subtype:void 0,iswhile:!1,varname:"",maxloops:"100",click:!1,navigate:!0,gen:!0,cap:!0,errname:"cfginvalid",wait:"-1",id:"0"},action_loopdowhile_templstr:{type:void 0,subtype:void 0,iswhile:!1,varname:"",maxloops:"100",id:"0"},action_loopfor:{type:void 0,from:void 0,to:void 0,step:"1",roundtype:"ceil",varname:void 0,maxloops:"0",errname:"ignore",id:"0"},action_loopineles:{type:void 0,varname:"",maxloops:"0",start:"1",end:"0",step:"1",errname:"ignore",id:"0"},action_loopinstr:{type:void 0,list:void 0,split:defaultSplitOfParases,varname:void 0,maxloops:"0",trim:!0,errname:"ignore",id:"0"},action_scroll_by:{type:void 0,subtype:void 0,height:"100",unit:"windowpct",maxtimes:"1",interval:"800",gen:!1,cap:!1,datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_scroll_intoview:{type:void 0,subtype:void 0,gen:!1,cap:!1,errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_scroll_to:{type:void 0,subtype:void 0,height:"100",unit:"bodypct",gen:!1,cap:!1,datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},action_select:{type:void 0,selecttype:"value",selectval:"",gen:!1,cap:!1,try:"1",wait:"-1",errname:"cfginvalid",datapage:"current",popupsubtask:!1,login:!1,captcha:!1,eurl:"",eloc:"",pn1:"",pv1:"",pn2:"",pv2:"",id:"0"},dbquery:{dbname:void 0,sql:void 0,sqlparas:"",parasplit:defaultSeperatorOfParas,join:newLineChar},action_setvar_dbquery:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_element:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",try:"1",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},file:{path:void 0,encoding:"utf8"},action_setvar_file:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",proxy:!0,valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_get:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_ocr:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_subtask:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},action_setvar_templstr:{type:void 0,subtype:void 0,varname:void 0,defaultval:"",valerrname:"ignore",pattern:"",flags:"",path:"",id:"0"},subtask:{tid:void 0,parasstr:void 0,idx:"",minlen:"1",errname:"cfginvalid",popuppage:!1},action_subtask:{type:void 0,subtasks:"",sameasparent:!1,id:"0"},action_wait_element:{type:void 0,subtype:void 0,timeout:"30000",state:"visible",errname:"ignore",wait:"-1",id:"0"},action_wait_navigation:{type:void 0,subtype:void 0,timeout:"30000",waituntil:"load",url:"",errname:"cfginvalid",wait:"-1",id:"0"},action_wait_sleep:{type:void 0,subtype:void 0,minms:"5000",maxms:"10000",errname:"ignore",wait:"-1",id:"0"},fonts:{},fontsvg:{exloc:"head link[rel='stylesheet']",inloc:"",csmaptype:"general",bsfilter:""},fontttf:{exloc:"head link[rel='stylesheet']",inloc:"",minuc:"57344",maxuc:"63744",startidx:"-1",fsfilter:"",fufilter:"",parsetype:""},fontselector:{name:void 0,fontfamily:void 0},fontfamily:{name:void 0,fontcodes:void 0,fontchars:void 0},fontcodes:{name:void 0,codes:void 0},fontchars:{name:void 0,chars:void 0},myfuns:{},myfun:{name:void 0,arg2:"",arg3:"",arg4:"",arg5:"",funbody:void 0},captcha_amazon:{type:void 0},captcha_funcaptcha:{type:void 0},captcha_geetest:{type:void 0},captcha_keycaptcha:{type:void 0},captcha_mtcaptcha:{type:void 0},captcha_recaptcha:{type:void 0,minscore:"0.3"},captcha_turnstile:{type:void 0},image_element:{type:void 0},comment_element:{type:void 0,attr:"textContent"},input_element:{type:void 0,enter:!1},submit_element:{type:void 0},check_result:{type:void 0,attr:"",failedstr:""},captcha_text:{type:void 0,case:!1,lang:"en",math:!1,maxlen:"0",minlen:"0",question:!1,space:!1,texttype:"any"},captcha_coordinate:{type:void 0,lang:"en",maxclicks:"0",minclicks:"1"},transform:{defaultval:"",usevar:!1},fun_c_attr:{type:void 0,subtype:void 0,loc:void 0,idx:"0",attrname:void 0},fun_c_data:{type:void 0,subtype:void 0,loc:void 0,idx:"0",attrname:void 0},fun_c_html:{type:void 0,subtype:void 0,loc:void 0,idx:"0"},fun_c_length:{type:void 0,subtype:void 0,loc:void 0},fun_c_text:{type:void 0,subtype:void 0,loc:void 0,idx:"-9999"},fun_ceil:{type:void 0},fun_closingsubstr:{type:void 0,openchar:void 0,startstr:"",position:"0"},fun_compress:{type:void 0,method:"gzip",flush:"",finishflush:"",chunksize:"0",windowbits:"0",level:"",memlevel:"",strategy:"",sourceencoding:"utf8",targetencoding:"base64"},fun_concat:{type:void 0,str1:void 0,str2:"",str3:"",str4:"",str5:""},fun_convertencoding:{type:void 0,sourceencoding:"base64",targetencoding:"utf8",start:"0",end:"0"},fun_decompress:{type:void 0,method:"gunzip",flush:"",finishflush:"",chunksize:"0",windowbits:"0",sourceencoding:"base64",targetencoding:"utf8",start:"0",end:"0"},fun_decode:{type:void 0,str1:"",val1:"",str2:"",val2:"",str3:"",val3:"",str4:"",val4:"",str5:"",val5:"",defaultval:""},fun_floor:{type:void 0},fun_hashcode:{type:void 0,hashmethod:"MD5"},fun_includes:{type:void 0,searchstr:void 0},fun_insert:{type:void 0,str:void 0},fun_itemstoobj:{type:void 0,split:defaultSplitOfParases,kvsplit:":",keytrim:!0,keynows:!1,valuetrim:!0,valuenows:!1},fun_jsonparse:{type:void 0,key1:"",key2:"",key3:"",key4:"",key5:"",key6:"",stringify:!1,keys:!1,length:!1,srctype:"json"},fun_length:{type:void 0},fun_matchall:{type:void 0,pattern:void 0,flags:"g",itemvalue:"match[0]",filter:"",start:"0",end:"0",resulttype:"all",join:newLineChar},fun_max:{type:void 0,split:defaultSplitOfParases},fun_min:{type:void 0,split:defaultSplitOfParases},fun_numeq:{type:void 0,num:void 0},fun_numge:{type:void 0,num:void 0},fun_numgt:{type:void 0,num:void 0},fun_numle:{type:void 0,num:void 0},fun_numlt:{type:void 0,num:void 0},fun_notincludes:{type:void 0,searchstr:void 0},fun_padend:{type:void 0,targetlen:void 0,padstr:" "},fun_padstart:{type:void 0,targetlen:void 0,padstr:" "},fun_parsefloat:{type:void 0},fun_parseint:{type:void 0},fun_randomarryidxes:{type:void 0,join:newLineChar},fun_replace:{type:void 0,substr:void 0,newstr:void 0,regexp:!1,flags:""},fun_round:{type:void 0},fun_setcomplement:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setdifference:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setintersection:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_setunion:{type:void 0,array2:void 0,split1:defaultSplitOfParases,split2:defaultSplitOfParases,sorttype:"nosort",valtype:"number",limit:"0",join:newLineChar},fun_slice:{type:void 0,beginidx:void 0,endidx:""},fun_shuffle:{type:void 0,split:defaultSplitOfParases},fun_sort:{type:void 0,split:defaultSplitOfParases,valtype:"number",sorttype:"asc",join:newLineChar},fun_specialconvert:{type:void 0,method:void 0,para1:"",para2:"",para3:"",para4:"",para5:""},fun_split:{type:void 0,split:defaultSplitOfParases,limit:"0",filter:"",start:"0",end:"0",resulttype:"all",val:"",insertstr:"",join:newLineChar},fun_streq:{type:void 0,str:void 0},fun_substrafter:{type:void 0,substr:void 0,position:"0",last:!1,include:!1},fun_substrbefore:{type:void 0,substr:void 0,position:"0",last:!1,include:!1},fun_tobool:{type:void 0},fun_tolowercase:{type:void 0},fun_tonum:{type:void 0},fun_touppercase:{type:void 0},fun_trim:{type:void 0},fun_trimend:{type:void 0},fun_trimstart:{type:void 0},fun_uniq:{type:void 0,split:defaultSplitOfParases,join:newLineChar},fun_urldelparams:{type:void 0,param1:void 0,param2:"",param3:""},fun_urldecode:{type:void 0,method:"URI",encoding:"utf8",percent:!0},fun_urlencode:{type:void 0,method:"URI",encoding:"utf8",percent:!0},fun_urlget:{type:void 0,attrorpath:void 0,hashmethod:"MD5",hostnameparts:"0",params:"",param:"",extname:"",validextnames:"",title:""},fun_urlsetparams:{type:void 0,param1:void 0,val1:"",param2:"",val2:"",param3:"",val3:"",replace:!0},fun_myfun1:Object.assign({},myfunCfg),fun_myfun2:Object.assign({},myfunCfg),fun_myfun3:Object.assign({},myfunCfg),fun_myfun4:Object.assign({},myfunCfg),fun_myfun5:Object.assign({},myfunCfg),get_addedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_api:{type:void 0,method:"GET",url:void 0,encodeuri:!1,session:!1,headerssource:"headers",headers:"",referer:"",proxy:!0,timeout:"30000",context:"default",data:"",datatype:"json"},get_content:{type:void 0},get_cookies:{type:void 0,urls:"",domain:"",name:"",path:"",value:!1},get_datetime:{type:void 0,format:"yyyy-MM-ddThh:mm:ss"},get_execedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_file:{type:void 0,url:void 0,path:"",basedir:"file",pathtype:"path_name",hashmethod:"MD5",hostnameparts:"0",params:"",extname:"",validextnames:"",pathvarname:"",proxy:!0,headers:"",referer:"",setvar:!1},httpbrowser:{name:void 0,minversion:"0",maxversion:"0",httpversion:"0"},httpdevice:{device:void 0},httpheader:{name:void 0,source:"value",value:void 0,delete:!1},httpos:{os:void 0},get_httpheaders:{type:void 0,headermethod:"genLatestWindowsChrome",origheaders:"",browserlist:"",httpversion:"0"},get_mhtml:{type:void 0,path:"",basedir:"mhtml",pathtype:"title",hashmethod:"MD5",hostnameparts:"0",pathvarname:""},get_pdf:{type:void 0,path:"",basedir:"pdf",pathtype:"title",hashmethod:"MD5",hostnameparts:"0",scale:"1",displayheaderfooter:!1,headertemplate:"",footertemplate:"",printbackground:!1,landscape:!1,pageranges:"",format:"Letter",width:"",height:"",top:"",right:"",bottom:"",left:"",screen:!1,onepage:!1,hmargin:"",setvar:!1,pathvarname:""},get_queuedcntwithparas:{type:void 0,tid:void 0,parasstr:void 0,operator:"eq"},get_queuedtaskparas:{type:void 0,tid:void 0,limit:"10000",join:newLineChar,interval:"3600"},get_random:{type:void 0,min:void 0,max:void 0},get_response:{type:void 0,requrl:"",idx:"-1",length:!1},get_screenshot:{type:void 0,basedir:"screenshot",path:"",extname:"png",quality:"-1",fullpage:!1,x:"0",y:"0",width:"0",height:"0",omitbackground:!1,setvar:!1,pathvarname:""},get_special:{type:void 0,method:void 0,parasstr:""},get_title:{type:void 0,aspath:!1},get_window:{type:void 0,key1:void 0,key2:"",key3:"",key4:"",key5:"",key6:"",keys:!1,length:!1}},appConfig={XML_ATTR_KEY:"$$",XML_CHILDREN_KEY:"children",GEN_CREDITS_BITS:16,GEN_CREDITS_AND:65535,CHECK_DNS_DOMAIN:"example.com",CHECK_POPUP_PAGE_TIMEOUT1:35e3,CHECK_POPUP_PAGE_TIMEOUT2:5e3,RESERVED_PARA_NAMES:["authInfo","inParas","sysParas"],EVENT_TYPES:{pagePopup:"pagePopup",pageClose:"pageClose",taskComplete:"taskComplete"},OPERATE_ACTIONS:["click","goto","hover","input","scroll","select","wait","captcha"],performOneTask:console.log,cfg:{exitWhenCompleted:!1,useParasstrInXmlIfNeeded:!1,loadUnfinishedTasks:!1,baseDir:"",baseDownloadDir:"",baseFileDir:"",templateDir:"",taskDir:"",exportsDir:"",inputFileDirPart:"LetsScrapeData",useStealthPlugin:!1,browserControllerType:"patchright",browserType:"chromium",lsdLaunchOptions:{headless:!1},lsdConnectOptions:{browserUrl:""},browserConfigs:[],captcha:{clientKey:""},maxConcurrency:1,readCode:"",templateParas:[],urlPrefix:"https://api.letsscrapedata.com/nologin",totalMaxConcurrency:10,minMiliseconds:2e3,moveDataWhenStart:!1,dataFileFormat:"jsonl",useNickName:!0,columnSeperator:"::"}},TaskParser=class t{static#t(e,r){try{if(!e||!r)throw new Error(`##te Invalid paras, parentEleName ${r} in _applyDefaultElementCfg`);if("object"!=typeof e)return!0;const a=appConfig.XML_ATTR_KEY,s=appConfig.XML_CHILDREN_KEY;e[a]||(e[a]={});let i=r;e[a].type&&(i=i+"_"+e[a].type),e[a].subtype&&(i=i+"_"+e[a].subtype);const n=defaultElementCfg[i];if(!n)throw new Error(`##te No default Cfg for eleName ${i} in _applyDefaultElementCfg`);const o=Object.assign({},n);if(Object.keys(n).forEach(t=>{let r=e[a][t];if("boolean"==typeof o[t])"true"===r||""===r?o[t]=!0:"false"===r&&(o[t]=!1);else if(r||""===r)o[t]=r;else if(void 0===o[t])throw new Error(`##te Attribute ${t} of Element ${i} is undefined`)}),e[a]=o,!e[s])return!0;const c=Array.from(Object.keys(e[s]));for(const r of c){const a=e[s][r];1===a.length&&""===a[0]?e[s][r]=[]:a.forEach(e=>t.#t(e,r))}return!0}catch(t){throw logerr(`##te Error parentEleName ${r} in _applyDefaultElementCfg`),new Error(t)}}static async convertXmlToJson(e,r=!0){try{const a=function(t){const e=t.indexOf("_");return e>0?t.slice(0,e):t},s={attrkey:appConfig.XML_ATTR_KEY,childkey:appConfig.XML_CHILDREN_KEY,explicitChildren:!0,tagNameProcessors:[a]},i=appConfig.XML_CHILDREN_KEY,n=await import_xml2js_lsd.default.parseStringPromise(e,s);if(!(n&&n.template&&n.template[i]&&Array.isArray(n.template[i].actions)))throw new Error(`##te Invalid XML config string: ${e}`);return r&&t.#t(n.template,"template"),n.template}catch(t){throw logerr("##te Exeception in convertXmlToJson"),logerr(`##te xmlCfgStr: ${e}`),new Error(t)}}static getPartOfJsonCfg(t,e,r=!1){try{const r=t[appConfig.XML_CHILDREN_KEY][e][0];return r||null}catch(t){if(r)return null;throw logerr(`##te Exeception when try to get ${e} in getPartOfJsonCfg`),new Error(t)}}static#e(e){try{const r=t.getPartOfJsonCfg(e,"attrs");return!Array.isArray(r?.children?.attr)||r.children.attr.length<=0?[]:r.children.attr.map(t=>t[appConfig.XML_ATTR_KEY])}catch(t){return logerr(t),[]}}static getParaCfgsFromJsonCfg(e){try{const r=t.getPartOfJsonCfg(e,"paras");return!Array.isArray(r?.children?.para)||r.children.para.length<=0?[]:r.children.para.map(t=>t[appConfig.XML_ATTR_KEY])}catch(t){return logerr(t),[]}}static getAttrsInXml(e){const r=t.#e(e),a={};return r.forEach(t=>a[t.name]=t.value),a}static#r(e,r){const a=[],s=appConfig.XML_CHILDREN_KEY;for(const i of e)if(i?.[appConfig.XML_ATTR_KEY]?.type===r)a.push(i);else if(i&&i[s]&&(i[s].action||i[s].condition)){t.#r(i[s].action?i[s].action:i[s].condition,r).forEach(t=>{a.push(t)})}return a}static getCaptchTypes(e){const r=appConfig.XML_CHILDREN_KEY,a=e[r].actions[0][r].action,s=t.#r(a,"captcha"),i=[];for(const t of s){const e=t[appConfig.XML_CHILDREN_KEY]?.captcha;Array.isArray(e)&&e.forEach(t=>{const e=t[appConfig.XML_ATTR_KEY].type;e&&i.push(e)})}return i}static getDatableMapFromJsonCfg(e){try{const r=appConfig.XML_CHILDREN_KEY,a=e[r].actions[0][r].action,s=t.#r(a,"extract"),i=new Map;for(const t of s){const e=t[appConfig.XML_ATTR_KEY].tabname;let a=i.get(e);a||(a=new Map,i.set(e,a));for(const e of t[r].column)if(!e[appConfig.XML_ATTR_KEY].discarded){const t=e[appConfig.XML_ATTR_KEY].colname,r=e[appConfig.XML_ATTR_KEY].nickname?e[appConfig.XML_ATTR_KEY].nickname:t;a.set(t,r)}}return i}catch(t){return null}}static getInParas(t,e=[],r="::"){const a={},s=t.split("@@"),i=["para","apara","rpara"];for(let t=0;t<3;t++){let n=i[t],o=e.filter(t=>t.paraname.startsWith(n)).length;if(s.length>t&&s[t].length>0){let e=s.length>t&&s[t].length>0?s[t].split(r):[];for(let t=1;t<=e.length;t++)a[`${n}${t}`]=e[t-1];for(let t=e.length+1;t<=o;t++)a[`${n}${t}`]=""}}return a}static convertExecData(t,e){if(!e)return t;const r={};for(const a of Object.keys(t)){const s=t[a],i=e.get(a);if(i&&0!==s.length){const t={};Object.keys(s[0]).forEach(e=>{const r=i.get(e);return t[e]=r||e});const e=s.map(e=>{const r={};for(const a of Object.keys(e))r[t[a]]=e[a];return r});r[a]=e}else r[a]=s}return r}},fs=__toESM(require("fs"),1),path=__toESM(require("path"),1),import_utils2=require("@letsscrapedata/utils"),TemplateManagerInScraper=class t{static#a=new Map;static#s=new Set;static#i(t){const e=appConfig.cfg.templateParas.find(e=>e.templateId===t);return e?.readCode?e.readCode:appConfig.cfg.readCode}static async#n(t,e){try{if(!e){if(!appConfig.cfg.templateDir)return null;const r=await(0,import_utils2.filesInDir)(appConfig.cfg.templateDir,"general",`templ${t}_`,"xml");let a=1===r.length?r[0]:`templ${t}.xml`;a=path.join(appConfig.cfg.templateDir,a),e=fs.readFileSync(a,"utf8")}if(!e)return null;return{templateId:t,domainId:1,defaultElementSource:"browser",maxExecutionDuration:600,configDetail:e}}catch(t){return null}}static async#o(e,r){try{if(t.#s.has(e))return null;let a=`${appConfig.cfg.urlPrefix}/template`;a=`${a}?tid=${e}&code=${r}`;const s=await fetch(a);if(200!==s.status)return 403===s.status?logwarn(`Failed to get template ${e}: No privilege`):logwarn(`Failed to get template ${e}, status: ${s.status}`),t.#s.add(e),null;const i=await s.json();if(!i||0!==i.code||!i.data)return logwarn(`Failed to get template ${e}: ${i.message}`),null;if(i.data.templateId!==e)return logerr(`Different templateId ${e} / ${i.templateId}`),null;const{domainId:n=0,defaultElementSource:o,configDetail:c="",maxExecutionDuration:l}=i.data;if(!c)return logerr(`Invalid content in template ${e}`),null;return{templateId:e,domainId:n,defaultElementSource:o,maxExecutionDuration:l,configDetail:c}}catch(t){return logdbg(t),null}}static async parseXmlTemplate(t,e=appConfig.cfg.useNickName){const r=await TaskParser.convertXmlToJson(t),a=TaskParser.getParaCfgsFromJsonCfg(r),s=TaskParser.getPartOfJsonCfg(r,"fonts",!0),i=TaskParser.getPartOfJsonCfg(r,"actions",!1),n=TaskParser.getAttrsInXml(r),o=TaskParser.getCaptchTypes(r),c=e?TaskParser.getDatableMapFromJsonCfg(r):null,l=(0,import_utils2.getCurrentUnixTime)();return{actionConfigs:i,paraCfgs:a,fontsConfig:s||null,attrsInXml:n,captchaTypes:o,datatableMap:c,lastUsedTime:l,lastCheckTime:l}}static async getTemplateConfig(e,r=""){if(!r){const r=t.#a.get(e);if(r)return r}let a=await t.#n(e,r);if(!a){const r=t.#i(e);a=await t.#o(e,r)}if(!a)throw new Error(`Cannot get template config of templateId ${e}`);const s=await t.parseXmlTemplate(a.configDetail),i=Object.assign({},s,{template:a});return t.#a.set(e,i),i}static clearTemplateConfig(e=0){return e>0?t.#a.delete(e):t.#a.clear(),!0}},import_utils11=require("@letsscrapedata/utils"),import_node_events=__toESM(require("events"),1),import_node_events2=require("events"),fs4=__toESM(require("fs"),1),path4=__toESM(require("path"),1),import_utils9=require("@letsscrapedata/utils"),import_controller2=require("@letsscrapedata/controller"),uuid=__toESM(require("uuid"),1),SysParas=class{#c;#l;#p;#g;constructor(t,e,r,a){this.#c=t,this.#l=e,this.#p=r,this.#g=a}get version(){return this.#g}get templateId(){return this.#l}get taskId(){return this.#p}get url(){try{return this.#c?.page?this.#c.page.url():""}catch{return""}}get hasPopupPage(){return this.#c?.popupPage?"1":"0"}get subtaskNum(){const t=this.#c?.subtasks;return Array.isArray(t)?String(t.length):"0"}get datatableNum(){const t=this.#c?.execData;return t&&"object"==typeof t?String(Object.keys(t).length):"0"}get responses(){try{const t=this.#c?.responses;return Array.isArray(t)?JSON.stringify(t):""}catch(t){return""}}get currentDate(){return(new Date).toISOString().slice(0,10)}get currentDateTime(){return(new Date).toISOString().slice(0,19)}get currentIsoTime(){return(new Date).toISOString()}get currentLocalTime(){const t=new Date;return`${t.getFullYear()}-${String(t.getMonth()+1).padStart(2,"0")}-${String(t.getDate()).padStart(2,"0")} ${String(t.getHours()).padStart(2,"0")}:${String(t.getMinutes()).padStart(2,"0")}:${String(t.getSeconds()).padStart(2,"0")}`}get currentUtcTime(){return(new Date).toISOString().slice(0,19).replace("T"," ")}get currentTime(){return(new Date).toISOString().slice(11,19)}get currentUnixMs(){return String(Date.now())}get currentUnixSec(){return String(Math.floor(Date.now()/1e3))}get random10(){return String(Math.floor(10*Math.random()))}get random100(){return String(Math.floor(100*Math.random()))}get random1000(){return String(Math.floor(1e3*Math.random()))}get random10000(){return String(Math.floor(1e4*Math.random()))}get uuidv1(){return uuid.v1()}get uuidv4(){return uuid.v4()}},iconv=__toESM(require("iconv-lite"),1),zlib=__toESM(require("zlib"),1),cheerio=__toESM(require("cheerio"),1),import_utils3=require("@letsscrapedata/utils"),LsdString=class t{static specialConvertFunObj={};static setSpecialConvertFunObj(e){return t.specialConvertFunObj=e,!0}static transformStr(e,r,a,s){let i=e;try{if("string"!=typeof e||!Array.isArray(r))throw new Error("Invalid paras in transformStr");for(const e of r){if("object"!=typeof e)throw new Error("Invalid funCfg in transformStr");const r="c"===e.type?`${e.type}_${e.subtype}`:e.type;let a=t._funPerformers[r];!a&&e.type.match(/^myfun[0-9]$/)&&"object"==typeof s&&(a=s[e.type]),a&&"function"==typeof a?i=a(i,e):logerr(`Invalid funName ${e.type} in transformStr`)}}catch(t){loginfo(t)}return i||a}static __getElementBySelector(t,e,r=0){try{const a="root"===e?t.root():t(e);return a.length<=0||-9999===r?a:a.eq(r)}catch(t){return logerr("##te cheerio invalid $ or loc"),null}}static __cheerioOptions={xmlMode:!0,decodeEntities:!1};static _c_attr(e,r){try{const{loc:a,idx:s,attrname:i}=r,n=parseInt(s),o=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(o,a,n).attr(i)}catch(t){return logerr(`##tmp cheerio c_attr err: ${t}`),""}}static _c_data(e,r){try{const{loc:a,idx:s,attrname:i}=r,n=parseInt(s),o=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(o,a,n).data(i)}catch(t){return logerr(`##tmp cheerio c_data err: ${t}`),""}}static _c_html(e,r){try{const{loc:a,idx:s}=r,i=parseInt(s),n=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(n,a,i).html()}catch(t){return logerr(`##tmp cheerio c_html err: ${t}`),""}}static _c_length(e,r){try{const{loc:a}=r,s=-9999,i=cheerio.load(e,t.__cheerioOptions),n=t.__getElementBySelector(i,a,s);return String(n.length)}catch(t){return logerr(`##tmp cheerio c_length err: ${t}`),"0"}}static _c_text(e,r){try{const{loc:a,idx:s}=r,i=parseInt(s),n=cheerio.load(e,t.__cheerioOptions);return t.__getElementBySelector(n,a,i).text()}catch(t){return logerr(`##tmp cheerio c_text err: ${t}`),""}}static _ceil(t){return String(Math.ceil(Number(t)))}static _closingsubstr(t,e){const{openchar:r,startstr:a,position:s}=e;if("string"!=typeof t||"string"!=typeof r||!["{","[","(","lt"].includes(r)||"string"!=typeof a||"number"!=typeof parseInt(s)||parseInt(s)<0)return"";const i="lt"===r?"<":r,n={"{":"}","[":"]","(":")","<":">"}[i];let o=1,c=-1;if(a){if(c=t.indexOf(a),c<0)return""}else c=parseInt(s);const l=t.indexOf(i,c);if(l<0)return-1;let p=l+1;for(;p<t.length;p++){let e=t[p];if(e===i)o++;else if(e===n&&(o--,0===o))break}const g=p<=t.length?p+1:-1;return g>l?t.substring(l,g):""}static __getCompressOptions(t,e){return t||e?void 0:{}}static _compress(e,r){try{const{method:a,sourceencoding:s,targetencoding:i,start:n,end:o}=r,c=t.__getCompressOptions(r,!0),l=Buffer.from(e,s);let p=null;return"gzip"===a?p=zlib.gzipSync(l,c):"deflate"===a?p=zlib.deflateSync(l,c):"deflateraw"===a?p=zlib.deflateRawSync(l,c):"brotli"===a&&(p=zlib.brotliCompressSync(l,c)),p?t.__convertBufferToString(p,i,n,o):""}catch(t){return""}}static _decompress(e,r){try{const{method:a,sourceencoding:s,targetencoding:i,start:n,end:o}=r,c=t.__getCompressOptions(r,!1),l=Buffer.from(e,s);let p=Buffer.from(e,s);return"gunzip"===a?p=zlib.gunzipSync(l,c):"inflate"===a?p=zlib.inflateSync(l,c):"inflateraw"===a?p=zlib.inflateRawSync(l,c):"unzip"===a?p=zlib.unzipSync(l,c):"brotli"===a&&(p=zlib.brotliDecompressSync(l,c)),p?t.__convertBufferToString(p,i,n,o):""}catch(t){return""}}static _concat(t,e){let r=t;const{str1:a,str2:s,str3:i,str4:n,str5:o}=e;return[a,s,i,n,o].forEach(t=>{"string"==typeof t&&(r=r.concat(t))}),r}static _specialconvert(e,r){let a=e;try{const{method:s}=r;return"function"==typeof t.specialConvertFunObj[s]&&(a=t.specialConvertFunObj[s](e,r)),a}catch(t){return a}}static _decode(t,e){const{str1:r,val1:a,str2:s,val2:i,str3:n,val3:o,str4:c,val4:l,str5:p,val5:g,defaultval:h}=e;try{return t===r?a:t===s?i:t===n?o:t===c?l:t===p?g:h}catch(t){return""}}static __convertBufferToString(t,e,r,a){const s=r?Number(r):0,i=a?Number(a):0;return t instanceof Buffer?i>0?t.toString(e,s,i):s>0?t.toString(e,s):t.toString(e):""}static _convertencoding(e,r){const{sourceencoding:a,targetencoding:s,start:i,end:n}=r;try{const r=Buffer.from(e,a);return t.__convertBufferToString(r,s,i,n)}catch(t){return""}}static _floor(t){return String(Math.floor(Number(t)))}static _hashcode(t,e){const{hashmethod:r}=e;return import_utils3.LsdUrl.getHashCode(t,r)}static _includes(t,e){const{searchstr:r}=e;if("string"!=typeof r)throw new Error("Invalid searchstr in _contains");return String(Number(t.includes(r)))}static _insert(t,e){const{str:r}=e;if("string"!=typeof r)throw new Error("Invalid str in _insert");return r+t}static _itemstoobj(t,e){const{split:r,kvsplit:a,keytrim:s,keynows:i,valuetrim:n,valuenows:o}=e;if(!r||!a)return"{}";try{let e={};const c=t.split(new RegExp(r)),l=new RegExp(a);for(const t of c){const r=t.split(l);if(2!==r.length)continue;let a=s?r[0].trim():r[0];i&&(a=a.split("").filter(t=>t.trim()).join(""));let c=n?r[1].trim():r[1];o&&(c=c.split("").filter(t=>t.trim()).join("")),a&&(e[a]=c)}return JSON.stringify(e)}catch(t){return"{}"}}static _jsonparse(t,e){const{key1:r,key2:a,key3:s,key4:i,key5:n,key6:o,keys:c,length:l,srctype:p}=e;let g=null;try{if("script"===p)return"";if(g=JSON.parse(t),"object"!=typeof g||!g)return""}catch(t){return""}return g=(0,import_utils3.getMemberOfObject)(g,r,a,s,i,n,o),null==g?"":"string"==typeof g?g:"number"==typeof g?String(g):(c&&(g=Array.from(Object.keys(g))),l&&Array.isArray(g)?String(g.length):JSON.stringify(g))}static _length(t){return String(t.length)}static _matchall(t,e){try{const{pattern:r,flags:a,itemvalue:s,filter:i,start:n,end:o,resulttype:c,join:l}=e,p=new RegExp(r,a),g=t.matchAll(p);if("boolean"===c)return g?"1":"0";if(!g)return"";let h=[];for(const t of g)if("match[0]"===s)h.push(t[0]);else{let e=s;for(let r=0;r<t.length;r++)e=e.replaceAll(`match[${r}]`,t[r]);t.index>=0&&(e=e.replaceAll("match.index",String(t.index))),h.push(e)}if(i){const t=new RegExp(i);h=h.filter(e=>e.match(t))}const d=parseInt(n),f=parseInt(o);return 0===d&&0===f||(h=0===f?h.slice(d):h.slice(d,f)),"all"===c?h.join(l):"length"===c?String(h.length):""}catch(t){return loginfo(t),""}}static _max(t,e){try{const{split:r}=e,a=t.split(new RegExp(r)).filter(t=>t.trim()).map(t=>Number(t));return a.length>0?String(a.sort((t,e)=>t-e).pop()):""}catch(t){return loginfo(t),""}}static _min(t,e){try{const{split:r}=e,a=t.split(new RegExp(r)).filter(t=>t.trim()).map(t=>Number(t));return a.length>0?String(a.sort((t,e)=>t-e)[0]):""}catch(t){return loginfo(t),""}}static _numeq(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)===Number(r)?"1":"0"}static _numge(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)>=Number(r)?"1":"0"}static _numgt(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)>Number(r)?"1":"0"}static _numle(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)<=Number(r)?"1":"0"}static _numlt(t,e){const{num:r}=e;return"string"!=typeof t||"string"!=typeof r?"0":t.trim().match(/^\d+$/)&&r.trim().match(/^\d+$/)&&Number(t)<Number(r)?"1":"0"}static _notincludes(t,e){const{searchstr:r}=e;if("string"!=typeof r)throw new Error("Invalid str in _notcontains");return String(Number(!t.includes(r)))}static _padEnd(t,e){const{targetlen:r,padstr:a}=e,s=parseInt(r),i="string"==typeof a&&a.length>0?a:" ";return t.padEnd(s,i)}static _padStart(t,e){const{targetlen:r,padstr:a}=e,s=parseInt(r),i="string"==typeof a&&a.length>0?a:" ";return t.padStart(s,i)}static _parseFloat(t){return String(parseFloat(t))}static _parseInt(t){return String(parseInt(t))}static _randomarryidxes(t,e){const{join:r}=e;try{const e=parseInt(t);return[...Array(e).keys()].sort(()=>Math.random()-.5).join(r)}catch(t){return""}}static _replace(t,e){const{substr:r,newstr:a,regexp:s,flags:i}=e;try{const e=s;if("string"!=typeof r)throw new Error("Invalid substr in _replace");if("string"!=typeof a)throw new Error("Invalid newstr in _replace");return e?t.replace(RegExp(r,i),a):t.replace(r,a)}catch(e){return t}}static _round(t){return String(Math.round(Number(t)))}static __sort(t,e,r){let a;return a="string"===e?"asc"===r?t.sort((t,e)=>t.localeCompare(e)):t.sort((t,e)=>e.localeCompare(t)):"asc"===r?t.sort((t,e)=>parseFloat(t)-parseFloat(e)):t.sort((t,e)=>parseFloat(e)-parseFloat(t)),a}static _setcomplement(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getComplementOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _setdifference(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getDifferenceOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _setintersection(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getIntersectionOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _setunion(e,r){const{array2:a,split1:s,split2:i,sorttype:n,valtype:o,limit:c,join:l}=r;let p,g;try{p=new RegExp(s),g=new RegExp(i)}catch(t){throw new Error(`Invalid regexp split1: ${s}, split2: ${i} in _setcomplement`)}try{const r=parseInt(c);if(!["nosort","asc","desc"].includes(n)||!["string","number"].includes(o)||r<0||"string"!=typeof a||"string"!=typeof l)throw new Error(`Invalid limit: ${c}, sorttype: ${n}, or array2 in _setcomplement`);const s=e.split(p),i=a.split(g);let h=(0,import_utils3.getUnionOfTwoArray)(s,i);return["asc","desc"].includes(n)&&(h=t.__sort(h,o,n)),r>0?h.slice(0,r).join(l):h.join(l)}catch(t){return loginfo(t),e}}static _shuffle(t,e){try{const{split:r}=e;if("string"!=typeof r)throw new Error("Invalid split in _shuffle");const a=t.split(new RegExp(r));return!Array.isArray(a)||a.length<=1?t:a.sort(()=>Math.random()-.5).join(r)}catch(e){return loginfo(e),t}}static _slice(t,e){const{beginidx:r,endidx:a}=e,s=parseInt(r),i="string"==typeof a&&a.length>0?parseInt(a):void 0;return t.slice(s,i)}static _sort(t,e){try{const{split:r,valtype:a,sorttype:s,join:i}=e;if("string"!=typeof r||!["string","number"].includes(a)||!["asc","desc"].includes(s))throw new Error("Invalid split or valtype in _sort");const n=t.split(new RegExp(r));return!Array.isArray(n)||n.length<=1?t:"string"===a?"asc"===s?n.sort((t,e)=>t.localeCompare(e)).join(i):n.sort((t,e)=>e.localeCompare(t)).join(i):"asc"===s?n.sort((t,e)=>parseFloat(t)-parseFloat(e)).join(i):n.sort((t,e)=>parseFloat(e)-parseFloat(t)).join(i)}catch(t){return loginfo(t),""}}static _split(t,e){try{const{split:r,limit:a,filter:s,start:i,end:n,resulttype:o,val:c,insertstr:l,join:p}=e;if("string"!=typeof r)throw new Error("Invalid split in _split");const g=new RegExp(r),h=parseInt(a);let d=h>0?t.split(g,h):t.split(g);if(!Array.isArray(d))return"";if(s){const t=new RegExp(s);d=d.filter(e=>e.match(t))}const f=parseInt(i),u=parseInt(n);if(0===f&&0===u||(d=0===u?d.slice(f):d.slice(f,u)),l&&(d=d.map(t=>`${l}${t}`)),"all"===o)return d.join(p);if("length"===o)return String(d.length);if("index"===o){const t=d.findIndex(t=>t.trim()===c);return t>=0?String(t+1):""}return""}catch(t){return loginfo(t),""}}static _streq(t,e){const{str:r}=e;return"string"!=typeof r?"0":t===r?"1":"0"}static _substrAfter(t,e){const{substr:r,position:a,last:s,include:i}=e,n=i,o=parseInt(a);if("string"!=typeof r)throw new Error("Invalid substr in _substrAfter");let c=0;return c=s?o?t.lastIndexOf(r,o):t.lastIndexOf(r):o?t.indexOf(r,o):t.indexOf(r),-1===c?t:String(n?t.substring(c):t.substring(c+r.length))}static _substrBefore(t,e){const{substr:r,position:a,last:s,include:i}=e,n=parseInt(a),o=i;if("string"!=typeof r)throw new Error("Invalid substr in substrBefore");let c=0;return c=s?n?t.lastIndexOf(r,n):t.lastIndexOf(r):n?t.indexOf(r,n):t.indexOf(r),-1===c?t:o?t.substring(0,c+r.length):t.substring(0,c)}static _toBool(t){try{let e=t.replace(/true/g,"1");return e=e.replace(/false/g,"0"),e.search(/^[\s\d.+\-*/()<>=!|&:?]*$/)>-1&&-1===e.search(/=>/)?String(Number(Boolean((0,eval)(e)))):(loginfo(`Invalid logic expression ${t} in _toBool`),!1)}catch(t){return!1}}static _toLowerCase(t){return t.toLowerCase()}static _toNum(t){let e=t;return e.search(/^[\s\d.+\-*/()<>=!|&:?]+$/)>-1&&-1===e.search(/=>/)&&e.trim().length>0?String(Number((0,eval)(e))):(loginfo(`Invalid arithmetic expression ${e} in _toNum`),"0")}static _toUpperCase(t){return t.toUpperCase()}static _trim(t){return t.trim()}static _trimEnd(t){return t.trimEnd()}static _trimStart(t){return t.trimStart()}static _uniq(t,e){try{const{split:r,join:a}=e;if("string"!=typeof r||"string"!=typeof a)throw new Error("Invalid split in _uniq");const s=t.split(new RegExp(r));return!Array.isArray(s)||s.length<=1?t:[...new Set(s)].join(a)}catch(e){return loginfo(e),t}}static _urlencode(t,e){try{const{method:r,encoding:a,percent:s}=e;if("URI"===r)return encodeURI(t);if("URIComponent"===r)return encodeURIComponent(t);if("iconv"===r){const e=iconv.encode(t,a).toString("hex").toUpperCase();if(!e||e.length%2!=0)return logerr("Invalid str length in _urlencode"),"";if(s){const t=e.match(/../g);return t?t.map(t=>`%${t}`).join(""):""}return e}return logerr(`Invalid method ${r} in _urlencode`),""}catch(t){return logerr(t),""}}static _urldecode(t,e){try{const{method:r,encoding:a,percent:s}=e;if("URI"===r)return decodeURI(t);if("URIComponent"===r)return decodeURIComponent(t);if("iconv"===r){const e=s?/.../g:/../g,r=t.match(e);if(!r)return"";const i=r.map(t=>parseInt(t.slice(1),16));return iconv.decode(Buffer.from(i),a)}return logerr(`Invalid method ${r} in _urlencode`),""}catch(t){return logerr(t),""}}static _urldelparams(t,e){const{param1:r,param2:a,param3:s}=e;if("string"!=typeof r)throw new Error("Invalid params in _urldel");let i="";try{const e=new URL(t);if(!e||!e.searchParams||!e.searchParams.delete)return"";r&&e.searchParams.delete(r),a&&e.searchParams.delete(a),s&&e.searchParams.delete(s),i=e.href}catch(t){i=""}return i||""}static _urlget(t,e){const{attrorpath:r,hashmethod:a,param:s,hostnameparts:i,params:n,title:o,extname:c,validextnames:l}=e;if("string"!=typeof r)throw new Error("Invalid attr in _urlget");let p="";try{p=["href","origin","protocol","username","password","host","search","hash","hostname","port","hrefwithoutsearch","param"].includes(r)?import_utils3.LsdUrl.getUrlAttribute(t,r,i,s):import_utils3.LsdUrl.getFilePathFromUrl(t,r,a,i,n,c,l,o)}catch(t){p=""}return"string"==typeof p?p:""}static __setUrlParam(t,e,r,a){if(!t||!t.searchParams||!t.searchParams.get||"string"!=typeof e||"string"!=typeof r)return!1;if(!e)return!0;if(!a){if(null!==t.searchParams.get(e))return!0}return t.searchParams.set(e,r),!0}static _urlsetparams(e,r){const{param1:a,val1:s,param2:i,val2:n,param3:o,val3:c,replace:l}=r,p=l;if("string"!=typeof a)throw new Error("Invalid params in _urlset");let g="";try{const r=new URL(e);t.__setUrlParam(r,a,s,p),t.__setUrlParam(r,i,n,p),t.__setUrlParam(r,o,c,p),g=r.href}catch(t){g=""}return g||""}static _funPerformers={c_attr:t._c_attr,c_data:t._c_data,c_html:t._c_html,c_length:t._c_length,c_text:t._c_text,ceil:t._ceil,closingsubstr:t._closingsubstr,compress:t._compress,concat:t._concat,convertencoding:t._convertencoding,decode:t._decode,decompress:t._decompress,floor:t._floor,hashcode:t._hashcode,includes:t._includes,insert:t._insert,itemstoobj:t._itemstoobj,jsonparse:t._jsonparse,length:t._length,matchall:t._matchall,max:t._max,min:t._min,numeq:t._numeq,numge:t._numge,numgt:t._numgt,numle:t._numle,numlt:t._numlt,notincludes:t._notincludes,padend:t._padEnd,padstart:t._padStart,parsefloat:t._parseFloat,parseint:t._parseInt,randomarryidxes:t._randomarryidxes,replace:t._replace,round:t._round,setcomplement:t._setcomplement,setdifference:t._setdifference,setintersection:t._setintersection,setunion:t._setunion,shuffle:t._shuffle,slice:t._slice,sort:t._sort,specialconvert:t._specialconvert,split:t._split,streq:t._streq,substrafter:t._substrAfter,substrbefore:t._substrBefore,tobool:t._toBool,tolowercase:t._toLowerCase,tonum:t._toNum,touppercase:t._toUpperCase,trim:t._trim,trimend:t._trimEnd,trimstart:t._trimStart,uniq:t._uniq,urldecode:t._urldecode,urldelparams:t._urldelparams,urlencode:t._urlencode,urlget:t._urlget,urlsetparams:t._urlsetparams}};function getTransformExpStr(t,e,r="",a={}){return LsdString.transformStr(t,e,r,a)}var fs2=__toESM(require("fs"),1),path2=__toESM(require("path"),1),import_got_scraping=require("got-scraping"),import_header_generator=require("header-generator"),import_dateformat=__toESM(require("dateformat"),1),uuid2=__toESM(require("uuid"),1),import_utils4=require("@letsscrapedata/utils"),import_hpagent=require("hpagent"),download=__toESM(require("download"),1);async function lsdDownload(t,e="",r=null){const a={};if(e){const t=new import_hpagent.HttpsProxyAgent({keepAlive:!0,keepAliveMsecs:1e3,maxSockets:256,maxFreeSockets:256,scheduling:"lifo",proxy:e});a.agent={https:t}}let s;return r&&(a.headers=r),s=e||r?await download.default(t,void 0,a):await download.default(t),s}var GetPara=class t{static getSpecialFunObj={};static setGetSpecialFunObj(e){return t.getSpecialFunObj=e,!0}static getOtherFunObj={};static setGetOtherFunObj(e){return t.getOtherFunObj=e,!0}static noNeedToReturnContent="";static async getPara(e,r){if(!e||!e.type||!r)return"";const a=e.type;let s="";switch(a){case"addedtaskparas":"function"==typeof t.getOtherFunObj.getAddedTaskParasFun&&(s=await t.getOtherFunObj.getAddedTaskParasFun(e));break;case"api":s=await t.getApi(e,r);break;case"datetime":s=t.getDatetime(e);break;case"execedtaskparas":"function"==typeof t.getOtherFunObj.getExecedTaskParasFun&&(s=await t.getOtherFunObj.getExecedTaskParasFun(e));break;case"file":s=await t.getFile(e,r);break;case"httpheaders":s=await t.getHttpHeaders(e,r);break;case"queuedcntwithparas":"function"==typeof t.getOtherFunObj.getQueuedCntWithParasCfgFun&&(s=await t.getOtherFunObj.getQueuedCntWithParasCfgFun(e));break;case"queuedtaskparas":"function"==typeof t.getOtherFunObj.getQueuedTaskParasFun&&(s=await t.getOtherFunObj.getQueuedTaskParasFun(e));break;case"random":s=t.getRandom(e);break;case"response":s=await t.getResponse(e,r);break;case"special":s=await t.getSpecial(e,r);break;case"sysparas":s="please use ${sysParas.paraname}!!!";break;default:(0,import_utils4.unreachable)(a)}return s}static async getParaInElement(e,r,a){if(!e||!e.type||!a)return"";const s=e.type,i=a.paras;let n="";if("screenshot"===s)n=await t.getScreenshot(r,e,i);else(0,import_utils4.unreachable)(s);return n}static async getParaInPage(e,r,a){if(!e||!e.type||!a)return"";const s=e.type,i=a.paras;let n="";switch(s){case"content":n=await t.getContent(r);break;case"cookies":n="";break;case"mhtml":n=await t.getMhtml(r,e,i);break;case"pdf":n=await t.getPdf(r,e,i);break;case"screenshot":n=await t.getScreenshot(r,e,i);break;case"title":n=await t.getTitle(r,e);break;case"window":n=await t.getWindow(r,e);break;default:(0,import_utils4.unreachable)(s)}return n}static async getApi(e,r){const{method:a,url:s,encodeuri:i,session:n,headerssource:o,headers:c,referer:l,proxy:p,data:g,datatype:h}=e,d=e.context,f=e.timeout?Number(e.timeout):0;let u="";if(!["GET","DELETE","POST","PUT"].includes(a.toUpperCase()))return logerr(`##task invalid method ${a} in getApi`),"";const m=s.startsWith("/");try{const e=i?encodeURI(s):s;if(m){if("function"==typeof t.getOtherFunObj.getDataFromServerFun){const r=f>0?{timeout:f}:void 0;return await t.getOtherFunObj.getDataFromServerFun(a,e,g,r)}return""}{const t={method:a.toUpperCase()};if(p){const e=r.proxyUrl?r.proxyUrl:"";e&&(t.proxyUrl=e)}n&&r.proxyUrl&&"default"===d&&(t.sessionToken=r.proxyUrl);const s="Content-Length";let i=!1,m={};if("headers"!==o||"headers"===o&&c||l){let a="";const n=r.interceptionData;if("headers"===o&&c?(m=JSON.parse(c),i="0"===m[s]):"cachedRequestHeaders"===o?(a=n.cachedRequestUrl,m=Object.assign({},n.cachedRequestHeaders),i=void 0!==m[s]):"cachedResponseHeaders"===o?(a=n.cachedResponseUrl,m=Object.assign({},n.cachedResponseHeaders),i=void 0!==m[s]):"origStateHeaders"===o?(a=n.origStateUrl,m=Object.assign({},n.origStateHeaders),i=void 0!==m[s]):"updatedRequestHeaders"===o?(a=n.updatedRequestUrl,m=Object.assign({},n.updatedRequestHeaders),i=void 0!==m[s]):"updatedResponseHeaders"===o?(a=n.updatedResponseUrl,m=Object.assign({},n.updatedResponseHeaders),i=void 0!==m[s]):"updatedStateHeaders"===o&&(a=n.updatedStateUrl,m=Object.assign({},n.updatedStateHeaders),i=void 0!==m[s]),o.endsWith("Headers")){const t=new URL(e),r=new URL(a);if(t.origin!==r.origin){if(!["origStateHeaders","updatedStateHeaders"].includes(o))return logerr(`Differenct url origins in getApi: req ${t.origin}, orig ${r.origin} `),"";if(!t.hostname.endsWith(r.hostname))return logerr(`Invalid url hostname in getApi: req ${t.hostname}, orig ${r.hostname} `),""}}l&&(m.referer=l),t.headers=m}if(g)if(i&&(m[s]=String(g.length)),"browser"===d||"state"===d||"task"===d||"standalone"===d)"json"===h?t.data=JSON.parse(g):"form"===h?t.form=JSON.parse(g):t.data=g;else if("default"===d)"json"===h?t.data=JSON.parse(g):"form"===h?t.form=JSON.parse(g):t.body=g;else if("fetch"===d)if("json"===h)t.data=JSON.parse(g);else if("form"===h){const e=new FormData,r=JSON.parse(g);for(const t of Object.keys(r))e.set(t,r[t]);t.data=e}else t.body=g;else logerr(`Invalid context ${d} in getApi`);if("browser"===d||"state"===d||"task"===d||"standalone"===d){const a=r.apiContext;if(!a)throw new Error("Null ApiContext in getApi");f>0&&(t.timeout=f);u=(await a.fetch(e,t)).text}else if("default"===d){f>0&&(t.timeout={request:f});u=(await(0,import_got_scraping.gotScraping)(e,t)).body}else if("fetch"===d){const r=await fetch(e,t);u=await r.text()}else logerr(`Invalid context ${d} in getApi`);return"string"==typeof u?u:JSON.stringify(u)}}catch(t){return loginfo(t),""}}static async getContent(t){try{if(!t)return"";return await t.content()}catch(t){return""}}static async getCookies(t,e){try{if(!t)return"";const{urls:r,domain:a,name:s,path:i,value:n}=e;let o="";r.length>0&&r.split(";");let c=await t.cookies();return a&&c.filter(t=>t.domain===a),s&&c.filter(t=>t.name===s),i&&c.filter(t=>t.path===i),o=n?c.length>0?c[0].value:"":JSON.stringify(c),o}catch(t){return""}}static getDatetime(t){const{format:e}=t;return(0,import_dateformat.default)(new Date,e)}static async getFile(e,r){const a=r.paras;try{const{url:s,pathtype:i,hashmethod:n,hostnameparts:o,params:c,extname:l,validextnames:p,basedir:g,setvar:h,pathvarname:d,proxy:f,headers:u,referer:m}=e;let y=e.path.trim();if(y)y=path2.join(y);else{let t=import_utils4.LsdUrl.getFilePathFromUrl(s,i,n,Number(o),c,l,p);y=path2.join(g,t)}if(y){const t=path2.dirname(y);if(!aw