UNPKG

cejs

Version:

A JavaScript module framework that is simple to use.

1,574 lines (1,419 loc) 118 kB
/** * @name CeL function for Ajax (Asynchronous JavaScript and XML) * @fileoverview 本檔案包含了模擬WWW客戶端發送HTTP/HTTPS請求用的 functions。 * @since 2015/1/1 */ 'use strict'; // 'use asm'; // -------------------------------------------------------------------------------------------- // 不採用 if 陳述式,可以避免 Eclipse JSDoc 與 format 多縮排一層。 typeof CeL === 'function' && CeL.run({ // module name name : 'application.net.Ajax', // Promise for fetch() require : 'data.code.compatibility.' // library_namespace.copy_properties() + '|data.native' // MIME_of() + '|application.net.MIME.' // for CeL.to_file_name(), CeL.URI, CeL.Search_parameters + '|application.net.', // 設定不匯出的子函式。 // no_extend : '*', // 為了方便格式化程式碼,因此將 module 函式主體另外抽出。 code : module_code }); function module_code(library_namespace) { function time_message(millisecond) { if (library_namespace.age_of) { return library_namespace.age_of(0, millisecond, { digits : 1 }); } return millisecond % 1000 === 0 ? millisecond / 1000 + 's' : millisecond + 'ms'; } // "Error: socket hang up" {code: 'ECONNRESET'} // "Error: connect ETIMEDOUT 1.1.1.1:80" // {errno:'ETIMEDOUT', code: 'ETIMEDOUT', address: '125.89.70.31', port:80 } // Error: connect ECONNREFUSED 127.0.0.1:443 // "Error: read ECONNRESET" // {errno: 'ECONNRESET', code: 'ECONNRESET', syscall: 'read'} // Error: getaddrinfo ENOTFOUND domain // ERROR_BAD_STSTUS // Error: Timeout 30s function localize_error(error) { var message = String(error); if (library_namespace.gettext) { // 處理特別的錯誤訊息。 var matched = message .match(/^(Error: (?:(?:connect|getaddrinfo) E[A-Z]+|Timeout) )(.+)$/); if (matched) { message = [ matched[1] + '%1', matched[2] ]; } message = Array.isArray(message) // gettext_config:{"id":"error-connect-etimedout-$1","mark_type":"combination_message_id"} // gettext_config:{"id":"error-connect-econnrefused-$1","mark_type":"combination_message_id"} // gettext_config:{"id":"error-getaddrinfo-enotfound-$1","mark_type":"combination_message_id"} // gettext_config:{"id":"error-timeout-$1","mark_type":"combination_message_id"} ? library_namespace.gettext.apply(null, message) // gettext_config:{"id":"error-socket-hang-up","mark_type":"combination_message_id"} // gettext_config:{"id":"error-read-econnreset","mark_type":"combination_message_id"} // gettext_config:{"id":"error-write-econnaborted","mark_type":"combination_message_id"} // gettext_config:{"id":"error-unexpected-end-of-file","mark_type":"combination_message_id"} : library_namespace.gettext(message); } return message; } var /** {Number}未發現之index。 const: 基本上與程式碼設計合一,僅表示名義,不可更改。(=== -1) */ NOT_FOUND = ''.indexOf('_'); /** * null module constructor * * @class web Ajax 的 functions */ var _// JSDT:_module_ = function() { // null module constructor }; /** * for JSDT: 有 prototype 才會將之當作 Class */ _// JSDT:_module_ .prototype = {}; // ---------------------------------------------------------------------// // XMLHttp set ajax通信処理ライブラリ ================== /** * <code> to use: include in front: way1(good: 以reg代替functionPath!): // [function.js]_iF // [function.js]End way2(old): // [function.js]getU,functionPath,'eval(getU(functionPath));' // [function.js]End old: function getU(p){var o;try{o=new ActiveXObject('Microsoft.XMLHTTP');}catch(e){o=new XMLHttpRequest();}if(o)with(o){open('GET',p,false),send(null);return responseText;}} </code> */ /** * JScript or .wsh only, 能 encode. * * @param {String}page_url * page url * @param {String}[charset] * character encoding of HTML web page. e.g., 'UTF-8', big5, * euc-jp, ... * @param POST_text * POST text * * @returns {String} * @see http://neural.cs.nthu.edu.tw/jang/books/asp/getWebPage.asp?title=10-1%20%E6%8A%93%E5%8F%96%E7%B6%B2%E9%A0%81%E8%B3%87%E6%96%99 */ function get_page(page_url, charset, POST_text) { try { // may cause error var X = new ActiveXObject('Microsoft.XMLHTTP'), AS; X.open(POST_text ? 'POST' : 'GET', page_url, false); // POST need this X.setRequestHeader("Content-Type", "application/x-www-form-urlencoded"); // Download the file X.send(POST_text || null); AS = new ActiveXObject("ADODB.Stream"); // 可同時進行讀寫 AS.Mode = 3; // 以二進位方式操作 AS.Type = 1; // 開啟物件 AS.Open(); // 將 binary 的資料寫入物件內 may error AS.Write(X.responseBody); AS.Position = 0; // 以文字模式操作 AS.Type = 2; // 設定編碼方式 if (charset) AS.Charset = charset; // 將物件內的文字讀出 X = AS.ReadText(); // Release memory. 釋放被占用的記憶體. AS = null; return X; } catch (e) { library_namespace.warn('get_page: ' + e.message); } } // ---------------------------------------------------------------------// var KEY_URL = typeof Symbol === 'function' ? Symbol('URL') : '\0URL to fetch'; /** * * @param URL_to_fetch * @param search * @param hash * @returns * * @inner */ function set_parameters_and_hash(URL_to_fetch, search, hash) { // URL_to_fetch = library_namespace.URI(URL_to_fetch); // assert: library_namespace.is_URI(URL_to_fetch) if (hash || hash === '') { if (Object.defineProperty[KEY_not_native] && !/^#/.test(hash)) hash = '#' + hash; URL_to_fetch.hash = hash; } URL_to_fetch.search_params.set_parameters(search); // console.trace(URL_to_fetch.toString(charset)); return URL_to_fetch; } function normalize_URL_to_fetch(URL_to_fetch, charset, options) { // console.trace(URL_to_fetch); // https://developer.mozilla.org/en-US/docs/Web/API/URL // [ origin + pathname, search, hash ] // hrer = [].join('') if (Array.isArray(URL_to_fetch)) { URL_to_fetch = set_parameters_and_hash(library_namespace .URI(URL_to_fetch[0]), URL_to_fetch[1], URL_to_fetch[2]); if (charset) URL_to_fetch.charset = charset; } else { // 當輸入 {URL} 時,node_https.request() 會將 {URL} 轉成 // {Object}options,不會考慮額外選項 (headers, ...)。 // 且必須處理 charset,乾脆直接將 {URL} 轉成尋常 plain object / {URI}。 // https://nodejs.org/api/http.html#http_http_request_url_options_callback // If url is a string, it is automatically parsed with new URL(). If // it is a URL object, it will be automatically converted to an // ordinary options object. // console.trace([ URL_to_fetch, charset ]); URL_to_fetch = library_namespace.URI(URL_to_fetch, null, { charset : charset }); // console.trace(URL_to_fetch); } // assert: library_namespace.is_URI(URL_to_fetch) // console.trace(URL_to_fetch); if (options.search || options.hash) { URL_to_fetch = set_parameters_and_hash(URL_to_fetch, options.search, options.hash); } library_namespace.debug({ T : [ 'Fetching URL: %1', '{' + (typeof URL_to_fetch) + '} [' // + (typeof URL_to_fetch === 'string' ? URL_to_fetch // : URL_to_fetch && URL_to_fetch[KEY_URL] // || URL_to_fetch.toString(charset)) + ']' ] }, 1, 'normalize_URL_to_fetch'); return URL_to_fetch; } if (false) // default arguments var get_URL_arguments = { URL : '', charset : '', // HTTP方法,如"GET", "POST", HEAD, "PUT", "DELETE"等。 method : 'GET', post_data : {}, async : true, // user name. 驗證用使用者名稱。 user : '', // 驗證用密碼。 password : '', // header headers : { contentType : 'text/xml' }, // location.search search : { contentType : 'text/xml' }, // location.hash hash : '', mime : 'text/xml', // onreadystatechange onchange : function() { }, timeout : 0, onfail : function(error) { this.status; }, onload : function() { } }; // XMLHttp.readyState 所有可能的值如下: // 0 還沒開始 // 1 讀取中 Sending Data // 2 已讀取 Data Sent // 3 資訊交換中 interactive: getting data // 4 一切完成 Completed var readyState_done = 4, // document_head = library_namespace.is_WWW(true) && (document.head || document.getElementsByTagName('head')[0]); /** * 讀取 URL via XMLHttpRequest。 * * @param {String|Object}URL_to_fetch * 欲請求之目的 URL or options * @param {Function}[onload] * callback when successful loaded * @param {String}[charset] * character encoding of HTML web page. e.g., 'UTF-8', big5, * euc-jp, ... * @param {String|Object}[post_data] * text data to send when method is POST * @param {Object}[options] * 附加參數/設定選擇性/特殊功能與選項 * * TODO: 代理伺服器 using proxy server * * @see https://developer.mozilla.org/zh-TW/docs/DOM/XMLHttpRequest * http://msdn.microsoft.com/en-us/library/ie/ms535874.aspx */ function get_URL(URL_to_fetch, onload, charset, post_data, options) { // 前導作業。 if (library_namespace.is_Object(charset) && !options) { options = post_data; post_data = charset; charset = null; } // 正規化並提供可隨意改變的同內容參數,以避免修改或覆蓋附加參數。 options = library_namespace.new_options(options); // ------------------------------------------------------ if (library_namespace.is_Object(URL_to_fetch) && URL_to_fetch[KEY_URL]) { Object.assign(options, URL_to_fetch); // 注意: options.onload 另有用途! // https://xhr.spec.whatwg.org/#handler-xhr-onloadstart // onload = options.onload || onload; post_data = options.post || post_data; charset = options.charset || charset; URL_to_fetch = options[KEY_URL]; } URL_to_fetch = normalize_URL_to_fetch(URL_to_fetch, charset, options); // assert: library_namespace.is_URI(URL_to_fetch) if (typeof onload === 'object') { library_namespace.debug( 'Trying to JSONP, insert page, need callback.', 3, 'get_URL'); // library_namespace.run(URL_to_fetch); for ( var callback_param in onload) { library_namespace.debug('Trying (' + (typeof onload[callback_param]) + ') [' + callback_param + '] = [' + onload[callback_param] + ']', 3, 'get_URL'); if (callback_param && typeof onload[callback_param] === 'function') { var callback_name, node = document.createElement('script'); for (charset = 0; (callback_name = 'cb' + charset) in library_namespace;) charset++; library_namespace[callback_name] = function(data) { library_namespace.debug('[' + URL_to_fetch + ']: callback 完自動移除 .js。', 2, 'get_URL'); document_head.removeChild(node); // Release memory. 釋放被占用的記憶體. node = null; delete library_namespace[callback_name]; onload[callback_param](data); }; // callback_param: callback parameter URL_to_fetch.search_params[callback_param] = library_namespace.Class + '.' + callback_name; node.src = URL_to_fetch.toString(); library_namespace.debug('Use script node: [' + node.src + ']', 3, 'get_URL'); document_head.appendChild(node); return; } } library_namespace.debug('Skip JSONP. No callback specified.', 3, 'get_URL'); } if (post_data && !options.form_data) { post_data = library_namespace.Search_parameters(post_data) .toString(charset); } if (!onload && typeof options.onchange === 'function') { onload = function() { options.onchange(readyState_done, XMLHttp); }; } if (options.async === false && onload || typeof onload !== 'function') { onload = false; } /** * The XMLHttpRequest object can't be cached. */ var XMLHttp = library_namespace.new_XMLHttp(); try { // IE:404 會 throw error, timeout 除了 throw error, // 還會 readystatechange; // Gecko 亦會 throw error // IE 10 中,local file 光 .open() 就 throw 了。 XMLHttp.open(options.method || (post_data ? 'POST' : 'GET'), URL_to_fetch.toString(), !!onload, options.user || '', options.password || ''); // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/response // XMLHttp.responseType = 'blob'; if (options.timeout > 0 && !onload) { // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/timeout XMLHttp.timeout = options.timeout; if (typeof options.onfail === 'function') XMLHttp.ontimeout = function(e) { options.onfail.call(XMLHttp, e || 'Timeout'); }; } // TODO: 處理有 onload 下之 timeout 逾時ms數 // Ajax 程式應該考慮到 server 沒有回應時之處置 if (library_namespace.is_Object(options.headers) // https://developer.mozilla.org/zh-TW/docs/Web/API/XMLHttpRequest/setRequestHeader // `!!XMLHttp.setRequestHeader` will throw @ HTA (HTML Application) && ('setRequestHeader' in XMLHttp)) { Object.keys(options.headers).forEach(function(key) { XMLHttp.setRequestHeader(key, options.headers[key]); }); } if (options.mime) { // ignore charset! charset = options.mime; } else if (charset) { // old: 'text/xml;charset=' + charset // 但這樣會被當作 XML 解析,產生語法錯誤。 // TODO: try: // 'text/'+(/\.x(ht)?ml$/i.test(URL_to_fetch)?'xml':'plain')+';charset=' // + charset; charset = 'application/json;charset=' + charset; } // 有些版本的 Mozilla 瀏覽器在伺服器送回的資料未含 XML mime-type // 檔頭(header)時會出錯。為了避免這個問題,可以用下列方法覆寫伺服器傳回的檔頭,以免傳回的不是 text/xml。 // http://squio.nl/blog/2006/06/27/xmlhttprequest-and-character-encoding/ // http://www.w3.org/TR/XMLHttpRequest/ search encoding if (charset && XMLHttp.overrideMimeType) XMLHttp.overrideMimeType(charset); if (onload) { XMLHttp.onreadystatechange = function() { if (XMLHttp.readyState === readyState_done) return onload(XMLHttp); if (0 < XMLHttp.readyState && XMLHttp.readyState < readyState_done) { if (typeof options.onchange === 'function') options.onchange(XMLHttp.readyState, XMLHttp); } else if (typeof options.onfail === 'function') { options.onfail(XMLHttp); } }; } // 若檔案不存在,會 throw。 XMLHttp.send(post_data || null); if (!onload) { // XMLHttp.response blob // XMLHttp.responseText 會把傳回值當字串用 // XMLHttp.responseXML 會把傳回值視為 XMLDocument 物件,而後可用 JavaScript // DOM 相關函式處理 // IE only(?): // XMLHttp.responseBody 以unsigned array格式表示binary data // try{responseBody=(new // VBArray(XMLHttp.responseBody)).toArray();}catch(e){} // http://aspdotnet.cnblogs.com/archive/2005/11/30/287481.html // XMLHttp.responseStream return AdoStream return XMLHttp.responseText; } } catch (e) { library_namespace.error(e); if (typeof options.onfail === 'function') { options.onfail(XMLHttp, e); } else if (onload) { onload(undefined, e); } } } _.get_URL = get_URL; // TODO: 處理 multiple requests function get_URLs() { } // ---------------------------------------------------- var is_nodejs = library_namespace.platform.nodejs; /** * <code> // "file": keyword for "Content-Disposition: file;" {type:'jpg',image:{file:'fn1.jpg'}} // will fetch url first. {type:'jpg',image:{url:'http://host/'}} {type:'jpg',image:{file:'fn1.jpg',type:'image/jpeg'}} // Array: use "Content-Type: multipart/mixed;" {type:'jpg',images:[{file:'fn1.jpg'},{file:'fn2.jpg'}]} {type:'jpg',images:[{file:'fn1.jpg'},{file:'fn2.jpg'}],docs:[{file:'fn1.txt'},{file:'fn2.txt'}]} {type:'jpg',images:[{file:'fn1.jpg',type:'image/jpeg'},{file:'fn1.txt',type:'text/plain'}]} </code> */ // should be CRLF // @see https://tools.ietf.org/html/rfc7578#section-4.1 var form_data_new_line = '\r\n'; function form_data_to_Array(is_slice) { if (this.generated) { return this.generated; } var boundary = '--' + this.boundary + form_data_new_line, // generated raw post data generated = this.generated = [ boundary ], content_length = boundary.length; boundary = form_data_new_line + boundary; this.forEach(function(chunk, index) { if (Array.isArray(chunk)) { // chunk = chunk.to_Array(true); if (!chunk.content_length) { console.log(chunk); throw new Error( // gettext_config:{"id":"the-chunk-do-not-has-regular-.content_length"} 'The chunk do not has regular .content_length!'); } content_length += chunk.content_length; } else { // chunk: {String} or {Buffer} content_length += chunk.length; } generated.push(chunk); if (index < this.length - 1) { generated.push(boundary); content_length += boundary.length; } }, this); if (!(content_length > 0)) { console.log(this); // gettext_config:{"id":"illegal-chunk.content_length"} throw new Error('Illegal chunk.content_length!'); } boundary = form_data_new_line + '--' + this.boundary; if (!is_slice) { boundary += '--'; } generated.push(boundary); content_length += boundary.length; generated.content_length = content_length; // console.log(generated); return generated; } // 選出 data.generated 不包含之 string function give_boundary(data_Array) { function not_includes_in(item) { // console.trace([ typeof item, item ]); return Array.isArray(item) ? item.every(not_includes_in) // item: Should be {String} or {Buffer} : !item.includes(boundary); } var boundary, retry_count = 0; while (retry_count++ < 8) { boundary = (Number.MAX_SAFE_INTEGER * Math.random()) .toString(10 + 26); // console.log('test boundary: [' + boundary + ']'); for (var i = 1; i < boundary.length / 2 | 0; i++) { var slice = boundary.slice(0, i); if (boundary.lastIndexOf(slice) > 0) { boundary = null; break; } } // assert: boundary 不自包含,例如 'aa'自包含'a','asas'自包含'as' if (boundary) { if (not_includes_in(data_Array)) { data_Array.boundary = boundary; return boundary; } } } throw new Error('give_boundary: ' // gettext_config:{"id":"retry-too-many-times"} + 'Retry too many times!'); } var to_form_data_generated = { form_data_generated : true }; // https://github.com/form-data/form-data/blob/master/lib/form_data.js // https://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.2 // https://tools.ietf.org/html/rfc7578 // https://tools.ietf.org/html/rfc2046#section-5.1 // The "multipart" boundary delimiters and header fields are always // represented as 7bit US-ASCII // https://tools.ietf.org/html/rfc2049#appendix-A // http://stackoverflow.com/questions/4238809/example-of-multipart-form-data function to_form_data(parameters, callback, options) { function get_file_object(value, callback, key, slice) { var is_url, MIME_type; if (typeof value === 'string') { is_url = value.includes('://'); } else // else: assert: library_namespace.is_Object(value) if (is_url = value.url) { value = is_url; // is_url = true; } else { // .type: MIME type MIME_type = value.type; // value: file_path value = value.file; } function push_and_callback(MIME_type, content) { // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition var headers = 'Content-Disposition: ' + (slice ? 'file' : 'form-data; name="' + key + '"') + '; filename="' + encodeURIComponent(value) + '"' + "; filename*=UTF-8''" + encodeURIComponent(value) + form_data_new_line; if (MIME_type) { headers += 'Content-Type: ' + MIME_type + form_data_new_line; } if (is_nodejs && Buffer.isBuffer(content)) { headers += 'Content-Transfer-Encoding: binary' + form_data_new_line; } headers += form_data_new_line; var chunk = [ headers, content ]; // 手動設定 Content-Length。 chunk.content_length = headers.length + content.length; // TODO: use stream (slice || root_data).push(chunk); callback(); } if (!is_url) { var content; // read file contents if (is_nodejs) { try { // get {Buffer} content = node_fs.readFileSync(value); } catch (e) { // When we cannot read, it will throw now. // e.g., Error: ENOENT: no such file or directory, open // 'value' } } else { // node.js 之下此方法不能處理 binary data。 content = library_namespace .get_file(value/* , 'binary' */); } if (!content) { library_namespace.error([ 'to_form_data: ', { // gettext_config:{"id":"failed-to-get-file-$1"} T : [ 'Failed to get file: [%1]', value ] } ]); // Skip this one. callback(); return; } if (options && options.file_post_processor) { options.file_post_processor(value, content); } // value: file path → file name value = value.match(/[^\\\/]*$/)[0]; if (!MIME_type) { MIME_type = library_namespace.MIME_of(value); } push_and_callback(MIME_type, content); return; } library_namespace.debug({ // gettext_config:{"id":"fetching-url-$1"} T : [ '自網路取得 URL:%1', value ] }, 1, 'to_form_data'); _.get_URL(value, function(XMLHttp, error) { if (options && options.url_post_processor) { options.url_post_processor(value, XMLHttp, error); } if (error) { library_namespace.error([ 'to_form_data: ', { // gettext_config:{"id":"got-error-when-retrieving-$1-$2"} T : [ 'Got error when retrieving [%1]: %2', // value, localize_error(error) ] } ]); // Skip this one. callback(); return; } // value: url → file name value = value.replace(/[?#].*$/, '') .match(/([^\\\/]*)[\\\/]?$/)[1]; // console.log('-'.repeat(79)); // console.log(value); library_namespace.debug({ // gettext_config:{"id":"got-url-from-the-network-$1-$2-bytes"} T : [ '自網路取得 URL:%1,%2{{PLURAL:%2|位元組}}。', value, XMLHttp.buffer.length ] }, 1, 'to_form_data'); push_and_callback(XMLHttp.type, XMLHttp.buffer); }, 'buffer'); } parameters = library_namespace.Search_parameters(parameters); var root_data = [], keys = Object.keys(parameters), index = 0; root_data.to_Array = form_data_to_Array; // console.log('-'.repeat(79)); // console.log(keys); // 因為在遇到fetch url時需要等待,因此採用async。 function process_next() { if (false) { console.log('-'.repeat(60)); console.log('process_next: ' + index + '/' + keys.length); console.log(root_data); } if (index === keys.length) { // 決定 boundary give_boundary(root_data); // WARNING: 先結束作業: 生成 .to_Array(), // 才能得到 root_data.to_Array().content_length。 root_data.to_Array(); if (false) { console.log('-'.repeat(79)); console.log(root_data); console.log('-'.repeat(79)); console.log(root_data.to_Array().content_length); console.log(root_data.to_Array().join('')); throw 5; } callback(root_data); return; } var key = keys[index++], value = parameters[key]; // console.log(key + ': ' + JSON.stringify(value)); if (Array.isArray(value)) { // assert: is files/urls var slice = [], item_index = 0, // next_item = function() { if (item_index === value.length) { give_boundary(slice); var headers = 'Content-Disposition: form-data; name="' + key + '"' + form_data_new_line + 'Content-Type: multipart/mixed; boundary=' + slice.boundary + form_data_new_line + form_data_new_line; slice = form_data_to_Array.call(slice, true); slice.unshift(headers); slice.content_length += headers.length; root_data.push(slice); process_next(); } else { get_file_object(value[item_index++], next_item,/* key */ undefined, slice); } }; next_item(); return; } if (library_namespace.is_Object(value)) { // assert: is file/url get_file_object(value, process_next, key); return; } // 預防有 null, undefined, {Number}, true 之類。 if (!value || typeof value === 'number' || typeof value === 'boolean') { value = String(value); } var headers; if (is_nodejs && typeof value === 'string') { value = Buffer.from(value, 'utf8'); headers = 'Content-Type: text/plain; charset=UTF-8' + form_data_new_line; } // 非檔案,屬於普通的表單資料。 if (!key) { throw new Error('No key for value: ' + value); } if (!value && value !== '') { // e.g., token=undefined throw new TypeError('Invalid type of ' + key + ': ' + typeof value); } // @see function push_and_callback(MIME_type, content) headers = 'Content-Disposition: form-data; name="' + key + '"' + form_data_new_line + (headers || '') + form_data_new_line; var chunk = [ headers, value ]; // 手動設定 Content-Length。 chunk.content_length = headers.length + value.length; root_data.push(chunk); process_next(); } process_next(); return root_data; } _.to_form_data = to_form_data; // ---------------------------------------------------------------------// /** * <code> 讀取URL by XMLHttpRequest http://jck11.pixnet.net/blog/post/11630232 * 若有多行程或為各URL設定個別XMLHttp之必要,請在一開始便設定deprecated_get_URL.multi_request,並且別再更改。 ** 在此情況下,單一URL仍只能有單一個request! ** 設定 handle_function 須注意程式在等待回應時若無執行其他程式碼將自動中止! 可設定: while(deprecated_get_URL.doing)WScript.Sleep(1); //||timeout arguments f:{ URL:'', // The same origin policy prevents document or script loaded from one origin, from getting or setting properties from a of a document from a different origin.(http://www.mozilla.org/projects/security/components/jssec.html#sameorigin) enc:'UTF-8', // charset: big5, euc-jp,.. fn:(handle_function), // onLoad:function(){}, method:'GET', // POST,.. sendDoc:'text send in POST,..' async:ture/false, // true if want to asynchronous(非同期), false if synchronous(同期的,會直到readyState==4才return) http://jpspan.sourceforge.net/wiki/doku.php?id=javascript:xmlhttprequest:behaviour user:'userName', passwd:'****', // password //TODO: parameters:'~=~&~=~', // {a:1,b:2} header:{contentType:'text/xml'}, contentType:'text/xml', run:true/false, // do eval update:DOMDocument, // use onLoad/onFailed to 加工 return text. onFailed(){throw;} will abort change. interval:\d, decay:\d, // wait decay*interval when no change maxInterval::\d, //insertion:top/bottom,.. onFailed:function(error){this.status;}, // onFailed.apply(XMLHttp,[XMLHttp.status]) onStateChange:function(){}, } handle_function: 自行處理 typeof handle_function=='function': function handle_function(error){..} 代為處理 handle_function=[d_func,0: responseText,1: responseXML]: responseXML: http://msdn2.microsoft.com/en-us/library/ms757878.aspx function d_func(content,headers[,XMLHttp,URL]){ if(headers){ // content,headers各為XMLHttp.responseText內容及XMLHttp.getAllResponseHeaders(),其他皆可由XMLHttp取得。 }else{ // content為error } } e.g., the simplest: [function(c,h){h&&alert(c);}] ) </code> */ // (URL,fn) or flag URL, handle_function handle result, // method,sendDoc,asyncFlag,userName,password function deprecated_get_URL(f) { var _f = arguments.callee; if (typeof _f.XMLHttp === 'object') { // try{_f.XMLHttp.abort();}catch(e){} // 此時可能衝突或lose?! _f.XMLHttp = null; } // 處理 arguments if (!library_namespace.is_Object(f)) a = arguments, f = { URL : f, fn : a[1], method : a[2], sendDoc : a[3] }; if (f.post) f.method = 'POST', f.sendDoc = f.post; if (!f[KEY_URL] || !(_f.XMLHttp = library_namespace.new_XMLHttp(f.enc, !/\.x(?:ht)?ml$/i.test(f[KEY_URL])))) // throw return; // try{_f.XMLHttp.overrideMimeType('text/xml');}catch(e){} if (typeof f.async !== 'boolean') // 設定f.async f.async = f.fn ? true : false; else if (!f.async) f.fn = null; else if (!f.fn) if (typeof _f.HandleStateChange !== 'function' || typeof _f.HandleContent !== 'function') // 沒有能處理的function // throw return; else // =null; f.fn = _f.HandleContent; if (/* typeof _f.multi_request!=='undefined'&& */_f.multi_request) { if (!_f.q) // queue _f.i = {}, _f.q = []; // ** 沒有考慮到 POST 時 URL 相同的情況! _f.i[f[KEY_URL]] = _f.q.length; _f.q.push({ uri : f[KEY_URL], XMLHttp : _f.XMLHttp, func : f.fn, start : _f.startTime = new Date }); } else if (_f.q && typeof _f.clean === 'function') _f.clean(); // for Gecko Error: uncaught exception: Permission denied to call method // XMLHttpRequest.open if (f[KEY_URL].indexOf('://') !== NOT_FOUND && typeof netscape === 'object') if (_f.asked > 2) { _f.clean(f[KEY_URL]); return; } else try { if (typeof _f.asked === 'undefined') { _f.asked = 0; alert('我們需要一點權限來使用 XMLHttpRequest.open。\n* 請勾選記住這項設定的方格。'); } netscape.security.PrivilegeManager // UniversalBrowserAccess .enablePrivilege('UniversalXPConnect'); } catch (e) { _f.asked++; _f.clean(f[KEY_URL]); return; } // if(isNaN(_f.timeout))_f.timeout=300000;//5*60*1000; try { // IE:404會throw error, timeout除了throw error, 還會readystatechange; // Gecko亦會throw error try { _f.XMLHttp.setRequestHeader("Accept-Encoding", "gzip, deflate, br"); } catch (e) { } // Set header so the called script knows that it's an XMLHttpRequest if (false) _f.XMLHttp.setRequestHeader("X-Requested-With", "XMLHttpRequest"); // Set the If-Modified-Since header, if ifModified mode. if (false) _f.XMLHttp.setRequestHeader("If-Modified-Since", "Thu, 01 Jan 1970 00:00:00 GMT"); if (f.method === 'POST' // &&_f.XMLHttp.setRequestHeader ) { // use .getAttribute('method') to get 長度不一定如此 if (false) _f.XMLHttp.setRequestHeader("Content-Length", f.sendDoc.length); // 有些CGI會用Content-Type測試是XMLHttp或是regular form // It may be necessary to specify // "application/x-www-form-urlencoded" or "multipart/form-data" // for posted XML data to be interpreted on the server. _f.XMLHttp.setRequestHeader('Content-Type', Array.isArray(f.fn) && f.fn[1] ? 'text/xml' // application/x-www-form-urlencoded; charset=utf-8 : 'application/x-www-form-urlencoded'); } _f.XMLHttp.abort(); _f.XMLHttp.open(f.method || 'GET', f[KEY_URL], f.async, f.user || null, f.passwd || null); // alert((f.method||'GET')+','+f[KEY_URL]+','+f.async); /** * @see http://www.javaworld.com.tw/jute/post/view?bid=49&id=170177&sty=3&age=0&tpg=1&ppg=1 * 根據 W3C的 XMLHttpRequest 規格書上說,①在呼叫 open * 時,如果readyState是4(Loaded) ②呼叫abort之後 * ③發生其他錯誤,如網路問題,無窮迴圈等等,則會重設所有的值。使用全域的情況就只有第一次可以執行,因為之後的readyState是4,所以onreadystatechange * 放在open之前會被清空,因此,onreadystatechange 必須放在open之後就可以避免這個問題。 * * 每使用一次XMLHttpRequest,不管成功或失敗,都要重設onreadystatechange一次。onreadystatechange * 的初始值是 null * * @see http://www.xulplanet.com/references/objref/XMLHttpRequest.html * After the initial response, all event listeners will be * cleared. Call open() before setting new event listeners. */ if (f.async) { _f.doing = (_f.doing || 0) + 1; _f.XMLHttp.onreadystatechange = typeof f.fn === 'function' // ? f.fn : function(e) { _f.HandleStateChange(e, f[KEY_URL], f.fn); } // ||null ; // 應加 clearTimeout( ) setTimeout('try{deprecated_get_URL.' // + (_f.multi_request ? 'q[' + _f.i[f[KEY_URL]] + ']' : 'XMLHttp') + '.onreadystatechange();}catch(e){}', // 5*60*1000; _f.timeout || 3e5); } _f.XMLHttp.send(f.sendDoc || null); if (!f.fn) { /** * 非async(異步的)能在此就得到 response。Safari and Konqueror cannot * understand the encoding of text files! * * @see http://www.kawa.net/works/js/jkl/parsexml.html */ // responseXML: responseXML.loadXML(text) return _f.XMLHttp.responseText; } } catch (e) { if (typeof f.fn === 'function') f.fn(e); else if (typeof window === 'object') window.status = e.message; return e; } } deprecated_get_URL.timeoutCode = -7732147; /** * agent handle function * * e: object Error, handle_function: function(return text, headers, * XMLHttpRequest object, URL) | [ function, (default|NULL:responseText, * others:responseXML) ] */ deprecated_get_URL.HandleStateChange = function(e, URL, handle_function) { var _t = 0, isOKc, m = deprecated_get_URL.multi_request, _oXMLH; if (m) m = deprecated_get_URL.q[isNaN(URL) ? deprecated_get_URL.i[URL] : URL], _oXMLH = m.XMLHttp, handle_function = m.func, URL = m.uri; else _oXMLH = deprecated_get_URL.XMLHttp; if (Array.isArray(handle_function)) _t = handle_function[1], handle_function = handle_function[0]; if (!handle_function || typeof handle_function !== 'function') { deprecated_get_URL.doing--; deprecated_get_URL.clean(URL); return; } // http://big5.chinaz.com:88/book.chinaz.com/others/web/web/xml/index1/21.htm if (!e) if (typeof _oXMLH === 'object' && _oXMLH) { if (_oXMLH.parseError && _oXMLH/* .responseXML */.parseError.errorCode !== 0) e = _oXMLH.parseError, e = new Error(e.errorCode, e.reason); else if (_oXMLH.readyState === 4) { // only if XMLHttp shows "loaded" // condition is OK? isOKc = _oXMLH.status; isOKc = isOKc >= 200 && isOKc < 300 || isOKc === 304 || !isOKc && (location.protocol === "file:" || location.protocol === "chrome:"); if (handle_function === deprecated_get_URL.HandleContent) // handle_function.apply() handle_function(0, isOKc, _oXMLH, URL); else { // handle_function.apply() handle_function(isOKc ? _t ? _oXMLH.responseXML // JKL.ParseXML: Safari and Konqueror cannot // understand the encoding of text files. : typeof window === 'object' && window.navigator.appVersion .indexOf("KHTML") !== NOT_FOUND && !(e = escape(_oXMLH.responseText)) .indexOf("%u") !== NOT_FOUND // ? e : _oXMLH.responseText : 0, // isOKc ? _oXMLH.getAllResponseHeaders() : 0, _oXMLH, URL); } // URL之protocol==file: // 可能需要重新.loadXML((.responseText+'').replace(/<\?xml[^?]*\?>/,"")) // 用 .responseXML.documentElement 可調用 deprecated_get_URL.doing--; deprecated_get_URL.clean(URL); return; } } else if (new Date - (m ? m.start : deprecated_get_URL.startTime) > deprecated_get_URL.timeout) // timeout & timeout function // http://www.stylusstudio.com/xmldev/199912/post40380.html // _oXMLH.abort(); e = new Error(deprecated_get_URL.timeoutCode, 'Timeout'); // alert(URL+'\n'+_t+'\n'+e+'\n'+_oXMLH.readyState+'\n'+handle_function); if (e) { handle_function(e, 0, _oXMLH, URL); deprecated_get_URL.doing--; deprecated_get_URL.clean(URL); }// handle_function.apply(e,URL); }; /** * agent content handle function<br /> * 有headers時content包含回應,否則content表error */ deprecated_get_URL.HandleContent = function(content, headers, _oXMLHttp, URL) { if (headers) { // _oXMLHttp.getResponseHeader("Content-Length") alert("URL: " + URL + "\nHeaders:\n" + _oXMLHttp.getAllResponseHeaders() + "\n------------------------\nLastModified: " + _oXMLHttp.getResponseHeader("Last-Modified") + "\nResult:\n" + _oXMLHttp.responseText.slice(0, 200));// _oXMLHttp.responseXML.xml } else { // error // test時,可用deprecated_get_URL.XMLHttp.open("HEAD","_URL_",true);,deprecated_get_URL(url,handle_function,'HEAD',true)。 if (content instanceof Error) alert('Error occured!\n' + (typeof e === 'object' && e.number ? e.number + ':' + e.message : e || '')); else if (typeof _oXMLHttp === 'object' && _oXMLHttp) alert((_oXMLHttp.status === 404 ? "URL doesn't exist!" : 'Error occured!') + '\n\nStatus: ' + _oXMLHttp.status + '\n' + _oXMLHttp.statusText); } }; // 在MP模式下清乾淨queue deprecated_get_URL.clean = function(i, force) { // multiple requests if (force || deprecated_get_URL.multi_request) if (!i && isNaN(i)) { if (deprecated_get_URL.q) for (i in deprecated_get_URL.i) try { deprecated_get_URL.q[deprecated_get_URL.i[i]].XMLHttp .abort(); // deprecated_get_URL.q[deprecated_get_URL.i[i]].XMLHttp=null; } catch (e) { } deprecated_get_URL.q = deprecated_get_URL.i // =null = 0; } else if (!isNaN(i) || !isNaN(i = deprecated_get_URL.i[typeof i === 'object' ? i.uri : i])) { try { deprecated_get_URL.q[i].XMLHttp.abort(); } catch (e) { } // deprecated_get_URL.q[i].XMLHttp=0; delete deprecated_get_URL.i[deprecated_get_URL.q[i].uri]; deprecated_get_URL.q[i] = 0; } }; // ↑XMLHttp set ================== // ---------------------------------------------------------------------// // @see https://github.com/request/request var node_url, node_http, node_http2, node_https, // reuse the sockets (keep-alive connection). node_http_agent, node_http2_agent, node_https_agent, // node_zlib; /** * 快速 merge cookie: 只檢查若沒有重複的 key,則直接加入。不檢查 path 也不處理 expires, domain, * secure。<br /> * 為增加效率,不檢查 agent.last_cookie 本身之重複的 cookie。 * * TODO: create class Cookie, Cookie.prototype.merge(), * Cookie.prototype.clone() * * @param {Object}agent * node_http_agent / node_https_agent * @param {Array}cookie * new cookie to merge * * @returns {Object}agent.last_cookie * * @inner */ function merge_cookie(agent, cookie) { // 初始化 initialization + 正規化 normalization var last_cookie = agent.last_cookie; if (!Array.isArray(last_cookie)) { last_cookie = agent.last_cookie = agent.last_cookie ? [ agent.last_cookie ] : []; } if (!cookie) { cookie = []; } else if (typeof cookie === 'string') { cookie = cookie.split(';'); } // assert: Array.isArray(cookie) // console.log(agent); // console.log(last_cookie.cookie_hash); // console.trace(cookie); // cookie_index_of[key] = index of last_cookie var cookie_index_of = last_cookie.cookie_index_of; if (!cookie_index_of) { if (last_cookie.length > 0) { // regenerate agent.last_cookie delete agent.last_cookie; last_cookie = merge_cookie(agent, last_cookie); // assert: last_cookie === agent.last_cookie } else { last_cookie.cookie_index_of = Object.create(null); last_cookie.cookie_hash = Object.create(null); } cookie_index_of = last_cookie.cookie_index_of; } var cookie_hash = last_cookie.cookie_hash; // assert: !!cookie_hash === true cookie.forEach(function for_each_cookie_piece(piece) { piece = piece.trim(); if (!piece) return; // [ cookie value without path / domain / expires, // key, value, extra ] var matched = piece.match(/^([^=;]+)(?:=([^;]+))?(.*)$/); library_namespace.debug('last_cookie: ' + last_cookie, 3, 'merge_cookie'); // console.log(matched); var key, value; if (matched) { key = matched[1]; value = matched[2]; } else { library_namespace.warn([ 'merge_cookie: ', { // gettext_config:{"id":"invalid-cookie"} T : 'Invalid cookie?' }, ' [' + piece + ']' ]); // treat cookie piece as key key = piece; value = ''; } if (!key) return; cookie_hash[key] = value; if (key in cookie_index_of) { // assert: (key in cookie_hash) === true library_namespace.debug([ { // gettext_config:{"id":"duplicate-cookie-name!-the-later-newcomer-will-prevail"} T : 'cookie 名稱重複!以後來/新出現者為準。' }, ' [' + last_cookie[cookie_index_of[key]] // + ']→[' + piece + ']' ], 3, 'merge_cookie'); // remove duplicate cookie: 直接取代。 last_cookie[cookie_index_of[key]] = piece; } else { // assert: (key in cookie_hash) === false library_namespace.debug([ { T : '正常情況。登記已存在之 cookie。' } ], 3, 'merge_cookie'); // console.trace(matched); cookie_index_of[key] = last_cookie.length; last_cookie.push(piece); } }); // console.trace(cookie_hash); // console.trace(last_cookie); library_namespace.debug('array: ' + JSON.stringify(last_cookie), 3, 'merge_cookie'); library_namespace.debug('hash: ' + JSON.stringify(cookie_hash), 3, 'merge_cookie'); return last_cookie; } _.merge_cookie = merge_cookie; function set_cookie_to_URL_object(URL_options_to_fetch, agent) { // console.trace('agent.last_cookie:'); // console.log(agent.last_cookie); if (agent.last_cookie) { // 使用 cookie library_namespace.debug('agent.last_cookie: ' + JSON.stringify(agent.last_cookie), 3, 'set_cookie_to_URL_object'); library_namespace.debug('agent.last_cookie.cookie_hash: ' + JSON.stringify(agent.last_cookie.cookie_hash), 3, 'set_cookie_to_URL_object'); var cookie = (URL_options_to_fetch.headers.Cookie ? URL_options_to_fetch.headers.Cookie + ';' : '') // cookie is Array @ Wikipedia + (Array.isArray(agent.last_cookie) ? agent.last_cookie // 去掉 expires=...; path=/; domain=...; HttpOnly // 這個動作不做也可以,不影響結果。 .map(function(cookie) { return cookie.replace(/;.*/, ''); }).join('; ') : agent.last_cookie); if (cookie) { URL_options_to_fetch.headers.Cookie = cookie; } else { delete URL_options_to_fetch.headers.Cookie; } } library_namespace.debug('Set cookie: ' + JSON.stringify(URL_options_to_fetch.headers.Cookie), 3, 'set_cookie_to_URL_object'); library_namespace.debug('Set protocol: ' + URL_options_to_fetch.protocol, 3, 'set_cookie_to_URL_object'); library_namespace.debug('Set headers: ' + JSON.stringify(URL_options_to_fetch.headers), 3, 'set_cookie_to_URL_object'); } // ---------------------------------------------------------------------// // 正處理中之 connections var get_URL_node_connections = 0, // 所有 requests get_URL_node_requests = 0; // 強制使用POST傳送。 var FORCE_POST = { FORCE_POST : true }; var ERROR_BAD_STSTUS = 'BAD STATUS'; var KEY_not_native = library_namespace.env.not_native_keyword; var has_native_URL = typeof URL === "function" && !URL[KEY_not_native]; /** * 讀取 URL via node http/https。<br /> * assert: arguments 必須與 get_URL() 相容! * * @param {String|Object}URL_to_fetch * 欲請求之目的 URL or options * @param {Function}[onload] * callback when successful loaded. For failure handling, using * options.onfail(error); * @param {String}[charset] * character encoding of HTML web page. e.g., 'UTF-8', big5, * euc-jp,.. * @param {String|Object}[post_data] * text data to send when method is POST * @param {Object}[options] * 附加參數/設定選擇性/特殊功能與選項 * * @see https://nodejs.org/api/http.html#http_http_request_options_callback * https://nodejs.org/api/https.html#https_https_request_options_callback * * @since 2015/1/13 23:23:38 */ function get_URL_node(URL_to_fetch, onload, charset, post_data, options) { if (!URL_to_fetch) { onload(undefined, new SyntaxError('No URL input.')); return; } get_URL_node_requests++; if (get_URL_node_connections >= get_URL_node.connects_limit) { library_namespace.debug({ // gettext_config:{"id":"waiting-$1-$2-connections-$3"} T : [ 'Waiting %1/%2 {{PLURAL:%1|connection|connections}}: %3', // 避免同時開過多 connections 的機制。 get_URL_node_connections, get_URL_node_requests, String(URL_to_fetch) ] }, 3, 'get_URL_node'); var _arguments = arguments; setTimeout(function() { get_URL_node_requests--; get_URL_node.apply(null, _arguments); }, 500); return; } // 進入 request 程序 get_URL_node_connections++; // 前導作業。 if (library_namespace.is_Object(charset) && !options) { options = post_data; post_data = charset; charset = null; } // 正規化並提供可隨意改變的同內容參數,以避免修改或覆蓋附加參數。 options = library_namespace.new_options(options); // console.log('-'.repeat(79)); // console.log(JSON.stringify(options)); // console.log(options.form_data); if (options.form_data && options.form_data !== to_form_data_generated) { // console.trace(options); // TODO: charset for post_data to_form_data(post_data, function(data) { if (false) { console.log('>> ' // library_namespace.data.code.string_digest() + library_namespace.string_digest(data.toString(), 200)); } options.form_data = to_form_data_generated; get_URL_node(URL_to_fetch, onload, charset, data, options); }, options.form_data); return; } // ------------------------------------------------------ if (library_namespace.is_Object(URL_to_fetch) && URL_to_fetch[KEY_URL]) { Object.assign(options, URL_to_fetch); // 注意: options.onload 另有用途! // https://xhr.spec.whatwg.org/#handler-xhr-onloadstart // onload = options.onload || onload; post_data = options.post || post_data; charset = options.charset || charset; URL_to_fetch = options[KEY_URL]; } // 不改變 options。 var agent = options.agent; if (typeof URL_to_fetch === 'string' && URL_to_fetch.startsWith('//')) { // 處理 '//domain.org/path' 的情況。 URL_to_fetch = (agent && agent.protocol || 'https:') + URL_to_fetch; } var URL_options_to_fetch = normalize_URL_to_fetch(URL_to_fetch, charset, options); // assert: library_namespace.is_URI(URL_options_to_fetch) // console.trace([ URL_to_fetch, URL_options_to_fetch ]); if (typeof onload === 'object') { library_namespace.debug( 'Trying to JSONP, insert page, need callback.', 3, 'get_URL_node'); // library_namespace.run(URL_options_to_fetch); for ( var callback_param in onload) { if (callback_param && typeof onload[callback_param] === 'function') { // 模擬 callback。 // callback_param: callback parameter URL_options_to_fetch.search_params[callback_param] = 'cb'; onload = onload[callback_param]; break; } } } // assert: 自此開始不會改變 URL,也不會中途 exit 本函數。 if (post_data && !options.form_data) { if (library_namespace.is_Object(post_data) && options.headers && typeof options.headers['Content-Type'] === 'string' && options.headers['Content-Type'] .includes('application/json')) { post_data = JSON.stringify(post_data) || FORCE_POST; } else { post_data = library_namespace.Search_parameters(post_data) .toString(options.post_data_charset || charset) || FORCE_POST; } } if (!onload && typeof options.onchange === 'function') { onload = function() { options.onchange(readyState_done); }; } if (options.async === false && onload || typeof onload !== 'function') { onload = false; } // console.trace(URL_options_to_fetch); // node_http.request(options): options needs a .path // https://nodejs.org/dist/latest/docs/api/http.html#http_http_request_options_callback URL_options_to_fetch.path = URL_options_to_fetch.pathname + URL_options_to_fetch.search; var URL_is_https = /^https:?$/i.test(URL_options_to_fetch.protocol); // console.trace([ URL_to_fetch, URL_options_to_fetch.toString() ]); URL_to_fetch = URL_options_to_fetch.toString(); // assert: {String}URL_to_fetch, // library_namespace.is_URI(URL_options_to_fetch) /** * https://stackoverflow.com/questions/53593182/client-network-socket-disconnected-before-secure-tls-connection-was-established * mh160.js 必須使用 request,https-proxy-agent 才能正常工作 TODO: * socks-proxy-agent * * <code> // https://techcult.com/free-proxy-software-for-windows-10/#1_Ultrasurf > SET http_proxy=http://127.0.0.1:9666 > node u17.js 镇魂街 </code> */ /** * <code> // http://anonproxyserver.sourceforge.net/ // https://www.proxynova.com/proxy-server-list/country-tw/ var http = require("http"); var options = { host: "211.22.233.69", port: 3128, path: "http://dict.revised.moe.edu.tw/cgi-bin/cbdic/gsweb.cgi?ccd=9gW4am&o=e0&sec=sec11&option=linkout001&actice=layout", //method: 'GET', headers: { Host: "dict.revised.moe.edu.tw" } }; var request=http.request(options, function(response) { console.log(response.statusCode); console.log(response.headers); var data = [], length = 0; response.on('data', function(chunk) {length += chunk.length;data.push(chunk);}); response.on('end', function() {data = Buffer.concat(data, length);console.log(data+'')}); }); request.end(); require('./work_crawler_loader.js'); var PROXY='localhost:8080'; CeL.get_URL('https://zh.wikipedia.org/wiki/Special:%E6%9C%80%E8%BF%91%E6%9B%B4%E6%94%B9',function(X){console.log(X.responseText)},null,null,{proxy:PROXY}); CeL.get_URL('https://zh.wikipedia.org/wiki/Special:%E6%9C%80%E8%BF%91%E6%9B%B4%E6%94%B9',function(X){console.log(X.responseText)}); CeL.get_URL('http://dict.revised.moe.edu.tw/cgi-bin/cbdic/gsweb.cgi?c