shubhadownloader
Version:
There is large amount of information available in market place. The markets are always in sync. In today's world investors find it very difficult to make effective use of information available to them. Shubha Downloader is great tool which resolves this problem and helps investor to increase his productivity and stay focused on decision making. Shubha Downloader enable end user to download market data from available sources and organize it. Shubha Downloader is Open source & FREE utility for end users. Shubha Downloader have main features as follows End of the day market data from web to your favorite charting application . Fundamental market data from web to your favorite charting application. Market reports from web to your favorite charting application.
494 lines (424 loc) • 22.8 kB
JavaScript
function process_yahooieod(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,yahoo_min)
{
var fs = require('fs');
var csv = require('ya-csv');
var async = require('async');
//var path = require('./node_modules/path');
//var writeFileStream = fs.createWriteStream('./tempData/output.txt',{flag:'a',encoding:'utf-8'});
try{
async.parallel({
data1 : function(callback){
if (fs.existsSync('yahoo.csv')) {
fs.readFile('yahoo.csv','utf-8', function(err, data){
callback(err, data);
});
}else
{
// Here readdir's functionaliity has no use. It is written for only to send
// '0' data in its callback function because we cannot call "callback(err, data);"
// function out side a callback function
fs.readdir('./', function(err, files){
callback(err,'0');
});
}
}
}, function(err, result){
if(err) throw err;
if (fs.existsSync('yahoo.csv')) {
var arrRow = new Array();
arrRow = result['data1'].split('\n');
if(arrRow.length>0)
{
var arrFileData = new Array();
// arrRow.lenght-3 is for last 3 lines of no use in file
for(var i=0 ; i<arrRow.length-1;i++)
{
var tempArr = arrRow[i].split(',');
arrFileData.push(tempArr);
}
if(arrFileData.length > 0)
{
downloadAndProcessYahooIEOD(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,arrFileData, yahoo_min, 0);
}else
{
document.getElementById("om").innerHTML+="<br/><font color=lightcoral>Please add yahoo symbols to download yahoo data</font>";
sn++;
callfun(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn);
}
}
else
{
document.getElementById("om").innerHTML+="<br/><font color=lightcoral>Please add yahoo symbols to download yahoo data</font>";
sn++;
callfun(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn);
}
}
else
{
document.getElementById("om").innerHTML+="<br/><font color=lightcoral>Please add yahoo symbols to download yahoo data</font>";
sn++;
callfun(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn);
}
});
}catch(err){
console.error(err);
}
}
function downloadAndProcessYahooIEOD(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,arrYahooSymbols,yahoo_min,n)
{
var urlStockinfo = "";
if(yahoo_min == "1")
urlStockinfo = "http://chartapi.finance.yahoo.com/instrument/1.0/"+arrYahooSymbols[n][0]+"/chartdata;type=quote;range=1d/csv/";
if(yahoo_min == "5")
urlStockinfo = "http://chartapi.finance.yahoo.com/instrument/5.0/"+arrYahooSymbols[n][0]+"/chartdata;type=quote;range=5d/csv/";
//console.log(urlStockinfo);
var http = require('http'),
fs = require('fs'),
request = require('request');
request
(
{
method: 'POST',
uri: urlStockinfo,
headers:
{
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
//"Referer": "http://download.finance.yahoo.com",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Cookie": "cookie"
}
}
,function(error, response, body)
{
if(response != null)
{
if(response.statusCode != null || response.statusCode != "")
{
if (!error && response.statusCode == 200)
{
downloadyahooieod(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,response.statusCode, urlStockinfo, arrYahooSymbols, yahoo_min, n);
}
else
{
downloadyahooieod(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,404, urlStockinfo, arrYahooSymbols, yahoo_min, n);
}
}else
{
downloadyahooieod(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,404, urlStockinfo, arrYahooSymbols, yahoo_min, n);
}
}else
{
downloadyahooieod(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,404, urlStockinfo, arrYahooSymbols, yahoo_min, n);
}
});
}
function downloadyahooieod(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,statuscode, urlStockinfo, arrYahooSymbols, yahoo_min, n)
{
var fileName = "";
if(yahoo_min == "1")
{
fileName = downloadpath+"temp_yahoo/yahooieod_1min_"+arrYahooSymbols[n][0]+".csv";
//document.getElementById("om").innerHTML+="<br/>yahooieod_1min_"+arrYahooSymbols[n][0]+".csv is processed";
}
if(yahoo_min == "5")
fileName = downloadpath+"temp_yahoo/yahooieod_5min_"+arrYahooSymbols[n][0]+".csv";
//document.getElementById("om").innerHTML+="<br/>yahooieod_5min_"+arrYahooSymbols[n][0]+".csv is processed";
var http = require('http'),
fs = require('fs'),
request = require('request');
var out = fs.createWriteStream(fileName);
if(statuscode==200){
var req = request(
{
method: 'GET',
uri: urlStockinfo,
headers:
{
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
//"Referer": "http://download.finance.yahoo.com",
"Accept-Encoding": "text/html",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Cookie": "cookie"
}
}
);//.pipe(out);
req.pipe(out);
req.on('error',function(err){
console.log('\n err'+err.stack);
});
req.on('end', function()
{
Ext.getCmp('pbar3').updateText(fileName.split("/")[fileName.split("/").length-1]+' downloaded...');
document.getElementById("om").innerHTML+="<br/>"+fileName.split('/')[fileName.split('/').length-1]+" downloaded...";
n++;
if(arrYahooSymbols.length > n)
{
setTimeout(function(){
downloadAndProcessYahooIEOD(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,arrYahooSymbols, yahoo_min, n);
},100);
}else
{
setTimeout(function(){
processYahooIEODFiles(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,arrYahooSymbols, yahoo_min, fileName, 0);
},1000);
}
});
} //if 200
else
{
console.log('\n ERROR while downloading '+arrYahooSymbols[n][0]);
}
}
function processYahooIEODFiles(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,arrYahooSymbols, yahoo_min, fileName, n)
{
var fileName1 = "";
if(yahoo_min == "1")
{
fileName1 = downloadpath+"temp_yahoo/yahooieod_1min_"+arrYahooSymbols[n][0]+".csv";
}
if(yahoo_min == "5")
fileName1 = downloadpath+"temp_yahoo/yahooieod_5min_"+arrYahooSymbols[n][0]+".csv";
var fs = require('fs');
var csv = require('ya-csv');
var async = require('async');
async.parallel({
data1 : function(callback){
if (fs.existsSync(fileName1)) {
fs.readFile(fileName1,'utf-8', function(err, data){
if(err) {
console.log(err.stack);
}
callback(err, data);
});
}else
{
// Here readdir's functionaliity has no use. It is written for only to send
// '0' data in its callback function because we cannot call "callback(err, data);"
// function out side a callback function
fs.readdir(downloadpath, function(err, files){
if(err) {
console.log(err.stack);
}
callback(err,'0');
});
}
}
}, function(err, result){
if(err) {
console.log(err.stack);
}
//console.log('filename-'+fileName1);
if (fs.existsSync(fileName1)) {
var arrRow = new Array();
var strSymbol = arrYahooSymbols[n][0].toString();
var strCompanyName = arrYahooSymbols[n][1].toString();
arrRow = result['data1'].split('\n');
var yahoohr = "0";
var yahoomin = "0";
for(var i=0;i<baseurl.length;i++)
{
if(baseurl[i].toString().split(',')[0] == "yahoohr")
yahoohr = baseurl[i].toString().split(',')[1].toString();
if(baseurl[i].toString().split(',')[0] == "yahoomin")
yahoomin = baseurl[i].toString().split(',')[1].toString();
}
var arrFileData = new Array();
if(yahoo_min == "5")
{
for(var i=20 ; i<arrRow.length-1;i++)
{
var tempArr = arrRow[i].split(',');
arrFileData.push(tempArr);
}
filename = "yahooieod_5min"+strSymbol+".csv";
if(arrRow.length > 19 && !fs.existsSync(outputpath+filename))
{
var header='TICKER,NAME,DATE,TIME,OPEN,HIGH,LOW,CLOSE,VOLUME,OPENINT';
var range7 = arrRow[7].toString().split(':')[1].toString().split(',');
var range8 = arrRow[8].toString().split(':')[1].toString().split(',');
var range9 = arrRow[9].toString().split(':')[1].toString().split(',');
var range10 = arrRow[10].toString().split(':')[1].toString().split(',');
var range11 = arrRow[11].toString().split(':')[1].toString().split(',');
fs.writeFile(outputpath+filename, header+'\n', function (err)
{
if (err) throw err;
for(var i=0 ; i<arrFileData.length;i++)
{
if(parseInt(arrFileData[i][0]) <= range7[1] && parseInt(arrFileData[i][0]) >= range7[1] )
{
var d = new Date();
//console.log('\n yahoo 5min tt : '+d.getTimezoneOffset());
var today = new Date(parseInt(arrFileData[i][0]) * 1000);
var dd = today.getDate();
var mm = today.getMonth()+1; //January is 0!
var yyyy = today.getFullYear();
if(dd<10){
dd='0'+dd
}
if(mm<10){
mm='0'+mm
}
today.setHours(today.getHours()+parseInt(yahoohr));
var hr = today.getHours();
today.setMinutes(today.getMinutes()+parseInt(yahoomin));
var min = today.getMinutes();
var sec = today.getSeconds();
fs.appendFile(outputpath+filename,strSymbol+','+strCompanyName+','+yyyy+mm+dd+','+(hr+':'+min+':'+sec).toString()+','+arrFileData[i][4]+','+arrFileData[i][2]+','+arrFileData[i][3]+','+arrFileData[i][1]+','+arrFileData[i][5]+','+'0'+'\n', function (err) {
if (err) throw err;
});
}else
if(parseInt(arrFileData[i][0]) >= parseInt(range8[1]) && parseInt(arrFileData[i][0]) <= parseInt(range8[2]) )
{
var d = new Date();
var today = new Date(parseInt(arrFileData[i][0]) * 1000);
var dd = today.getDate();
var mm = today.getMonth()+1; //January is 0!
var yyyy = today.getFullYear();
if(dd<10){
dd='0'+dd
}
if(mm<10){
mm='0'+mm
}
today.setHours(today.getHours()+parseInt(yahoohr));
var hr = today.getHours();
today.setMinutes(today.getMinutes()+parseInt(yahoomin));
var min = today.getMinutes();
var sec = today.getSeconds();
fs.appendFile(outputpath+filename,strSymbol+','+strCompanyName+','+yyyy+mm+dd+','+(hr+':'+min+':'+sec).toString()+','+arrFileData[i][4]+','+arrFileData[i][2]+','+arrFileData[i][3]+','+arrFileData[i][1]+','+arrFileData[i][5]+','+'0'+'\n', function (err) {
if (err) throw err;
});
}else
if(parseInt(arrFileData[i][0]) >= parseInt(range9[1]) && parseInt(arrFileData[i][0]) <= parseInt(range9[2]) )
{
var d = new Date();
var today = new Date(parseInt(arrFileData[i][0]) * 1000 );
var dd = today.getDate();
var mm = today.getMonth()+1; //January is 0!
var yyyy = today.getFullYear();
if(dd<10){
dd='0'+dd
}
if(mm<10){
mm='0'+mm
}
today.setHours(today.getHours()+parseInt(yahoohr));
var hr = today.getHours();
today.setMinutes(today.getMinutes()+parseInt(yahoomin));
var min = today.getMinutes();
var sec = today.getSeconds();
fs.appendFile(outputpath+filename,strSymbol+','+strCompanyName+','+yyyy+mm+dd+','+(hr+':'+min+':'+sec).toString()+','+arrFileData[i][4]+','+arrFileData[i][2]+','+arrFileData[i][3]+','+arrFileData[i][1]+','+arrFileData[i][5]+','+'0'+'\n', function (err) {
if (err) throw err;
});
}else
if(parseInt(arrFileData[i][0]) >= parseInt(range10[1]) && parseInt(arrFileData[i][0]) <= parseInt(range10[2]) )
{
var d = new Date();
var today = new Date(parseInt(arrFileData[i][0]) * 1000 );
var dd = today.getDate();
var mm = today.getMonth()+1; //January is 0!
var yyyy = today.getFullYear();
if(dd<10){
dd='0'+dd
}
if(mm<10){
mm='0'+mm
}
today.setHours(today.getHours()+parseInt(yahoohr));
var hr = today.getHours();
today.setMinutes(today.getMinutes()+parseInt(yahoomin));
var min = today.getMinutes();
var sec = today.getSeconds();
fs.appendFile(outputpath+filename,strSymbol+','+strCompanyName+','+yyyy+mm+dd+','+(hr+':'+min+':'+sec).toString()+','+arrFileData[i][4]+','+arrFileData[i][2]+','+arrFileData[i][3]+','+arrFileData[i][1]+','+arrFileData[i][5]+','+'0'+'\n', function (err) {
if (err) throw err;
});
}else
if(parseInt(arrFileData[i][0]) >= parseInt(range11[1]) && parseInt(arrFileData[i][0]) <= parseInt(range11[2]) )
{
var d = new Date();
var today = new Date(parseInt(arrFileData[i][0]) * 1000);
var dd = today.getDate();
var mm = today.getMonth()+1; //January is 0!
var yyyy = today.getFullYear();
if(dd<10){
dd='0'+dd
}
if(mm<10){
mm='0'+mm
}
today.setHours(today.getHours()+parseInt(yahoohr));
var hr = today.getHours();
today.setMinutes(today.getMinutes()+parseInt(yahoomin));
var min = today.getMinutes();
var sec = today.getSeconds();
fs.appendFile(outputpath+filename,strSymbol+','+strCompanyName+','+yyyy+mm+dd+','+(hr+':'+min+':'+sec).toString()+','+arrFileData[i][4]+','+arrFileData[i][2]+','+arrFileData[i][3]+','+arrFileData[i][1]+','+arrFileData[i][5]+','+'0'+'\n', function (err) {
if (err) throw err;
});
}
}
});
}
}
if(yahoo_min == "1")
{
// console.log('\n For 1min');
for(var i=15 ; i<arrRow.length-1;i++)
{
var tempArr = arrRow[i].split(',');
arrFileData.push(tempArr);
}
var filename = "yahooieod_1min"+strSymbol+".csv";
if(arrRow.length > 14 && !fs.existsSync(outputpath+filename))
{
var header='TICKER,NAME,DATE,TIME,OPEN,HIGH,LOW,CLOSE,VOLUME,OPENINT';
fs.writeFile(outputpath+filename, header+'\n', function (err)
{
if(err)console.log(err)
for(var i=0;i<arrFileData.length; i++)
{
var d = new Date();
//console.log('\n yahooo 1min tt : '+d.getTimezoneOffset());
var today = new Date(parseInt(arrFileData[i][0])*1000);
var dd = today.getDate();
var mm = today.getMonth()+1; //January is 0!
var yyyy = today.getFullYear();
if(dd<10){
dd='0'+dd
}
if(mm<10){
mm='0'+mm
}
today.setHours(today.getHours()+parseInt(yahoohr));
var hr = today.getHours();
today.setMinutes(today.getMinutes()+parseInt(yahoomin));
var min = today.getMinutes();
var sec = today.getSeconds();
fs.appendFile(outputpath+filename,strSymbol+','+strCompanyName+','+yyyy+mm+dd+','+(hr+':'+min+':'+sec).toString()+','+arrFileData[i][4]+','+arrFileData[i][2]+','+arrFileData[i][3]+','+arrFileData[i][1]+','+arrFileData[i][5]+','+'0'+'\n', function (err) {
if (err) throw err;
});
}
});
}//if
}
n++;
Ext.getCmp('pbar3').updateText('Processing '+filename+' completed');
document.getElementById("om").innerHTML+='<br/>Processing '+filename+' completed';
if(arrYahooSymbols.length > n)
{
loadprogressbar();
setTimeout(function(){
processYahooIEODFiles(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn,arrYahooSymbols, yahoo_min, fileName, n);
},1000);
}else
{
//console.log('\n Finished All');
/************/
visit('yahooieod.html');
/***************/
sn++;
callfun(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn);
}
}
} );
}