shubhadownloader
Version:
There is large amount of information available in market place. The markets are always in sync. In today's world investors find it very difficult to make effective use of information available to them. Shubha Downloader is great tool which resolves this problem and helps investor to increase his productivity and stay focused on decision making. Shubha Downloader enable end user to download market data from available sources and organize it. Shubha Downloader is Open source & FREE utility for end users. Shubha Downloader have main features as follows End of the day market data from web to your favorite charting application . Fundamental market data from web to your favorite charting application. Market reports from web to your favorite charting application.
1,267 lines (1,131 loc) • 79.7 kB
JavaScript
/*****************************
*
*Author:JAYANT PATIL
*COMPANY NAME XNet Inc,Pune.
*last modified date:01-jan-2013
*
*
*
*Description:
*this function will create folder hierarchies if not found and
*
*variables :
*count --->contains array of date in which we take one date process files regarding to that
*n --->it is a counter variable
*baseurl-->it contians all url for nse equity
*downloadpath---> it contain download path of file for processing
*unzippath ----> it contains unzip folder path
*outputpah ----> it containds output older path
*root ----> it contains path of output folder to process file like C:/data
*
*Functions:
*downloadseclist-----> this fun download seclist.csv
*downloadfiles -----> this fun download nse equity, mto, file for each date available
*loadevent and download -----> these are the functions for start up
*performtask -----> this is used for calling downloadfiles fun and set count i.e no of date are present or to download files for that dates
*getMonth -----> this is used for the convertting from string month to integer month i.e. in number
* it get mon vaiable and conert it to interger reutrn number format
*getmnth -----> this is used for the convertting integer month to string month
* in this parameter mon
*
*
*copyright 2013,
******************************/
/********************************
<Shubha Downloader, It downloads end of day pricing data from publicly available sites and converts it different formats....>
Copyright (C) 2013 Shubhalabha.in
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
***********************************/
var http = require('http'),fs = require('fs'),
request = require('request'),
AdmZip = require('adm-zip');
function loadevent(fromd, tod)
{
Ext.getCmp('pbar3').updateText('Processing started please wait . . .');
Ext.getCmp('pbar3').updateProgress(0.002);
var baseurl=new Array();
var csv = require('ya-csv');
//var setting=new Array();
var reader = csv.createCsvFileReader('baseurl.csv',{
'separator': ','
});
reader.addListener('data', function(data) {
baseurl.push(data);
});
reader = csv.createCsvFileReader('settings.csv',{
'separator': ','
});
reader.addListener('data', function(data) {
baseurl.push(data);
if(data[0]=="options"){
download(fromd, tod, baseurl);
}
});
/*process.on('uncaughtException', function(err) {
console.log('\n In downloader'+err.stack);
$("#p3").hide();
document.getElementById("om").innerHTML+="<br/><font color='red'><b>An error has been occurred while processing. Please try again.</b>";
});*/
}
function downloadseclist(downloadpath,logpath)
{
var http = require('http'),fs = require('fs'), request = require('request');
//// Downloading NSE Security list////
var url_seclist='http://www.nseindia.com/content/equities/sec_list.csv';
request(
{
method: 'GET',
uri: url_seclist,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nseindia.com/products/content/all_daily_reports.htm",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Cookie": "cookie"
}
},
function(error, response, body)
{
if(response != null)
{
if(response.statusCode != null || response.statusCode != "")
{
if (!error && response.statusCode == 200)
{
downloadcsv(response.statusCode);
}
else
{
downloadcsv(404);
}
}else
{
downloadcsv(404);
}
}
else
{
downloadcsv(404);
}
}
);
function downloadcsv(statuscode)
{
if(statuscode==200){
var secout = fs.createWriteStream(downloadpath+'sec_list.csv'); // For saving NSE Equity Sec list
var req=request(
{
method: 'GET',
uri: url_seclist,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nseindia.com/products/content/all_daily_reports.htm",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Cookie": "cookie"
}
});
req.pipe(secout);
loadprogressbar();
req.on('end',function(){
document.getElementById("om").innerHTML+="<br/>sec_list is downloaded";
addlog(logpath,'sec_list is downloaded');
});
}
else
{
document.getElementById("om").innerHTML+="<br/><font color=red>sec_list.csv is not on server</font>";
addlog(logpath,'sec_list.csv is not on server');
}
}
/*process.on('uncaughtException', function(err) {
console.log('\n In downloader'+err.stack);
$("#p3").hide();
document.getElementById("om").innerHTML+="<br/><font color='red'><b>An error has been occurred while processing. Please try again.</b>";
});*/
return true;
}
function download(fromd, tod, baseurl)
{
Ext.getCmp('pbar3').updateText('Processing started please wait . . .');
Ext.getCmp('pbar3').updateProgress(0.04);
document.getElementById("om").innerHTML="loading..."
for(var i=0; i<baseurl.length;i++)
{
if(baseurl[i].toString().split(',')[0]=="datafolder")
{ //document.getElementById("om").innerHTML+="data-"+baseurl[i].toString().split(',')[1];
makedir(baseurl[i].toString().split(',')[1],baseurl,fromd,tod);
}
}
/*process.on('uncaughtException', function(err) {
console.log('\n In downloader'+err.stack);
$("#p3").hide();
document.getElementById("om").innerHTML+="<br/><font color='red'><b>An error has been occurred while processing. Please try again.</b>";
});*/
}
function downloadfiles(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)//(baseurl,mon,m,d,yy)
{
try{
var mon=count[n].split('-')[0];
var m=count[n].split('-')[1];
var d=count[n].split('-')[2];
var yy=count[n].split('-')[3];
var y=yy.substr(2,2);
var month=mon.toUpperCase()
Ext.getCmp('pbar3').updateText('cm'+d+month+yy+'bhav.csv.zip downloading. . .');
Ext.getCmp('pbar3').updateProgress(0.2);
var fs = require('fs');
if(!fs.existsSync(downloadpath+'cm'+d+month+yy+'bhav.csv.zip')){
var file_url='',file_url1='';
for (var i=0;i<baseurl.length;i++)
{
if(baseurl[i].toString().split(',')[0]=='nsecmbhavcopy')
file_url=baseurl[i].toString().split(',')[1]+yy+'/'+month+'/cm'+d+''+month+''+yy+'bhav.csv.zip';
//if(baseurl[i].toString().split(',')[0]=='nsemto')
// file_url1=baseurl[i].toString().split(',')[1]+'MTO_'+d+''+m+''+yy+'.DAT';
}
var url = require('url');
var http = require('http'),
AdmZip = require('adm-zip'),
request = require('request');
// For saving NSE Equity Security-wise Delivery Position
// Downloading NSE Bhavcopy
request(
{
method: 'GET',
uri: file_url,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nseindia.com/products/content/all_daily_reports.htm",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Cookie": "cookie"
}
},
function(error, response, body) {
if(response != null)
{
if(response.statusCode != null || response.statusCode != "")
{
if (!error && response.statusCode == 200)
{
downloadcsvzip(response.statusCode);
}
else
{
downloadcsvzip(404);
}
}else
{
downloadcsvzip(404);
}
}else
{
downloadcsvzip(404);
}
}
);
function downloadcsvzip(statuscode)
{
if(statuscode==200){
Ext.getCmp('pbar3').updateText('cm'+d+month+yy+'bhav.csv.zip dwonloading started . . .');
var out = fs.createWriteStream(downloadpath+'cm'+d+month+yy+'bhav.csv.zip'); // For saving NSE Equity bhavcopy
var req = request(
{
method: 'GET',
uri: file_url,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nseindia.com/products/content/all_daily_reports.htm",
"Accept-Encoding": "gzip,deflate,sdch",
"encoding": "null",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*//*;q=0.8",
"Cookie": "cookie"
}
}
);
req.on('error',function(){
document.getElementById("om").innerHTML+='<br/>Error occured in downloading cm'+d+month+yy+'bhav.csv.zip';
addlog(logpath, 'Error occured in downloading cm'+d+month+yy+'bhav.csv.zip');
setTimeout(function (){
downloadfiles(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);
}, 1000);
});
req.pipe(out); //request pipeout
req.on('end', function() {
Ext.getCmp('pbar3').updateText('cm'+d+month+yy+'bhav.csv.zip is downloaded. . .');
document.getElementById("om").innerHTML+='<br>cm'+d+month+yy+'bhav.csv.zip is downloaded...'
addlog(logpath, 'cm'+d+month+yy+'bhav.csv.zip is downloaded');
var zip = new AdmZip(downloadpath+'cm'+d+month+yy+'bhav.csv.zip');
zip.extractAllTo(unzippath, /*overwrite*/true);
Ext.getCmp('pbar3').updateText('cm'+d+month+yy+'bhav.csv.zip is extracted. . .');
Ext.getCmp('pbar3').updateProgress(1);
document.getElementById("om").innerHTML+='<br>cm'+d+month+yy+'bhav.csv.zip is extracted...';
addlog(logpath, 'cm'+d+month+yy+'bhav.csv.zip is extracted');
n++;
if(count.length>n)
{
loadprogressbar();
setTimeout(function (){
downloadfiles(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
//document.getElementById("om").innerHTML+="<br/>nse processing.."+n;
loadprogressbar();
downloadmto(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);
}
});
}
else
{
document.getElementById("om").innerHTML+='<br/><font color=red>cm'+d+month+yy+'bhav.csv.zip is not present on server</font>';
addlog(logpath, 'cm'+d+month+yy+'bhav.csv.zip is not present on server');
n++;
loadprogressbar();
/*if(count.length==1){
$("#p3").hide();
}
else
{*/if(count.length>n)
{
setTimeout(function (){
downloadfiles(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
//document.getElementById("om").innerHTML+="<br/>nse processing.."+n;
downloadmto(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);
}
}
}
/**************end of download bhav csv*************/
}
else
{
n++;
if(count.length>n)
{
loadprogressbar();
setTimeout(function (){
downloadfiles(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
//document.getElementById("om").innerHTML+="<br/>nse processing.."+n;
loadprogressbar();
downloadmto(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);
}
}
}
catch(err){
document.getElementById("om").innerHTML+="<br/>File is not found for this date: "+d+m+yy;
addlog(logpath, 'File is not found for this date: '+d+m+yy);
setTimeout(function (){
downloadfiles(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
}
/*process.on('uncaughtException', function(err) {
console.log('\n In downloader'+err.stack);
$("#p3").hide();
document.getElementById("om").innerHTML+="<br/><font color='red'><b>An error has been occurred while processing. Please try again.</b>";
});*/
}
function performtask(baseurl,downloadpath,unzippath,outputpath,logpath,fromd, tod)
{
/******************************/
var ld=new Date(),fs=require('fs');
var logfilename=ld.getTime().toString()+'.txt';
fs.writeFile(logpath+logfilename,'log date:-'+ld+'\n',function (err) {
if (err)
if(err){
document.getElementById("om").innerHTML+="<br/>"+err;
}
});
logpath=logpath+logfilename;
/******creating logs here*****/
var opPath= ""; //outputpath.replace(/std_csv/g,'events');
var csv = require('ya-csv');
var fs =require('fs');
if(fs.existsSync('settings.csv'))
{
var reader = csv.createCsvFileReader('settings.csv',{
'separator': ','
});
reader.addListener('data', function(data)
{
if(data[0]=='datafolder')
{
opPath = data[1];
}
if(data[0]=='options')
{
if(data[1].search('Delete_All_Events')>=0)
{
// delete events file
if(fs.existsSync(opPath+'/events/CA_ALL_FORTHCOMING.csv'))
{
fs.unlinkSync(opPath+'/events/CA_ALL_FORTHCOMING.csv');
}
if(fs.existsSync(opPath+'/events/BM_All_Forthcoming.csv'))
{
fs.unlinkSync(opPath+'/events/BM_All_Forthcoming.csv');
}
}else
{
calendarEventCADownload(opPath+'/events/CA_ALL_FORTHCOMING.csv');
calendarEventBMDownload(opPath+'/events/BM_All_Forthcoming.csv');
}
}
});
}
Ext.getCmp('pbar3').updateText('Processing started please wait . . .');
Ext.getCmp('pbar3').updateProgress(0.006);
var datefrom=fromd;//document.getElementById("datefield-1029-inputEl").value;
var dateto=tod;//document.getElementById("datefield-1031-inputEl").value;
var fromdate=datefrom.split('/');
var todate=dateto.split('/');
var count=new Array();
$(document).ready(function(){
var bd,beginDate = new Date(fromdate[0]+','+fromdate[1]+','+fromdate[2]);//MM DD YYYY
var ed,endDate = new Date(todate[0]+','+todate[1]+','+todate[2]);
var mnth,dy;
while (beginDate <= endDate) {
if(!beginDate.toDateString().match('Sat')&& !beginDate.toDateString().match('Sun'))
{
if((beginDate.getMonth()+1)<10)
mnth=0+''+(beginDate.getMonth()+1);
else
mnth=beginDate.getMonth()+1;
if(beginDate.getDate()<10)
dy=0+''+beginDate.getDate();
else
dy=beginDate.getDate();
var mon,m,d,yy;
mon=beginDate.toDateString().split(" ")[1];
m=mnth;
d=dy;
yy=beginDate.getFullYear();
// document.getElementById("om").innerHTML+='<br/>performing task for'+d+mon+yy;
var dm=d+'-'+m;
count.push(mon+'-'+m+'-'+d+'-'+yy);
}
beginDate.setTime(beginDate.getTime() + 86400000);
}
});
if(count.length>0)
{
Ext.getCmp('pbar3').updateText('Cash Market dwonloading is started . . .');
Ext.getCmp('pbar3').updateProgress(0.02);
// callfun(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,0);
downloadseclist(downloadpath,logpath);
downloadfiles(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);
/* //Here we check for selection option
var ds;
for (var i=0;i<baseurl.length;i++)
{
if(baseurl[i].toString().split(',')[0]=='options')
ds=baseurl[i].toString().split(',')[1];
}
if(ds.search("Reports for all reports item except nse combined report")>=0)
{
//Note:- ****This is for NSE Bulk block vol openint ****
process_nse_bulk(baseurl,downloadpath,unzippath,outputpath);
process_nse_blockdeal(baseurl,downloadpath,unzippath,outputpath);
nse_equity_vol(baseurl,downloadpath,unzippath,outputpath);
nse_equity_openint(baseurl,downloadpath,unzippath,outputpath);
process_bse_bulk(baseurl,downloadpath,unzippath,outputpath);
process_bse_blockdeal(baseurl,downloadpath,unzippath,outputpath);
process_nse_bc(count,count.length-1,baseurl,downloadpath,unzippath,outputpath,logpath)
}
//Note:- *****This is for BSE BULK and Block ****
//Note:- *****This is for BSE index data at the end of this function next function is called............**
if(ds.search("Bse Index")>=0)
process_bse_indexdata(baseurl,downloadpath,unzippath,outputpath,fromd,tod,logpath,count);
else if(ds.search("Yahoo Fundamental")>=0)
{
Ext.getCmp('pbar3').updateText('Yahoo fundamental Processing is started . . .');
Ext.getCmp('pbar3').updateProgress(0.03);
process_yahoodata(outputpath,logpath);
}
else if(ds.search("MCX Commodity Futures")>=0)
{
Ext.getCmp('pbar3').updateText('MCX bhavcopy Processing is started . . .');
Ext.getCmp('pbar3').updateProgress(0.03);
mcx_bhavcopy(count,0,baseurl,downloadpath,unzippath,outputpath,logpath);
}
else if(ds.search("National Spot Exchange")>=0)
{
Ext.getCmp('pbar3').updateText('NSX EOD Processing is started . . .');
Ext.getCmp('pbar3').updateProgress(0.04);
process_nsxeod(count,0,baseurl,downloadpath,unzippath,outputpath,logpath);
}
else if("MCX Index")
{
Ext.getCmp('pbar3').updateText('MCX Index Processing is started . . .');
Ext.getCmp('pbar3').updateProgress(0.04);
mcx_Future_IndexHistory(count,0,baseurl,downloadpath,unzippath,outputpath,logpath)
}
//else
/**************
downloadseclist(downloadpath,logpath)) //calling function to downoad seclist file
//downloadfiles(count,0,baseurl,downloadpath,unzippath,outputpath,logpath);
process_yahoodata(outputpath,logpath);
mcx_bhavcopy(count,0,baseurl,downloadpath,unzippath,outputpath,logpath);
****************/
}
else
$("#p3").hide();
/*process.on('uncaughtException', function(err) {
console.log('\n In downloader'+err.stack);
$("#p3").hide();
document.getElementById("om").innerHTML+="<br/><font color='red'><b>An error has been occurred while processing. Please try again.</b>";
});*/
}
function getmonth(mon){
switch(parseInt(mon))
{
case 1:
mon='JAN';
break;
case 2:
mon='FEB';
break;
case 3:
mon='MAR';
break;
case 4:
mon='APR';
break;
case 5:
mon='MAY';
break;
case 6:
mon='JUN';
break;
case 7:
mon='JUL';
break;
case 8:
mon='AUG';
break;
case 9:
mon='SEP';
break;
case 10:
mon='OCT';
break;
case 11:
mon='NOV';
break;
case 12:
mon='DEC';
break;
}
return mon;
}
function getMonth(mon){
var month='';
switch(mon.toUpperCase())
{
case 'JAN':
month='01';
break;
case 'FEB':
month='02';
break;
case 'MAR':
month='03';
break;
case 'APR':
month='04';
break;
case 'MAY':
month='05';
break;
case 'JUN':
month='06';
break;
case 'JUL':
month='07';
break;
case 'AUG':
month='08';
break;
case 'SEP':
month='09';
break;
case 'OCT':
month='10';
break;
case 'NOV':
month='11';
break;
case 'DEC':
month='12';
break;
}
return month;
}
// Downloading NSE Security-wise Delivery Position
function downloadmto(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
{
/* var ds;
for (var i=0;i<baseurl.length;i++)
{
if(baseurl[i].toString().split(',')[0]=='options')
ds=baseurl[i].toString().split(',')[1];
}
if(ds.search("MCX Index")>=0)
{
var om=323;
while(om<327)
{
process_mcxindex(count,om,baseurl,downloadpath,unzippath,outputpath,logpath);
om++;
}
if(ds.search("MCX Commodity Futures")>=0)
process_mcxeodc(count,0,baseurl,downloadpath,unzippath,outputpath,logpath);
}*/
var mon=count[n].split('-')[0];
var m=count[n].split('-')[1];
var d=count[n].split('-')[2];
var yy=count[n].split('-')[3];
var y=yy.substr(2,2);
var month=mon.toUpperCase()
Ext.getCmp('pbar3').updateText('MTO_'+d+''+m+''+yy+'.DAT downloading. . .');
Ext.getCmp('pbar3').updateProgress(0.02);
var fs = require('fs');
if(!fs.existsSync(downloadpath+'MTO_'+d+''+m+''+yy+'.DAT')){
var file_url='',file_url1='';
for (var i=0;i<baseurl.length;i++)
{
if(baseurl[i].toString().split(',')[0]=='nsemto')
file_url1=baseurl[i].toString().split(',')[1]+'MTO_'+d+''+m+''+yy+'.DAT';
}
var request = require('request');
request(
{
method: 'GET',
uri: file_url1,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nseindia.com/products/content/all_daily_reports.htm",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Cookie": "cookie"
}
},
function(error, response, body) {
if(response != null)
{
if(response.statusCode != null || response.statusCode != "")
{
if (!error && response.statusCode == 200)
{
download_dat(response.statusCode);
}
else
{
download_dat(404);
}
}else
{
download_dat(404);
}
}else
{
download_dat(404);
}
}
);
function download_dat(statuscode)
{
var fs=require('fs');
if(statuscode==200){
Ext.getCmp('pbar3').updateText('MTO_'+d+''+m+''+yy+'.DAT downloading started. . .');
var datout = fs.createWriteStream(downloadpath+'MTO_'+d+''+m+''+yy+'.DAT');
var req1= request(
{
method: 'GET',
uri: file_url1, //'http://nseindia.com/archives/equities/mto/MTO_'+d+''+m+''+yy+'.DAT',
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nseindia.com/products/content/all_daily_reports.htm",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Cookie": "cookie"
}
}
);
req1.on('error',function(){
document.getElementById("om").innerHTML+='<br/>Error occured in downloading MTO_'+d+''+m+''+yy+'.DAT';
addlog(logpath, 'Error occured in downloading MTO_'+d+''+m+''+yy+'.DAT');
setTimeout(function (){
downloadmto(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
});
req1.pipe(datout);
req1.on('end', function() {
Ext.getCmp('pbar3').updateText('MTO_'+d+''+m+''+yy+'.DAT download completed. . .');
document.getElementById("om").innerHTML+='<br>MTO_'+d+''+m+''+yy+'.DAT is downloaded ..' ;
addlog(logpath, 'MTO_'+d+''+m+''+yy+'.DAT is downloaded ..');
n++;
if(count.length>n)
{
loadprogressbar();
setTimeout(function (){
downloadmto(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
//document.getElementById("om").innerHTML+="<br/>nse processing.."+n;
//process_nse(count,0,baseurl,downloadpath,unzippath,outputpath);
loadprogressbar();
download_przip(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}
});
}
else
{
document.getElementById("om").innerHTML+='<br/><font color=red>MTO_'+d+''+m+''+yy+'.DAT is not present on server</font>';
addlog(logpath, 'MTO_'+d+''+m+''+yy+'.DAT is not present on server');
loadprogressbar();
n++;
if(count.length>n)
{
setTimeout(function (){
downloadmto(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
//document.getElementById("om").innerHTML+="<br/>nse processing.."+n;
// process_nse(count,0,baseurl,downloadpath,unzippath,outputpath);
download_przip(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);
}
}
}
}
else
{
n++;
if(count.length>n)
{
setTimeout(function (){
downloadmto(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
//document.getElementById("om").innerHTML+="<br/>nse processing.."+n;
// process_nse(count,0,baseurl,downloadpath,unzippath,outputpath);
download_przip(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);
}
}
/*process.on('uncaughtException', function(err) {
console.log('\n In downloader'+err.stack);
$("#p3").hide();
document.getElementById("om").innerHTML+="<br/><font color='red'><b>An error has been occurred while processing. Please try again.</b>";
});*/
}
function download_przip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)//(baseurl,mon,m,d,yy)
{
try{
var mon=count[n].split('-')[0];
var m=count[n].split('-')[1];
var d=count[n].split('-')[2];
var yy=count[n].split('-')[3];
var y=yy.substr(2,2);
var month=mon.toUpperCase();
Ext.getCmp('pbar3').updateText('PR'+d+m+y+'.zip is downloading. . .');
Ext.getCmp('pbar3').updateProgress(0.1);
var fs = require('fs');
if(!fs.existsSync(downloadpath+'PR'+d+m+y+'.zip')){
document.getElementById("om").innerHTML+='<br/>PR'+d+m+y+'.zip is downloading..';
var file_url;//'http://nseindia.com/archives/equities/bhavcopy/pr/PR'+m+d+y+'.zip';//,file_url1='';
for (var i=0;i<baseurl.length;i++)
{
if(baseurl[i].toString().split(',')[0]=='nseprbhavcopy')
file_url=baseurl[i].toString().split(',')[1]+'PR'+d+m+y+'.zip';
//if(baseurl[i].toString().split(',')[0]=='nsemto')
// file_url1=baseurl[i].toString().split(',')[1]+'MTO_'+d+''+m+''+yy+'.DAT';
}
var url = require('url');
var http = require('http'),fs = require('fs'),AdmZip = require('adm-zip'),request = require('request');
//Download PR Bhavcopy File for NSE
request(
{
method: 'GET',
uri: file_url,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nseindia.com/products/content/all_daily_reports.htm",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Cookie": "cookie"
}
},
function(error, response, body) {
if(response != null)
{
if(response.statusCode != null || response.statusCode != "")
{
if (!error && response.statusCode == 200)
{
downloadcsvzip(response.statusCode);
}
else
{
downloadcsvzip(404);
}
}else
{
downloadcsvzip(404);
}
}else
{
downloadcsvzip(404);
}
}
);
function downloadcsvzip(statuscode)
{
//here we takeing value of the downloading options.......
//
var ds;
for (var i=0;i<baseurl.length;i++)
{
if(baseurl[i].toString().split(',')[0]=='options')
ds=baseurl[i].toString().split(',')[1];
}
//this alert boe to show value selected//Ext.Msg.alert('Message', ds.search("AmiBroker"));
//
if(statuscode==200){
Ext.getCmp('pbar3').updateText('PR'+d+m+y+'.zip is downloading started. . .');
var out = fs.createWriteStream(downloadpath+'PR'+d+m+y+'.zip'); // For saving NSE Equity bhavcopy
var req = request(
{
method: 'GET',
uri: file_url,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nseindia.com/products/content/all_daily_reports.htm",
"Accept-Encoding": "gzip,deflate,sdch",
"encoding": "null",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*//*;q=0.8",
"Cookie": "cookie"
}
}
);
req.on('error',function(){
document.getElementById("om").innerHTML+='<br/>Error occured in downloading PR'+d+m+y+'.zip';
addlog(logpath, 'Error occured in downloading PR'+d+m+y+'.zip');
download_przip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);//(baseurl,mon,m,d,yy)
});
req.pipe(out); //request pipeout
req.on('end', function() {
Ext.getCmp('pbar3').updateText('PR'+d+m+y+'.zip is downloaded. . .');
document.getElementById("om").innerHTML+='<br>PR'+d+m+y+'.zip is downloaded...';
addlog(logpath, 'PR'+d+m+y+'.zip is downloaded...');
var zip = new AdmZip(downloadpath+'PR'+d+m+y+'.zip');
zip.extractAllTo(unzippath, /*overwrite*/true);
Ext.getCmp('pbar3').updateText('PR'+d+m+y+'.zip is extracting. . .');
document.getElementById("om").innerHTML+='<br>PR'+d+m+y+'.zip is extracted...'
addlog(logpath, 'PR'+d+m+y+'.zip is extracted...');
if (fs.existsSync(unzippath+'fo'+d+m+yy+'.zip')){
zip = new AdmZip(unzippath+'fo'+d+m+yy+'.zip');
zip.extractAllTo(unzippath+'unzip1/', /*overwrite*/true);
//document.getElementById("om").innerHTML+='<br>fo'+d+m+yy+'.zip is extracted...';
}
if (fs.existsSync(unzippath+'cd'+d+m+yy+'.zip')){
zip = new AdmZip(unzippath+'cd'+d+m+yy+'.zip');
zip.extractAllTo(unzippath+'unzip1/', /*overwrite*/true);
//document.getElementById("om").innerHTML+='<br>cd'+d+m+yy+'.zip is extracted...'
Ext.getCmp('pbar3').updateText('PR'+d+m+y+'.zip is extracted. . .');
}
n++;
if(count.length>n)
{
loadprogressbar();
setTimeout(function (){
download_przip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
loadprogressbar();
process_nse(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);
// callfun(baseurl,0);
}
});
}
else
{
document.getElementById("om").innerHTML+='<br/><font color=red>PR'+m+d+y+'.zip is not present on server</font>';
addlog(logpath, 'PR'+m+d+y+'.zip is not present on server');
n++;
loadprogressbar();
/*if(count.length==1){
$("#p3").hide();
}
else
{*/if(count.length>n)
{
setTimeout(function (){
download_przip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
process_nse(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);
//callfun(baseurl,0);
}
}
}
/**************end of download bhav csv*************/
}
else
{
n++;
if(count.length>n)
{
loadprogressbar();
setTimeout(function (){
download_przip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
loadprogressbar();
process_nse(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod);
// callfun(baseurl,0);
}
}
}
catch(err){
document.getElementById("om").innerHTML+="<br/>File is not found for this date: <br/>"+err;
setTimeout(function (){
download_przip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod)
}, 1000);
}
/*process.on('uncaughtException', function(err) {
console.log('\n In downloader'+err.stack);
$("#p3").hide();
document.getElementById("om").innerHTML+="<br/><font color='red'><b>An error has been occurred while processing. Please try again.</b>";
});*/
}
function download_nse_cmbnd_rpt_zip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)//(baseurl,mon,m,d,yy)
{
try{
var mon=count[n].split('-')[0];
var m=count[n].split('-')[1];
var d=count[n].split('-')[2];
var yy=count[n].split('-')[3];
var y=yy.substr(2,2);
var month=mon.toUpperCase();
document.getElementById("om").innerHTML+='<br/>combined_report'+d+m+yy+'.zip is downloading..';
var file_url;//'http://nseindia.com/archives/equities/bhavcopy/pr/PR'+m+d+y+'.zip';//,file_url1='';
for (var i=0;i<baseurl.length;i++)
{
if(baseurl[i].toString().split(',')[0]=='nsecmbndrpt')
file_url=baseurl[i].toString().split(',')[1]+'combined_report'+d+m+yy+'.zip';
//if(baseurl[i].toString().split(',')[0]=='nsemto')
// file_url1=baseurl[i].toString().split(',')[1]+'MTO_'+d+''+m+''+yy+'.DAT';
}
var url = require('url');
var http = require('http'),fs = require('fs'),AdmZip = require('adm-zip'),request = require('request');
//Download PR Bhavcopy File for NSE
request(
{
method: 'GET',
uri: file_url,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nseindia.com/products/content/all_daily_reports.htm",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Cookie": "cookie"
}
},
function(error, response, body) {
if(response != null)
{
if(response.statusCode != null || response.statusCode != "")
{
if (!error && response.statusCode == 200)
{
downloadcsvzip(response.statusCode);
}
else
{
downloadcsvzip(404);
}
}else
{
downloadcsvzip(404);
}
}else
{
downloadcsvzip(404);
}
}
);
function downloadcsvzip(statuscode)
{
if(statuscode==200){
var out = fs.createWriteStream(downloadpath+'combined_report'+d+m+yy+'.zip'); // For saving NSE Equity bhavcopy
var req = request(
{
method: 'GET',
uri: file_url,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nseindia.com/products/content/all_daily_reports.htm",
"Accept-Encoding": "gzip,deflate,sdch",
"encoding": "null",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*//*;q=0.8",
"Cookie": "cookie"
}
}
);
req.on('error',function(){
document.getElementById("om").innerHTML+='<br/>Error occured in downloading combined_report'+d+m+yy+'.zip';
addlog(logpath, 'Error occured in downloading combined_report'+d+m+yy+'.zip');
setTimeout(function (){
download_nse_cmbnd_rpt_zip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)
}, 1000);
});
req.pipe(out); //request pipeout
req.on('end', function() {
document.getElementById("om").innerHTML+='<br>combined_report'+d+m+yy+'.zip is downloaded...';
addlog(logpath, 'combined_report'+d+m+yy+'.zip is downloaded...');
var zip = new AdmZip(downloadpath+'combined_report'+d+m+yy+'.zip');
var outputpath1=outputpath.replace(/std_csv/g,'reports');
zip.extractAllTo(unzippath, /*overwrite*/true);
document.getElementById("om").innerHTML+='<br>combined_report'+d+m+yy+'.zip is extracted...';
addlog(logpath, 'combined_report'+d+m+yy+'.zip is extracted...');
fs.createReadStream(unzippath+'combined_report'+d+m+yy+'.xls').pipe(fs.createWriteStream(outputpath1+'combined_report'+d+m+yy+'.xls'));
document.getElementById("om").innerHTML+='<br>combined_report'+d+m+yy+'.xls is processed...';
addlog(logpath, 'combined_report'+d+m+yy+'.xls is processed...');
n++;
if(count.length>n)
{
loadprogressbar();
setTimeout(function (){
download_nse_cmbnd_rpt_zip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
Ext.getCmp('pbar3').updateProgress(0);
/************/
visit('nsecombine.html');
/***************/
sn++;
callfun(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn);
}
});
}
else
{
document.getElementById("om").innerHTML+='<br/><font color=red>combined_report'+d+m+yy+'.zip is not present on server</font>';
addlog(logpath, 'combined_report'+d+m+yy+'.zip is not present on server');
n++;
loadprogressbar();
/*if(count.length==1){
$("#p3").hide();
}
else
{*/if(count.length>n)
{
setTimeout(function (){
download_nse_cmbnd_rpt_zip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)
}, 1000);
//document.getElementById("om").innerHTML+='<br/>Loading next....'+n;
}
else
{
Ext.getCmp('pbar3').updateProgress(0);
/************/
visit('nsecombine.html');
/***************/
sn++;
callfun(count,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn);
}
}
}
/**************end of download bhav csv*************/
}
catch(err){
document.getElementById("om").innerHTML+="<br/>File is not found for this date: <br/>"+err;
addlog(logpath, "File is not found for this date: <br/>"+err);
setTimeout(function (){
download_nse_cmbnd_rpt_zip(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)
}, 1000);
}
/*process.on('uncaughtException', function(err) {
console.log('\n In downloader'+err.stack);
$("#p3").hide();
document.getElementById("om").innerHTML+="<br/><font color='red'><b>An error has been occurred while processing. Please try again.</b>";
});*/
}
//******************--BSE -----****************/
//
// This is for the bse equity downloading
//
/********************=-------=****************/
function download_bseequity(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)//(baseurl,mon,m,d,yy)
{
try{
var mon=count[n].split('-')[0];
var m=count[n].split('-')[1];
var d=count[n].split('-')[2];
var yy=count[n].split('-')[3];
var y=yy.substr(2,2);
var month=mon.toUpperCase()
var file_url='',file_url1='';
for (var i=0;i<baseurl.length;i++)
{
if(baseurl[i].toString().split(',')[0]=='bseequity')
file_url=baseurl[i].toString().split(',')[1]+'eq'+d+m+y+'_csv.zip';
//if(baseurl[i].toString().split(',')[0]=='bsedata')
// file_url1=baseurl[i].toString().split(',')[1]+yy+'/SCBSEALL'+d+m+'.zip';//http://www.bseindia.com/BSEDATA/gross/2012/SCBSEALL2311.zip
}
var url = require('url');
var http = require('http'),
fs = require('fs'),
AdmZip = require('adm-zip'),
request = require('request');
// For saving NSE Equity Security-wise Delivery Position
// Downloading NSE Bhavcopy
request(
{
method: 'GET',
uri: file_url,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.bseindia.com/markets/equity/EQReports/BhavCopyDebt.aspx?expanda