shubhadownloader
Version:
There is large amount of information available in market place. The markets are always in sync. In today's world investors find it very difficult to make effective use of information available to them. Shubha Downloader is great tool which resolves this problem and helps investor to increase his productivity and stay focused on decision making. Shubha Downloader enable end user to download market data from available sources and organize it. Shubha Downloader is Open source & FREE utility for end users. Shubha Downloader have main features as follows End of the day market data from web to your favorite charting application . Fundamental market data from web to your favorite charting application. Market reports from web to your favorite charting application.
391 lines (313 loc) • 12.5 kB
JavaScript
/********************************
<Shubha Downloader, It downloads end of day pricing data from publicly available sites and converts it different formats....>
Copyright (C) 2013 Shubhalabha.in
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
***********************************/
function process_nsxeod(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)
{
try
{
var mon=count[n].split('-')[0];
var m=count[n].split('-')[1];
var d=count[n].split('-')[2];
var yy=count[n].split('-')[3];
var y=yy.substr(2,2);
var currdate1=new Date();
var year1=currdate1.getFullYear();
if(year1==yy){
var csv = require('ya-csv');
var async = require('async');
var htmlparser = require("htmlparser");
var DomHandler = require("DomHandler");
var DomUtils = require("DomUtils");
var DomElementType = require('DomElementType');
var http = require('http'), fs = require('fs'), request = require('request');
var urlStockinfo = "http://www.nationalspotexchange.com/Sitepages/BhavCopyDateWise.aspx"; // url to which request have to post
var r = request.get(urlStockinfo); // This request is for to get __VIEWSTATE and __EVENTVALIDATION values
var resData = "";
var resData1 = "";
r.end();
r.on('response', function(response) {
response.on('data', function(chunk) {
resData += chunk;
});
});
r.on('end', function ()
{
//console.log('\n data = '+resData);
var viewState_value = "";
var eventValidation_value = "";
var mImgBtnGo_X = "";
try{
// Parsing html for getting value of __VIEWSTATE
var parser = new htmlparser.Parser(
{
onopentag: function(name, attribs){
if(name === "input" && attribs["id"] === "__VIEWSTATE"){
viewState_value = attribs["value"];
}
},
ontext:function(text){
},
onclosetag: function(tagname){
}
});
parser.write(resData);
parser.done();
// Parsing html for getting value of __EVENTVALIDATION
var parser1 = new htmlparser.Parser(
{
onopentag: function(name, attribs){
if(name === "input" && attribs["id"] === "__EVENTVALIDATION"){
eventValidation_value = attribs["value"];
}
},
ontext:function(text){
},
onclosetag: function(tagname){
}
});
parser1.write(resData);
parser1.done();
new htmlparser.FeedHandler(function(error, feed){
console.log('\n Error : '+error);
});
//console.log('\n Viewstate: '+viewState_value);
//console.log('\n EventVal: '+eventValidation_value);
}catch(err)
{
if(err){document.getElementById("om").innerHTML+="<br/>"+err;}
}
var r1 = request // This request to load data of given date and set x and y co-ordinates to mImgBtnGo using __VIEWSTATE and __EVENTVALIDATION values
(
{
method: 'POST',
uri: urlStockinfo,
headers: {"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
//"Referer": "http://www.mcxindia.com",
"Accept-Encoding": "utf-8",
"Accept": "text/plain,application/csv,*/*;q=0.3",
"Cookie": "cookie"
},
form:{'__VIEWSTATE':viewState_value.trim(),'__EVENTVALIDATION':eventValidation_value.trim(),'mTbdate':m+'/'+d+'/'+yy,
'mBtnGo':'GO'} //,'__EVENTTARGET':'btnLink_Excel'}//,'__EVENTARGUMENT':'Page$2'}
}
);
r1.on('response', function(response) {
response.on('data', function(chunk) {
resData1 += chunk;
});
});
var viewState_value1 = "";
var eventValidation_value1 = "";
var downloadLink = "";
r1.on('end', function()
{
//console.log('\n'+resData1);
try{
// Parsing html for getting value of __VIEWSTATE
var parser2 = new htmlparser.Parser(
{
onopentag: function(name, attribs){
if(name === "input" && attribs["id"] === "__VIEWSTATE"){
viewState_value1 = attribs["value"];
}
},
ontext:function(text){
},
onclosetag: function(tagname){
}
});
parser2.write(resData1);
parser2.done();
// Parsing html for getting value of __EVENTVALIDATION
var parser3 = new htmlparser.Parser(
{
onopentag: function(name, attribs)
{
if(name === "input" && attribs["id"] === "__EVENTVALIDATION"){
eventValidation_value1 = attribs["value"];
}
},
ontext:function(text){
},
onclosetag: function(tagname){
}
});
parser3.write(resData1);
parser3.done();
// Parsing html for getting href
var parser4 = new htmlparser.Parser(
{
onopentag: function(name, attribs)
{
if(name === "a" && attribs["class"] === "bluelinks"){
downloadLink = attribs["href"];
}
},
ontext:function(text){
},
onclosetag: function(tagname){
}
});
parser4.write(resData1);
parser4.done();
// console.log('\nhref = '+downloadLink);
new htmlparser.FeedHandler(function(error, feed){
console.log('\n Error : '+error);
});
}catch(err)
{
if(err){document.getElementById("om").innerHTML+="<br/> An error occured while downloading NSX_EOD file.";}
}
//console.log('\n viewstate 1 value is : '+viewState_value1);
//console.log('\n eventvalidation 1 value is : '+eventValidation_value1);
/********cheking for status code here****/
//var finalurl = "http://www.nationalspotexchange.com/"+downloadLink.substring(4);
request // This request to load data of given date and set x and y co-ordinates to mImgBtnGo using __VIEWSTATE and __EVENTVALIDATION values
(
{
method: 'POST',
uri: urlStockinfo,
headers: {"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
//"Referer": "http://www.mcxindia.com",
"Accept-Encoding": "utf-8",
"Accept": "text/plain,application/csv,*/*;q=0.3",
"Cookie": "cookie"
},
form:{'__VIEWSTATE':viewState_value.trim(),'__EVENTVALIDATION':eventValidation_value.trim(),'mTbdate':m+'/'+d+'/'+yy,
'mBtnGo':'GO'} //,'__EVENTTARGET':'btnLink_Excel'}//,'__EVENTARGUMENT':'Page$2'}
},
function(error, response, body) {
if(response != null)
{
if(response.statusCode != null || response.statusCode != "")
{
if (!error && response.statusCode == 200)
{
downloadnsxcsv(response.statusCode);
}
else
{
downloadnsxcsv(404);
}
}else
{
downloadnsxcsv(404);
}
}else
{
downloadnsxcsv(404);
}
}
);
/**************************end of checking status code****/
function downloadnsxcsv(code)
{
var currdate=new Date();
var year=currdate.getFullYear();
if(code==200 && yy==year)
{
document.getElementById("om").innerHTML+="<br/>nsxbhavcopy"+d+m+yy+".csv is dwonloading..";
var fileNameCsv = downloadpath+"nsxbhavcopy"+d+m+yy+".csv" //Download file to folder
var outCsv = fs.createWriteStream(fileNameCsv);
// This request is for to download loaded data (ie csv) by above request using __VIEWSTATE and __EVENTVALIDATION values
var finalurl = "http://www.nationalspotexchange.com/"+downloadLink.substring(4);
//console.log('\n Final url = '+finalurl);
var r2 = request
(
{
method: 'GET',
uri: finalurl,
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.97 Safari/537.11",
"Referer": "http://www.nationalspotexchange.com",
"Accept-Encoding": "utf-8",
"Accept": "text/plain,application/csv,*/*;q=0.3",
"Cookie": "cookie"
}
//form:{'__VIEWSTATE':viewState_value1.trim(),'__EVENTVALIDATION':eventValidation_value1.trim()}
}
);
r2.pipe(outCsv);
r2.on('end',function(){
document.getElementById("om").innerHTML+="<br/>nsxbhavcopy"+d+m+yy+".csv is dwonloaded..";
addlog(logpath, 'nsxbhavcopy'+d+m+yy+'.csv is downloaded..');
n++;
if(count.length>n)
{
loadprogressbar();
setTimeout(function (){process_nsxeod(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)}, 1000);
document.getElementById("om").innerHTML+='..';
}
else
{
Ext.getCmp('pbar3').updateProgress(0);
setTimeout(function(){process_nesl(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)},2000);
}
});
r2.on('error',function(){
document.getElementById("om").innerHTML+="<br/>Problem in dwonloading file nsxbhavcopy"+d+m+yy+".csv ";
setTimeout(function (){process_nsxeod(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)}, 1000);
});
}
else
{
document.getElementById("om").innerHTML+="<br/><font color=red>nsxbhavcopy"+d+m+yy+".csv file is not found on sever.</font>";
n++;
if(count.length>n)
{
loadprogressbar();
setTimeout(function (){process_nsxeod(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)}, 1000);
document.getElementById("om").innerHTML+='..';
}
else
{
Ext.getCmp('pbar3').updateProgress(0);
setTimeout(function(){process_nesl(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)},1000);
}
}
}
});
});
}
else
{
n++;
if(count.length>n)
{
process_nsxeod(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn);
//document.getElementById("om").innerHTML+='..';
}
else
{
Ext.getCmp('pbar3').updateProgress(0);
setTimeout(function(){process_nesl(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)},1000);
}
}
}catch(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing nsxbhavcopy"+d+m+yy+".csv";
n++;
if(count.length>n)
{
loadprogressbar();
process_nsxeod(count,n,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn);
//document.getElementById("om").innerHTML+='..';
}
else
{
Ext.getCmp('pbar3').updateProgress(0);
setTimeout(function(){process_nesl(count,0,baseurl,downloadpath,unzippath,outputpath,logpath,fromd,tod,sn)},1000);
}
}
}