shubhadownloader
Version:
There is large amount of information available in market place. The markets are always in sync. In today's world investors find it very difficult to make effective use of information available to them. Shubha Downloader is great tool which resolves this problem and helps investor to increase his productivity and stay focused on decision making. Shubha Downloader enable end user to download market data from available sources and organize it. Shubha Downloader is Open source & FREE utility for end users. Shubha Downloader have main features as follows End of the day market data from web to your favorite charting application . Fundamental market data from web to your favorite charting application. Market reports from web to your favorite charting application.
526 lines (452 loc) • 32.1 kB
JavaScript
/*****************************
*
*Author:JAYANT PATIL
*COMPANY NAME XNet Inc,Pune.
*last modified date:04-jan-2013
*
*
*
*Description:
*This will process MCX FOREX file and generate output file
*
*variables :
*count --->contains array of date in which we take one date process files regarding to that
*n --->it is a counter variable
*baseurl-->it contians all url for nse equity
*downloadpath---> it contain download path of file for processing
*unzippath ----> it contains unzip folder path
*outputpah ----> it containds output older path
*
*
*copyright 2013
******************************/
/********************************
<Shubha Downloader, It downloads end of day pricing data from publicly available sites and converts it different formats....>
Copyright (C) 2013 Shubhalabha.in
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>
***********************************/
function process_mcxforex(count,n,baseurl,downloadpath,unzippath,outputpath,logpath)
{
try
{
//document.getElementById("om").innerHTML+="loading";
/*var mon=count[n].split('-')[0];
var m=count[n].split('-')[1];
var d=count[n].split('-')[2];
var yy=count[n].split('-')[3];
var y=yy.substr(2,2);*/
//import files from modules
var fs = require('fs');
var csv = require('ya-csv');
var async = require('async');
//var path = require('./node_modules/path');
//var writeFileStream = fs.createWriteStream('./tempData/output.txt',{flag:'a',encoding:'utf-8'});
async.parallel({
data1 : function(callback){
if (fs.existsSync(downloadpath+'CD_MCXSX_20130107c.csv')) {
fs.readFile(downloadpath+'CD_MCXSX_20130107c.csv','utf-8', function(err, data){
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while reading "+downloadpath+"CD_MCXSX_20130107c.csv";
}
callback(err, data);
});
}else
{
// Here readdir's functionaliity has no use. It is written for only to send
// '0' data in its callback function because we cannot call "callback(err, data);"
// function out side a callback function
fs.readdir('html', function(err, files){
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while reading "+downloadpath+"CD_MCXSX_20130107c.csv";
}
callback(err,'0');
});
}
}
}, function(err, result){
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
if (fs.existsSync(downloadpath+'CD_MCXSX_20130107c.csv')) {
var arrRow = new Array();
arrRow = result['data1'].split('\n');
var arrFileData = new Array();
// arrRow.lenght-3 is for last 3 lines of no use in file
for(var i=0 ; i<arrRow.length-1;i++)
{
var tempArr = arrRow[i].split(',');
arrFileData.push(tempArr);
}
// add blank space where cell of company name col not have value
var outputpath1=outputpath;
var filename = outputpath1+'MCX_Com_mcxsx_07012013.csv';
var header='TICKER,NAME,DATE,OPEN,HIGH,LOW,CLOSE,VOLUME,OPENINT';
if(!fs.existsSync(filename)){
fs.writeFile(filename, header+'\n', function (err)
{
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
//console.log(err);
// console.log(filename);
//processing actual file data
var dates = new Array();
for(i=1;i<arrFileData.length;i++)
{
if(typeof arrFileData[i][3]!=="undefined" && arrFileData[i][3]!=="" && arrFileData[i][3]!==null && arrFileData[i][3]!= "\"N/A\"" && arrFileData[i][3]!= "N/A" && arrFileData[i][3]!= "NaN" && arrFileData[i][3]!= "undefined")
dates.push(new Date(arrFileData[i][3].split('-')[2],getMonth(arrFileData[i][3].split('-')[1]),arrFileData[i][3].split('-')[0]));
}
function sortDates(a, b)
{
return a.getTime() - b.getTime();
}
var sorted = dates.sort(sortDates);
console.log(sorted);
var m1=sorted[0].getMonth()+1;
var day=sorted[0].getFullYear()+'/'+m1+'/'+sorted[0].getDate();
var date ='20130107';
// console.log(date);
// Writing data to output file
for(i=1;i<arrFileData.length;i++)
{
var date1 = new Date(day);
var date2 = new Date(arrFileData[i][3].split('-')[2]+'/'+getMonth(arrFileData[i][3].split('-')[1])+'/'+arrFileData[i][3].split('-')[0]);
m1=monthDiff(date1,date2);
if(m1==0)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-I'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-I'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==1)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-II'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-II'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==2)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-III'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-III'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==3)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-IV'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-IV'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==4)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-V'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-V'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==5)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-VI'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-VI'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==6)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-VII'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-VII'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==7)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-VIII'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-VIII'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==8)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-IX'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-IX'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==9)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-X'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-X'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==10)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-XI'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-XI'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
if(m1==11)
{
if(arrFileData[i][4]!='' && arrFileData[i][5]!=''){
fs.appendFileSync(filename,arrFileData[i][2]+'-XII'+arrFileData[i][4]+arrFileData[i][5]+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
else
{
fs.appendFileSync(filename,arrFileData[i][2]+'-XII'+','+arrFileData[i][2]+','+arrFileData[i][3].split('-')[2]+getMonth(arrFileData[i][3].split('-')[1])+arrFileData[i][3].split('-')[0]+','+arrFileData[i][6]+','+arrFileData[i][7]+','+arrFileData[i][8]+','+arrFileData[i][9]+','+arrFileData[i][12]+','+arrFileData[i][15]+'\n','UTF-8', function (err) {
if(err)
{
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
// console.log('| Appended to file!');
});
}
}
}// for
});
}
}
});
document.getElementById("om").innerHTML+="<br/>mcxsx_07012013.csv is processed..";
function monthDiff(d1, d2) {
var months;
months = (d2.getFullYear() - d1.getFullYear()) * 12;
months -= d1.getMonth() + 1;
months += d2.getMonth();
return months+1;
}
function getMonth(mon){
switch(mon.toUpperCase())
{
case 'JAN':
mon='01';
break;
case 'FEB':
mon='02';
break;
case 'MAR':
mon='03';
break;
case 'APR':
mon='04';
break;
case 'MAY':
mon='05';
break;
case 'JUN':
mon='06';
break;
case 'JUL':
mon='07';
break;
case 'AUG':
mon='08';
break;
case 'SEP':
mon='09';
break;
case 'OCT':
mon='10';
break;
case 'NOV':
mon='11';
break;
case 'DEC':
mon='12';
break;
}
return mon;
}
}
catch(err){
// console.error(err);
document.getElementById("om").innerHTML+="<br/>An error occured while processing "+downloadpath+"CD_MCXSX_20130107c.csv";
}
}