@idealic/poker-engine
Version:
Professional poker game engine and hand evaluator with built-in iterator utilities
604 lines • 19.8 kB
JavaScript
import * as I from '../lib/iterators';
import { Command } from '@commander-js/extra-typings';
import * as fs from 'fs';
import * as glob from 'glob';
import { Pool } from 'pg';
import { from as copyFrom } from 'pg-copy-streams';
import { parseHand, parseHandIdentifiers } from '../formats/pokerstars';
import { Game } from '../Game';
const HEADERS = [
'timestamp',
'player_id',
'game_id',
'hand_idx',
'table_id',
'street',
'aggressions',
'all_ins',
'balance',
'bets',
'big_blind',
'calls',
'cbet',
'cbet_challenges',
'cbet_defenses',
'cbet_folds',
'cbet_opportunities',
'check_raise_challenges',
'check_raise_defenses',
'check_raise_folds',
'check_raise_opportunities',
'check_raises',
'checks',
'currency',
'currency_rate',
'decision_duration',
'decisions',
'donk_bet_challenges',
'donk_bet_defenses',
'donk_bet_folds',
'donk_bet_opportunities',
'donk_bets',
'first_aggressions',
'folds',
'four_bet_challenges',
'four_bet_defenses',
'four_bet_folds',
'four_bet_opportunities',
'four_bets',
'investments',
'last_aggressions',
'limp_opportunities',
'limps',
'losses',
'lost',
'open_shove_challenges',
'open_shove_defenses',
'open_shove_folds',
'open_shove_opportunities',
'open_shoves',
'passivities',
'profits',
'raises',
'rake',
'shove_challenges',
'shove_defenses',
'shove_folds',
'stack_after',
'stack_before',
'steal_challenges',
'steal_defenses',
'steal_folds',
'steal_opportunities',
'steals',
'success',
'three_bet_challenges',
'three_bet_defenses',
'three_bet_folds',
'three_bet_opportunities',
'three_bets',
'voluntary_put_money_in_pot_times',
'went_to_showdown',
'winnings',
'won',
'won_at_showdown',
'won_without_showdown',
];
const program = new Command()
.name('parse-hands')
.description('Parse poker hand history and output stats as CSV or PostgreSQL table')
.argument('<output>', 'output file path (for CSV) or table name (for PostgreSQL)')
.option('-i, --input <pattern>', 'input file glob pattern (e.g., "files/**/*.txt")', String)
.option('-l, --limit <number>', 'limit number of hands to parse', parseInt, Infinity)
.option('-p, --progress', 'show progress while parsing')
.option('-v, --verbose', 'output extra information')
.option('-e, --error-file <path>', 'file to save hand texts that caused parsing errors')
.option('--skip-timeouts', 'skip hands that include timeout messages or incomplete showdowns')
.option('-c, --connection <string>', 'PostgreSQL connection string (e.g., postgresql://user:pass@host:port/db)')
.option('-m, --method <string>', 'PostgreSQL import method (copy or insert)', 'insert')
.option('--max-concurrency <number>', 'maximum number of concurrent database operations', parseInt, 2)
.version('1.0.0');
program.parse();
const options = program.opts();
const [outputPath] = program.args;
// Constants
const PROCESS_BATCH_SIZE = 5000;
const OUTPUT_BATCH_SIZE = 5000;
// Setup error file stream if option provided
let errorStream = null;
if (options.errorFile) {
errorStream = fs.createWriteStream(options.errorFile);
}
// Setup PostgreSQL client if table output is requested
let pool = null;
// Add after other variable declarations
let isInterrupted = false;
let readCount = 0;
let parsedCount = 0;
let analyzedCount = 0;
let skippedCount = 0;
let errorCount = 0;
let duplicateCount = 0;
let writtenCount = 0;
let processedFilesCount = 0;
let totalInputSize = 0;
const startTime = Date.now();
const seenGameIds = new Set();
// Helper function to escape CSV values
function escapeCSV(value) {
if (value === null || value === undefined)
return '';
const str = String(value);
return `"${str.replaceAll('"', `""`)}"`;
}
// Helper function to get stats row
function getStatsRow(table, playerId, street, streetStats) {
return [
new Date(table.gameTimestamp).toISOString(),
playerId,
table.gameId,
table.hand,
table.tableId,
street,
streetStats.aggressions,
streetStats.allIns,
streetStats.balance,
streetStats.bets,
streetStats.bigBlind,
streetStats.calls,
streetStats.cbet,
streetStats.cbetChallenges,
streetStats.cbetDefenses,
streetStats.cbetFolds,
streetStats.cbetOpportunities,
streetStats.checkRaiseChallenges,
streetStats.checkRaiseDefenses,
streetStats.checkRaiseFolds,
streetStats.checkRaiseOpportunities,
streetStats.checkRaises,
streetStats.checks,
streetStats.currency,
streetStats.currencyRate,
streetStats.decisionDuration,
streetStats.decisions,
streetStats.donkBetChallenges,
streetStats.donkBetDefenses,
streetStats.donkBetFolds,
streetStats.donkBetOpportunities,
streetStats.donkBets,
streetStats.firstAggressions,
streetStats.folds,
streetStats.fourBetChallenges,
streetStats.fourBetDefenses,
streetStats.fourBetFolds,
streetStats.fourBetOpportunities,
streetStats.fourBets,
streetStats.investments,
streetStats.lastAggressions,
streetStats.limpOpportunities,
streetStats.limps,
streetStats.losses,
streetStats.lost,
streetStats.openShoveChallenges,
streetStats.openShoveDefenses,
streetStats.openShoveFolds,
streetStats.openShoveOpportunities,
streetStats.openShoves,
streetStats.passivities,
streetStats.profits,
streetStats.raises,
streetStats.rake,
streetStats.shoveChallenges,
streetStats.shoveDefenses,
streetStats.shoveFolds,
streetStats.stackAfter,
streetStats.stackBefore,
streetStats.stealChallenges,
streetStats.stealDefenses,
streetStats.stealFolds,
streetStats.stealOpportunities,
streetStats.steals,
streetStats.success,
streetStats.threeBetChallenges,
streetStats.threeBetDefenses,
streetStats.threeBetFolds,
streetStats.threeBetOpportunities,
streetStats.threeBets,
streetStats.voluntaryPutMoneyInPotTimes,
streetStats.wentToShowdown,
streetStats.winnings,
streetStats.won,
streetStats.wonAtShowdown,
streetStats.wonWithoutShowdown,
];
}
// Function to check if games exist in database
let filterBatchProcessing = 0;
async function filterBatch(gameBatch) {
filterBatchProcessing++;
console.log('filter batch', gameBatch.length, filterBatchProcessing, 'in flight');
//await new Promise(resolve => setTimeout(resolve, Math.random() * 1000));
if (!pool)
throw new Error('PostgreSQL pool not initialized');
if (gameBatch.length === 0 || isInterrupted)
return [];
// First pass: collect all game IDs and check for duplicates
const ids = [];
const uniqueBatch = gameBatch.filter((handText, index) => {
const { id } = parseHandIdentifiers(handText.substring(0, handText.indexOf('\n') - 1));
if (seenGameIds.has(id)) {
duplicateCount++;
return false;
}
else {
ids.push(id);
seenGameIds.add(id);
return true;
}
});
if (uniqueBatch.length === 0) {
filterBatchProcessing--;
console.log('filtered', 0);
return [];
}
const { venue } = parseHandIdentifiers(uniqueBatch[0].substring(0, uniqueBatch[0].indexOf('\n') - 1));
const result = await pool.query('SELECT id FROM games WHERE id = ANY($1) AND venue = $2', [
ids,
venue,
]);
const existingGameIds = new Set(result.rows.map(row => row.id));
const filtered = uniqueBatch.filter((_, index) => {
const id = ids[index];
seenGameIds.delete(id);
if (existingGameIds.has(id)) {
duplicateCount++;
return false;
}
return true;
});
filterBatchProcessing--;
//console.log('filtered', filtered.length);
return filtered;
}
// Helper function to format file size
function formatFileSize(bytes) {
if (bytes < 1024)
return bytes + ' B';
if (bytes < 1024 * 1024)
return (bytes / 1024).toFixed(2) + ' KB';
if (bytes < 1024 * 1024 * 1024)
return (bytes / (1024 * 1024)).toFixed(2) + ' MB';
return (bytes / (1024 * 1024 * 1024)).toFixed(2) + ' GB';
}
// Function to output final stats and clean up
function outputFinalStats() {
try {
// Get output file size for CSV
let outputSize = 0;
if (!options.connection) {
try {
const stats = fs.statSync(outputPath);
outputSize = stats.size;
}
catch (error) {
console.error(`Error getting output file size: ${error}`);
}
}
const duration = (Date.now() - startTime) / 1000; // Convert to seconds
const inputPerSecond = readCount / duration;
const outputPerSecond = writtenCount / duration;
const message = `\nSummary:
- Read: ${readCount} hands
- Parsed: ${parsedCount} hands
- Analyzed: ${analyzedCount} hands
- Skipped: ${skippedCount} hands
- Duplicates: ${duplicateCount} hands
- Written: ${writtenCount} hands
- Files processed: ${processedFilesCount}
- Input size: ${formatFileSize(totalInputSize)}
- Errored: ${errorCount} hands
${!options.connection ? `- Output size: ${formatFileSize(outputSize)}` : ''}
- Duration: ${duration.toFixed(2)} seconds
- Input: ${inputPerSecond.toFixed(2)} hands/second
- Output: ${outputPerSecond.toFixed(2)} hands/second
`;
process.stdout.write(message + '\n');
console.timeEnd('Total time');
}
catch (error) {
console.error('Error during final stats output:', error);
}
}
// Function to handle graceful shutdown
async function shutdown() {
if (isInterrupted)
return;
isInterrupted = true;
try {
// Close database connection
if (pool) {
await Promise.all(Object.values(connections).map(client => client.release()));
}
await pool?.end();
// Close error stream if it exists
if (errorStream) {
errorStream.end();
await new Promise(resolve => errorStream.once('finish', resolve));
}
// Output final stats
outputFinalStats();
}
catch (error) {
console.error('Error during shutdown:', error);
}
}
// Handle graceful shutdown on SIGINT (Ctrl+C)
process.on('SIGINT', () => {
console.log(1);
if (isInterrupted) {
console.error('\nForce quitting...');
process.exit(1);
}
process.stdout.write('\nProcess interrupted. Shutting down gracefully...\n');
shutdown().then(() => process.exit(0));
return false;
});
process.on('SIGTERM', () => {
console.log(1);
if (isInterrupted) {
console.error('\nForce quitting...');
process.exit(1);
}
process.stdout.write('\nProcess interrupted. Shutting down gracefully...\n');
shutdown().then(() => process.exit(0));
return false;
});
// Handle process exit to ensure clean shutdown
process.on('exit', async () => {
if (!isInterrupted) {
await shutdown();
}
});
function processHand(handText) {
if (options.skipTimeouts &&
(handText.includes('has timed out') || handText.includes(' showed and '))) {
skippedCount++;
return null;
}
//console.log('processHand', handText.length);
let hand = null;
try {
hand = parseHand(handText);
}
catch (error) {
errorCount++;
if (errorStream) {
errorStream.write(handText + '\n\n');
}
return null;
}
if (!hand)
return null;
parsedCount++;
let game = null;
try {
game = Game.create(hand);
analyzedCount++;
}
catch (error) {
errorCount++;
if (errorStream) {
errorStream.write(JSON.stringify(hand, null, 2) + '\n\n');
}
return null;
}
if (!game)
return null;
return [hand, game];
}
// Function to write batch of games
async function writeGames(hands) {
console.log('writeGames', options.method);
if (!options.connection)
return;
//const client = await pool!.connect();
try {
if (options.method === 'copy') {
const copyStream = client.query(copyFrom(`COPY games (
id, venue, table_name, hand, seat_count, started_at,
variant, min_bet, small_bet, big_bet, bring_in, log
) FROM STDIN CSV`));
for await (const game of games) {
writtenCount++;
const gameData = [
game.id,
game.venue,
game.table,
game.hand,
game.seatCount,
new Date(game.timestamp || 0).toISOString(),
game.variant,
game.minBet || 0,
game.smallBet || 0,
game.bigBet || 0,
game.bringIn || 0,
'{}',
];
copyStream.write(gameData.map(escapeCSV).join(',') + '\n');
}
console.log('writeGames done', writtenCount);
copyStream.end();
await new Promise(resolve => copyStream.once('finish', resolve));
}
else {
const values = [];
for await (const game of games) {
writtenCount++;
values.push(`(${[
`'${game.id}'`,
`'${game.venue}'`,
`'${game.table}'`,
`'${game.hand}'`,
game.seatCount,
`'${new Date(game.timestamp || 0).toISOString()}'`,
`'${game.variant}'`,
game.minBet || 0,
game.smallBet || 0,
game.bigBet || 0,
game.bringIn || 0,
"'{}'",
].join(',')})`);
}
await pool.query(`INSERT INTO games (
id, venue, table_name, hand, seat_count, started_at,
variant, min_bet, small_bet, big_bet, bring_in, log
) VALUES ${values.join(',')}
ON CONFLICT (id) DO NOTHING`);
}
}
finally {
//client.release();
}
}
// Function to write batch of stats
async function writeStats(tables) {
console.log('writeStats', options.method);
//const client = await pool!.connect();
try {
if (options.connection) {
if (options.method === 'copy') {
const copyStream = client.query(copyFrom(`COPY ${outputPath} (${HEADERS.join(',')}) FROM STDIN CSV`));
for await (const table of tables) {
for (const stat of table.stats) {
const row = getStatsRow(table, stat.playerId, stat.street, stat);
copyStream.write(row.map(escapeCSV).join(',') + '\n');
}
}
copyStream.end();
await new Promise(resolve => copyStream.once('finish', resolve));
console.log('writeStats done', writtenCount);
}
else {
const values = [];
for await (const table of tables) {
for (const stat of table.stats) {
const row = getStatsRow(table, stat.playerId, stat.street, stat);
values.push(`(${row.map(v => (v === null ? 'NULL' : `'${String(v).replace(/'/g, "''")}'`)).join(',')})`);
}
}
if (values.length > 0) {
await pool.query(`INSERT INTO ${outputPath} (${HEADERS.join(',')}) VALUES ${values.join(',')} ON CONFLICT DO NOTHING`);
}
}
}
else {
for await (const table of tables) {
for (const stat of table.stats) {
process.stdout.write(getStatsRow(table, stat.playerId, stat.street, stat).map(escapeCSV).join(',') + '\n');
}
}
}
}
finally {
// client.release();
}
}
// Generator function for file paths
async function* getFilePaths(pattern) {
const files = await glob.glob(pattern);
if (files.length === 0) {
console.error(`Error: No files found matching pattern "${pattern}"`);
process.exit(1);
}
if (options.verbose) {
console.log(`Processing ${files.length} files matching "${pattern}"`);
}
for (const file of files) {
yield file;
}
}
// Function to read file contents
async function readFile(file) {
processedFilesCount++;
console.log('readFile');
try {
const stats = await fs.promises.stat(file);
totalInputSize += stats.size;
console.log('File', file, formatFileSize(stats.size));
}
catch (error) {
console.error(`Error getting file size for ${file}: ${error}`);
}
return fs.createReadStream(file, { encoding: 'utf8' });
}
// Generator function to split into hands
async function* splitHands(chunks) {
//console.log('splitHands');
let buffer = '';
const hands = [];
let count = 0;
for await (const chunk of chunks) {
buffer += chunk.replace(/\r/g, '');
let start = 0;
let pos = 0;
while ((pos = buffer.indexOf('PokerStars Hand #', start + 1)) !== -1) {
if (start < pos) {
const hand = buffer.substring(start, pos).trim();
if (hand) {
count++;
yield hand;
readCount++;
}
}
start = pos;
}
buffer = buffer.substring(start);
}
if (buffer) {
count++;
yield buffer;
hands.push(buffer);
}
console.log('splitHands done', count);
return hands;
}
var connections = {};
// Main processing function
async function processFiles() {
console.time('Total time');
if (options.connection) {
pool = new Pool({ connectionString: options.connection, max: 10, keepAlive: true });
connections.default = await pool.connect();
}
try {
if (!options.input) {
throw new Error('Input pattern is required');
}
// Create the processing pipeline using rotery
const result = I.pipe(
// Get file paths
getFilePaths(options.input), I.map(readFile, 4), I.map(splitHands, 4), I.concat(4, 4), // Find new hands in batches in parallel
I.chunk(5000), I.map(filterBatch, 5), I.concat(5, 1), I.map(processHand), I.filter(I.identity), I.take(options.limit), I.dispatch(([game, stats]) => ({ game, stats }), {
game: I.pipe(I.chunk(2500), I.map(writeGames, 5)),
stats: I.pipe(I.chunk(500), I.map(writeStats, 5)),
}));
var i = 0;
for await (const stream of result) {
if (++i % 1000 === 0) {
console.log('>>>>>>>>>>>>>>>>>', i, process.memoryUsage());
}
}
console.log('DONE');
}
finally {
console.log('Complete');
await shutdown();
}
}
// Start processing
processFiles().catch(error => {
console.error('Error during processing:', error);
process.exit(1);
});
//# sourceMappingURL=parse-hands.js.map