@idealic/poker-engine
Version:
Professional poker game engine and hand evaluator with built-in iterator utilities
711 lines • 24.6 kB
JavaScript
#!/usr/bin/env node
import { Command } from 'commander';
import * as fs from 'fs';
import * as glob from 'glob';
import { Client } from 'pg';
import { from as copyFrom } from 'pg-copy-streams';
import { parseHand, parseHandIdentifiers } from '../formats/pokerstars';
import { Game } from '../Game';
const HEADERS = [
'timestamp',
'player_id',
'game_id',
'hand_idx',
'table_id',
'street',
'aggressions',
'all_ins',
'balance',
'bets',
'big_blind',
'calls',
'cbet',
'cbet_challenges',
'cbet_defenses',
'cbet_folds',
'cbet_opportunities',
'check_raise_challenges',
'check_raise_defenses',
'check_raise_folds',
'check_raise_opportunities',
'check_raises',
'checks',
'currency',
'currency_rate',
'decision_duration',
'decisions',
'donk_bet_challenges',
'donk_bet_defenses',
'donk_bet_folds',
'donk_bet_opportunities',
'donk_bets',
'first_aggressions',
'folds',
'four_bet_challenges',
'four_bet_defenses',
'four_bet_folds',
'four_bet_opportunities',
'four_bets',
'investments',
'last_aggressions',
'limp_opportunities',
'limps',
'losses',
'lost',
'open_shove_challenges',
'open_shove_defenses',
'open_shove_folds',
'open_shove_opportunities',
'open_shoves',
'passivities',
'profits',
'raises',
'rake',
'shove_challenges',
'shove_defenses',
'shove_folds',
'stack_after',
'stack_before',
'steal_challenges',
'steal_defenses',
'steal_folds',
'steal_opportunities',
'steals',
'success',
'three_bet_challenges',
'three_bet_defenses',
'three_bet_folds',
'three_bet_opportunities',
'three_bets',
'voluntary_put_money_in_pot_times',
'went_to_showdown',
'winnings',
'won',
'won_at_showdown',
'won_without_showdown',
];
const program = new Command();
program
.name('parse-hand-history')
.description('Parse poker hand history and output stats as CSV or PostgreSQL table')
.argument('<output>', 'output file path (for CSV) or table name (for PostgreSQL)')
.option('-i, --input <pattern>', 'input file glob pattern (e.g., "files/**/*.txt")')
.option('-l, --limit <number>', 'limit number of hands to parse')
.option('-p, --progress', 'show progress while parsing')
.option('-v, --verbose', 'output extra information')
.option('-e, --error-file <path>', 'file to save hand texts that caused parsing errors')
.option('--skip-timeouts', 'skip hands that include timeout messages or incomplete showdowns')
.option('-c, --connection <string>', 'PostgreSQL connection string (e.g., postgresql://user:pass@host:port/db)')
.option('-m, --method <string>', 'PostgreSQL import method (copy or insert)', 'insert')
.option('--max-concurrency <number>', 'maximum number of concurrent database operations', '100')
.version('1.0.0');
program.parse();
const options = program.opts();
const [outputPath] = program.args;
const limit = options.limit ? parseInt(options.limit) : Infinity;
const maxConcurrency = parseInt(options.maxConcurrency);
// Setup error file stream if option provided
let errorStream = null;
if (options.errorFile) {
errorStream = fs.createWriteStream(options.errorFile);
}
if (options.limit && isNaN(limit)) {
console.error('Error: --limit requires a number argument');
process.exit(1);
}
// Setup PostgreSQL client if table output is requested
let pgClient = null;
let copyStream = null;
// Add concurrency control variables
let activeQueries = [];
let isPaused = false;
// Function to manage concurrency
async function waitForConcurrencySlot() {
if (activeQueries.length >= maxConcurrency) {
isPaused = true;
await Promise.all(activeQueries);
activeQueries = [];
isPaused = false;
}
}
// Function to add a query to active queries
function trackQuery(promise) {
activeQueries.push(promise);
promise.finally(() => {
const index = activeQueries.indexOf(promise);
if (index !== -1) {
activeQueries.splice(index, 1);
}
});
}
function escapeCSV(value) {
if (value === null || value === undefined)
return '';
const str = String(value);
return `"${str.replace('"', `""`)}"`;
}
function getStatsRow(table, playerId, street, streetStats) {
const values = [
new Date(table.gameTimestamp).toISOString(),
playerId,
table.gameId,
table.hand,
table.tableId,
street,
streetStats.aggressions,
streetStats.allIns,
streetStats.balance,
streetStats.bets,
streetStats.bigBlind,
streetStats.calls,
streetStats.cbet,
streetStats.cbetChallenges,
streetStats.cbetDefenses,
streetStats.cbetFolds,
streetStats.cbetOpportunities,
streetStats.checkRaiseChallenges,
streetStats.checkRaiseDefenses,
streetStats.checkRaiseFolds,
streetStats.checkRaiseOpportunities,
streetStats.checkRaises,
streetStats.checks,
streetStats.currency,
streetStats.currencyRate,
streetStats.decisionDuration,
streetStats.decisions,
streetStats.donkBetChallenges,
streetStats.donkBetDefenses,
streetStats.donkBetFolds,
streetStats.donkBetOpportunities,
streetStats.donkBets,
streetStats.firstAggressions,
streetStats.folds,
streetStats.fourBetChallenges,
streetStats.fourBetDefenses,
streetStats.fourBetFolds,
streetStats.fourBetOpportunities,
streetStats.fourBets,
streetStats.investments,
streetStats.lastAggressions,
streetStats.limpOpportunities,
streetStats.limps,
streetStats.losses,
streetStats.lost,
streetStats.openShoveChallenges,
streetStats.openShoveDefenses,
streetStats.openShoveFolds,
streetStats.openShoveOpportunities,
streetStats.openShoves,
streetStats.passivities,
streetStats.profits,
streetStats.raises,
streetStats.rake,
streetStats.shoveChallenges,
streetStats.shoveDefenses,
streetStats.shoveFolds,
streetStats.stackAfter,
streetStats.stackBefore,
streetStats.stealChallenges,
streetStats.stealDefenses,
streetStats.stealFolds,
streetStats.stealOpportunities,
streetStats.steals,
streetStats.success,
streetStats.threeBetChallenges,
streetStats.threeBetDefenses,
streetStats.threeBetFolds,
streetStats.threeBetOpportunities,
streetStats.threeBets,
streetStats.voluntaryPutMoneyInPotTimes,
streetStats.wentToShowdown,
streetStats.winnings,
streetStats.won,
streetStats.wonAtShowdown,
streetStats.wonWithoutShowdown,
];
return values;
}
// Helper function to format file size
function formatFileSize(bytes) {
if (bytes < 1024)
return bytes + ' B';
if (bytes < 1024 * 1024)
return (bytes / 1024).toFixed(2) + ' KB';
if (bytes < 1024 * 1024 * 1024)
return (bytes / (1024 * 1024)).toFixed(2) + ' MB';
return (bytes / (1024 * 1024 * 1024)).toFixed(2) + ' GB';
}
// Add after other variable declarations
let isInterrupted = false;
let insertBatch = [];
const BATCH_SIZE = 5000;
let totalRowsProcessed = 0;
let totalRowsInserted = 0;
let gameBatch = [];
// Function to check if games exist in database
async function filterOutExistingGames(gameBatch) {
if (!pgClient)
throw new Error('PostgreSQL client not initialized');
if (gameBatch.length === 0)
return [];
const { venue } = parseHandIdentifiers(gameBatch[0].substring(0, gameBatch[0].indexOf('\n') - 1));
const gameIds = gameBatch.map(handText => {
const { id } = parseHandIdentifiers(handText.substring(0, handText.indexOf('\n') - 1));
return id;
});
console.log('Wtf');
const r = await pgClient.query("SELECT id FROM games WHERE id = 1 and venue = 'PokerStars'");
console.log('Wtf?');
const result = await pgClient
.query('SELECT id FROM games WHERE id = ANY($1) AND venue = $2', [gameIds, venue])
.catch(e => {
console.log('Error filtering out existing games', e);
});
const existingGameIds = new Set(result.rows.map(row => row.id));
return gameBatch.filter((_, index) => {
return !existingGameIds.has(gameIds[index]);
});
}
// Function to insert game into database
async function insertGame(game) {
if (!pgClient)
throw new Error('PostgreSQL client not initialized');
const gameData = {
id: game.id,
venue: game.venue,
table_id: game.table,
hand: game.hand,
seat_count: game.seatCount,
started_at: new Date(game.timestamp),
variant: game.variant,
min_bet: game.minBet,
small_bet: game.smallBet,
big_bet: game.bigBet,
bring_in: game.bringIn,
log: JSON.stringify(game),
};
await pgClient.query(`INSERT INTO games (
id, venue, table_name, hand, seat_count, started_at,
variant, min_bet, small_bet, big_bet, bring_in, log
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
ON CONFLICT (id) DO NOTHING`, [
gameData.id,
gameData.venue,
gameData.table_id,
gameData.hand,
gameData.seat_count,
gameData.started_at,
gameData.variant,
gameData.min_bet,
gameData.small_bet,
gameData.big_bet,
gameData.bring_in,
gameData.log,
]);
}
// Function to flush the current batch to the database
async function flushBatch() {
if (insertBatch.length === 0)
return;
if (!pgClient)
throw new Error('PostgreSQL client not initialized');
const values = insertBatch.map(row => '(' +
row.map(v => (v === null ? 'NULL' : `'${String(v).replace(/'/g, "''")}'`)).join(',') +
')');
insertBatch = [];
const columns = HEADERS.join(',');
await waitForConcurrencySlot();
const queryPromise = pgClient.query(`INSERT INTO ${outputPath} (${columns}) VALUES ${values.join(',')} ON CONFLICT DO NOTHING`);
trackQuery(queryPromise);
const result = await queryPromise;
totalRowsProcessed += values.length;
totalRowsInserted += result.rowCount || 0;
}
// Function to write a row to either CSV or PostgreSQL
function writeRow(row) {
if (options.connection) {
if (options.method === 'copy') {
if (!copyStream)
throw new Error('PostgreSQL copy stream not initialized');
if (!row[0]) {
console.log(row);
throw new Error('Row has no timestamp');
}
copyStream.write(row.map(escapeCSV).join(',') + '\n');
}
else {
if (!pgClient)
throw new Error('PostgreSQL client not initialized');
insertBatch.push(row);
if (insertBatch.length >= BATCH_SIZE) {
flushBatch();
}
}
}
else {
if (!writeStream)
throw new Error('Write stream not initialized');
writeStream.write(row.map(v => escapeCSV(v)).join(',') + '\n');
}
}
// Function to output final stats and clean up
async function outputFinalStats() {
try {
// Get output file size for CSV
let outputSize = 0;
if (!options.connection) {
try {
const stats = fs.statSync(outputPath);
outputSize = stats.size;
}
catch (error) {
console.error(`Error getting output file size: ${error}`);
}
}
const duration = (Date.now() - startTime) / 1000; // Convert to seconds
const handsPerSecond = handCount / duration;
const message = `\nSummary:
- Processed: ${handCount} hands
- Skipped: ${skippedCount} hands
- Errored: ${errorCount} hands
- Total hands: ${handCount + skippedCount} hands
- Files processed: ${processedFilesCount}
- Input size: ${formatFileSize(totalInputSize)}
${!options.connection ? `- Output size: ${formatFileSize(outputSize)}` : ''}
${options.connection && options.method === 'insert'
? `- Rows processed: ${totalRowsProcessed}
- Rows inserted: ${totalRowsInserted}
- Insertion success rate: ${((totalRowsInserted / totalRowsProcessed) * 100).toFixed(2)}%`
: ''}
- Duration: ${duration.toFixed(2)} seconds
- Rate: ${handsPerSecond.toFixed(2)} hands/second`;
process.stdout.write(message + '\n');
console.timeEnd('Total time');
}
catch (error) {
console.error('Error during final stats output:', error);
}
}
// Function to handle graceful shutdown
async function shutdown() {
if (isInterrupted)
return;
isInterrupted = true;
try {
// For INSERT method, wait for all active queries to complete
if (options.method === 'insert') {
await flushBatch();
}
// End the copy stream if it exists
if (copyStream) {
try {
if (options.verbose) {
console.log('Ending copy stream...');
}
// End the copy stream and wait for it to finish
copyStream.end();
await new Promise((resolve, reject) => {
copyStream.on('finish', () => {
if (options.verbose) {
console.log('Copy stream finished');
}
resolve();
});
copyStream.on('error', (err) => {
if (options.verbose) {
console.error('Copy stream error:', err);
}
reject(err);
});
});
}
catch (error) {
console.error('Error ending copy stream:', error);
// If there's an error, try to destroy the stream
if (copyStream) {
if (options.verbose) {
console.log('Destroying copy stream due to error');
}
copyStream.destroy();
}
}
}
if (writeStream) {
writeStream.end();
}
if (pgClient) {
try {
if (options.verbose) {
console.log('Closing PostgreSQL client...');
}
await pgClient.end();
if (options.verbose) {
console.log('PostgreSQL client closed');
}
}
catch (error) {
console.error('Error closing PostgreSQL client:', error);
}
}
// Close error stream if it exists
if (errorStream) {
errorStream.end();
}
// Output final stats
await outputFinalStats();
}
catch (error) {
console.error('Error during shutdown:', error);
}
}
// Handle graceful shutdown on SIGINT (Ctrl+C)
process.on('SIGINT', async () => {
console.log('sigint');
if (isInterrupted) {
console.error('\nForce quitting...');
process.exit(1);
}
process.stdout.write('\nProcess interrupted. Shutting down gracefully...\n');
try {
await shutdown();
}
catch (error) {
console.error('Error during shutdown:', error);
}
process.exit(0);
});
// Handle process exit to ensure clean shutdown
process.on('exit', async () => {
if (!isInterrupted) {
await shutdown();
}
});
// Helper function to parse hand and handle errors
function parseHandHelper(handText) {
try {
return parseHand(handText);
}
catch (error) {
errorCount++;
console.error(`Error parsing hand: ${handText.split('\n')[0]}`);
console.error(error);
// Save error text to file if option provided
if (errorStream) {
errorStream.write(handText + '\n\n');
}
return null;
}
}
// Process a single hand history and output stats as CSV
async function processHand(handText) {
// Skip hands with timeouts or incomplete showdowns if skip-timeouts option is enabled
if (options.skipTimeouts &&
(handText.includes('has timed out') || handText.includes(' showed and '))) {
skippedCount++;
return;
}
const game = parseHandHelper(handText);
if (!game)
return; // Skip if parsing failed
try {
// Insert game into database
await insertGame(game);
const table = Game.create(game);
// Write stats for each player and street directly from table.stats
for (const stat of table.stats) {
// Skip stats that don't have playerId or street
if (!stat.playerId || !stat.street)
continue;
// Add to stats array and write to CSV
const row = getStatsRow(table, stat.playerId, stat.street, stat);
writeRow(row);
}
}
catch (error) {
errorCount++;
console.error(`Error processing hand ${handCount}: ${handText.split('\n')[0]}`);
console.error(error);
// Save error text to file if option provided
if (errorStream) {
errorStream.write(handText + '\n\n');
}
}
}
console.time('Total time');
// Extract buffer processing logic to a separate function
async function processBuffer() {
// Process complete hands from the buffer
let handStartIndex = buffer.indexOf('PokerStars Hand #');
// If we found at least one hand header
while (handStartIndex !== -1) {
// Look for the next hand header
const nextHandIndex = buffer.indexOf('PokerStars Hand #', handStartIndex + 1);
if (nextHandIndex !== -1) {
// We found the next hand, so we can process the current one
const handText = buffer.substring(handStartIndex, nextHandIndex).trim();
if (!isFirstChunk) {
// Add to batch
gameBatch.push(handText);
// If batch is full, check existing games
if (gameBatch.length >= BATCH_SIZE) {
filterOutExistingGames(gameBatch).then(games => {
console.log('Wtf2!!!!!!!!!!!!');
// Process filtered games
for (const gameText of games) {
processHand(gameText);
if (handCount % 1000 === 0 && options.progress) {
process.stdout.write(handCount + '\n');
}
handCount++;
}
if (handCount >= limit) {
if (options.progress || options.verbose) {
process.stdout.write(`\nReached limit of ${limit} hands\n`);
}
// Start shutdown sequence
shutdown();
return;
}
});
gameBatch = [];
}
}
else {
isFirstChunk = false;
}
// Move to the next hand
handStartIndex = nextHandIndex;
// Keep only what's left in buffer
buffer = buffer.substring(handStartIndex);
handStartIndex = 0; // Reset for the next iteration
}
else {
// No more complete hands in the buffer
break;
}
}
}
// Setup output stream only for CSV mode
let writeStream = null;
// Add at the top with other variables
const startTime = Date.now();
// Main processing
let isFirstChunk = true;
let handCount = 0;
let skippedCount = 0;
let errorCount = 0;
let processedFilesCount = 0;
let totalInputSize = 0;
let buffer = ''; // Buffer to accumulate text across chunks
async function start() {
if (options.connection) {
pgClient = new Client(options.connection ? { connectionString: options.connection } : undefined);
// Handle PostgreSQL connection errors
pgClient.on('error', (err) => {
console.error('PostgreSQL client error:', err);
if (!isInterrupted) {
shutdown();
}
});
await pgClient.connect();
if (options.verbose) {
console.log('Connected to PostgreSQL');
}
if (options.method === 'copy') {
copyStream = pgClient.query(copyFrom(`COPY ${outputPath} (${HEADERS.join(',')}) FROM STDIN CSV`));
}
}
else {
writeStream = fs.createWriteStream(outputPath);
writeStream.write(HEADERS.join(',') + '\n');
}
// If input glob pattern is provided, use it to read files
if (options.input) {
const files = glob.sync(options.input).sort();
if (files.length === 0) {
console.error(`Error: No files found matching pattern "${options.input}"`);
process.exit(1);
}
if (options.verbose) {
console.log(`Processing ${files.length} files matching "${options.input}"`);
}
// Process files sequentially
let fileIndex = 0;
function processNextFile() {
if (fileIndex >= files.length || handCount >= limit || isInterrupted) {
if (options.progress || options.verbose) {
process.stdout.write(`\nFinished processing ${handCount} hands from ${fileIndex} files\n`);
}
return;
}
const filePath = files[fileIndex++];
// Get file size
try {
const stats = fs.statSync(filePath);
totalInputSize += stats.size;
}
catch (error) {
console.error(`Error getting file size for ${filePath}: ${error}`);
}
if (options.verbose) {
console.log(`Reading file: ${filePath}`);
}
const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' });
fileStream.on('data', async (chunk) => {
if (isInterrupted) {
fileStream.destroy();
return;
}
if (isPaused) {
fileStream.pause();
await waitForConcurrencySlot();
fileStream.resume();
}
buffer += chunk;
fileStream.pause();
processBuffer();
fileStream.resume();
});
fileStream.on('end', () => {
// Process any remaining hand in the buffer
if (buffer.indexOf('PokerStars Hand #') === 0 && handCount < limit && !isInterrupted) {
processHand(buffer);
handCount++;
buffer = '';
}
processedFilesCount++;
processNextFile();
});
fileStream.on('error', err => {
console.error(`Error reading file ${filePath}: ${err.message}`);
processNextFile();
});
}
processNextFile();
}
else {
// For stdin input, we can't calculate input size
process.stdin.setEncoding('utf8');
process.stdin.on('data', chunk => {
if (isInterrupted) {
process.stdin.destroy();
return;
}
buffer += chunk;
processBuffer();
});
process.stdin.on('end', () => {
// Process the last hand if there's anything left in the buffer
if (buffer.indexOf('PokerStars Hand #') === 0 && handCount < limit && !isInterrupted) {
processHand(buffer);
handCount++;
}
processedFilesCount = 1; // Count stdin as one file
if (options.progress || options.verbose) {
process.stdout.write(`\nFinished processing ${handCount} hands\n`);
}
});
}
}
setTimeout(start, 1);
//# sourceMappingURL=parse-hand-history.js.map