@turingnova/robots
Version:
Next.js robots.tsx generator - Automatically create and serve robots.txt for Next.js applications
195 lines (193 loc) • 7.17 kB
JavaScript
#!/usr/bin/env node
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const express_1 = __importDefault(require("express"));
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const app = (0, express_1.default)();
const PORT = parseInt(process.env.PORT || '3001'); // Ensure it's a number
// Middleware
app.use(express_1.default.json());
app.use(express_1.default.static('public'));
// Serve robots.txt
app.get('/robots.txt', (req, res) => {
try {
const robotsPath = path.join(process.cwd(), 'robots.txt');
if (fs.existsSync(robotsPath)) {
const content = fs.readFileSync(robotsPath, 'utf8');
res.set('Content-Type', 'text/plain');
res.send(content);
}
else {
// Generate default robots.txt
const defaultContent = `User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/
Sitemap: https://example.com/sitemap.xml`;
res.set('Content-Type', 'text/plain');
res.send(defaultContent);
}
}
catch (error) {
res.status(500).send('Error serving robots.txt');
}
});
// Serve a simple web interface
app.get('/', (req, res) => {
try {
const robotsPath = path.join(process.cwd(), 'robots.txt');
let robotsContent = '';
if (fs.existsSync(robotsPath)) {
robotsContent = fs.readFileSync(robotsPath, 'utf8');
}
else {
robotsContent = `User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/
Sitemap: https://example.com/sitemap.xml`;
}
res.send(`
<!DOCTYPE html>
<html>
<head>
<title>Robots.txt Viewer</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<style>
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
min-height: 100vh;
padding: 20px;
margin: 0;
}
.container {
max-width: 800px;
margin: 0 auto;
background: white;
border-radius: 12px;
box-shadow: 0 20px 40px rgba(0,0,0,0.1);
overflow: hidden;
}
.header {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
padding: 30px;
text-align: center;
}
.header h1 {
font-size: 2.5rem;
margin-bottom: 10px;
font-weight: 300;
}
.content {
padding: 40px;
}
.robots-content {
background: #f8f9fa;
border: 1px solid #e9ecef;
border-radius: 8px;
padding: 20px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
font-size: 14px;
line-height: 1.6;
white-space: pre-wrap;
overflow-x: auto;
}
.info {
background: #e3f2fd;
border: 1px solid #2196f3;
border-radius: 8px;
padding: 20px;
margin-bottom: 20px;
}
.info h3 {
margin-top: 0;
color: #1976d2;
}
.info p {
margin: 10px 0;
color: #424242;
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>🤖 Robots.txt Viewer</h1>
<p>Your robots.txt file content</p>
</div>
<div class="content">
<div class="info">
<h3>📄 robots.txt File</h3>
<p><strong>URL:</strong> <a href="/robots.txt" target="_blank">http://localhost:${PORT}/robots.txt</a></p>
<p><strong>Status:</strong> ✅ File is being served correctly</p>
<p><strong>Note:</strong> This robots.txt will be accessible at your-domain.com/robots.txt when deployed</p>
</div>
<h3>📝 Current robots.txt Content:</h3>
<div class="robots-content">${robotsContent}</div>
</div>
</div>
</body>
</html>
`);
}
catch (error) {
res.status(500).send('Error loading robots.txt content');
}
});
// Handle port conflicts
const server = app.listen(PORT, () => {
console.log(`🤖 Robots.txt Server running at http://localhost:${PORT}`);
console.log(`📄 View robots.txt at http://localhost:${PORT}/robots.txt`);
console.log(`🌐 Web interface at http://localhost:${PORT}`);
console.log(`💡 To serve robots.txt on your domain, make sure this server is running`);
}).on('error', (err) => {
if (err.code === 'EADDRINUSE') {
const nextPort = PORT + 1;
console.log(`⚠️ Port ${PORT} is in use. Trying port ${nextPort}...`);
server.listen(nextPort);
}
else {
console.error('Server error:', err);
}
});
//# sourceMappingURL=simple-server.js.map