skills-ws
Version:
87 agent skills for AI coding assistants — marketing, growth, web3, dev, design & operations. Built for OpenClaw, Claude Code, Cursor, and Codex.
237 lines (200 loc) • 9.12 kB
Markdown
## Contents
- 2. Data Source Strategies
- 2.1 APIs (Best for Fresh Data)
- 2.2 Database (Best for Scale + Control)
- 2.3 Scraping + Enrichment Pipeline
- 2.4 CSV / Spreadsheet (Quick Start)
## 2. Data Source Strategies
### 2.1 APIs (Best for Fresh Data)
```typescript
// lib/data-sources/api.ts
import pThrottle from 'p-throttle';
// Always throttle API calls during build
const throttle = pThrottle({ limit: 5, interval: 1000 });
const fetchWithRetry = throttle(async (url: string, retries = 3): Promise<any> => {
for (let attempt = 0; attempt < retries; attempt++) {
try {
const res = await fetch(url, {
headers: { Authorization: `Bearer ${process.env.API_KEY}` },
next: { revalidate: 86400 }, // ISR: rebuild daily
});
if (!res.ok) throw new Error(`${res.status}: ${res.statusText}`);
return res.json();
} catch (e) {
if (attempt === retries - 1) throw e;
await new Promise(r => setTimeout(r, 2 ** attempt * 1000));
}
}
});
// Cache API results to avoid hammering during build/dev.
// `unstable_cache` still ships in Next 15/16 (note the `unstable_` prefix) and is
// fine to use today. The forward-looking replacement is the `'use cache'`
// directive + cacheLife/cacheTag (see below). Pick ONE and be consistent.
import { unstable_cache } from 'next/cache';
export const getProductData = unstable_cache(
async (productSlug: string) => {
const data = await fetchWithRetry(`https://api.example.com/products/${productSlug}`);
return transformProductData(data);
},
['product-data'],
{ revalidate: 86400, tags: ['products'] }
);
```
**Modern alternative: `'use cache'` (stable in Next 16 via Cache Components; experimental behind a flag in 15.x).** The new cache model is opt-in via the `cacheComponents` flag in `next.config.js` (this flag was named `dynamicIO` in earlier 15.x canaries; check your version). Inside a cached scope you set freshness with `cacheLife` and invalidation keys with `cacheTag`. Verify the directive's stability for your exact version at `nextjs.org/docs`.
```typescript
import { cacheLife, cacheTag } from 'next/cache';
export async function getProductData(productSlug: string) {
'use cache';
cacheLife('days'); // preset: seconds|minutes|hours|days|weeks|max, or { stale, revalidate }
cacheTag(`product-${productSlug}`); // revalidateTag(`product-${slug}`) busts just this entry
const data = await fetchWithRetry(`https://api.example.com/products/${productSlug}`);
return transformProductData(data);
}
```
### 2.2 Database (Best for Scale + Control)
```typescript
// lib/data-sources/db.ts
import { prisma } from '@/lib/prisma';
export async function getLocationData(slug: string) {
return prisma.location.findUnique({
where: { slug },
include: {
stats: true,
providers: { where: { active: true }, orderBy: { rating: 'desc' }, take: 20 },
nearbyLocations: { take: 8 },
},
});
}
// For generateStaticParams — paginate to avoid memory issues
export async function* getAllLocationSlugs() {
let cursor: string | undefined;
while (true) {
const batch = await prisma.location.findMany({
select: { slug: true },
take: 1000,
...(cursor ? { skip: 1, cursor: { slug: cursor } } : {}),
orderBy: { slug: 'asc' },
});
if (batch.length === 0) break;
for (const item of batch) yield item.slug;
cursor = batch[batch.length - 1].slug;
}
}
```
### 2.3 Scraping + Enrichment Pipeline
> **Scraping compliance checklist — do this before writing the scraper.** Scraping for *commercial republishing* (which pSEO is) carries legal and contractual risk; the naive "open page, grab pricing" loop is a liability. Treat this as the minimum bar:
> 1. **Prefer an official API or licensed feed.** Almost every "scrape pricing" task has an API, partner feed, or affiliate data export that is faster, cleaner, and contractually safe. Scrape only as a last resort.
> 2. **Honor `robots.txt`.** Fetch and parse it per origin; skip disallowed paths. Robots is not a law, but ignoring it is the first thing cited against you.
> 3. **Read the Terms of Service.** Many sites' ToS forbid scraping and especially *republishing* their data. Republishing facts you scraped can implicate copyright, database rights (EU `sui generis`), and unfair-competition claims. **Get legal review before commercial reuse**, and prefer attribution + linking back.
> 4. **Identify yourself.** Set a descriptive `User-Agent` with a contact URL (`MyBot/1.0 (+https://example.com/bot)`). No spoofing real browsers to evade blocks.
> 5. **Rate-limit per domain** and add jittered exponential backoff; back off hard on `429`/`503`. Never run unbounded concurrency against one host.
> 6. **Record provenance.** Store `sourceUrl` + `fetchedAt` for every scraped value so you can show "as of <date>", expire stale data, and audit disputes.
> 7. **Cache politely.** Re-fetch on a schedule (e.g. weekly), not on every build. Conditional requests (ETag/If-Modified-Since) save everyone bandwidth.
```typescript
// scripts/enrich-data.ts
// Run as a scheduled job (cron), NOT at build time. tsx scripts/enrich-data.ts
import { chromium, type Browser, type Page } from 'playwright';
import pThrottle from 'p-throttle';
import robotsParser from 'robots-parser';
import { prisma } from '@/lib/prisma';
const USER_AGENT =
'MyCompanyEnrichBot/1.0 (+https://example.com/bot-info; bot@example.com)';
// Per-domain throttle: at most 1 request / 2s to any single host.
const throttlesByHost = new Map<string, ReturnType<typeof pThrottle>>();
function hostThrottle(host: string) {
if (!throttlesByHost.has(host)) {
throttlesByHost.set(host, pThrottle({ limit: 1, interval: 2000 }));
}
return throttlesByHost.get(host)!;
}
// Cache robots.txt per origin so we fetch it once.
const robotsByOrigin = new Map<string, Awaited<ReturnType<typeof loadRobots>>>();
async function loadRobots(origin: string) {
const robotsUrl = `${origin}/robots.txt`;
try {
const res = await fetch(robotsUrl, { headers: { 'User-Agent': USER_AGENT } });
const body = res.ok ? await res.text() : '';
return robotsParser(robotsUrl, body);
} catch {
// Fail CLOSED on robots fetch error: if we can't confirm we're allowed, skip.
return robotsParser(robotsUrl, 'User-agent: *\nDisallow: /');
}
}
async function isAllowed(url: string) {
const origin = new URL(url).origin;
if (!robotsByOrigin.has(origin)) robotsByOrigin.set(origin, await loadRobots(origin));
return robotsByOrigin.get(origin)!.isAllowed(url, USER_AGENT) ?? false;
}
async function withBackoff<T>(fn: () => Promise<T>, retries = 3): Promise<T> {
for (let attempt = 0; ; attempt++) {
try {
return await fn();
} catch (e) {
if (attempt >= retries) throw e;
const wait = 2 ** attempt * 1000 + Math.random() * 500; // jittered backoff
await new Promise(r => setTimeout(r, wait));
}
}
}
async function enrichOne(browser: Browser, product: { id: string; name: string; pricingUrl: string }) {
const url = product.pricingUrl;
if (!(await isAllowed(url))) {
console.warn(`robots.txt disallows ${url} — skipping ${product.name}`);
return;
}
const host = new URL(url).host;
await hostThrottle(host)(async () => {
let page: Page | undefined;
try {
page = await browser.newPage({ userAgent: USER_AGENT });
await withBackoff(() => page!.goto(url, { waitUntil: 'domcontentloaded', timeout: 30_000 }));
const pricing = await extractPricing(page);
await prisma.product.update({
where: { id: product.id },
// Provenance: record WHERE and WHEN, so the page can say "as of <date>".
data: { pricing, sourceUrl: url, lastEnriched: new Date() },
});
} catch (e) {
console.error(`Failed to enrich ${product.name} (${url}):`, e);
} finally {
await page?.close(); // ALWAYS close — otherwise pages leak and the run OOMs.
}
})();
}
async function enrichProductData() {
const browser = await chromium.launch();
try {
const products = await prisma.product.findMany({
where: {
OR: [
{ lastEnriched: null },
{ lastEnriched: { lt: new Date(Date.now() - 7 * 86_400_000) } }, // > 7 days old
],
},
take: 100,
});
// Sequential per host via throttle; products on different hosts still interleave.
for (const product of products) await enrichOne(browser, product);
} finally {
await browser.close();
}
}
enrichProductData().catch((e) => {
console.error(e);
process.exit(1);
});
```
### 2.4 CSV / Spreadsheet (Quick Start)
Good for prototyping. Use a CMS or database for production.
```typescript
// lib/data-sources/csv.ts
import { parse } from 'csv-parse/sync';
import { readFileSync } from 'fs';
import path from 'path';
const dataDir = path.join(process.cwd(), 'data');
export function loadLocations() {
const raw = readFileSync(path.join(dataDir, 'locations.csv'), 'utf-8');
return parse(raw, { columns: true, cast: true }) as Location[];
}
```
---