@atproto/ozone
Version:
Backend service for moderating the Bluesky network.
695 lines • 28.3 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ModerationViews = void 0;
exports.getSelfLabels = getSelfLabels;
const kysely_1 = require("kysely");
const api_1 = require("@atproto/api");
const common_1 = require("@atproto/common");
const lexicon_1 = require("@atproto/lexicon");
const syntax_1 = require("@atproto/syntax");
const lexicons_1 = require("../lexicon/lexicons");
const defs_1 = require("../lexicon/types/com/atproto/label/defs");
const defs_2 = require("../lexicon/types/com/atproto/moderation/defs");
const defs_3 = require("../lexicon/types/tools/ozone/moderation/defs");
const util_1 = require("../lexicon/util");
const logger_1 = require("../logger");
const status_1 = require("./status");
const subject_1 = require("./subject");
const util_2 = require("./util");
const isValidSelfLabels = (0, util_1.asPredicate)(defs_1.validateSelfLabels);
const ifString = (val) => typeof val === 'string' ? val : undefined;
const ifBoolean = (val) => typeof val === 'boolean' ? val : undefined;
const ifNumber = (val) => typeof val === 'number' ? val : undefined;
class ModerationViews {
constructor(db, signingKey, signingKeyId, appviewAgent, appviewAuth, idResolver, devMode) {
Object.defineProperty(this, "db", {
enumerable: true,
configurable: true,
writable: true,
value: db
});
Object.defineProperty(this, "signingKey", {
enumerable: true,
configurable: true,
writable: true,
value: signingKey
});
Object.defineProperty(this, "signingKeyId", {
enumerable: true,
configurable: true,
writable: true,
value: signingKeyId
});
Object.defineProperty(this, "appviewAgent", {
enumerable: true,
configurable: true,
writable: true,
value: appviewAgent
});
Object.defineProperty(this, "appviewAuth", {
enumerable: true,
configurable: true,
writable: true,
value: appviewAuth
});
Object.defineProperty(this, "idResolver", {
enumerable: true,
configurable: true,
writable: true,
value: idResolver
});
Object.defineProperty(this, "devMode", {
enumerable: true,
configurable: true,
writable: true,
value: devMode
});
}
async getAccoutInfosByDid(dids) {
if (dids.length === 0)
return new Map();
const auth = await this.appviewAuth(lexicons_1.ids.ComAtprotoAdminGetAccountInfos);
if (!auth)
return new Map();
try {
const res = await this.appviewAgent.com.atproto.admin.getAccountInfos({
dids: (0, common_1.dedupeStrs)(dids),
}, auth);
return res.data.infos.reduce((acc, cur) => {
return acc.set(cur.did, cur);
}, new Map());
}
catch (err) {
logger_1.httpLogger.error({ err, dids }, 'failed to resolve account infos from appview');
return new Map();
}
}
async repos(dids) {
if (dids.length === 0)
return new Map();
const [infos, subjectStatuses] = await Promise.all([
this.getAccoutInfosByDid(dids),
this.getSubjectStatus(dids),
]);
return dids.reduce((acc, did) => {
const info = infos.get(did);
if (!info)
return acc;
const status = subjectStatuses.get(did);
return acc.set(did, {
// No email or invite info on appview
did,
handle: info.handle,
relatedRecords: info.relatedRecords ?? [],
indexedAt: info.indexedAt,
moderation: {
subjectStatus: status ? this.formatSubjectStatus(status) : undefined,
},
});
}, new Map());
}
formatEvent(row) {
const eventView = {
id: row.id,
event: {
$type: row.action,
comment: row.comment ?? undefined,
},
subject: (0, subject_1.subjectFromEventRow)(row).lex(),
subjectBlobCids: row.subjectBlobCids ?? [],
createdBy: row.createdBy,
createdAt: row.createdAt,
subjectHandle: row.subjectHandle ?? undefined,
creatorHandle: row.creatorHandle ?? undefined,
modTool: row.modTool
? {
name: row.modTool.name,
meta: row.modTool.meta,
}
: undefined,
};
const { event } = eventView;
const meta = row.meta || {};
if ((0, defs_3.isModEventMuteReporter)(event) ||
(0, defs_3.isModEventTakedown)(event) ||
(0, defs_3.isModEventLabel)(event) ||
(0, defs_3.isModEventMute)(event)) {
event.durationInHours = row.durationInHours ?? undefined;
}
if (((0, defs_3.isModEventTakedown)(event) || (0, defs_3.isModEventAcknowledge)(event)) &&
meta.acknowledgeAccountSubjects) {
event.acknowledgeAccountSubjects = ifBoolean(meta.acknowledgeAccountSubjects);
}
if ((0, defs_3.isModEventPriorityScore)(event)) {
event.score = ifNumber(meta?.priorityScore) ?? 0;
}
if ((0, defs_3.isModEventTakedown)(event) ||
(0, defs_3.isModEventEmail)(event) ||
(0, defs_3.isModEventReverseTakedown)(event)) {
if (typeof meta.policies === 'string' && meta.policies.length > 0) {
event.policies = meta.policies.split(',');
}
event.strikeCount = ifNumber(row.strikeCount);
event.severityLevel = ifString(row.severityLevel);
if ((0, defs_3.isModEventTakedown)(event) || (0, defs_3.isModEventEmail)(event)) {
event.strikeExpiresAt = ifString(row.strikeExpiresAt);
}
}
if ((0, defs_3.isModEventTakedown)(event)) {
if (typeof meta.targetServices === 'string' &&
meta.targetServices.length > 0) {
event.targetServices = meta.targetServices.split(',');
}
}
if ((0, defs_3.isModEventLabel)(event)) {
event.createLabelVals = row.createLabelVals?.length
? row.createLabelVals.split(' ')
: [];
event.negateLabelVals = row.negateLabelVals?.length
? row.negateLabelVals.split(' ')
: [];
}
else if ((0, defs_3.isModEventAcknowledge)(event) ||
(0, defs_3.isModEventTakedown)(event) ||
(0, defs_3.isModEventEscalate)(event)) {
// This is for legacy data only, for new events, these types of events
// won't have labels attached:
if (row.createLabelVals?.length) {
// @ts-expect-error legacy
event.createLabelVals = row.createLabelVals.split(' ');
}
if (row.negateLabelVals?.length) {
// @ts-expect-error legacy
event.negateLabelVals = row.negateLabelVals.split(' ');
}
}
if ((0, defs_3.isModEventReport)(event)) {
event.isReporterMuted = !!meta.isReporterMuted;
event.reportType = ifString(meta.reportType);
}
if ((0, defs_3.isModEventEmail)(event)) {
event.content = ifString(meta.content);
event.subjectLine = ifString(meta.subjectLine);
event.isDelivered = ifBoolean(meta.isDelivered);
}
if ((0, defs_3.isModEventComment)(event) && meta.sticky) {
event.sticky = true;
}
if ((0, defs_3.isModEventTag)(event)) {
event.add = row.addedTags || [];
event.remove = row.removedTags || [];
}
if ((0, defs_3.isAccountEvent)(event)) {
event.active = !!meta.active;
event.timestamp = ifString(meta.timestamp);
event.status = ifString(meta.status);
}
if ((0, defs_3.isIdentityEvent)(event)) {
event.timestamp = ifString(meta.timestamp);
event.handle = ifString(meta.handle);
event.pdsHost = ifString(meta.pdsHost);
event.tombstone = !!meta.tombstone;
}
if ((0, defs_3.isRecordEvent)(event)) {
event.op = ifString(meta.op);
event.cid = ifString(meta.cid);
event.timestamp = ifString(meta.timestamp);
}
if ((0, defs_3.isAgeAssuranceEvent)(event)) {
event.status = ifString(meta.status);
event.access = ifString(meta.access);
event.createdAt = ifString(meta.createdAt);
event.attemptId = ifString(meta.attemptId);
event.initIp = ifString(meta.initIp);
event.initUa = ifString(meta.initUa);
event.completeIp = ifString(meta.completeIp);
event.completeUa = ifString(meta.completeUa);
}
if ((0, defs_3.isAgeAssuranceOverrideEvent)(event)) {
event.status = ifString(meta.status);
event.access = ifString(meta.access);
}
if ((0, defs_3.isScheduleTakedownEvent)(event)) {
event.executeAt = ifString(meta.executeAt);
event.executeAfter = ifString(meta.executeAfter);
event.executeUntil = ifString(meta.executeUntil);
}
return eventView;
}
async eventDetail(result) {
const subjectId = result.subjectType === 'com.atproto.admin.defs#repoRef'
? result.subjectDid
: result.subjectUri;
if (!subjectId) {
throw new Error(`Bad subject: ${result.id}`);
}
const subject = await this.subject(subjectId);
const eventView = this.formatEvent(result);
const allBlobs = 'value' in subject ? findBlobRefs(subject.value) : [];
const subjectBlobs = await this.blob(allBlobs.filter((blob) => eventView.subjectBlobCids.includes(blob.ref.toString())));
return {
...eventView,
subject,
subjectBlobs,
};
}
async repoDetails(dids, labelers) {
const results = new Map();
if (!dids.length) {
return results;
}
const [repos, localLabels, externalLabels] = await Promise.all([
this.repos(dids),
this.labels(dids),
this.getExternalLabels(dids, labelers),
]);
repos.forEach((repo, did) => {
const labels = [
...(localLabels.get(did) || []),
...(externalLabels.get(did) || []),
];
const repoView = {
...repo,
labels,
moderation: {
...repo.moderation,
},
};
results.set(did, repoView);
});
return results;
}
async fetchRecord(params, appviewAuth) {
try {
const record = await this.appviewAgent.com.atproto.repo.getRecord(params, appviewAuth);
return record;
}
catch (err) {
if (err instanceof api_1.ComAtprotoRepoGetRecord.RecordNotFoundError) {
// If pds fetch fails, just return null regardless of the error
try {
const { agent: pdsAgent } = await (0, util_2.getPdsAgentForRepo)(this.idResolver, params.repo, this.devMode);
if (!pdsAgent) {
return null;
}
const record = await pdsAgent.com.atproto.repo.getRecord(params);
return record;
}
catch (error) {
return null;
}
}
return null;
}
}
async fetchRecords(subjects) {
const appviewAuth = await this.appviewAuth(lexicons_1.ids.ComAtprotoRepoGetRecord);
if (!appviewAuth)
return new Map();
const fetched = await Promise.all(subjects.map(async (subject) => {
const uri = new syntax_1.AtUri(subject.uri);
const params = {
repo: uri.hostname,
collection: uri.collection,
rkey: uri.rkey,
cid: subject.cid,
};
return this.fetchRecord(params, appviewAuth);
}));
return fetched.reduce((acc, cur) => {
if (!cur)
return acc;
const data = cur.data;
const indexedAt = new Date().toISOString();
return acc.set(data.uri, { ...data, cid: data.cid ?? '', indexedAt });
}, new Map());
}
async records(subjects) {
const uris = subjects.map((record) => new syntax_1.AtUri(record.uri));
const dids = uris.map((u) => u.hostname);
const [repos, subjectStatuses, records] = await Promise.all([
this.repos(dids),
this.getSubjectStatus(subjects.map((s) => s.uri)),
this.fetchRecords(subjects),
]);
const map = new Map();
for (const uri of uris) {
const repo = repos.get(uri.hostname);
if (!repo)
continue;
const record = records.get(uri.toString());
if (!record)
continue;
const subjectStatus = subjectStatuses.get(uri.toString());
map.set(uri.toString(), {
uri: uri.toString(),
cid: record.cid,
value: record.value,
blobCids: findBlobRefs(record.value).map((blob) => blob.ref.toString()),
indexedAt: record.indexedAt,
repo,
moderation: {
subjectStatus: subjectStatus
? this.formatSubjectStatus(subjectStatus)
: undefined,
},
});
}
return map;
}
async recordDetails(subjects, labelers) {
const results = new Map();
if (!subjects.length) {
return results;
}
const subjectUris = subjects.map((s) => s.uri);
const [records, subjectStatusesResult, localLabels, externalLabels] = await Promise.all([
this.records(subjects),
this.getSubjectStatus(subjectUris),
this.labels(subjectUris),
this.getExternalLabels(subjectUris, labelers),
]);
await Promise.all(Array.from(records.entries()).map(async ([uri, record]) => {
const selfLabels = getSelfLabels({
uri: record.uri,
cid: record.cid,
record: record.value,
});
const status = subjectStatusesResult.get(uri);
const blobs = await this.blob(findBlobRefs(record.value));
results.set(uri, {
...record,
blobs,
moderation: {
...record.moderation,
subjectStatus: status
? this.formatSubjectStatus(status)
: undefined,
},
labels: [
...(localLabels.get(uri) || []),
...selfLabels,
...(externalLabels.get(uri) || []),
],
});
}));
return results;
}
async getExternalLabels(subjects, labelers) {
const results = new Map();
if (!labelers?.dids.length && !labelers?.redact.size)
return results;
try {
const { data: { labels }, } = await this.appviewAgent.com.atproto.label.queryLabels({
uriPatterns: subjects,
sources: labelers.dids,
});
labels.forEach((label) => {
if (!results.has(label.uri)) {
results.set(label.uri, [label]);
return;
}
results.get(label.uri)?.push(label);
});
return results;
}
catch (err) {
logger_1.httpLogger.error({ err, subjects, labelers }, 'failed to resolve labels from appview');
return results;
}
}
formatReport(report) {
return {
id: report.id,
createdAt: report.createdAt,
// Ideally, we would never have a report entry that does not have a reasonType but at the schema level
// we are not guarantying that so in whatever case, if we end up with such entries, default to 'other'
reasonType: report.meta?.reportType
? report.meta?.reportType
: defs_2.REASONOTHER,
reason: report.comment ?? undefined,
reportedBy: report.createdBy,
subject: (0, subject_1.subjectFromEventRow)(report).lex(),
};
}
// Partial view for subjects
async subject(subject) {
if (subject.startsWith('did:')) {
const repos = await this.repos([subject]);
const repo = repos.get(subject);
if (repo) {
return {
...repo,
$type: 'tools.ozone.moderation.defs#repoView',
};
}
else {
return {
$type: 'tools.ozone.moderation.defs#repoViewNotFound',
did: subject,
};
}
}
else {
const records = await this.records([{ uri: subject }]);
const record = records.get(subject);
if (record) {
return {
...record,
$type: 'tools.ozone.moderation.defs#recordView',
};
}
else {
return {
$type: 'tools.ozone.moderation.defs#recordViewNotFound',
uri: subject,
};
}
}
}
// Partial view for blobs
async blob(blobs) {
if (!blobs.length)
return [];
const { ref } = this.db.db.dynamic;
const modStatusResults = await (0, status_1.moderationSubjectStatusQueryBuilder)(this.db.db)
.where((0, kysely_1.sql) `${ref('moderation_subject_status.blobCids')} @> ${JSON.stringify(blobs.map((blob) => blob.ref.toString()))}`)
.executeTakeFirst();
const statusByCid = (modStatusResults?.blobCids || [])?.reduce((acc, cur) => Object.assign(acc, { [cur]: modStatusResults }), {});
// Intentionally missing details field, since we don't have any on appview.
// We also don't know when the blob was created, so we use a canned creation time.
const unknownTime = new Date(0).toISOString();
return blobs.map((blob) => {
const cid = blob.ref.toString();
const subjectStatus = statusByCid[cid]
? this.formatSubjectStatus(statusByCid[cid])
: undefined;
return {
cid,
mimeType: blob.mimeType,
size: blob.size,
createdAt: unknownTime,
moderation: {
subjectStatus,
},
};
});
}
async labels(subjects, includeNeg) {
const now = new Date().toISOString();
const labels = new Map();
const res = await this.db.db
.selectFrom('label')
.where('label.uri', 'in', subjects)
.where((qb) => qb.where('label.exp', 'is', null).orWhere('label.exp', '>', now))
.if(!includeNeg, (qb) => qb.where('neg', '=', false))
.selectAll()
.execute();
await Promise.all(res.map(async (labelRow) => {
const signedLabel = await this.formatLabelAndEnsureSig(labelRow);
const current = labels.get(labelRow.uri);
if (current)
current.push(signedLabel);
else
labels.set(labelRow.uri, [signedLabel]);
}));
return labels;
}
async formatLabelAndEnsureSig(row) {
const formatted = (0, util_2.formatLabel)(row);
if (!!row.sig && row.signingKeyId === this.signingKeyId) {
return formatted;
}
const signed = await (0, util_2.signLabel)(formatted, this.signingKey);
try {
await this.db.db
.updateTable('label')
.set({ sig: Buffer.from(signed.sig), signingKeyId: this.signingKeyId })
.where('id', '=', row.id)
.execute();
}
catch (err) {
logger_1.dbLogger.error({ err, label: row }, 'failed to update resigned label');
}
return signed;
}
async getSubjectStatus(subjects) {
if (!subjects.length)
return new Map();
const parsedSubjects = subjects.map(parseSubjectId);
const builder = (0, status_1.moderationSubjectStatusQueryBuilder)(this.db.db)
//
.where((qb) => {
for (const sub of parsedSubjects) {
qb = qb.orWhere((qb) => qb
.where('moderation_subject_status.did', '=', sub.did)
.where('moderation_subject_status.recordPath', '=', sub.recordPath ?? ''));
}
return qb;
});
const [statusRes, accountsByDid] = await Promise.all([
builder.execute(),
this.getAccoutInfosByDid(parsedSubjects.map((s) => s.did)),
]);
return new Map(statusRes.map((row) => {
const subjectId = formatSubjectId(row.did, row.recordPath);
const handle = accountsByDid.get(row.did)?.handle ?? syntax_1.INVALID_HANDLE;
return [subjectId, { ...row, handle }];
}));
}
formatSubjectStatus(status) {
const statusView = {
id: status.id,
reviewState: status.reviewState,
createdAt: status.createdAt,
updatedAt: status.updatedAt,
comment: status.comment ?? undefined,
lastReviewedBy: status.lastReviewedBy ?? undefined,
lastReviewedAt: status.lastReviewedAt ?? undefined,
lastReportedAt: status.lastReportedAt ?? undefined,
lastAppealedAt: status.lastAppealedAt ?? undefined,
muteUntil: status.muteUntil ?? undefined,
muteReportingUntil: status.muteReportingUntil ?? undefined,
suspendUntil: status.suspendUntil ?? undefined,
takendown: status.takendown ?? undefined,
appealed: status.appealed ?? undefined,
subjectRepoHandle: status.handle ?? undefined,
subjectBlobCids: status.blobCids || [],
tags: status.tags || [],
priorityScore: status.priorityScore,
ageAssuranceState: status.ageAssuranceState ?? undefined,
ageAssuranceUpdatedBy: status.ageAssuranceUpdatedBy ?? undefined,
subject: (0, subject_1.subjectFromStatusRow)(status).lex(),
accountStats: {
// Explicitly typing to allow for easy manipulation (e.g. to strip from tests snapshots)
$type: 'tools.ozone.moderation.defs#accountStats',
// account_events_stats
reportCount: status.reportCount ?? undefined,
appealCount: status.appealCount ?? undefined,
suspendCount: status.suspendCount ?? undefined,
takedownCount: status.takedownCount ?? undefined,
escalateCount: status.escalateCount ?? undefined,
},
recordsStats: {
// Explicitly typing to allow for easy manipulation (e.g. to strip from tests snapshots)
$type: 'tools.ozone.moderation.defs#recordsStats',
// account_record_events_stats
totalReports: status.totalReports ?? undefined,
reportedCount: status.reportedCount ?? undefined,
escalatedCount: status.escalatedCount ?? undefined,
appealedCount: status.appealedCount ?? undefined,
// account_record_status_stats
subjectCount: status.subjectCount ?? undefined,
pendingCount: status.pendingCount ?? undefined,
processedCount: status.processedCount ?? undefined,
takendownCount: status.takendownCount ?? undefined,
},
accountStrike: status.strikeCount !== null || status.totalStrikeCount !== null
? {
$type: 'tools.ozone.moderation.defs#accountStrike',
activeStrikeCount: status.strikeCount ?? undefined,
totalStrikeCount: status.totalStrikeCount ?? undefined,
firstStrikeAt: status.firstStrikeAt ?? undefined,
lastStrikeAt: status.lastStrikeAt ?? undefined,
}
: undefined,
};
if (status.recordPath !== '') {
statusView.hosting = {
$type: 'tools.ozone.moderation.defs#recordHosting',
updatedAt: status.hostingUpdatedAt ?? undefined,
deletedAt: status.hostingDeletedAt ?? undefined,
status: status.hostingStatus ?? 'unknown',
};
}
else {
statusView.hosting = {
$type: 'tools.ozone.moderation.defs#accountHosting',
updatedAt: status.hostingUpdatedAt ?? undefined,
deletedAt: status.hostingDeletedAt ?? undefined,
status: status.hostingStatus ?? 'unknown',
deactivatedAt: status.hostingDeactivatedAt ?? undefined,
reactivatedAt: status.hostingReactivatedAt ?? undefined,
};
}
return statusView;
}
async fetchAuthorFeed(actor) {
const auth = await this.appviewAuth(lexicons_1.ids.AppBskyFeedGetAuthorFeed);
if (!auth)
return [];
const { data: { feed }, } = await this.appviewAgent.app.bsky.feed.getAuthorFeed({ actor }, auth);
return feed;
}
async getProfiles(dids) {
const profiles = new Map();
const auth = await this.appviewAuth(lexicons_1.ids.AppBskyActorGetProfiles);
if (!auth)
return profiles;
for (const actors of (0, common_1.chunkArray)(dids, 25)) {
const { data } = await this.appviewAgent.getProfiles({ actors }, auth);
data.profiles.forEach((profile) => {
profiles.set(profile.did, profile);
});
}
return profiles;
}
}
exports.ModerationViews = ModerationViews;
function parseSubjectId(subject) {
if (subject.startsWith('did:')) {
return { did: subject };
}
const uri = new syntax_1.AtUri(subject);
return { did: uri.hostname, recordPath: `${uri.collection}/${uri.rkey}` };
}
function formatSubjectId(did, recordPath) {
return recordPath ? `at://${did}/${recordPath}` : did;
}
function findBlobRefs(value, refs = []) {
if (value instanceof lexicon_1.BlobRef) {
refs.push(value);
}
else if (Array.isArray(value)) {
value.forEach((val) => findBlobRefs(val, refs));
}
else if (value && typeof value === 'object') {
Object.values(value).forEach((val) => findBlobRefs(val, refs));
}
return refs;
}
function getSelfLabels(details) {
const { uri, cid, record } = details;
if (!uri || !cid || !record)
return [];
if (!isValidSelfLabels(record.labels))
return [];
const src = new syntax_1.AtUri(uri).host; // record creator
const cts = typeof record.createdAt === 'string'
? (0, syntax_1.normalizeDatetimeAlways)(record.createdAt)
: new Date(0).toISOString();
return record.labels.values.map(({ val }) => {
return { src, uri, cid, val, cts };
});
}
//# sourceMappingURL=views.js.map