@forzalabs/remora
Version:
A powerful CLI tool for seamless data translation.
308 lines • 11.5 kB
JSON
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Producer Schema",
"description": "Schema for defining data producers",
"type": "object",
"properties": {
"$schema": {
"type": "string",
"format": "uri"
},
"name": {
"type": "string",
"description": "The name of the producer"
},
"description": {
"type": "string",
"description": "Optional description of the producer"
},
"source": {
"type": "string",
"description": "The name of the Source"
},
"dimensions": {
"type": "array",
"description": "Dimensions for the producer",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "The name of the dimension. This is the output name of this dimension."
},
"description": {
"type": "string",
"description": "Optional description of the dimension"
},
"type": {
"type": "string",
"enum": [
"string",
"number",
"datetime",
"boolean"
],
"description": "The data type of the dimension"
},
"alias": {
"type": "string",
"description": "The SQL column or field key that corresponds to this dimension. If left empty, the column name is assumed to be the same as the dimension name."
},
"pk": {
"type": "boolean",
"description": "Whether this is a primary key column or not"
},
"classification": {
"type": "array",
"items": {
"type": "string",
"enum": [
"PHI",
"PII",
"GDPR"
],
"description": "Classification categories for this dimension"
}
},
"mask": {
"type": "string",
"description": "Masking type to apply to this dimension. Predefined values: 'hash' (replaces with a hashed value), 'mask' (replaces characters with a mask character), 'crypt' (encrypts the value), 'random' (replaces with a random value), 'seeded-random' (replaces with a random value generated from a seed), 'none' (no masking). You can also use environment variables by using the {your-env-var} notation or any custom string value.",
"examples": [
"hash",
"mask",
"crypt",
"random",
"seeded-random",
"none",
"{REMORA_MASK_IN_DEV}"
]
},
"sourceFilename": {
"type": "boolean",
"description": "When true, this dimension will be populated with the source filename. Only valid for file-based producers (local, aws-s3) and only one dimension per producer can have this set to true. Useful when reading multiple files with wildcard patterns to track which file each row came from."
}
},
"required": [
"name",
"type"
],
"additionalProperties": false
},
"minItems": 1
},
"measures": {
"type": "array",
"description": "Optional measures for the producer",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "The name of the measure"
},
"description": {
"type": "string",
"description": "Optional description of the measure"
},
"sql": {
"type": "string",
"description": "SQL command used to create a metric"
}
},
"required": [
"name",
"sql"
],
"additionalProperties": false
}
},
"settings": {
"type": "object",
"compressionType": {
"type": "string",
"enum": [
"GZ",
"TAR",
"ZIP"
],
"description": "The compression type used from the file to read"
},
"description": "Settings for the producer",
"properties": {
"sqlTable": {
"type": "string",
"description": "The SQL table name, which is a concise way of setting the sql property to 'SELECT * FROM <sql table name>'"
},
"direct": {
"type": "boolean",
"description": "If true, no view is created on the producer side due to permission limitations"
},
"sql": {
"type": "string",
"description": "The name of the file that has the SQL statement to run to create this producer"
},
"fileKey": {
"type": "string",
"description": "For S3/local sources: the file key/path that identifies the file to read. For HTTP API sources: the API endpoint path (e.g., '/api/v1/users')"
},
"fileType": {
"type": "string",
"enum": [
"JSON",
"JSONL",
"CSV",
"TXT",
"XLS",
"XLSX",
"XML",
"PARQUET"
],
"description": "The type of file to read."
},
"delimiter": {
"type": "string",
"description": "The column delimiter for CSV or TXT files if different from the default (,)."
},
"hasHeaderRow": {
"type": "boolean",
"description": "For TXT files, specifies whether the file has a header row containing column names. Defaults to true."
},
"sheetName": {
"type": "string",
"description": "For Excel files (.xls/.xlsx), specifies the name of the sheet to read data from. If not specified, the first sheet will be used."
}
},
"additionalProperties": false
},
"_version": {
"type": "number",
"description": "Version number of the producer configuration"
}
},
"required": [
"name",
"source",
"dimensions",
"settings"
],
"additionalProperties": false,
"examples": [
{
"name": "CustomerData",
"description": "Producer for customer data from the main database",
"source": "Production PostgreSQL Database",
"dimensions": [
{
"name": "customer_id",
"type": "string",
"pk": true
},
{
"name": "full_name",
"type": "string",
"classification": [
"PII",
"GDPR"
],
"mask": "hash"
},
{
"name": "email",
"type": "string",
"format": {
"type": "string",
"description": "Format for datetype of the source ('YYYYMMDD', 'DD-MM-YYYY')"
},
"classification": [
"PII",
"GDPR"
],
"mask": "mask"
},
{
"name": "signup_date",
"type": "datetime"
}
],
"measures": [
{
"name": "total_orders",
"description": "Total number of orders by this customer",
"sql": "COUNT(order_id)"
},
{
"name": "total_spent",
"description": "Total amount spent by this customer",
"sql": "SUM(order_amount)"
}
],
"settings": {
"sqlTable": "customers"
},
"_version": 1
},
{
"name": "SalesData",
"description": "Producer for sales data from S3 bucket",
"source": "Data Lake",
"dimensions": [
{
"name": "transaction_id",
"type": "string",
"pk": true
},
{
"name": "product_id",
"type": "string"
},
{
"name": "sale_amount",
"type": "number"
},
{
"name": "sale_date",
"type": "datetime"
}
],
"settings": {
"fileKey": "sales/daily_sales.csv",
"fileType": "CSV"
},
"_version": 2
},
{
"name": "APIUsers",
"description": "Producer for user data from REST API",
"source": "REST API with Bearer Token",
"dimensions": [
{
"name": "user_id",
"type": "string",
"pk": true
},
{
"name": "username",
"type": "string"
},
{
"name": "email",
"type": "string",
"classification": [
"PII",
"GDPR"
],
"mask": "mask"
},
{
"name": "created_at",
"type": "datetime"
}
],
"settings": {
"fileKey": "/api/v1/users",
"fileType": "JSON"
},
"_version": 1
}
]
}