@sanity/import/src/importFromArray.ts

Import documents to a Sanity dataset

import createDebug from 'debug'

import {absolutifyPaths, getAssetRefs, unsetAssetRefs} from './assetRefs.js'
import {assignArrayKeys} from './assignArrayKeys.js'
import {assignDocumentId} from './assignDocumentId.js'
import {batchDocuments} from './batchDocuments.js'
import {documentHasError} from './documentHasErrors.js'
import {importBatches} from './importBatches.js'
import {
  cleanupReferences,
  getStrongRefs,
  strengthenReferences,
  type StrongRefsTask,
  weakenStrongRefs,
} from './references.js'
import {type ImportOptions, type ImportResult, type SanityDocument} from './types.js'
import {uploadAssets} from './uploadAssets.js'
import {ensureUniqueIds} from './util/ensureUniqueIds.js'
import {validateAssetMapForReplacementChars} from './util/validateReplacementCharacters.js'
import {validateAssetDocuments} from './validateAssetDocuments.js'
import {validateCdrDatasets} from './validateCdrDatasets.js'

const debug = createDebug('sanity:import:array')

async function importDocuments(
  documents: SanityDocument[],
  options: ImportOptions,
): Promise<ImportResult> {
  options.onProgress({step: 'Reading/validating data file'})
  for (const [i, doc] of documents.entries()) {
    documentHasError(doc, i)
  }

  // Validate assetMap for replacement characters
  if (options.assetMap && options.allowReplacementCharacters !== true) {
    validateAssetMapForReplacementChars(options.assetMap)
  }

  // Validate that there are no duplicate IDs in the documents
  ensureUniqueIds(documents)

  // Ensure that any cross-dataset references has datasets to point to
  if (!options.skipCrossDatasetReferences) {
    await validateCdrDatasets(documents, options)
  }

  let filteredDocuments = documents
  // Always filter out system documents unless explicitly allowed.
  // Release system documents are an exception to this flag.
  if (options.allowSystemDocuments !== true) {
    filteredDocuments = documents.filter(
      (doc) => doc._id?.startsWith('_.releases.') || !doc._id?.startsWith('_.'),
    )
  }

  // Replace relative asset paths if one is defined
  // (file://./images/foo-bar.png -> file:///abs/olute/images/foo-bar.png)
  const absPathed = filteredDocuments.map((doc) => absolutifyPaths(doc, options.assetsBase))

  // Assign document IDs for document that do not have one. This is necessary
  // for us to strengthen references and import assets properly.
  const ided = absPathed.map((doc) => assignDocumentId(doc))

  // User might not have applied `_key` on array elements which are objects;
  // if this is the case, generate random keys to help realtime engine
  const keyed = ided.map((doc) => assignArrayKeys(doc))

  // Sanity prefers to have a `_type` on every object. Make sure references
  // has `_type` set to `reference`, and that there are no `_projectId` keys
  const docs = keyed.map((doc) => cleanupReferences(doc, options))

  // Find references that will need strengthening when import is done
  const strongRefs = docs
    .map((doc) => getStrongRefs(doc))
    .filter((ref): ref is StrongRefsTask => ref !== null)

  // Extract asset references from the documents
  const assetRefs = docs.flatMap((doc) => getAssetRefs(doc))

  // Remove asset references from the documents
  const assetless = docs.map((doc) => unsetAssetRefs(doc))

  // Make strong references weak so they can be imported in any order
  const weakened = assetless.map((doc) => weakenStrongRefs(doc))

  // Create batches of documents to import. Try to keep batches below a certain
  // byte-size (since document may vary greatly in size depending on type etc)
  const batches = batchDocuments(weakened)

  // Ensure that we don't reference missing assets, or assets in different datasets
  debug('Validating asset documents')
  await validateAssetDocuments(docs, options)

  // Trigger actual import process
  debug('Starting import of documents')
  const {count: docsImported, importedIds} = await importBatches(batches, options)

  // When using createIfNotExists (--missing), only upload assets for documents
  // that were actually created. Documents that already existed in the dataset
  // are assumed to already have their assets in place.
  let filteredAssetRefs = assetRefs
  if (options.operation === 'createIfNotExists') {
    const importedIdSet = new Set(importedIds)
    filteredAssetRefs = assetRefs.filter((ref) => importedIdSet.has(ref.documentId))
    const skipped = assetRefs.length - filteredAssetRefs.length
    if (skipped > 0) {
      debug('Skipping %d asset refs for %d already-existing documents', skipped, skipped)
    }
  }

  // Documents are imported, now proceed with post-import operations
  debug('Uploading assets')
  const {failures: assetWarnings} = await uploadAssets(filteredAssetRefs, options)

  // Strengthen references
  debug('Strengthening references')
  await strengthenReferences(strongRefs, options)

  // Return number of documents imported
  return {
    numDocs: docsImported,
    warnings: assetWarnings.map((warning) => ({message: `Failed to upload asset: ${warning.url}`})),
  }
}

export {importDocuments}