marklogic/lib/documents.js

The official MarkLogic Node.js client API.

/*
* Copyright (c) 2015-2025 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
*/
'use strict';

const requester  = require('./requester.js');
const mlutil     = require('./mlutil.js');
const Operation  = require('./operation.js');
const qb         = require('./query-builder.js').lib;
const pathModule = require('path');
const fs         = require('fs');
const stream = require('stream');
const bldrbase = require('./plan-builder-base.js');
const duplexify = require('duplexify');

/** @ignore */
function addDocumentUri(documents, document) {
  if (document != null) {
    const uri = document.uri;
    if ((typeof uri === 'string' || uri instanceof String) && uri.length > 0) {
      documents.push(uri);
    }
  }
  return documents;
}
/** @ignore */
function getDocumentUris(documents) {
  if (!Array.isArray(documents) || documents.length === 0) {
    return [];
  }
  return documents.reduce(addDocumentUri, []);
}

/** @ignore */
function compareDocuments(firstDoc, secondDoc) {
  const hasFirstDoc  = (firstDoc  !== null);
  const hasSecondDoc = (secondDoc !== null);
  if (!hasFirstDoc && !hasSecondDoc) {return  0;}
  if (!hasFirstDoc && hasSecondDoc)  {return -1;}
  if (hasFirstDoc  && !hasSecondDoc) {return  1;}
  const firstUri  = firstDoc.uri;
  const secondUri = secondDoc.uri;
  const hasFirstUri  = ((typeof firstUri  === 'string' || firstUri  instanceof String) && firstUri.length  > 0);
  const hasSecondUri = ((typeof secondUri === 'string' || secondUri instanceof String) && secondUri.length > 0);
  if (!hasFirstUri && !hasSecondUri) {return  0;}
  if (!hasFirstUri && hasSecondUri)  {return -1;}
  if (hasFirstUri  && !hasSecondUri) {return  1;}
  if (firstUri < secondUri) {return -1;}
  if (firstUri > secondUri) {return  1;}
  return 0;
}

/** @ignore */
function uriErrorTransform(message) {
  /*jshint validthis:true */
  const operation = this;

  const uri = operation.uri;
  return (uri == null) ? message :
    (message+' (on '+uri+')');
}
/** @ignore */
function uriListErrorTransform(message) {
  /*jshint validthis:true */
  const operation = this;

  const uris = operation.uris;
  return ((!Array.isArray(uris)) || uris.length === 0) ?
    message : (message+' (on '+uris.join(', ')+')');
}

/** @ignore */
function Documents(client) {
  if (!(this instanceof Documents)) {
    return new Documents(client);
  }
  this.client = client;
}

/**
 * Provides functions to write, read, query, or perform other operations
 * on documents in the database. For operations that modify the database,
 * the client must have been created for a user with the rest-writer role.
 * For operations that read or query the database, the user need only have
 * the rest-reader role.
 * @namespace documents
 */

/** @ignore */
function probeOutputTransform(/*headers, data*/) {
  /*jshint validthis:true */
  const operation = this;

  const statusCode = operation.responseStatusCode;
  const exists     = (statusCode === 200) ? true : false;
  if (operation.contentOnly === true) {
    return exists;
  }

  const output = exists ? operation.responseHeaders : {};
  output.uri    = operation.uri;
  output.exists = exists;

  return output;
}

function protectOutputTransform(/*headers, data*/) {
  /*jshint validthis:true */
  const operation = this;

  const output = {
    uri:                operation.uri,
    temporalCollection: operation.temporalCollection,
    level:              operation.level
  };

  return output;
}

function wipeOutputTransform(/*headers, data*/) {
  /*jshint validthis:true */
  const operation = this;

  const output = {
    uri:                operation.uri,
    temporalCollection: operation.temporalCollection,
    wiped:              true
  };

  return output;
}

function advanceLsqtOutputTransform(headers) {
  /*jshint validthis:true */

  const output = {
    lsqt: headers.lsqt
  };

  return output;
}

/**
 * An object offering the alternative of a {@link ResultProvider#result} function
 * or a {@link ResultProvider#stream} function for receiving the results
 * @namespace ResultProvider
 */
/**
 * Accepts success and/or failure callbacks and returns a
 * {@link https://www.promisejs.org/|Promises} object for chaining
 * actions with then() functions.
 * @name ResultProvider#result
 * @since 1.0
 * @function
 * @param {function}  [success] - a callback invoked when the request succeeds
 * @param {function}  [failure] - a callback invoked when the request fails
 * @returns {object} a Promises object
 */
/**
 * Returns a ReadableStream object in object mode for receiving results as
 * complete objects.
 * @name ResultProvider#stream
 * @since 1.0
 * @function
 * @returns {object} a {@link http://nodejs.org/api/stream.html#stream_class_stream_readable|ReadableStream}
 * object
 */

/**
 * Provides a description of a document to write to the server, after reading
 * from the server, or for another document operation.  The descriptor may have
 * more or fewer properties depending on the operation.
 * @typedef {object} documents.DocumentDescriptor
 * @since 1.0
 * @property {string} uri - the identifier for the document in the database
 * @property {object|string|Buffer|ReadableStream} [content] - the content
 * of the document; when writing a ReadableStream for the content, first pause
 * the stream
 * @property {string[]} [collections] - the collections to which the document belongs
 * @property {object[]} [permissions] - the permissions controlling which users can read or
 * write the document
 * @property {object[]} [properties] - additional properties of the document
 * @property {number} [quality] - a weight to increase or decrease the rank of the document
 * @property {object[]} [metadataValues] - the metadata values of the document
 * @property {number} [versionId] - an identifier for the currently stored version of the
 * document
 * @property {string} [temporalDocument] - the collection URI for a temporal document;
 * use only when writing a document to a temporal collection
 */

/**
 * Categories of information to read or write for documents.
 * The possible values of the enumeration are
 * content|collections|metadataValues|permissions|properties|quality|metadata|rawContent|none where
 * metadata is an alias for all of the categories other than content.
 * @typedef {enum} documents.categories
 * @since 1.0
 */

/**
 * A success callback for {@link ResultProvider} that receives the result from
 * the {@link documents#probe}.
 * @callback documents#probeResult
 * @since 1.0
 * @param {documents.DocumentDescriptor} document - a sparse document descriptor with an exists
 * property that identifies whether the document exists
 */
/**
 * Probes whether a document exists; takes a configuration
 * object with the following named parameters or, as a shortcut,
 * a uri string.
 * @method documents#probe
 * @since 1.0
 * @param {string}  uri - the uri for the database document
 * @param {string|transactions.Transaction}  [txid] - a string
 * transaction id or Transaction object identifying an open
 * multi-statement transaction
 * @returns {ResultProvider} an object whose result() function takes
 * a {@link documents#probeResult} success callback.
 */
Documents.prototype.probe = function probeDocument() {
  return probeDocumentsImpl.call(this, false, mlutil.asArray.apply(null, arguments));
};
function probeDocumentsImpl(contentOnly, args) {
  /*jshint validthis:true */
  if (args.length !== 1 && args.length !== 2) {
    throw new Error('must supply uri for document check()');
  }

  const params = (args.length === 1 && typeof args[0] !== 'string' && !(args[0] instanceof String)) ? args[0] : null;

  let uri = null;
  let txid = null;
  let path = '/v1/documents?format=json';
  // params as list
  if (params === null) {
    uri = args[0];
    path += '&uri='+encodeURIComponent(uri);
    txid = mlutil.convertTransaction(args[1]);
    if (txid != null) {
      path += '&txid='+mlutil.getTxidParam(txid);
    }
  }
  // params as object
  else {
    uri = params.uri;
    if (uri == null) {
      throw new Error('must specify the uri parameter for the document to check');
    }
    path += '&uri='+encodeURIComponent(uri);
    txid = mlutil.convertTransaction(params.txid);
    if (txid != null) {
      path += '&txid='+mlutil.getTxidParam(txid);
    }
  }

  const requestOptions = mlutil.newRequestOptions(this.client.getConnectionParams(), path, 'HEAD');
  mlutil.addTxidHeaders(requestOptions, txid);

  const operation = new Operation(
      'probe document', this.client, requestOptions, 'empty', 'empty'
      );
  operation.uri              = uri;
  operation.validStatusCodes = [200, 404];
  operation.outputTransform  = probeOutputTransform;
  operation.errorTransform   = uriErrorTransform;
  operation.contentOnly      = (contentOnly === true);

  return requester.startRequest(operation);
}

/**
 * A success callback for {@link ResultProvider} that receives the result from
 * the {@link documents#protect}.
 * @callback documents#protectResult
 * @since 2.0.1
 * @param {documents.DocumentDescriptor} document - a sparse document descriptor
 * for the protected document
 */
/**
 * Protects a temporal document from temporal operations for a
 * period of time.
 * @method documents#protect
 * @since 2.0.1
 * @param {string} uri - the uri for the temporal document to protect
 * @param {string} temporalCollection - the temporal collection for the document
 * @param {string} [duration] - a protection duration; either a duration or an
 * expire time must be provided
 * @param {string} [expireTime] - an expiration time; either an expiration time
 * or a duration must be provided
 * @param {string} [level] - a protection level of 'noWipe'|'noDelete'|'noUpdate'
 * (default is 'noDelete')
 * @param {string} [archivePath] - an archive path
 * @returns {ResultProvider} an object whose result() function takes
 * a {@link documents#protectResult} success callback.
 */
Documents.prototype.protect = function protectDocument() {
  /*jshint validthis:true */
  const args = mlutil.asArray.apply(null, arguments);
  const argLen = args.length;

  let uri           = null;
  let tempColl      = null;
  let duration      = null;
  let expireTime    = null;
  let level         = 'noDelete';
  let archivePath   = null;

  // Params as single object
  if (argLen === 1) {
    const obj = args[0];
    if (obj.uri === void 0) {
      throw new Error('must specify uri');
    } else {
      uri = obj.uri;
    }
    if (obj.temporalCollection === void 0) {
      throw new Error('must specify temporalCollection');
    } else {
      tempColl = obj.temporalCollection;
    }
    if (obj.expireTime !== void 0) {
      expireTime = obj.expireTime;
    } else if (obj.duration !== void 0) {
      duration = obj.duration;
    } else {
      throw new Error('must specify duration or expireTime');
    }
    if (obj.level !== void 0) {
      level = obj.level;
    }
    if (obj.archivePath !== void 0) {
      archivePath = obj.archivePath;
    }
  }
  // Multiple params
  else {
    if (argLen < 3) {
      throw new Error('must specify uri, temporalCollection, and duration or expireTime');
    }
    uri = args[0];
    tempColl = args[1];
    // see: https://www.w3.org/TR/xmlschema-2/#duration
    if (args[2].charAt(0) === 'P' || args[2].substring(0, 2) === '-P') {
      duration = args[2];
    } else {
      expireTime = args[2];
    }
    const levels = ['noWipe', 'noDelete', 'noUpdate'];
    if (levels.indexOf(args[3]) !== -1) {
      level = args[3];
    } else {
      archivePath = args[3] || null;
    }
    if (args[4] && archivePath === null) {
      archivePath = args[4];
    }
  }

  if (archivePath !== null) {
    try {
      fs.accessSync(pathModule.dirname(archivePath));
    } catch (e) {
      throw new Error('archive directory does not exist: ' + archivePath);
    }
  }

  let path = '/v1/documents/protection?uri=' + encodeURIComponent(uri);
  path += '&temporal-collection=' + encodeURIComponent(tempColl);
  if (duration !== null) {
    path += '&duration=' + encodeURIComponent(duration);
  } else {
    path += '&expireTime=' + encodeURIComponent(expireTime);
  }
  path += '&level=' + encodeURIComponent(level);
  if (archivePath !== null) {
    path += '&archivePath=' + encodeURIComponent(archivePath);
  }

  const requestOptions = mlutil.newRequestOptions(this.client.getConnectionParams(), path, 'POST');

  const operation = new Operation(
      'protect document', this.client, requestOptions, 'empty', 'empty'
      );
  operation.uri                = uri;
  operation.temporalCollection = tempColl;
  operation.level              = level;
  operation.validStatusCodes   = [204];
  operation.outputTransform    = protectOutputTransform;
  operation.errorTransform     = uriErrorTransform;

  return requester.startRequest(operation);
};

/**
 * A success callback for {@link ResultProvider} that receives the result from
 * the {@link documents#wipe}.
 * @callback documents#wipeResult
 * @since 2.0.1
 * @param {documents.DocumentDescriptor} document - a sparse document descriptor
 * for the wipe command
 */
/**
 * Deletes all versions of a temporal document.
 * @method documents#wipe
 * @since 2.0.1
 * @param {string} uri - the uri for the temporal document to wipe
 * @param {string} temporalCollection - the name of the temporal collection
 * @returns {ResultProvider} an object whose result() function takes
 * a {@link documents#wipeResult} success callback.
 */
Documents.prototype.wipe = function wipeDocument() {
  /*jshint validthis:true */
  const args = mlutil.asArray.apply(null, arguments);
  const argLen = args.length;

  let uri           = null;
  let tempColl      = null;

  // Params as single object
  if (argLen === 1) {
    const obj = args[0];
    if (obj.uri === void 0) {
      throw new Error('must specify uri');
    } else {
      uri = obj.uri;
    }
    if (obj.temporalCollection === void 0) {
      throw new Error('must specify temporalCollection');
    } else {
      tempColl = obj.temporalCollection;
    }
  }
  // Multiple params
  else {
    if (argLen < 2) {
      throw new Error('must specify uri and temporalCollection');
    }
    uri = args[0];
    tempColl = args[1];
  }

  let path = '/v1/documents?uri=' + encodeURIComponent(uri);
  path += '&temporal-collection=' + encodeURIComponent(tempColl);
  path += '&result=wiped';

  const requestOptions = mlutil.newRequestOptions(this.client.getConnectionParams(), path, 'DELETE');

  const operation = new Operation(
      'wipe document', this.client, requestOptions, 'empty', 'empty'
      );
  operation.uri                = uri;
  operation.temporalCollection = tempColl;
  operation.validStatusCodes   = [204];
  operation.outputTransform    = wipeOutputTransform;
  operation.errorTransform     = uriErrorTransform;

  return requester.startRequest(operation);
};

/**
 * Advances the LSQT (Last Stable Query Time) of a temporal collection.
 * @method documents#advanceLsqt
 * @since 2.1.1
 * @param {string} temporalCollection - The name of the temporal collection
 * for which to advance the LSQT.
 * @param {string} [lag] - The lag (in seconds (???)) to subtract from the
 * maximum system start time in the temporal collection to determine the LSQT.
 * @returns {ResultProvider} an object whose result() function takes
 * an object with the new LSQT as an 'lsqt' property.
 */
Documents.prototype.advanceLsqt = function temporalAdvanceLsqt() {
  /*jshint validthis:true */
  const args = mlutil.asArray.apply(null, arguments);

  let tempColl = null;
  let lag      = null;

  // Positional case
  if (typeof args[0] === 'string' || args[0] instanceof String) {
    tempColl = args[0];
    if (args[1] !== void 0) {
      if (typeof args[1] === 'number' || args[0] instanceof Number) {
        lag = args[1];
      } else {
        throw new Error('lag parameter takes a number in seconds');
      }
    }
  }
  // Object case
  else {
    const obj = args[0];
    if (obj.temporalCollection === void 0) {
      throw new Error('must specify temporalCollection');
    } else {
      tempColl = obj.temporalCollection;
    }
    if (obj.lag !== void 0) {
      if (typeof obj.lag === 'number' || obj.lag instanceof Number) {
        lag = obj.lag;
      } else {
        throw new Error('lag parameter takes a number in seconds');
      }
    }
  }

  let path = '/v1/temporal/collections/' + encodeURIComponent(tempColl);
  path += '?result=advance-lsqt';
  if (lag !== null) {
    path += '&lag=' + encodeURIComponent(lag);
  }

  const requestOptions = mlutil.newRequestOptions(this.client.getConnectionParams(), path, 'POST');

  const operation = new Operation(
      'advance LSQT', this.client, requestOptions, 'empty', 'empty'
      );
  // operation.temporalCollection = tempColl;
  operation.validStatusCodes   = [204];
  operation.outputTransform    = advanceLsqtOutputTransform;
  operation.errorTransform     = uriErrorTransform;

  return requester.startRequest(operation);
};

/** @ignore */
function readStatusValidator(statusCode) {
  return (statusCode < 400 || statusCode === 404) ?
      null : 'response with invalid '+statusCode+' status';
}
/** @ignore */
function singleReadOutputTransform(headers, data) {
  /*jshint validthis:true */
  const operation = this;

  const hasData = (data != null);
  if (hasData &&
      (data.errorResponse != null) &&
      data.errorResponse.statusCode === 404
      ) {
    return [];
  }

  const content = hasData ? data : null;

  if (operation.contentOnly === true) {
    return [content];
  }

  const categories = operation.categories;

  const document = (categories.length === 1 && categories[0] === 'content') ?
      {content: content} : collectMetadata(content);

  if(operation.uris){
    document.uri      = operation.uris[0];
  }
  document.category = categories;

  const format = headers.format;
  if (typeof format === 'string' || format instanceof String) {
    document.format = format;
    if (format !== 'json') {
      const contentLength = headers.contentLength;
      if (contentLength != null) {
        document.contentLength = contentLength;
      }
    }
  }

  const headerList  = ['contentType', 'versionId'];
  let headerKey   = null;
  let headerValue = null;
  let i = 0;
  for (i = 0; i < headerList.length; i++) {
    headerKey = headerList[i];
    headerValue = headers[headerKey];
    if (headerValue != null) {
      document[headerKey] = headerValue;
    }
  }

  return [document];
}

/**
 * A success callback for {@link ResultProvider} that receives the result from
 * the {@link documents#read}.
 * @callback documents#resultList
 * @since 1.0
 * @param {documents.DocumentDescriptor[]} documents - an array of
 * {@link documents.DocumentDescriptor} objects with the requested
 * metadata and/or content for the documents
 */
/**
 * Reads one or more documents; takes a configuration object with
 * the following named parameters or, as a shortcut, one or more
 * uri strings or an array of uri strings.
 * @method documents#read
 * @since 1.0
 * @param {string|string[]}  uris - the uri string or an array of uri strings
 * for the database documents
 * @param {documents.categories|documents.categories[]}  [categories] - the categories of information
 * to retrieve for the documents
 * @param {string|transactions.Transaction}  [txid] - a string
 * transaction id or Transaction object identifying an open
 * multi-statement transaction
 * @param {string|mixed[]}  [transform] - the name of a transform extension to apply to each document
 * or an array with the name of the transform extension and an object of parameter values; the
 * transform must have been installed using the {@link transforms#write} function.
 * @param {number[]}  [range] - the range of bytes to extract
 * from a binary document; the range is specified with a zero-based
 * start byte and the position after the end byte as in Array.slice()
 * @param {DatabaseClient.Timestamp}  [timestamp] - a Timestamp object for point-in-time
 * operations.
 * @returns {ResultProvider} an object whose result() function takes
 * a {@link documents#resultList} success callback.
 */
Documents.prototype.read = function readDocuments() {
  return readDocumentsImpl.call(this, false, mlutil.asArray.apply(null, arguments));
};
function readDocumentsImpl(contentOnly, args) {
  /*jshint validthis:true */
  if (args.length === 0) {
    throw new Error('must specify at least one document to read');
  }

  let uris = null;
  let categories = null;
  let txid = null;
  let transform = null;
  let contentType = null;
  let range = null;
  let timestamp = null;

  const arg = args[0];
  if (Array.isArray(arg)) {
    uris = arg;
  } else if (typeof arg === 'string' || arg instanceof String) {
    uris = args;
  } else {
    uris = arg.uris;
    if (uris == null) {
      throw new Error('must specify the uris parameters with at least one document to read');
    }
    if (!Array.isArray(uris)) {
      uris = [uris];
    }
    categories = arg.categories;
    txid = mlutil.convertTransaction(arg.txid);
    transform = arg.transform;
    contentType = arg.contentType;
    range = arg.range;
    timestamp = arg.timestamp;
  }

  if (categories == null) {
    categories = ['content'];
  } else if (typeof categories === 'string' || categories instanceof String) {
    categories = [categories];
  }

  if (categories != null) {
    let i = 0;
    for (i = 0; i < categories.length; i++) {
      if(categories[i] === 'rawContent'){
        if(categories.length>1) {
          throw new Error('Categories should not have other option(s) if rawContent is needed.');
        } else {
          categories = ['content'];
          contentOnly = true;
        }
      }
      categories[i] = categories[i] === 'metadataValues' ? 'metadata-values' : categories[i];
    }
  }

  let path = '/v1/documents?format=json&uri='+
    uris.map(encodeURIComponent).join('&uri=');
  path += '&category=' + categories.join('&category=');
  if (txid != null) {
    path += '&txid='+mlutil.getTxidParam(txid);
  }
  if (transform != null) {
    path += '&'+mlutil.endpointTransform(transform);
  }

  if (timestamp !== null && timestamp !== void 0) {
    if (timestamp.value !== null) {
      path += '&timestamp='+timestamp.value;
    }
  }

  const isSinglePayload = (
      uris.length === 1 && (
          (categories.length === 1 && categories[0] === 'content') ||
          categories.indexOf('content') === -1
      ));

  const requestOptions = mlutil.newRequestOptions(this.client.getConnectionParams(), path, 'GET');
  if (!isSinglePayload) {
    requestOptions.headers = {
        Accept: 'multipart/mixed; boundary='+mlutil.multipartBoundary
    };
  } else {
    let hasContentType = false;
    if (contentType != null) {
      if (typeof contentType === 'string' || contentType instanceof String) {
        hasContentType = true;
      } else {
        throw new Error('contentType is not string: '+contentType);
      }
    }

    if (range != null) {
      if (!Array.isArray(range)) {
        throw new Error('byte range parameter for reading binary document is not an array: '+range);
      }
      let bytes = null;
      switch (range.length) {
      case 0:
        throw new Error('no start length for byte range parameter for reading binary document');
      case 1:
        if (typeof range[0] !== 'number' && !(range[0] instanceof Number)) {
          throw new Error('start length for byte range parameter is not integer: '+range[0]);
        }
        bytes = 'bytes=' + range[0] + '-';
        break;
      case 2:
        if (typeof range[0] !== 'number' && !(range[0] instanceof Number)) {
          throw new Error('start length for byte range parameter is not integer: '+range[0]);
        }
        if (typeof range[1] !== 'number' && !(range[1] instanceof Number)) {
          throw new Error('end length for byte range parameter is not integer: '+range[1]);
        }
        if (range[0] >= range[1]) {
          throw new Error('start length greater than or equal to end length for byte range: '+range);
        }
        bytes = 'bytes=' + range[0] + '-' + (range[1] - 1);
        break;
      default:
        throw new Error('byte range parameter has more than start and end length: '+range);
      }

      if (!hasContentType) {
        requestOptions.headers = {
            Range: bytes
        };
      } else if (contentType.search(/^(application\/([^+]+\+)?(json|xml)|text\/.*)$/) > -1) {
        throw new Error('cannot request byte range for JSON, text, or XML document: '+contentType);
      } else {
        requestOptions.headers = {
            Range:          bytes,
            'Content-Type': contentType
        };
      }
    } else if (hasContentType) {
      requestOptions.headers = {
          'Content-Type': contentType
      };
    }
  }
  mlutil.addTxidHeaders(requestOptions, txid);

  const operation = new Operation(
      'read documents', this.client, requestOptions, 'empty',
        (isSinglePayload ? 'single' : 'multipart')
      );
  operation.uris           = uris;
  operation.categories     = categories;
  operation.errorTransform = uriListErrorTransform;
  if (isSinglePayload) {
    operation.contentOnly         = (contentOnly === true);
    operation.outputTransform     = singleReadOutputTransform;
    operation.statusCodeValidator = readStatusValidator;
  } else if (contentOnly === true) {
    operation.subdata = ['content'];
  }
  operation.timestamp = (timestamp !== null) ? timestamp : null;

  return requester.startRequest(operation);
}

/**
 * Writes a large document (typically a binary) in incremental chunks with
 * a stream; takes a {@link documents.DocumentDescriptor} object with the
 * following properties (but not a content property).
 * @method documents#createWriteStream
 * @since 1.0
 * @param {string} uri - the identifier for the document to write to the database
 * @param {string[]} [collections] - the collections to which the document should belong
 * @param {object[]} [permissions] - the permissions controlling which users can read or
 * write the document
 * @param {object[]} [properties] - additional properties of the document
 * @param {number} [quality] - a weight to increase or decrease the rank of the document
 * @param {object[]} [metadataValues] - the metadata values of the document
 * @param {number} [versionId] - an identifier for the currently stored version of the
 * document (when enforcing optimistic locking)
 * @param {string|transactions.Transaction}  [txid] - a string
 * transaction id or Transaction object identifying an open
 * multi-statement transaction
 * @param {string|mixed[]}  [transform] - the name of a transform extension to apply to each document
 * or an array with the name of the transform extension and an object of parameter values; the
 * transform must have been installed using the {@link transforms#write} function.
 * @returns {WritableStream} a stream for writing the database document; the
 * stream object also has a result() function that takes
 * a {@link documents#writeResult} success callback.
 */
Documents.prototype.createWriteStream = function createWriteStream(document) {
  if ((document == null) ||
      (document.uri == null)) {
    throw new Error('must specify document for write stream');
  }
  if (document.content != null) {
    throw new Error('must write to stream to supply document content');
  }

  let categories = document.categories;
  const hasCategories = Array.isArray(categories) && categories.length > 0;
  if (!hasCategories && (typeof categories === 'string' || categories instanceof String)) {
    categories = [categories];
  }

  if (document.properties == null) {
    return writeContent.call(this, false, document, document, categories, 'chunked');
  }

  return writeStreamImpl.call(this, document, categories);
};
/** @ignore */
function writeStreamImpl(document, categories) {
  /*jshint validthis:true */
  let endpoint = '/v1/documents';

  const txid = getTxid(document);

  const writeParams = addWriteParams(document, categories, txid);
  if (writeParams.length > 0) {
    endpoint += writeParams;
  }

  const multipartBoundary = mlutil.multipartBoundary;
  const requestOptions = mlutil.newRequestOptions(this.client.getConnectionParams(), endpoint, 'POST');
  requestOptions.headers = {
      'Content-Type': 'multipart/mixed; boundary='+multipartBoundary,
      'Accept': 'application/json'
  };
  mlutil.addTxidHeaders(requestOptions, txid);

  const operation = new Operation(
      'write document stream', this.client, requestOptions, 'chunkedMultipart', 'single'
      );
  operation.isReplayable = false;
  operation.uri = document.uri;

  // TODO: treat as chunked single document if no properties
  const requestPartList = [];
  addDocumentParts(operation, requestPartList, document, true);
  operation.requestDocument = requestPartList;

  operation.multipartBoundary = mlutil.multipartBoundary;
  operation.errorTransform    = uriErrorTransform;

  return requester.startRequest(operation);
}

/** @ignore */
function singleWriteOutputTransform(headers, data) {
  /*jshint validthis:true */
  const operation = this;

  let uri = operation.uri;

  if (uri == null) {
    const location = headers.location;
    if (location != null) {
      const startsWith = '/v1/documents?uri=';
      if (location.length > startsWith.length &&
          location.substr(0, startsWith.length) === startsWith) {
        uri = location.substr(startsWith.length);
      }
    }
  }

  if (operation.contentOnly === true) {
    return [uri];
  }

  const document = {uri: uri};

  const categories = operation.categories;
  if (categories == null) {
    document.categories = categories;
  }

  const contentType = (data == null) ? null : data['mime-type'];
  if (contentType == null) {
    document.contentType = contentType;
  }

  const wrapper = {documents: [document]};

  const systemTime = headers.systemTime;
  if (systemTime != null) {
    wrapper.systemTime = systemTime;
  }

  return wrapper;
}
/** @ignore */
function writeListOutputTransform(headers, data) {
//  var operation = this;

  const systemTime = headers.systemTime;
  if (systemTime == null) {
    return data;
  }

  return {
    documents:  data.documents,
    systemTime: systemTime
  };
}

/**
 * A success callback for {@link ResultProvider} that receives the result from
 * the {@link documents#write} or the {@link documents#createWriteStream}
 * functions.
 * @callback documents#writeResult
 * @since 1.0
 * @param {object} response - a response with a documents property providing
 * a sparse array of array of {@link documents.DocumentDescriptor} objects
 * providing the uris of the written documents.
 */
/**
 * Writes one or more documents; takes a configuration object with
 * the following named parameters or, as a shortcut, a document descriptor.
 * @method documents#write
 * @since 1.0
 * @param {DocumentDescriptor|DocumentDescriptor[]} documents - one descriptor
 * or an array of document descriptors to write
 * @param {documents.categories|documents.categories[]}  [categories] - the categories of information
 * to write for the documents
 * @param {string|transactions.Transaction}  [txid] - a string
 * transaction id or Transaction object identifying an open
 * multi-statement transaction
 * @param {string|mixed[]}  [transform] - the name of a transform extension to apply to each document
 * or an array with the name of the transform extension and an object of parameter values; the
 * transform must have been installed using the {@link transforms#write} function.
 * @param {string}  [forestName] - the name of a forest in which to write
 * the documents.
 * @param {string} [temporalCollection] - the name of the temporal collection;
 * use only when writing temporal documents that have the JSON properties or XML elements
 * specifying the valid and system start and end times as defined by the valid and
 * system axis for the temporal collection
 * @param {string|Date} [systemTime] - a datetime to use as the system start time
 * instead of the current time of the database server; can only be supplied
 * if the temporalCollection parameter is also supplied
 * @returns {ResultProvider} an object whose result() function takes
 * a {@link documents#writeResult} success callback.
 */
Documents.prototype.write = function writeDocuments() {
  return writeDocumentsImpl.call(this, false, mlutil.asArray.apply(null, arguments));
};
function writeDocumentsImpl(contentOnly, args) {
  /*jshint validthis:true */
  if (args.length < 1) {
    throw new Error('must provide uris for document write()');
  }

  const arg = args[0];

  let documents = arg.documents;
  const params = (documents == null) ? null : arg;
  if (params !== null) {
    if (!Array.isArray(documents)) {
      documents = [documents];
    }
  } else if (Array.isArray(arg)) {
    documents = arg;
  } else {
    documents = args;
  }

  const isSingleDoc = (documents.length === 1);

  const document      = isSingleDoc ? documents[0] : null;
  const hasDocument   = (document != null);
  const hasContent    = hasDocument && (document.content != null);

  const requestParams  =
    (params !== null) ? params   :
    (hasDocument)     ? document :
    null;

  let categories = (requestParams == null) ? null : requestParams.categories;
  if (typeof categories === 'string' || categories instanceof String) {
    categories = [categories];
  }

  if (categories != null) {
    for (let i = 0; i < categories.length; i++) {
      categories[i] = categories[i] === 'metadataValues' ? 'metadata-values' : categories[i];
    }
  }

  if (hasDocument) {
    if (hasContent && (document.properties == null)) {
      return writeContent.call(this, contentOnly, document, requestParams, categories, 'single');
    } else if (!hasContent && (document.uri != null)) {
      return writeMetadata.call(this, document, categories);
    }
  }

  return writeDocumentList.call(this, contentOnly, documents, requestParams, categories);
}
/** @ignore */
function writeMetadata(document, categories) {
  /*jshint validthis:true */
  const uri = document.uri;

  let endpoint = '/v1/documents?uri='+encodeURIComponent(uri);

  if (!Array.isArray(categories)) {
    categories = [];
  }

  let hasCategories = (categories.length > 0);
  if (!hasCategories) {
    const categoryCheck = ['collections', 'permissions', 'quality', 'properties', 'metadataValues'];
    let category = null;
    let i = 0;
    for (i = 0; i < categoryCheck.length; i++) {
      category = categoryCheck[i];
      if (document[category] != null) {
        category = category === 'metadataValues' ? 'metadata-values' : category;
        categories.push(category);
        if (!hasCategories) {
          hasCategories = true;
        }
      }
    }
  }

  if (hasCategories) {
    endpoint += '&category='+categories.join('&category=');
  }

  const txid = mlutil.convertTransaction(document.txid);
  if (txid != null) {
    endpoint += '&txid='+mlutil.getTxidParam(txid);
  }

  const requestHeaders = {
      'Accept':       'application/json',
      'Content-Type': 'application/json'
  };

  const requestOptions     = mlutil.newRequestOptions(this.client.getConnectionParams(), endpoint, 'PUT');
  requestOptions.headers = requestHeaders;
  mlutil.addTxidHeaders(requestOptions, txid);

  const operation = new Operation(
      'write single metadata', this.client, requestOptions, 'single', 'empty'
      );
  operation.uri = uri;
  if (hasCategories) {
    operation.categories = categories;
  }
  operation.requestBody     = JSON.stringify(collectMetadata(document));
  operation.outputTransform = singleWriteOutputTransform;

  return requester.startRequest(operation);
}
/** @ignore */
function writeContent(contentOnly, document, requestParams, categories, requestType) {
  /*jshint validthis:true */
  const content     = document.content;
  const hasContent  = (content != null);

  let endpoint = '/v1/documents';

  let sep = '?';

  const txid = getTxid(requestParams);

  const writeParams = addWriteParams(requestParams, categories, txid);
  if (writeParams.length > 0) {
    endpoint += writeParams;
    sep = '&';
  }

  const uri    = document.uri;
  const hasUri = (uri != null);

  if (hasUri) {
    endpoint += sep+'uri='+encodeURIComponent(uri);
    if (sep === '?') { sep = '&'; }
  }

  let i = 0;

  const collections = document.collections;
  if (collections != null) {
    if (Array.isArray(collections)) {
      for (i=0; i < collections.length; i++) {
        endpoint += sep+'collection='+encodeURIComponent(collections[i]);
        if (i === 0 && sep === '?') { sep = '&'; }
      }
    } else {
      endpoint += sep+'collection='+encodeURIComponent(collections);
      if (sep === '?') { sep = '&'; }
    }
  }

  const permissions = document.permissions;
  if (permissions != null) {
    let permission   = null;
    let roleName     = null;
    let capabilities = null;
    let j = 0;
    if (Array.isArray(permissions)) {
      for (i=0; i < permissions.length; i++) {
        permission   = permissions[i];
        roleName     = permission['role-name'];
        capabilities = permission.capabilities;
        if (Array.isArray(capabilities)) {
          for (j=0; j < capabilities.length; j++) {
            endpoint += sep+'perm:'+roleName+'='+capabilities[j];
            if (i === 0 && j === 0 && sep === '?') { sep = '&'; }
          }
        } else {
          endpoint += sep+'perm:'+roleName+'='+capabilities;
          if (i === 0 && sep === '?') { sep = '&'; }
        }
      }
    } else {
      roleName     = permissions['role-name'];
      capabilities = permissions.capabilities;
      if (Array.isArray(capabilities)) {
        for (j=0; j < capabilities.length; j++) {
          endpoint += sep+'perm:'+roleName+'='+capabilities[j];
          if (j === 0 && sep === '?') { sep = '&'; }
        }
      } else {
        endpoint += sep+'perm:'+roleName+'='+capabilities;
        if (sep === '?') { sep = '&'; }
      }
    }
  }

  const quality = document.quality;
  if (quality != null) {
    endpoint += sep+'quality='+quality;
    if (sep === '?') { sep = '&'; }
  }

  const metadataValues = document.metadataValues;
  for (const key in metadataValues) {
    endpoint += sep+'value:'+key+'='+encodeURIComponent(metadataValues[key]);
  }

  const temporalDocument = document.temporalDocument;
  if (temporalDocument != null) {
    endpoint += sep+'temporal-document='+temporalDocument;
    if (sep === '?') { sep = '&'; }
  }

  const requestHeaders = {
    'Accept': 'application/json'
  };

  const writeConfig = addWriteConfig(document, hasUri, content, requestHeaders, sep);
  if (writeConfig.length > 0) {
    endpoint += writeConfig;
    if (sep === '?') { sep = '&'; }
  }

  const requestOptions     = mlutil.newRequestOptions(this.client.getConnectionParams(), endpoint, hasUri ? 'PUT' : 'POST');
  requestOptions.headers = requestHeaders;
  mlutil.addTxidHeaders(requestOptions, txid);
  const operation = new Operation(
      'write single document', this.client, requestOptions, requestType, 'empty'
      );
  if (hasUri) {
    operation.uri = uri;
  }
  if (Array.isArray(categories) && categories.length > 0) {
    operation.categories = categories;
  }
  if (hasContent) {
    operation.requestBody = mlutil.marshal(content, operation);
  } else {
    operation.isReplayable = false;
  }
  operation.outputTransform = singleWriteOutputTransform;
  operation.contentOnly     = (contentOnly === true);

  return requester.startRequest(operation);
}
/** @ignore */
function writeDocumentList(contentOnly, documents, requestParams, categories) {
  /*jshint validthis:true */
  if (documents.length > 1) {
    documents = documents.sort(compareDocuments);
  }

  let endpoint = '/v1/documents';

  const txid = getTxid(requestParams);

  const writeParams = addWriteParams(requestParams, categories, txid);
  if (writeParams.length > 0) {
    endpoint += writeParams;
  }

  const multipartBoundary = mlutil.multipartBoundary;

  const requestHeaders = {
      'Accept':       'application/json',
      'Content-Type': 'multipart/mixed; boundary='+multipartBoundary
  };

  const requestOptions     = mlutil.newRequestOptions(this.client.getConnectionParams(), endpoint, 'POST');
  requestOptions.headers = requestHeaders;
  mlutil.addTxidHeaders(requestOptions, txid);

  const operation = new Operation(
      'write document list', this.client, requestOptions, 'multipart', 'single'
      );
  operation.uris = getDocumentUris(documents);
  if (Array.isArray(categories) && categories.length > 0) {
    operation.categories = categories;
  }

  operation.multipartBoundary = multipartBoundary;

  const requestPartList = [];
  for (let i=0; i < documents.length; i++) {
    addDocumentParts(operation, requestPartList, documents[i], false);
  }
  operation.requestPartList = requestPartList;

  operation.errorTransform    = uriListErrorTransform;
  if (contentOnly === true) {
    operation.subdata = ['documents', 'uri'];
  } else if ((requestParams != null) &&
      (requestParams.temporalCollection != null)) {
    operation.outputTransform = writeListOutputTransform;
  }

  return requester.startRequest(operation);
}
function getTxid(requestParams) {
  return (requestParams == null) ? null : requestParams.txid;
}
/** @ignore */
function addWriteParams(requestParams, categories, txidRaw) {
  let writeParams = '';
  const txid = mlutil.convertTransaction(txidRaw);
  let sep = '?';
  if (requestParams != null) {
    if (Array.isArray(categories) && categories.length > 0) {
      writeParams += sep+'category='+categories.join('&category=');
      if (sep !== '&') { sep = '&'; }
    }
    if (txid != null) {
      writeParams += sep+'txid='+mlutil.getTxidParam(txid);
      if (sep !== '&') { sep = '&'; }
    }
    const transform = mlutil.endpointTransform(requestParams.transform);
    if (transform != null) {
      writeParams += sep+transform;
      if (sep !== '&') { sep = '&'; }
    }
    const forestName = requestParams.forestName;
    if (forestName != null) {
      writeParams += sep+'forest-name='+encodeURIComponent(forestName);
      if (sep !== '&') { sep = '&'; }
    }
    const temporalCollection = requestParams.temporalCollection;
    if (temporalCollection != null) {
      writeParams += sep+'temporal-collection='+encodeURIComponent(temporalCollection);
      if (sep !== '&') { sep = '&'; }
      const systemTime = requestParams.systemTime;
      if (typeof systemTime === 'string' || systemTime instanceof String) {
        writeParams += '&system-time='+encodeURIComponent(systemTime);
      } else if (Object.prototype.toString.call(systemTime) === '[object Date]') {
        writeParams += '&system-time='+encodeURIComponent(systemTime.toISOString());
      }
    }
  }

  return writeParams;
}
/** @ignore */
function addWriteConfig(document, hasUri, content, headers, sep) {
  let writeConfig = '';

  const isBody = (sep !== '; ');

  if (!hasUri) {
    const extension = document.extension;
    if (extension != null) {
      writeConfig += sep+'extension='+extension;
      if (isBody && sep === '?') { sep = '&'; }
      const directory = document.directory;
      if (directory != null) {
        writeConfig += sep+'directory='+
            (isBody ? encodeURIComponent(directory) : directory);
      }
    }
  }

  const versionId = document.versionId;
  if (versionId != null) {
    if (isBody) {
      headers['If-Match'] = versionId;
    } else {
      writeConfig += sep+'versionId='+versionId;
    }
  }

  let contentType    = document.contentType;
  let hasContentType = (contentType != null);
  let format         = document.format;
  let hasFormat      = (format != null);

  if (hasContentType) {
    if (!hasFormat) {
      if (/^(application|text)\/([^+]+\+)?json$/.test(contentType)) {
        format    = 'json';
        hasFormat = true;
      } else if (/^(application|text)\/([^+]+\+)?xml$/.test(contentType)) {
        format    = 'xml';
        hasFormat = true;
      } else if (/^(text)\//.test(contentType)) {
        format    = 'text';
        hasFormat = true;
      }
    }
  } else if (!hasFormat && (
      Array.isArray(content) ||
      ((typeof content === 'object' && content !== null) &&
        (typeof content !== 'string' && !(content instanceof String)) &&
        !Buffer.isBuffer(content) &&
        (typeof content !== 'function')))) {
    contentType    = 'application/json';
    hasContentType = true;
    format         = 'json';
    hasFormat      = true;
  }

  if (hasFormat) {
    switch(format) {
    case 'binary':
      var extract = document.extract;
      if (extract != null) {
        if (extract === 'document' || extract === 'properties') {
          writeConfig += sep+'extract='+extract;
          if (isBody && sep === '?') { sep = '&'; }
        } else {
          throw new Error('extract must be "document" or "properties": '+extract.toString());
        }
      }
      break;
    case 'json':
      if (!hasContentType) {
        contentType = 'application/json';
      }
      break;
    case 'text':
      if (!hasContentType) {
        contentType = 'text/plain';
      }
      break;
    case 'xml':
      var repair = document.repair;
      if (repair != null) {
        if (repair === 'full' || repair === 'none') {
          writeConfig += sep+'repair='+repair;
          if (isBody && sep === '?') { sep = '&'; }
        } else {
          throw new Error('repair must be "full" or "none": '+repair.toString());
        }
      }
      if (!hasContentType) {
        contentType = 'application/xml';
      }
      break;
    }
  }

  if (hasContentType) {
    headers['Content-Type'] = contentType;
  }

  return writeConfig;
}
/** @ignore */
function addDocumentParts(operation, partList, document, isContentOptional) {
  const uri    = document.uri;
  const hasUri = (uri != null);

  let disposition = '';
  if (hasUri) {
    disposition = 'attachment; filename="'+uri+'"';
    if (document.temporalDocument != null) {
      disposition += '; temporal-document="'+document.temporalDocument+'"';
    }
  } else {
    disposition = 'inline';
  }

  const metadata = collectMetadata(document);
  if (metadata != null) {
    partList.push({
      headers:{
      'Content-Type'       : 'application/json',
      'Content-Disposition': disposition+'; category=metadata'
      },
      content: JSON.stringify(metadata)
    });
  }

  const content    = document.content;
  const hasContent = (content != null);
  if (hasContent || isContentOptional) {
    const headers = {};
    const part    = {headers: headers};

    const writeConfig = addWriteConfig(document, hasUri, content, headers, '; ');
    if (writeConfig.length > 0) {
      disposition += writeConfig;
    }

    headers['Content-Disposition'] = disposition+'; category=content';

    if (hasContent) {
      part.content = mlutil.marshal(content, operation);
    }

    partList.push(part);
  }
}

/** @ignore */
function collectMetadata(document) {
  let metadata = null;
  // TODO: create array wrapper for collections, capabilities
  const metadataCategories = ['collections', 'permissions', 'quality', 'properties', 'metadataValues'];
  for (let i = 0; i < metadataCategories.length; i++) {
    const category = metadataCategories[i];
    if (document !== null) {
      if (document[category] != null) {
        if (metadata === null) {
          metadata = {};
        }
        metadata[category] = document[category];
      }
    }
  }
  return metadata;
}

/** @ignore */
function removeOutputTransform(headers/*, data*/) {
  /*jshint validthis:true */
  const operation = this;

  if (operation.contentOnly === true) {
    return operation.uris;
  }

  const wrapper = {
    uris:    operation.uris,
    removed: true
  };

  const systemTime = headers.systemTime;
  if (systemTime != null) {
    wrapper.systemTime = systemTime;
  }

  return wrapper;
}

/**
 * A success callback for {@link ResultProvider} that receives the result from
 * the {@link documents#remove}.
 * @callback documents#removeResult
 * @since 1.0
 * @param {documents.DocumentDescriptor} document - a sparse document descriptor
 * for the removed document
 */
/**
 * Removes one or more database documents; takes a configuration
 * object with the following named parameters or, as a shortcut, one or more
 * uri strings or an array of uri strings.
 * @method documents#remove
 * @since 1.0
 * @param {string|string[]}  uris - the uri string or an array of uri strings
 * identifying the database documents
 * @param {string|transactions.Transaction}  [txid] - a string
 * transaction id or Transaction object identifying an open
 * multi-statement transaction
 * @param {string} [temporalCollection] - the name of the temporal collection;
 * use only when deleting a document created as a temporal document; sets the
 * system end time to record when the document was no longer active
 * @param {string|Date} [systemTime] - a datetime to use as the system end time
 * instead of the current time of the database server; can only be supplied
 * if the temporalCollection parameter is also supplied
 * @returns {ResultProvider} an object whose result() function takes
 * a {@link documents#removeResult} success callback.
 */
Documents.prototype.remove = function removeDocument() {
  return removeDocumentImpl.call(
      this, false, mlutil.asArray.apply(null, arguments)
      );
};
function removeDocumentImpl(contentOnly, args) {
  /*jshint validthis:true */
  if (args.length < 1) {
    throw new Error('must provide uris for document remove()');
  }

  let uris = null;
  let txid = null;
  let temporalCollection = null;
  let systemTime = null;
  let versionId = null;

  const arg = args[0];
  if (Array.isArray(arg)) {
    uris = arg;
  } else if (typeof arg === 'string' || arg instanceo