enketo-core/src/js/form-model.js

Extensible Enketo form engine

import MergeXML from 'mergexml/mergexml';
import config from 'enketo/config';
import bindJsEvaluator from 'enketo/xpath-evaluator-binding';
import { findMarkerComment } from './dom';
import { readCookie, parseFunctionFromExpression, stripQuotes } from './utils';
import {
    getSiblingElementsAndSelf,
    getXPath,
    getRepeatIndex,
    hasPreviousCommentSiblingWithContent,
    hasPreviousSiblingElementSameName,
} from './dom-utils';
import FormLogicError from './form-logic-error';
import types from './types';
import event from './event';
import { Nodeset } from './nodeset';

import './extend';

const REPEAT_COMMENT_PREFIX = 'repeat:/';
const INSTANCE = /instance\(\s*(["'])((?:(?!\1)[A-z0-9.\-_]+))\1\s*\)/g;
const OPENROSA =
    /(decimal-date-time\(|pow\(|indexed-repeat\(|format-date\(|coalesce\(|join\(|max\(|min\(|random\(|substr\(|int\(|uuid\(|regex\(|now\(|today\(|date\(|if\(|boolean-from-string\(|checklist\(|selected\(|selected-at\(|round\(|area\(|position\([^)])/;
const OPENROSA_XFORMS_NS = 'http://openrosa.org/xforms';
const JAVAROSA_XFORMS_NS = 'http://openrosa.org/javarosa';
const ENKETO_XFORMS_NS = 'http://enketo.org/xforms';
const ODK_XFORMS_NS = 'http://www.opendatakit.org/xforms';

const parser = new DOMParser();

/**
 * Class dealing with the XML Model of a form
 *
 * @class
 * @param {FormDataObj} data - data object
 * @param {object=} options - FormModel options
 * @param {string=} options.full - Whether to initialize the full model or only the primary instance.
 */
const FormModel = function (data, options) {
    if (typeof data === 'string') {
        data = {
            modelStr: data,
        };
    }

    data.external = data.external || [];
    data.submitted =
        typeof data.submitted !== 'undefined' ? data.submitted : true;
    options = options || {};
    options.full = typeof options.full !== 'undefined' ? options.full : true;

    this.events = document.createElement('div');
    this.convertedExpressions = new Map();
    this.templates = {};
    this.loadErrors = [];

    this.data = data;
    this.options = options;
    this.namespaces = {};
};

/**
 * Getter and setter functions
 */
FormModel.prototype = {
    /**
     * @type {string}
     */
    get version() {
        return this.evaluate('/*/@version', 'string', null, null, true);
    },
    /**
     * @type {string}
     */
    get instanceID() {
        return this.getMetaNode('instanceID').getVal();
    },
    /**
     * @type {string}
     */
    get deprecatedID() {
        return this.getMetaNode('deprecatedID').getVal() || '';
    },
    /**
     * @type {string}
     */
    get instanceName() {
        return this.getMetaNode('instanceName').getVal();
    },
};

/**
 * Initializes FormModel
 *
 * @return {Array<string>} list of initialization errors
 */
FormModel.prototype.init = function () {
    let id;
    let i;
    let instanceDoc;
    let secondaryInstanceChildren;
    const that = this;

    /**
     * Default namespaces (on a primary instance, instance child, model) would create a problem using the **native** XPath evaluator.
     * It wouldn't find any regular /path/to/nodes. The solution is to ignore these by renaming these attributes to data-xmlns.
     *
     * If the regex is later deemed too aggressive, it could target the model, primary instance and primary instance child only, after creating an XML Document.
     */
    this.data.modelStr = this.data.modelStr.replace(
        /\s(xmlns=("|')[^\s>]+("|'))/g,
        ' data-$1'
    );

    if (!this.options.full) {
        // Strip all secondary instances from string before parsing
        // This regex works because the model never includes itext in Enketo
        this.data.modelStr = this.data.modelStr.replace(
            /^(<model\s*><instance((?!<instance).)+<\/instance\s*>\s*)(<instance.+<\/instance\s*>)*/,
            '$1'
        );
    }

    // Create the model
    try {
        id = 'model';
        // The default model
        this.xml = parser.parseFromString(this.data.modelStr, 'text/xml');
        this.throwParserErrors(this.xml, this.data.modelStr);

        // Add external data to model
        this.data.external.forEach((instance) => {
            if (instance == null || !instance.xml) {
                return;
            }

            id = instance.id
                ? `instance "${instance.id}"`
                : 'instance "unknown"';
            instanceDoc = that.getSecondaryInstance(instance.id);
            // remove any existing content that is just an XLSForm hack to pass ODK Validate
            secondaryInstanceChildren = instanceDoc.children;
            for (i = secondaryInstanceChildren.length - 1; i >= 0; i--) {
                instanceDoc.removeChild(secondaryInstanceChildren[i]);
            }
            let rootEl;
            if (instance.xml instanceof XMLDocument) {
                // Create a clone of the root node
                rootEl = that.xml.importNode(
                    instance.xml.documentElement,
                    true
                );
            }
            if (rootEl) {
                instanceDoc.appendChild(rootEl);
            }
        });

        // TODO: in the future, we should search for jr://instance/session and
        // populate that one. This is just moving in that direction to implement preloads.
        this.createSession('__session', this.data.session);
    } catch (e) {
        console.error('parseXML error');
        this.loadErrors.push(`Error trying to parse XML ${id}. ${e.message}`);
    }

    // Initialize/process the model
    if (this.xml) {
        try {
            this.hasInstance = !!this.xml.querySelector('model > instance');
            this.rootElement =
                this.xml.querySelector('instance > *') ||
                this.xml.documentElement;
            this.setNamespaces();

            // Determine whether it is possible that this form uses incorrect absolute/path/to/repeat/node syntax when
            // it actually was supposed to use a relative ../node path (old issue with older pyxform-generated forms).
            // In the future, if there are more use cases for odk:xforms-version, we'll probably have to use a semver-parser
            // to do a comparison. In this case, the presence of the attribute is sufficient, as we know no older versions
            // than odk:xforms-version="1.0.0" exist. Previous versions had no number.
            this.noRepeatRefErrorExpected = this.evaluate(
                `/model/@${this.getNamespacePrefix(
                    ODK_XFORMS_NS
                )}:xforms-version`,
                'boolean',
                null,
                null,
                true
            );

            // Check if instanceID is present
            if (!this.getMetaNode('instanceID').getElement()) {
                that.loadErrors.push(
                    'Invalid primary instance. Missing instanceID node.'
                );
            }

            // Check if all secondary instances with an external source have been populated
            Array.prototype.slice
                .call(this.xml.querySelectorAll('model > instance[src]:empty'))
                .forEach((instance) => {
                    const src = instance.getAttribute('src');

                    const errorMessage =
                        src == null
                            ? `External instance "${instance.id}" is empty.`
                            : `Can't find ${src.replace(/.*\//, '')}.`;

                    that.loadErrors.push(errorMessage);
                });

            this.trimValues();
            this.extractTemplates();
        } catch (e) {
            console.error(e);
            this.loadErrors.push(`${e.name}: ${e.message}`);
        }
        // Merge an existing instance into the model, AFTER templates have been removed
        try {
            id = 'record';
            if (this.data.instanceStr) {
                this.mergeXml(this.data.instanceStr);
            }

            // Set the two most important meta fields before any field 'dataupdate' event fires.
            // The first dataupdate event will fire in response to the instance-first-load event.
            this.setInstanceIdAndDeprecatedId();

            if (!this.data.instanceStr) {
                // Only dispatch for newly created records
                this.events.dispatchEvent(event.InstanceFirstLoad());
            }
        } catch (e) {
            console.error(e);
            this.loadErrors.push(
                `Error trying to parse XML ${id}. ${e.message}`
            );
        }
    }

    return this.loadErrors;
};

/**
 * @param {Document} xmlDoc - XML Document
 * @param {string} xmlStr - XML string
 */
FormModel.prototype.throwParserErrors = (xmlDoc, xmlStr) => {
    if (!xmlDoc || xmlDoc.querySelector('parsererror')) {
        throw new Error(`Invalid XML: ${xmlStr}`);
    }
};

/**
 * @param {string} id - Instance ID
 * @param {object} [sessObj] - session object
 */
FormModel.prototype.createSession = function (id, sessObj) {
    let instance;
    let session;
    const model = this.xml.querySelector('model');
    const fixedProps = [
        'deviceid',
        'username',
        'email',
        'phonenumber',
        'simserial',
        'subscriberid',
    ];
    if (!model) {
        return;
    }

    sessObj = typeof sessObj === 'object' ? sessObj : {};
    instance = model.querySelector(`instance#${CSS.escape(id)}`);

    if (!instance) {
        instance = parser.parseFromString(
            `<instance id="${id}"/>`,
            'text/xml'
        ).documentElement;
        this.xml.adoptNode(instance);
        model.appendChild(instance);
    }

    // fixed: /sesssion/context properties
    fixedProps.forEach((prop) => {
        sessObj[prop] =
            sessObj[prop] ||
            readCookie(`__enketo_meta_${prop}`) ||
            `${prop} not found`;
    });

    session = parser.parseFromString(
        `<session><context>${fixedProps
            .map((prop) => `<${prop}>${sessObj[prop]}</${prop}>`)
            .join('')}</context></session>`,
        'text/xml'
    ).documentElement;

    // TODO: custom properties could be added to /session/user/data or to /session/data

    this.xml.adoptNode(session);
    instance.appendChild(session);
};

/**
 * Finds a secondary instance.
 *
 * @param  {string} id - DOM element id.
 * @return {Element|undefined} secondary instance XML element
 */
FormModel.prototype.getSecondaryInstance = function (id) {
    let instanceEl;

    [...this.xml.querySelectorAll('model > instance')].some((el) => {
        const idAttr = el.getAttribute('id');
        if (idAttr === id) {
            instanceEl = el;

            return true;
        }
        return false;
    });

    return instanceEl;
};

/**
 * Returns a new Nodeset instance
 *
 * @param {string|null} [selector] - simple path to node
 * @param {string|number|null} [index] - index of node
 * @param {NodesetFilter|null} [filter] - filter to apply
 * @return {Nodeset} Nodeset instance
 */
FormModel.prototype.node = function (selector, index, filter) {
    return new Nodeset(selector, index, filter, this);
};

/**
 * Merges an XML instance string into the XML Model
 *
 * @param {string} recordStr - The XML record as string
 */
FormModel.prototype.mergeXml = function (recordStr) {
    const that = this;

    if (!recordStr) {
        return;
    }

    const modelInstanceEl = this.xml.querySelector('instance');
    let modelInstanceChildEl = this.xml.querySelector('instance > *'); // do not use firstChild as it may find a #textNode

    if (!modelInstanceChildEl) {
        throw new Error(
            'Model is corrupt. It does not contain a childnode of instance'
        );
    }

    /**
     * A Namespace merge problem occurs when ODK decides to invent a new namespace for a submission
     * that is different from the XForm model namespace... So we just remove this nonsense.
     */
    recordStr = recordStr.replace(/\s(xmlns=("|')[^\s>]+("|'))/g, '');
    /**
     * Comments aren't merging in document order (which would be impossible also).
     * This may mess up repeat functionality, so until we actually need
     * comments, we simply remove them (multiline comments are probably not removed, but we don't care about them).
     */
    recordStr = recordStr.replace(/<!--[^>]*-->/g, '');
    const record = parser.parseFromString(recordStr, 'text/xml');

    /**
     * Normally records will not contain the special "jr:template" attribute. However, we should still be able to deal with
     * this if they do, including the old hacked non-namespaced "template" attribute.
     * https://github.com/enketo/enketo-core/issues/376
     *
     * The solution if these are found is to delete the node.
     *
     * Since the record is not a FormModel instance we revert to a very aggressive querySelectorAll that selects all
     * nodes with a template attribute name IN ANY NAMESPACE.
     */

    const templateEls = record.querySelectorAll('[*|template]');

    for (let i = 0; i < templateEls.length; i++) {
        templateEls[i].remove();
    }

    /**
     * To comply with quirky behaviour of repeats in XForms, we manually create the correct number of repeat instances
     * before merging. This resolves these two issues:
     *  a) Multiple repeat instances in record are added out of order when merged into a record that contains fewer
     *     repeat instances, see https://github.com/kobotoolbox/enketo-express/issues/223
     *  b) If a repeat node is missing from a repeat instance (e.g. the 2nd) in a record, and that repeat instance is not
     *     in the model, that node will be missing in the result.
     */
    // TODO: ES6 for (var node of record.querySelectorAll('*')){}
    Array.prototype.slice.call(record.querySelectorAll('*')).forEach((node) => {
        let path;
        let repeatIndex = 0;
        let positionedPath;
        let repeatParts;
        try {
            path = getXPath(node, 'instance', false);
            // If this is a templated repeat (check templates)
            // or a repeat without templates
            if (
                typeof that.templates[path] !== 'undefined' ||
                getRepeatIndex(node) > 0
            ) {
                positionedPath = getXPath(node, 'instance', true);
                if (!that.evaluate(positionedPath, 'node', null, null, true)) {
                    repeatParts = positionedPath.match(/([^[]+)\[(\d+)\]\//g);
                    // If the positionedPath has a non-0 repeat index followed by (at least) 1 node, avoid cloning out of order.
                    if (repeatParts && repeatParts.length > 0) {
                        // TODO: Does this work for triple-nested repeats. I don't really care though.
                        // repeatIndex of immediate parent repeat of deepest nested repeat in positionedPath
                        repeatIndex =
                            repeatParts[repeatParts.length - 1].match(
                                /\[(\d+)\]/
                            )[1] - 1;
                    }
                    that.addRepeat(path, repeatIndex, true);
                }
            }
        } catch (e) {
            console.warn('Ignored error:', e);
        }
    });

    /**
     * Any default values in the model, may have been emptied in the record.
     * MergeXML will keep those default values, which would be bad, so we manually clear defaults before merging.
     */
    // first find all empty leaf nodes in record
    Array.prototype.slice
        .call(record.querySelectorAll('*'))
        .filter((recordNode) => {
            const val = recordNode.textContent;

            return recordNode.children.length === 0 && val.trim().length === 0;
        })
        .forEach((leafNode) => {
            const path = getXPath(leafNode, 'instance', true);
            const instanceNode = that.node(path, 0).getElement();
            if (instanceNode) {
                if (instanceNode.children.length === 0) {
                    // Select all text nodes (excluding repeat COMMENT nodes!)
                    that.evaluate(
                        './text()',
                        'nodes-ordered',
                        path,
                        0,
                        true
                    ).forEach((node) => {
                        node.textContent = '';
                    });
                } else {
                    // If the node in the default instance is a group (empty in record, so appears to be a leaf node
                    // but isn't), empty all true leaf node descendants.
                    that.evaluate(
                        './/*[not(*)]',
                        'nodes-ordered',
                        path,
                        0,
                        true
                    ).forEach((node) => {
                        node.textContent = '';
                    });
                }
            }
        });

    const merger = new MergeXML({
        join: false,
    });

    const modelInstanceChildStr = new XMLSerializer().serializeToString(
        modelInstanceChildEl
    );
    recordStr = new XMLSerializer().serializeToString(record);

    // first the model, to preserve DOM order of that of the default instance
    merger.AddSource(modelInstanceChildStr);
    // then merge the record into the model
    merger.AddSource(recordStr);

    if (merger.error.code) {
        throw new Error(merger.error.text);
    }

    const mergeResultDoc = merger.dom;

    /**
     * To properly show 0 repeats, if the form definition contains multiple default instances
     * and the record contains none, we have to iterate trough the templates object, and
     * 1. check for each template path, whether the record contained more than 0 of these nodes
     * 2. remove all nodes on that path if the answer was no.
     *
     * Since this requires complex handcoded XForms it is unlikely to ever be needed, so I left this
     * functionality out.
     */

    // Remove the primary instance childnode from the original model
    this.xml.querySelector('instance').removeChild(modelInstanceChildEl);
    // adopt the merged instance childnode
    modelInstanceChildEl = this.xml.adoptNode(mergeResultDoc.documentElement);
    // append the adopted node to the primary instance
    modelInstanceEl.appendChild(modelInstanceChildEl);
    // reset the rootElement
    this.rootElement = modelInstanceChildEl;
};

/**
 * Trims values of all Form elements
 */
FormModel.prototype.trimValues = function () {
    this.node(null, null, {
        noEmpty: true,
    })
        .getElements()
        .forEach((element) => {
            element.textContent = element.textContent.trim();
        });
};

/**
 * Sets instance ID and deprecated ID
 */
FormModel.prototype.setInstanceIdAndDeprecatedId = function () {
    let instanceIdObj;
    let instanceIdEl;
    let deprecatedIdEl;
    let metaEl;
    let instanceIdExistingVal;

    instanceIdObj = this.getMetaNode('instanceID');
    instanceIdEl = instanceIdObj.getElement();
    instanceIdExistingVal = instanceIdObj.getVal();

    if (!instanceIdEl) {
        console.warn('Model has no instanceID element');

        return;
    }

    if (this.data.instanceStr && this.data.submitted) {
        deprecatedIdEl = this.getMetaNode('deprecatedID').getElement();

        // set the instanceID value to empty
        instanceIdEl.textContent = '';

        const namespace = instanceIdEl.namespaceURI;

        // add deprecatedID node if necessary
        if (!deprecatedIdEl) {
            const nsPrefix = namespace
                ? this.getNamespacePrefix(namespace)
                : '';
            const nsDeclaration = namespace
                ? `xmlns:${nsPrefix}="${namespace}"`
                : '';
            deprecatedIdEl = parser.parseFromString(
                `<${
                    nsPrefix ? `${nsPrefix}:` : ''
                }deprecatedID ${nsDeclaration}/>`,
                'text/xml'
            ).documentElement;
            this.xml.adoptNode(deprecatedIdEl);
            metaEl = this.xml.querySelector('instance > * > meta');
            metaEl.appendChild(deprecatedIdEl);
        }
    }

    if (!instanceIdObj.getVal()) {
        instanceIdObj.setVal(
            this.evaluate('concat("uuid:", uuid())', 'string')
        );
    }

    // after setting instanceID, give deprecatedID element the old value of the instanceId
    // ensure dataupdate event fires by using setVal
    if (deprecatedIdEl) {
        this.getMetaNode('deprecatedID').setVal(instanceIdExistingVal);
    }
};

/**
 * Creates a custom XPath Evaluator to be used for XPath Expresssions that contain custom
 * OpenRosa functions or for browsers that do not have a native evaluator.
 *
 * @type {Function}
 */
FormModel.prototype.bindJsEvaluator = bindJsEvaluator;

/**
 * @param {string} localName - node name without namespace
 * @return {Element} node
 */
FormModel.prototype.getMetaNode = function (localName) {
    const orPrefix = this.getNamespacePrefix(OPENROSA_XFORMS_NS);
    let n = this.node(`/*/${orPrefix}:meta/${orPrefix}:${localName}`);

    if (!n.getElement()) {
        n = this.node(`/*/meta/${localName}`);
    }

    return n;
};

/**
 * @param {string} path - path to repeat
 * @return {string} repeat comment text
 */
FormModel.prototype.getRepeatCommentText = (path) => {
    path = path.trim();

    return REPEAT_COMMENT_PREFIX + path;
};

/**
 * @param {string} repeatPath - path to repeat
 * @return {string} selector
 */
FormModel.prototype.getRepeatCommentSelector = function (repeatPath) {
    return `//comment()[self::comment()="${this.getRepeatCommentText(
        repeatPath
    )}"]`;
};

/**
 * @param {string} repeatPath - path to repeat
 * @param {number} repeatSeriesIndex - index of repeat series
 * @return {Node} node
 */
FormModel.prototype.getRepeatCommentNode = function (
    repeatPath,
    repeatSeriesIndex
) {
    return findMarkerComment(
        this.xml.documentElement,
        this.getRepeatCommentText(repeatPath),
        repeatSeriesIndex
    );
};

/**
 * Adds a <repeat>able instance node in a particular series of a repeat.
 *
 * @param  {string} repeatPath - absolute path of a repeat
 * @param  {number} repeatSeriesIndex - index of the repeat series that gets a new repeat (this is always 0 for non-nested repeats)
 * @param  {boolean} merge - whether this operation is part of a merge operation (won't send dataupdate event, clears all values and
 *                           will not add ordinal attributes as these should be provided in the record)
 */
FormModel.prototype.addRepeat = function (
    repeatPath,
    repeatSeriesIndex,
    merge
) {
    let templateClone;
    const that = this;

    if (!this.templates[repeatPath]) {
        // This allows the model itself without requiring the controller to cal call .extractFakeTemplates()
        // to extract non-jr:templates by assuming that addRepeat would only called for a repeat.
        this.extractFakeTemplates([repeatPath]);
    }

    const template = this.templates[repeatPath];
    const repeatSeries = this.getRepeatSeries(repeatPath, repeatSeriesIndex);
    const insertAfterNode = repeatSeries.length
        ? repeatSeries[repeatSeries.length - 1]
        : this.getRepeatCommentNode(repeatPath, repeatSeriesIndex);

    // if not exists and not a merge operation
    if (!merge) {
        repeatSeries.forEach((el) => {
            that.addOrdinalAttribute(el, repeatSeries[0]);
        });
    }

    /**
     * If templatenodes and insertAfterNode(s) have been identified
     */
    if (template && insertAfterNode) {
        templateClone = template.cloneNode(true);
        insertAfterNode.after(templateClone);

        this.removeOrdinalAttributes(templateClone);
        // We should not automatically add ordinal attributes for an existing record as the ordinal values cannot be determined.
        // They should be provided in the instanceStr (record).
        if (!merge) {
            this.addOrdinalAttribute(templateClone, repeatSeries[0]);
        }

        // If part of a merge operation (during form load) where the values will be populated from the record, defaults are not desired.
        if (merge) {
            Array.prototype.slice
                .call(templateClone.querySelectorAll('*'))
                .filter((node) => node.children.length === 0)
                .forEach((node) => {
                    node.textContent = '';
                });
        }

        // Note: the addrepeat eventhandler in Form.js takes care of initializing branches etc, so no need to fire an event here.
    } else {
        console.error(
            'Could not find template node and/or node to insert the clone after'
        );
    }
};

/**
 * @param {Element} repeat - Set ordinal attribue to this node
 * @param {Element} firstRepeatInSeries - Used to know what the next ordinal attribute value should be. Defaults to `repeat` node.
 */
FormModel.prototype.addOrdinalAttribute = function (
    repeat,
    firstRepeatInSeries
) {
    if (
        config.repeatOrdinals === true &&
        !repeat.getAttributeNS(ENKETO_XFORMS_NS, 'ordinal')
    ) {
        let lastUsedOrdinal;
        let newOrdinal;
        const enkNs = this.getNamespacePrefix(ENKETO_XFORMS_NS);
        firstRepeatInSeries = firstRepeatInSeries || repeat;
        // getAttributeNs and setAttributeNs results in duplicate namespace declarations on each repeat node in IE11 when serializing the model.
        // However, the regular getAttribute and setAttribute do not work properly in IE11.
        lastUsedOrdinal =
            firstRepeatInSeries.getAttributeNS(
                ENKETO_XFORMS_NS,
                'last-used-ordinal'
            ) || 0;
        newOrdinal = Number(lastUsedOrdinal) + 1;
        firstRepeatInSeries.setAttributeNS(
            ENKETO_XFORMS_NS,
            `${enkNs}:last-used-ordinal`,
            newOrdinal
        );

        repeat.setAttributeNS(ENKETO_XFORMS_NS, `${enkNs}:ordinal`, newOrdinal);
    }
};

/**
 * Removes all ordinal attriubetes from all applicable nodes
 *
 * @param {Element} el - Target node
 */
FormModel.prototype.removeOrdinalAttributes = (el) => {
    if (config.repeatOrdinals === true) {
        // Find all nested repeats first (this is only used for repeats that have no template).
        // The querySelector is actually too unspecific as it matches all ordinal attributes in ANY namespace.
        // However the proper [enk\\:ordinal] doesn't work if setAttributeNS was used to add the attribute.
        const repeats = Array.prototype.slice.call(
            el.querySelectorAll('[*|ordinal]')
        );
        repeats.push(el);
        for (let i = 0; i < repeats.length; i++) {
            repeats[i].removeAttributeNS(ENKETO_XFORMS_NS, 'last-used-ordinal');
            repeats[i].removeAttributeNS(ENKETO_XFORMS_NS, 'ordinal');
        }
    }
};

/**
 * Obtains a single series of repeat element;
 *
 * @param {string} repeatPath - The absolute path of the repeat.
 * @param {number} repeatSeriesIndex - The index of the series of that repeat.
 * @return {Array<Element>} Array of all repeat elements in a series.
 */
FormModel.prototype.getRepeatSeries = function (repeatPath, repeatSeriesIndex) {
    let pathSegments;
    let nodeName;
    let checkEl;
    const repeatCommentEl = this.getRepeatCommentNode(
        repeatPath,
        repeatSeriesIndex
    );
    const result = [];

    // RepeatCommentEl is null if the requested repeatseries is a nested repeat and its ancestor repeat
    // has 0 instances.
    if (repeatCommentEl) {
        pathSegments = repeatCommentEl.textContent
            .substr(REPEAT_COMMENT_PREFIX.length)
            .split('/');
        nodeName = pathSegments[pathSegments.length - 1];
        checkEl = repeatCommentEl.nextSibling;

        // then add all subsequent repeats
        while (checkEl) {
            // Ignore any sibling text and comment nodes (e.g. whitespace with a newline character)
            // also deal with repeats that have non-repeat siblings in between them, event though that would be a bug.
            if (checkEl.nodeName && checkEl.nodeName === nodeName) {
                result.push(checkEl);
            }
            checkEl = checkEl.nextSibling;
        }
    }

    return result;
};

/**
 * Determines the index of a repeated node amongst all nodes with the same XPath selector
 *
 * @param {Element} element - Target node
 * @return {number} Determined index
 */
FormModel.prototype.determineIndex = function (element) {
    if (element) {
        const { nodeName } = element;
        const path = getXPath(element, 'instance');
        const family = Array.prototype.slice
            .call(this.xml.querySelectorAll(nodeName.replace(/\./g, '\\.')))
            .filter((node) => path === getXPath(node, 'instance'));

        return family.length === 1 ? null : family.indexOf(element);
    }
    console.error(
        'no node, or multiple nodes, provided to determineIndex function'
    );

    return -1;
};

/**
 * Extracts all templates from the model and stores them in a Javascript object.
 */
FormModel.prototype.extractTemplates = function () {
    const that = this;

    // in reverse document order to properly deal with nested repeat templates
    this.getTemplateNodes()
        .reverse()
        .forEach((templateEl) => {
            const xPath = getXPath(templateEl, 'instance');
            that.addTemplate(xPath, templateEl);
            /*
             * Nested repeats that have a template attribute are correctly added to the templates object.
             * The template of the repeat ancestor of the nested repeat contains the correct comment.
             * However, since the ancestor repeat (template)
             */
            templateEl.remove();
        });
};

/**
 * @param {Array<string>} repeatPaths - repeat paths
 */
FormModel.prototype.extractFakeTemplates = function (repeatPaths) {
    const that = this;
    let repeat;

    repeatPaths.forEach((repeatPath) => {
        // Filter by elements that are the first in a series. This means that multiple instances of nested repeats
        // all get a comment insertion point.
        repeat = that.evaluate(repeatPath, 'node', null, null, true);
        if (repeat) {
            that.addTemplate(repeatPath, repeat, true);
        }
    });
};

/**
 * @param {string} repeatPath - path to repeat
 */
FormModel.prototype.addRepeatComments = function (repeatPath) {
    const comment = this.getRepeatCommentText(repeatPath);

    // Find all repeat series.
    this.evaluate(repeatPath, 'nodes-ordered', null, null, true).forEach(
        (repeat) => {
            if (
                !hasPreviousSiblingElementSameName(repeat) &&
                !hasPreviousCommentSiblingWithContent(repeat, comment)
            ) {
                // Add a comment to the primary instance that serves as an insertion point for each repeat series,
                repeat.before(document.createComment(comment));
            }
        }
    );
};

/**
 * @param {string} repeatPath - path to repeat
 * @param {Element} repeat - Target node
 * @param {boolean} empty - whether to empty values before adding the template
 */
FormModel.prototype.addTemplate = function (repeatPath, repeat, empty) {
    this.addRepeatComments(repeatPath);

    if (!this.templates[repeatPath]) {
        const clone = repeat.cloneNode(true);
        clone.removeAttribute('template');
        clone.removeAttribute('jr:template');
        if (empty) {
            Array.prototype.slice
                .call(clone.querySelectorAll('*'))
                .filter((node) => node.children.length === 0)
                .forEach((node) => {
                    node.textContent = '';
                });
        }
        // Add to templates object.
        this.templates[repeatPath] = clone;
    }
};

/**
 * @return {Array<Element>} template nodes list
 */
FormModel.prototype.getTemplateNodes = function () {
    const jrPrefix = this.getNamespacePrefix(JAVAROSA_XFORMS_NS);

    return this.evaluate(
        `/model/instance[1]/*//*[@${jrPrefix}:template]`,
        'nodes-ordered',
        null,
        null,
        true
    );
};

/**
 * Obtains a cleaned up string of the data instance
 *
 * @return {string} XML string
 */
FormModel.prototype.getStr = function () {
    let dataStr = new XMLSerializer().serializeToString(
        this.xml.querySelector('instance > *') || this.xml.documentElement,
        'text/xml'
    );
    // restore default namespaces
    dataStr = dataStr.replace(/\s(data-)(xmlns=("|')[^\s>]+("|'))/g, ' $2');
    // remove repeat comments
    dataStr = dataStr.replace(
        new RegExp(`<!--${REPEAT_COMMENT_PREFIX}\\/[^>]+-->`, 'g'),
        ''
    );
    // If not IE, strip duplicate namespace declarations. IE doesn't manage to add a namespace declaration to the root element.
    if (navigator.userAgent.indexOf('Trident/') === -1) {
        dataStr = this.removeDuplicateEnketoNsDeclarations(dataStr);
    }

    return dataStr;
};

/**
 * @param {string} xmlStr - XML string
 * @return {string} XML string without duplicates
 */
FormModel.prototype.removeDuplicateEnketoNsDeclarations = function (xmlStr) {
    let i = 0;
    const declarationExp = new RegExp(
        `( xmlns:${this.getNamespacePrefix(
            ENKETO_XFORMS_NS
        )}="${ENKETO_XFORMS_NS}")`,
        'g'
    );

    return xmlStr.replace(declarationExp, (match) => {
        i++;
        if (i > 1) {
            return '';
        }
        return match;
    });
};

/**
 * There is a huge historic issue (stemming from JavaRosa) that has resulted in the usage of incorrect formulae
 * on nodes inside repeat nodes.
 * Those formulae use absolute paths when relative paths should have been used. See more here:
 * http://opendatakit.github.io/odk-xform-spec/#a-big-deviation-with-xforms
 *
 * Tools such as pyxform also build forms in this incorrect manner. See https://github.com/modilabs/pyxform/issues/91
 * It will take time to correct this so makeBugCompliant() aims to mimic the incorrect
 * behaviour by injecting the 1-based [position] of repeats into the XPath expressions. The resulting expression
 * will then be evaluated in a way users expect (as if the paths were relative) without having to mess up
 * the XPath Evaluator.
 *
 * E.g. '/data/rep_a/node_a' could become '/data/rep_a[2]/node_a' if the context is inside
 * the second rep_a repeat.
 *
 * This function should be removed when we can reasonbly expect not many 'old XForms' to be in use any more.
 *
 * Already it should leave proper XPaths untouched.
 *
 * @param {string} expr - The XPath expression
 * @param {string} selector - Selector of the (context) node on which expression is evaluated
 * @param {number} index - Index of the instance node with that selector
 */
FormModel.prototype.makeBugCompliant = function (expr, selector, index) {
    if (this.noRepeatRefErrorExpected) {
        return expr;
    }

    let target = this.node(selector, index).getElement();

    // target is null for nested repeats if no repeats exist
    if (!target) {
        return expr;
    }

    const parents = [target];

    while (
        target &&
        target.parentElement &&
        target.nodeName.toLowerCase() !== 'instance'
    ) {
        target = target.parentElement;
        parents.push(target);
    }

    // traverse collection in reverse
    parents.forEach((element) => {
        // escape any dots in the node name
        const nodeName = element.nodeName.replace(/\./g, '\\.');
        const siblingsAndSelf = getSiblingElementsAndSelf(
            element,
            `${nodeName}:not([template])`
        );

        // if the node is a repeat node that has been cloned at least once (i.e. if it has siblings with the same nodeName)
        if (siblingsAndSelf.length > 1) {
            const parentSelector = getXPath(element, 'instance');
            const parentIndex = siblingsAndSelf.indexOf(element);
            // Add position to segments that do not have an XPath predicate.
            expr = expr.replace(
                new RegExp(`${parentSelector}/`, 'g'),
                `${parentSelector}[${parentIndex + 1}]/`
            );
        }
    });

    return expr;
};

/**
 * Set namespaces for all nodes
 */
FormModel.prototype.setNamespaces = function () {
    /**
     * Passing through all nodes would be very slow with an XForms model that contains lots of nodes such as large secondary instances.
     * (The namespace XPath axis is not support in native browser XPath evaluators unfortunately).
     *
     * For now it has therefore been restricted to only look at the top-level node in the primary instance and in the secondary instances.
     * We can always expand that later.
     */
    const start = this.hasInstance ? '/model/instance' : '';
    const nodes = this.evaluate(
        `${start}/*`,
        'nodes-ordered',
        null,
        null,
        true
    );
    const that = this;
    let prefix;

    nodes.forEach((node) => {
        if (node.hasAttributes()) {
            Array.from(node.attributes).forEach((attribute) => {
                if (attribute.name.indexOf('xmlns:') === 0) {
                    that.namespaces[attribute.name.substring(6)] =
                        attribute.value;
                }
            });
        }
        // add required namespaces to resolver and document if they are missing
        [
            ['orx', OPENROSA_XFORMS_NS, false],
            ['jr', JAVAROSA_XFORMS_NS, false],
            ['enk', ENKETO_XFORMS_NS, config.repeatOrdinals === true],
            ['odk', ODK_XFORMS_NS, false],
        ].forEach((arr) => {
            if (!that.getNamespacePrefix(arr[1])) {
                prefix = !that.namespaces[arr[0]] ? arr[0] : `__${arr[0]}`;
                // add to resolver
                that.namespaces[prefix] = arr[1];
                // add to document
                if (arr[2]) {
                    node.setAttributeNS(
                        'http://www.w3.org/2000/xmlns/',
                        `xmlns:${prefix}`,
                        arr[1]
                    );
                }
            }
        });
    });
};

/**
 * @param {string} namespace - Target namespace
 * @return {string|undefined} Namespace prefix
 */
FormModel.prototype.getNamespacePrefix = function (namespace) {
    const found = Object.entries(this.namespaces).find(
        (arr) => arr[1] === namespace
    );

    return found ? found[0] : undefined;
};

/**
 * Returns a namespace resolver with single `lookupNamespaceURI` method
 *
 * @return {{lookupNamespaceURI: Function}} namespace resolver
 */
FormModel.prototype.getNsResolver = function () {
    const namespaces =
        typeof this.namespaces === 'undefined' ? {} : this.namespaces;

    return {
        lookupNamespaceURI(prefix) {
            return namespaces[prefix] || null;
        },
    };
};

/**
 * Shift root to first instance for all absolute paths not starting with /model
 *
 * @param {string} expr - Original expression
 * @return {string} New expression
 */
FormModel.prototype.shiftRoot = function (expr) {
    const LITERALS = /"([^"]*)(")|'([^']*)(')/g;
    if (this.hasInstance) {
        // Encode all string literals in order to exclude them, without creating a monsterly regex
        expr = expr.replace(LITERALS, (m, p1, p2, p3, p4) => {
            const encoded =
                typeof p1 !== 'undefined'
                    ? encodeURIComponent(p1)
                    : encodeURIComponent(p3);
            const quote = p2 || p4;

            return quote + encoded + quote;
        });
        // Insert /model/instance[1]
        expr = expr.replace(
            /^(\/(?!model\/)[^/][^/\s,"']*\/)/g,
            '/model/instance[1]$1'
        );
        expr = expr.replace(
            /([^a-zA-Z0-9.\])/*_-])(\/(?!model\/)[^/][^/\s,"']*\/)/g,
            '$1/model/instance[1]$2'
        );
        // Decode string literals
        expr = expr.replace(LITERALS, (m, p1, p2, p3, p4) => {
            const decoded =
                typeof p1 !== 'undefined'
                    ? decodeURIComponent(p1)
                    : decodeURIComponent(p3);
            const quote = p2 || p4;

            return quote + decoded + quote;
        });
    }

    return expr;
};

/**
 * Replace instance('id') with an absolute path
 * Doing this here instead of adding an instance() function to the XPath evaluator, means we can keep using
 * the much faster native evaluator in most cases!
 *
 * @param {string} expr - Original expression
 * @return {string} New expression
 */
FormModel.prototype.replaceInstanceFn = function (expr) {
    let prefix;
    const that = this;

    // TODO: would be more consistent to use utils.parseFunctionFromExpression() and utils.stripQuotes
    return expr.replace(INSTANCE, (match, quote, id) => {
        prefix = `/model/instance[@id="${id}"]`;
        // check if referred instance exists in model
        if (that.evaluate(prefix, 'nodes-ordered', null, null, true).length) {
            return prefix;
        }
        throw new FormLogicError(`instance "${id}" does not exist in model`);
    });
};

/**
 * Replaces current() with /absolute/path/to/node to ensure the context is shifted to the primary instance
 *
 * Doing this here instead of adding a current() function to the XPath evaluator, means we can keep using
 * the much faster native evaluator in most cases!
 *
 * Root will be shifted later, and repeat positions are already injected into context selector.
 *
 * @param {string} expr - Original expression
 * @param {string} contextSelector - Context selector
 * @return {string} New expression
 */
FormModel.prototype.replaceCurrentFn = (expr, contextSelector) =>
    expr.replace(/current\(\)/g, `${contextSelector}`);

/**
 * Replaces indexed-repeat(node, path, position, path, position, etc) substrings by converting them
 * to their native XPath equivalents using [position() = x] predicates
 *
 * @param {string} expr - The XPath expression
 * @param {string} selector - context path
 * @param {number} index - index of context node
 * @return {string} Converted XPath expression
 */
FormModel.prototype.replaceIndexedRepeatFn = function (expr, selector, index) {
    const that = this;
    const indexedRepeats = parseFunctionFromExpression(expr, 'indexed-repeat');

    indexedRepeats.forEach((indexedRepeat) => {
        let i;
        let positionedPath;
        let position;
        const params = indexedRepeat[1];

        if (params.length % 2 === 1) {
            positionedPath = params[0];

            for (i = params.length - 1; i > 1; i -= 2) {
                // The position will become an XPath predicate. The context for an XPath predicate, is not the same
                // as the context for the complete expression, so we have to evaluate the position separately. Otherwise
                // relative paths would break.
                position = !isNaN(params[i])
                    ? params[i]
                    : that.evaluate(params[i], 'number', selector, index, true);
                positionedPath = positionedPath.replace(
                    params[i - 1],
                    `${params[i - 1]}[position() = ${position}]`
                );
            }

            expr = expr.replace(indexedRepeat[0], positionedPath);
        } else {
            throw new FormLogicError(
                `indexed repeat with incorrect number of parameters found: ${indexedRepeat[0]}`
            );
        }
    });

    return expr;
};

/**
 * @param {string} expr - The XPath expression
 * @return {string} Converted XPath expression
 */
FormModel.prototype.replaceVersionFn = function (expr) {
    const that = this;
    let version;
    const versions = parseFunctionFromExpression(expr, 'version');

    versions.forEach((versionPart) => {
        version =
            version || that.evaluate('/*/@version', 'string', null, 0, true);
        // ignore arguments
        expr = expr.replace(versionPart[0], `"${version}"`);
    });

    return expr;
};

/**
 * @param {string} expr - The XPath expression
 * @param {string} selector - context path
 * @param {number} index - index of context node
 * @return {string} Converted XPath expression
 */
FormModel.prototype.replacePullDataFn = function (expr, selector, index) {
    let pullDataResult;
    const that = this;
    const replacements = this.convertPullDataFn(expr, selector, index);

    for (const pullData in replacements) {
        if (Object.prototype.hasOwnProperty.call(replacements, pullData)) {
            // We evaluate this here, so we can use the native evaluator safely. This speeds up pulldata() by about a factor *740*!
            pullDataResult = that.evaluate(
                replacements[pullData],
                'string',
                selector,
                index,
                true
            );
            expr = expr.replace(pullData, `"${pullDataResult}"`);
        }
    }

    return expr;
};

/**
 * @param {string} expr - The XPath expression
 * @param {string} selector - context path
 * @param {number} index - index of context node
 * @return {string} Converted XPath expression
 */
FormModel.prototype.convertPullDataFn = function (expr, selector, index) {
    const that = this;
    const pullDatas = parseFunctionFromExpression(expr, 'pulldata');
    const replacements = {};

    if (!pullDatas.length) {
        return replacements;
    }

    pullDatas.forEach((pullData) => {
        let searchValue;
        let searchXPath;
        const params = pullData[1];

        if (params.length === 4) {
            // strip quotes
            params[1] = stripQuotes(params[1]);
            params[2] = stripQuotes(params[2]);

            // TODO: the 2nd and 3rd parameter could probably also be expressions.

            // The 4th argument will become an XPath predicate. The context for an XPath predicate, is not the same
            // as the context for the complete expression, so we have to evaluate the position separately. Otherwise
            // relative paths would break.
            searchValue = `'${that.evaluate(
                params[3],
                'string',
                selector,
                index,
                true
            )}'`;
            searchXPath = `instance(${params[0]})/root/item[${params[2]} = ${searchValue}]/${params[1]}`;

            replacements[pullData[0]] = searchXPath;
        } else {
            throw new FormLogicError(
                `pulldata with incorrect number of parameters found: ${pullData[0]}`
            );
        }
    });

    return replacements;
};

/**
 * Evaluates an XPath Expression using XPathJS_javarosa (not native XPath 1.0 evaluator)
 *
 * This function does not seem to work properly for nodeset resulttypes otherwise:
 * muliple nodes can be accessed by returned node.snapshotItem(i)(.textContent)
 * a single node can be accessed by returned node(.textContent)
 *
 * @param {string} expr - The expression to evaluate
 * @param {string} [resTypeStr] - "boolean", "string", "number", "node", "nodes" (best to always supply this)
 * @param {string} [selector] - Query selector which will be use to provide the context to the evaluator
 * @param {number} [index] - 0-based index of selector in document
 * @param {boolean} [tryNative] - Whether an attempt to try the Native Evaluator is safe (ie. whether it is
 *                                certain that there are no date comparisons)
 * @return {number|string|boolean|Array<Element>} The result
 */
FormModel.prototype.evaluate = function (
    expr,
    resTypeStr,
    selector,
    index,
    tryNative
) {
    let j;
    let context;
    let doc;
    let resTypeNum;
    let resultTypes;
    let result;
    let collection;
    let response;
    let repeats;
    let cacheKey;
    let original;
    let cacheable;

    // console.debug( 'evaluating expr: ' + expr + ' with context selector: ' + selector + ', 0-based index: ' +
    //    index + ' and result type: ' + resTypeStr );
    original = expr;
    tryNative = tryNative || false;
    resTypeStr = resTypeStr || 'any';
    index = index || 0;
    doc = this.xml;
    repeats = null;

    if (selector) {
        collection = this.node(selector).getElements();
        repeats = collection.length;
        context = collection[index];
    } else {
        // either the first data child of the first instance or the first child (for loaded instances without a model)
        context = this.rootElement;
    }

    if (!context) {
        console.error('no context element found', selector, index);
    }

    // cache key includes the number of repeated context nodes,
    // to force a new cache item if the number of repeated changes to > 0
    // TODO: these cache keys can get quite large. Would it be beneficial to get the md5 of the key?
    cacheKey = [expr, selector, index, repeats].join('|');

    // These functions need to come before makeBugCompliant.
    // An expression transformation with indexed-repeat or pulldata cannot be cached because in
    // "indexed-repeat(node, repeat nodeset, index)" the index parameter could be an expression.
    expr = this.replaceIndexedRepeatFn(expr, selector, index);
    expr = this.replacePullDataFn(expr, selector, index);
    cacheable = original === expr;

    // if no cached conversion exists
    const cachedExpr = this.convertedExpressions.get(cacheKey);

    if (cachedExpr === undefined) {
        expr = expr.trim();
        expr = this.replaceInstanceFn(expr);
        expr = this.replaceVersionFn(expr);
        expr = this.replaceCurrentFn(expr, getXPath(context, 'instance', true));
        // shiftRoot should come after replac