@kwiz/common/src/utils/sharepoint.rest/list.ts

KWIZ common utilities and helpers for M365 platform

import { jsonClone } from "../../exports-index";
import { PushNoDuplicate, firstOrNull, makeUniqueArray, toHash } from "../../helpers/collections.base";
import { jsonStringify } from "../../helpers/json";
import { NormalizeListName, SPBasePermissions, SchemaJsonToXml, SchemaXmlToJson, extendFieldInfos } from "../../helpers/sharepoint";
import { normalizeGuid } from "../../helpers/strings";
import { SafeIfElse, isBoolean, isNotEmptyArray, isNullOrEmptyArray, isNullOrEmptyString, isNullOrUndefined, isNumber, isPromise, isString, isValidGuid } from "../../helpers/typecheckers";
import { makeServerRelativeUrl, normalizeUrl } from "../../helpers/url";
import { IDictionary } from "../../types/common.types";
import { IRestOptions, contentTypes, jsonTypes } from "../../types/rest.types";
import { BaseTypes, FieldTypeAsString, FieldTypes, IFieldInfo, IFieldInfoEX, IFieldInfoExHash, IFieldJsonSchema, IFieldLookupInfo, ISPEventReceiver, ListTemplateTypes, PageType, SPBasePermissionKind } from "../../types/sharepoint.types";
import { GeListItemsFoldersBehaviour, IListWorkflowAssociation, IRestItem, ListExperienceOptions, iContentType, iList, iListVersionSettings, iListView } from "../../types/sharepoint.utils.types";
import { ConsoleLogger } from "../consolelogger";
import { GetJson, GetJsonSync, longLocalCache, shortLocalCache } from "../rest";
import { GetRestBaseUrl, GetSiteUrl, LIST_EXPAND, LIST_SELECT } from "./common";
import { __fixGetListItemsResults } from "./listutils/common";
import { GetContentTypes, GetContentTypesSync, GetListsSync, IGetContentTypesOptions } from "./web";

const logger = ConsoleLogger.get("SharePoint.Rest.List");

/** returns /_api/web/lists/getById() or /_api/web/lists/getByTitle() */
export function GetListRestUrl(siteUrl: string, listIdOrTitle: string): string {
    siteUrl = GetSiteUrl(siteUrl);

    let listId = GetListId(siteUrl, listIdOrTitle);

    let listPart = isValidGuid(listId) ? `getById('${normalizeGuid(listId)}')` : `getByTitle('${encodeURIComponent(listIdOrTitle)}')`;
    return GetRestBaseUrl(siteUrl) + `/web/lists/${listPart}`;
}

export function GetListId(siteUrl: string, listIdOrTitle: string): string {
    if (isNullOrEmptyString(listIdOrTitle)) return null;
    if (isValidGuid(listIdOrTitle)) return listIdOrTitle;
    //Issue 7508
    //When translation is enabled, and user changes list title but he is not on the same language as the site
    //he translates the list, but not changing its title
    //so REST api /lists/getByTitle will not work
    //instead, we need to get the list id from the web's lists collection.
    let lists = GetListsSync(siteUrl);
    var lower = listIdOrTitle.toLowerCase();
    var list = firstOrNull(lists, l => l.Title.toLowerCase() === lower);
    return list && list.Id || null;
}

/** get the list ID from a list page, such as a list view or an item form */
export function GetListIdFromPageSync(siteUrl: string, listPageUrl: string): string {
    let url = `${GetRestBaseUrl(siteUrl)}/web/getlist('${makeServerRelativeUrl(listPageUrl.split('?')[0].split('#')[0])}')?$select=id`;
    let response = GetJsonSync<{ Id: string; }>(url, null, {
        ...longLocalCache,
        jsonMetadata: jsonTypes.nometadata
    });
    if (!isNullOrUndefined(response) && response.success) {
        let listId = response.result.Id;
        return normalizeGuid(listId);
    }
    return null;
}

interface IGetSiteAssetLibraryResult { Id: string, Name: string, ServerRelativeUrl: string }
interface IGetSiteAssetLibraryReturnValue {
    value: {
        Id: string;
        RootFolder: {
            Name: string;
            ServerRelativeUrl: string;
            Exists: boolean;
        };
    }[];
}

/** ensures the site assets library exists and return its info. on errors - it will return null. */
export function EnsureAssetLibrary(siteUrl: string): Promise<IGetSiteAssetLibraryResult> {
    siteUrl = GetSiteUrl(siteUrl);

    let url = GetRestBaseUrl(siteUrl) +
        "/web/lists/EnsureSiteAssetsLibrary?$select=ID,RootFolder/Name,RootFolder/ServerRelativeUrl,RootFolder/Exists&$expand=RootFolder";
    return GetJson<{
        d: { Id: string; RootFolder: { Name: string; ServerRelativeUrl: string; Exists: boolean; }; };
    }>(url, null, { method: "POST", spWebUrl: siteUrl, ...longLocalCache }).then(result => {
        if (result && result.d) {
            return {
                Id: result.d.Id,
                Name: result.d.RootFolder.Name,
                ServerRelativeUrl: result.d.RootFolder.ServerRelativeUrl
            };
        } else return null;
    }).catch<IGetSiteAssetLibraryResult>(() => null);
}

interface IGetSitePagesLibrarResult { Id: string, Name: string, ServerRelativeUrl: string }

/** ensures the site pages library exists and return its info. on errors - it will return null. */
export async function EnsureSitePagesLibrary(siteUrl: string): Promise<IGetSitePagesLibrarResult> {
    let url = `${GetRestBaseUrl(siteUrl)}/web/lists/EnsureSitePagesLibrary`
        + `?$select=ID,RootFolder/Name,RootFolder/ServerRelativeUrl,RootFolder/Exists&$expand=RootFolder`;
    let response = await GetJson<iList>(url, null, {
        method: "POST",
        jsonMetadata: jsonTypes.nometadata,
        includeDigestInPost: true,
        ...longLocalCache
    });

    if (!isNullOrUndefined(response) && !isNullOrUndefined(response.RootFolder)) {
        return {
            Id: response.Id,
            Name: response.RootFolder.Name,
            ServerRelativeUrl: response.RootFolder.ServerRelativeUrl
        };
    }
    return null;
}

export function GetSiteAssetLibrary(siteUrl: string, sync?: false): Promise<IGetSiteAssetLibraryResult>;
export function GetSiteAssetLibrary(siteUrl: string, sync: true): IGetSiteAssetLibraryResult;
export function GetSiteAssetLibrary(siteUrl: string, sync?: boolean): IGetSiteAssetLibraryResult | Promise<IGetSiteAssetLibraryResult> {
    let reqUrl = `${GetRestBaseUrl(siteUrl)}/web/lists?`
        //Issue 1492: isSiteAssetsLibrary eq true does not work for reader users.
        //+ `$filter=isSiteAssetsLibrary eq true&$select=ID,RootFolder/Name,RootFolder/ServerRelativeUrl,RootFolder/Exists`
        + `$filter=EntityTypeName%20eq%20%27SiteAssets%27&$select=ID,RootFolder/Name,RootFolder/ServerRelativeUrl,RootFolder/Exists`
        + `&$expand=RootFolder`;

    let caller = sync ? GetJsonSync : GetJson;

    let result = caller<IGetSiteAssetLibraryReturnValue>(reqUrl, null, { ...longLocalCache, jsonMetadata: jsonTypes.nometadata });

    let transform: (v: IGetSiteAssetLibraryReturnValue) => IGetSiteAssetLibraryResult = (v) => {
        if (isNotEmptyArray(v && v.value)) {
            let assetLibrary = v.value[0];
            return {
                Id: assetLibrary.Id,
                Name: assetLibrary.RootFolder.Name,
                ServerRelativeUrl: assetLibrary.RootFolder.ServerRelativeUrl
            };
        }
        return null;
    };

    if (isPromise(result))
        return result.then(r => transform(r), () => null);
    else
        return result.success ? transform(result.result) : null;
}

/** Return the list Title */
export function GetListTitle(siteUrl: string, listIdOrTitle: string): Promise<string> {
    siteUrl = GetSiteUrl(siteUrl);

    return GetJson<{ d: { Title: string; }; }>(GetListRestUrl(siteUrl, listIdOrTitle) + `/Title`, null, { allowCache: true })
        .then(r => {
            return r.d.Title;
        })
        .catch<string>(() => null);
}

/** Return the list */
export function GetList(siteUrlOrId: string, listIdOrTitle: string, options?: {
    includeViews?: boolean;
    viewOptions?: IListViewOptions;
    includeContentTypes?: boolean;
    includeRootFolder?: boolean;
    includeEventReceivers?: boolean;
}, refreshCache = false): Promise<iList> {
    let siteUrl = GetSiteUrl(siteUrlOrId);

    if (isNullOrEmptyString(listIdOrTitle)) {
        return null;
    }

    return GetJson<{ d: iList; }>(GetListRestUrl(siteUrl, listIdOrTitle) + `?$select=${LIST_SELECT}&$expand=${LIST_EXPAND}`, null, { allowCache: true })
        .then(async r => {
            let list = r.d;
            if (options) {
                let promises = [];

                if (options.includeViews) {
                    promises.push(GetListViews(siteUrl, listIdOrTitle, options.viewOptions, refreshCache).then((r) => {
                        list.Views = r;
                    }))
                }
                if (options.includeContentTypes) {
                    promises.push(GetListContentTypes(siteUrl, listIdOrTitle, null, refreshCache).then((r) => {
                        list.ContentTypes = r;
                    }));
                }
                if (options.includeRootFolder) {
                    promises.push(GetListRootFolder(siteUrl, listIdOrTitle).then((r) => {
                        list.RootFolder = r;
                    }));
                }
                if (options.includeEventReceivers) {
                    promises.push(GetListEventReceivers(siteUrl, listIdOrTitle, refreshCache).then((r) => {
                        list.EventReceivers = r;
                    }));
                }

                if (promises.length > 0) {
                    await Promise.all(promises);
                }
            }

            if (list.EffectiveBasePermissions
                && (isString(list.EffectiveBasePermissions.High) || isString(list.EffectiveBasePermissions.Low))) {
                list.EffectiveBasePermissions = {
                    High: Number(list.EffectiveBasePermissions.High),
                    Low: Number(list.EffectiveBasePermissions.Low)
                };
            }

            return list;
        })
        .catch<iList>(() => null);
}
/** Return the list */
export function GetListSync(siteUrl: string, listIdOrTitle: string): iList {
    siteUrl = GetSiteUrl(siteUrl);

    if (isNullOrEmptyString(listIdOrTitle)) return null;

    let result = GetJsonSync<{ d: iList; }>(GetListRestUrl(siteUrl, listIdOrTitle) + `?$select=${LIST_SELECT}&$expand=${LIST_EXPAND}`, null, shortLocalCache);
    if (result && result.success) {
        let list = result.result.d;

        if (list.EffectiveBasePermissions
            && (isString(list.EffectiveBasePermissions.High)
                || isString(list.EffectiveBasePermissions.Low))) {
            list.EffectiveBasePermissions = {
                High: Number(list.EffectiveBasePermissions.High),
                Low: Number(list.EffectiveBasePermissions.Low)
            };
        }

        return list;
    }
    else return null;
}

export function GetListNameSync(webUrl: string, listIdOrTitle: string): string {
    let list = GetListSync(webUrl, listIdOrTitle);
    return NormalizeListName({ EntityTypeName: list.EntityTypeName, BaseType: list.BaseType });
}

export async function GetListName(webUrl: string, listIdOrTitle: string) {
    let list = await GetList(webUrl, listIdOrTitle);
    return NormalizeListName({ EntityTypeName: list.EntityTypeName, BaseType: list.BaseType });
}

export function GetListRootFolder(siteUrlOrId: string, listIdOrTitle: string): Promise<{ ServerRelativeUrl: string; Name: string; }> {
    let siteUrl = GetSiteUrl(siteUrlOrId);

    return GetJson<{
        d: { ServerRelativeUrl: string; Name: string; };
    }>(GetListRestUrl(siteUrl, listIdOrTitle) + `/RootFolder?$Select=Name,ServerRelativeUrl`,
        null, longLocalCache)
        .then(r => {
            return r.d;
        })
        .catch<{ ServerRelativeUrl: string; Name: string; }>(() => null);
}

export function GetListRootFolderSync(siteUrlOrId: string, listIdOrTitle: string): { ServerRelativeUrl: string; Name: string; } {
    let siteUrl = GetSiteUrl(siteUrlOrId);

    let result = GetJsonSync<{
        d: { ServerRelativeUrl: string; Name: string; };
    }>(GetListRestUrl(siteUrl, listIdOrTitle) + `/RootFolder?$Select=Name,ServerRelativeUrl`,
        null, longLocalCache);

    return SafeIfElse(() => result.result.d, null);
}

export function GetListField(siteUrlOrId: string, listIdOrTitle: string, fieldIdOrName: string, refreshCache?: boolean): Promise<IFieldInfo> {
    let siteUrl = GetSiteUrl(siteUrlOrId);

    var url = GetListRestUrl(siteUrl, listIdOrTitle) + `/fields`;

    if (isValidGuid(fieldIdOrName)) {
        url += `('${normalizeGuid(fieldIdOrName)}')`;
    } else {
        url += `/getbyinternalnameortitle(@u)?@u='${encodeURIComponent(fieldIdOrName)}'`;
    }

    let result = GetJson<{ d: IFieldInfo; }>(url, null, { allowCache: refreshCache !== true })
        .then(r => {
            return r.d;
        })
        .catch<IFieldInfo>(() => null);

    return result;
}

function _getListFieldsRequestUrl(siteUrl: string, listIdOrTitle: string) {
    return GetListRestUrl(siteUrl, listIdOrTitle) + `/fields`;
}

/** Gets ID, Title, ContentType Author, Editor, Created and Modified fields */
export function GetStandardListFields(siteUrlOrId: string, listIdOrTitle: string, refreshCache?: boolean) {
    let fieldNames = ["ID", "Title", "ContentType", "Author", "Editor", "Created", "Modified"];
    return GetListFields(siteUrlOrId, listIdOrTitle, { refreshCache: refreshCache, fieldNames: fieldNames });
}

export interface IGetListFieldsOptions {
    refreshCache?: boolean;
    /** fieldNames that should be returned with the request */
    fieldNames?: string[];
}

function _processGetListFields(fields: IFieldInfo[], fieldNames: string[]) {
    if (isNullOrEmptyArray(fields)) {
        return fields as IFieldInfoEX[];
    }
    let extendedFields = extendFieldInfos(fields);

    if (!isNullOrEmptyArray(fieldNames)) {
        return extendedFields.filter((extendedField) => {
            return fieldNames.includes(extendedField.InternalName);
        });
    }
    return extendedFields;
}

export function GetListFields(siteUrlOrId: string, listIdOrTitle: string, options: IGetListFieldsOptions = {}): Promise<IFieldInfoEX[]> {
    let siteUrl = GetSiteUrl(siteUrlOrId);
    let url = _getListFieldsRequestUrl(siteUrl, listIdOrTitle);

    let restOptions: IRestOptions = {
        allowCache: options.refreshCache !== true,
        jsonMetadata: jsonTypes.nometadata
    };

    return GetJson<{ value: IFieldInfo[]; }>(url, null, restOptions)
        .then((result) => {
            return _processGetListFields(result.value, options.fieldNames);
        }).catch<IFieldInfoEX[]>(() => {
            return null;
        });
}

export function GetListFieldsSync(siteUrlOrId: string, listIdOrTitle: string, options: IGetListFieldsOptions = {}): IFieldInfoEX[] {
    let siteUrl = GetSiteUrl(siteUrlOrId);
    let url = _getListFieldsRequestUrl(siteUrl, listIdOrTitle);

    let restOptions: IRestOptions = {
        allowCache: options.refreshCache !== true,
        jsonMetadata: jsonTypes.nometadata
    };

    let result = GetJsonSync<{ value: IFieldInfo[]; }>(url, null, restOptions);
    if (result.success && !isNullOrUndefined(result.result)) {
        return _processGetListFields(result.result.value, options.fieldNames);
    }
    return null;
}

export async function GetListFieldsAsHash(siteUrlOrId: string, listIdOrTitle: string, refreshCache?: boolean): Promise<IFieldInfoExHash> {
    let siteUrl = GetSiteUrl(siteUrlOrId);

    let fields = await GetListFields(siteUrl, listIdOrTitle, { refreshCache: refreshCache });
    let hash: IFieldInfoExHash = {};
    if (isNotEmptyArray(fields)) {
        hash = toHash(fields, f => f.InternalName);
    }
    return hash;
}

export function GetListFieldsAsHashSync(siteUrlOrId: string, listIdOrTitle: string, refreshCache?: boolean): IFieldInfoExHash {
    let siteUrl = GetSiteUrl(siteUrlOrId);

    let fields = GetListFieldsSync(siteUrl, listIdOrTitle, { refreshCache: refreshCache });
    let hash: IFieldInfoExHash = {};
    if (isNotEmptyArray(fields)) {
        fields.forEach(f => { hash[f.InternalName] = f; });
    }
    return hash;
}

export function GetListWorkflows(siteUrl: string, listIdOrTitle: string, refreshCache?: boolean): Promise<IListWorkflowAssociation[]> {
    siteUrl = GetSiteUrl(siteUrl);

    return GetJson<{
        d: { results: IListWorkflowAssociation[]; };
    }>(GetListRestUrl(siteUrl, listIdOrTitle) + `/workflowAssociations`,
        null, { allowCache: refreshCache !== true })
        .then(r => {
            if (r && r.d && isNotEmptyArray(r.d.results)) {
                r.d.results.forEach(wf => {
                    wf.BaseId = normalizeGuid(wf.BaseId);
                    wf.Id = normalizeGuid(wf.Id);
                    wf.ListId = normalizeGuid(wf.ListId);
                    wf.WebId = normalizeGuid(wf.WebId);
                });
                return r.d.results;
            }
            else return [];
        })
        .catch<IListWorkflowAssociation[]>(() => []);
}

export function UserHasManagePermissions(siteUrl: string, listIdOrTitle: string): Promise<boolean> {
    siteUrl = GetSiteUrl(siteUrl);

    return GetJson<{ d: { EffectiveBasePermissions: { High: number; Low: number; }; }; }>(GetListRestUrl(siteUrl, listIdOrTitle) + `/EffectiveBasePermissions`, null,
        { ...shortLocalCache })
        .then(r => {
            return new SPBasePermissions(r.d.EffectiveBasePermissions).has(SPBasePermissionKind.ManageLists);
        })
        .catch<boolean>(() => null);
}

export function UserHasEditPermissions(siteUrl: string, listIdOrTitle: string): Promise<boolean> {
    return UserHasPermissions(siteUrl, listIdOrTitle, SPBasePermissionKind.EditListItems);
}

export function UserHasPermissions(siteUrlOrId: string, listIdOrTitle: string, permissionKind: SPBasePermissionKind): Promise<boolean> {
    let siteUrl = GetSiteUrl(siteUrlOrId);

    return GetJson<{ d: { EffectiveBasePermissions: { High: number; Low: number; }; }; }>(GetListRestUrl(siteUrl, listIdOrTitle) + `/EffectiveBasePermissions`, null,
        { ...shortLocalCache })
        .then(r => {
            return new SPBasePermissions(r.d.EffectiveBasePermissions).has(permissionKind);
        })
        .catch<boolean>(() => null);
}

export function UserHasPermissionsSync(siteUrlOrId: string, listIdOrTitle: string, permissionKind: SPBasePermissionKind): boolean {
    let siteUrl = GetSiteUrl(siteUrlOrId);

    const res = GetJsonSync<{ d: { EffectiveBasePermissions: { High: number; Low: number; }; }; }>(GetListRestUrl(siteUrl, listIdOrTitle) + `/EffectiveBasePermissions`, null,
        { ...shortLocalCache });

    return new SPBasePermissions(res.result.d.EffectiveBasePermissions).has(permissionKind);
}

/** create a new column and try to add it to default view. Send either Title and Type, or SchemaXml. Create with SchemaXml also adds to all content types */
export async function CreateField(siteUrl: string, listIdOrTitle: string, options: {
    Title?: string;
    Type?: FieldTypes;
    Required?: boolean;
    Indexed?: boolean;
    SchemaXml?: string;
    /** requies Name and StaticName for the internal name */
    SchemaXmlSpecificInternalName?: boolean;
    SkipAddToDefaultView?: boolean;
    ClientSideComponentId?: string;
    ClientSideComponentProperties?: string;
    JSLink?: string;

}): Promise<IFieldInfoEX> {
    siteUrl = GetSiteUrl(siteUrl);

    let finish = async (result: IFieldInfo) => {
        if (!result) {
            return null;
        }

        let internalName = result.InternalName;
        //we need to clear and reload the list fields cache, so call it and return our field from that collection.
        let fields = await GetListFields(siteUrl, listIdOrTitle, { refreshCache: true });

        try {
            if (options.SkipAddToDefaultView !== true) {
                //try to add it to default view, don't wait for it
                GetListViews(siteUrl, listIdOrTitle).then(views => {
                    let defaultView = firstOrNull(views, v => v.DefaultView);
                    if (defaultView)
                        GetJson(GetListRestUrl(siteUrl, listIdOrTitle) + `/views('${defaultView.Id}')/ViewFields/addViewField('${internalName}')`, null, { method: "POST", spWebUrl: siteUrl });
                });
            }
        }
        catch (e) { }

        return firstOrNull(fields, f => f.InternalName === internalName);
    };

    if (!isNullOrEmptyString(options.SchemaXml)) {
        try {
            let updateObject: IDictionary<any> = {
                'parameters': {
                    '__metadata': { 'type': 'SP.XmlSchemaFieldCreationInformation' },
                    'SchemaXml': options.SchemaXml,
                    'Options': options.SchemaXmlSpecificInternalName !== true ?
                        4 ://SP.AddFieldOptions.addToAllContentTypes
                        4 | 8//SP.AddFieldOptions.addToAllContentTypes | addFieldInternalNameHint
                }
            };
            let url = `${GetListRestUrl(siteUrl, listIdOrTitle)}/fields/createFieldAsXml`;
            let newFieldResult = await GetJson<{ d: IFieldInfo; }>(url, JSON.stringify(updateObject));

            if (!isNullOrUndefined(newFieldResult)
                && !isNullOrUndefined(newFieldResult.d)) {
                if ((!isNullOrEmptyString(options.Title) && options.Title !== newFieldResult.d.Title)
                    || (isBoolean(options.Indexed) && options.Indexed !== newFieldResult.d.Indexed)) {
                    let updatedField = await UpdateField(siteUrl, listIdOrTitle, newFieldResult.d.InternalName, {
                        Title: options.Title,
                        Indexed: options.Indexed === true
                    });
                    return finish(updatedField);
                }
            }

            return finish(newFieldResult && newFieldResult.d);
        } catch {
        }
        return null;
    } else if (!isNullOrEmptyString(options.Title) && !isNullOrUndefined(options.Type)) {
        let updateObject: IDictionary<any> = {
            '__metadata': { 'type': 'SP.Field' },
            'Title': options.Title,
            'FieldTypeKind': options.Type,
            'Required': options.Required === true,
            'Indexed': options.Indexed === true
        };
        if (!isNullOrEmptyString(options.ClientSideComponentId)) {
            updateObject.ClientSideComponentId = options.ClientSideComponentId;
        }
        if (!isNullOrEmptyString(options.ClientSideComponentProperties)) {
            updateObject.ClientSideComponentProperties = options.ClientSideComponentProperties;
        }
        if (!isNullOrEmptyString(options.JSLink)) {
            updateObject.JSLink = options.JSLink;
        }

        try {
            let url = `${GetListRestUrl(siteUrl, listIdOrTitle)}/fields`;
            let newFieldResult = await GetJson<{ d: IFieldInfo; }>(url, JSON.stringify(updateObject));
            return finish(newFieldResult && newFieldResult.d);
        } catch {
        }
        return null;
    }
    else {
        console.error("You must send either SchemaXml or Title and Type");
        return null;
    }
}
/** Update field SchemaXml OR Title, only 1 update at a time supported. */
export async function UpdateField(siteUrlOrId: string, listIdOrTitle: string, fieldInternalName: string, options: {
    Title?: string;
    Indexed?: boolean;
    /** Update 'Choices' propertry on 'Choice' and 'MultiChoice' field types. */
    Choices?: string[];
    SchemaXml?: string;
    FieldType?: FieldTypeAsString;
    Required?: boolean;
    Hidden?: boolean;
    JSLink?: string;
    ClientSideComponentId?: string;
    ClientSideComponentProperties?: string;
}): Promise<IFieldInfoEX> {
    let siteUrl = GetSiteUrl(siteUrlOrId);

    let finish = async () => {
        //we need to clear and reload the list fields cache, so call it and return our field from that collection.
        let fields = await GetListFields(siteUrl, listIdOrTitle, { refreshCache: true });
        return firstOrNull(fields, f => f.InternalName === fieldInternalName);
    };

    let fields = await GetListFieldsAsHash(siteUrl, listIdOrTitle, true);
    let thisField = fields[fieldInternalName];

    //updates can either be SchemaXml, or others. Cannot be both.
    let updates: IDictionary<any> = {
        '__metadata': { 'type': 'SP.Field' }
    };

    if (!isNullOrEmptyString(options.SchemaXml)) {
        updates.SchemaXml = options.SchemaXml;
    }
    else {
        //cannot send schema updates with other updates.
        if (!isNullOrEmptyString(options.Title)) {
            updates.Title = options.Title;
        }
        if (!isNullOrEmptyString(options.FieldType)) {
            updates.TypeAsString = options.FieldType;
        }
        if (isBoolean(options.Required)) {
            updates.Required = options.Required === true;
        }
        if (isBoolean(options.Indexed)) {
            updates.Indexed = options.Indexed === true;
        }
        if (!isNullOrEmptyArray(options.Choices)) {
            let choiceType = options.FieldType || thisField.TypeAsString;
            if (choiceType === "Choice" || choiceType === "MultiChoice") {
                updates["__metadata"]["type"] = choiceType === "Choice" ? "SP.FieldChoice" : "SP.FieldMultiChoice"
                updates.Choices = { "results": options.Choices };
            } else {
                logger.warn("Can only update 'Choices' property on 'Choice' and 'MultiChoice' field types.");
            }
        }
        if (isBoolean(options.Hidden)) {
            //this requries the CanToggleHidden to be in the schema... if not - we will need to add it before we can update this.
            let fields = await GetListFieldsAsHash(siteUrl, listIdOrTitle, false);
            let thisField = fields[fieldInternalName];
            if (thisField.Hidden !== options.Hidden) {
                if (thisField) {
                    if (thisField.SchemaJson.Attributes.CanToggleHidden !== "TRUE") {
                        await UpdateField(siteUrl, listIdOrTitle, fieldInternalName, {
                            SchemaXml:
                                thisField.SchemaXml.replace("<Field ", `<Field CanToggleHidden="TRUE" `)
                        });
                    }
                }
                updates.Hidden = options.Hidden === true;
            }
        }

        if (!isNullOrEmptyString(options.ClientSideComponentId))
            updates.ClientSideComponentId = options.ClientSideComponentId;
        if (!isNullOrEmptyString(options.ClientSideComponentProperties))
            updates.ClientSideComponentProperties = options.ClientSideComponentProperties;
        if (!isNullOrEmptyString(options.JSLink))
            updates.JSLink = options.JSLink;
    }

    if (Object.keys(updates).length > 1) {
        return GetJson(GetListRestUrl(siteUrl, listIdOrTitle) + `/fields/getbyinternalnameortitle('${fieldInternalName}')`,
            JSON.stringify(updates), { xHttpMethod: "MERGE" })
            .then(r => {
                return finish();
            })
            .catch<IFieldInfoEX>(() => null);
    }
    else {
        console.error("You must send an option to update");
        return null;
    }
}

export async function ChangeTextFieldMode(
    siteUrlOrId: string,
    listIdOrTitle: string,
    textMode: "singleline" | "multiline" | "html",
    currentField: IFieldInfoEX
) {
    const newSchema = jsonClone(currentField.SchemaJson);
    const currentSchemaAttributes = newSchema.Attributes;

    switch (textMode) {
        case "singleline":
            let shouldIntermediateUpdate = false;

            if (currentSchemaAttributes.RichText === 'TRUE') {
                currentSchemaAttributes.RichText = 'FALSE';
                shouldIntermediateUpdate = true;
            };
            if (currentSchemaAttributes.RichTextMode === 'FullHTML') {
                currentSchemaAttributes.RichTextMode = 'Compatible';
                shouldIntermediateUpdate = true;
            };

            if (shouldIntermediateUpdate) {
                const intermediateSchema = SchemaJsonToXml(newSchema);
                const intermediateUpdatedField = await UpdateField(siteUrlOrId, listIdOrTitle, currentField.InternalName, {
                    SchemaXml: intermediateSchema
                });
                // Early exit if intermediate change failed.
                if (isNullOrUndefined(intermediateUpdatedField))
                    return false;
            };

            // Actual type update.
            currentSchemaAttributes.Type = 'Text';
            delete currentSchemaAttributes.RichTextMode;
            delete currentSchemaAttributes.RichText;
            break;
        case "multiline":
            currentSchemaAttributes.Type = 'Note';
            currentSchemaAttributes.RichText = 'FALSE';
            currentSchemaAttributes.RichTextMode = 'Compatible';
            break;
        case "html":
            currentSchemaAttributes.Type = 'Note';
            currentSchemaAttributes.RichText = 'TRUE';
            currentSchemaAttributes.RichTextMode = 'FullHTML';
            break;
    }

    const updatedSchema = SchemaJsonToXml(newSchema);
    const fieldUpdated = await UpdateField(siteUrlOrId, listIdOrTitle, currentField.InternalName, {
        SchemaXml: updatedSchema
    });

    // If object is null or undefined then request has failed.
    return !isNullOrUndefined(fieldUpdated);
}

export async function ChangeDatetimeFieldMode(
    siteUrlOrId: string,
    listIdOrTitle: string,
    includeTime: boolean,
    currentField: IFieldInfoEX
) {
    const dateTimeFormat = 'DateTime';
    const dateOnlyFormat = 'DateOnly';

    const newSchema = jsonClone(currentField.SchemaJson);
    const fieldAttributes = newSchema.Attributes;
    let needUpdate = false;
    if (includeTime && fieldAttributes.Format === dateOnlyFormat) {
        needUpdate = true;
        fieldAttributes.Format = dateTimeFormat;
    }
    else if (!includeTime && fieldAttributes.Format === dateTimeFormat) {
        needUpdate = true;
        fieldAttributes.Format = dateOnlyFormat;
    }

    if (needUpdate) {
        const updatedSchema = SchemaJsonToXml(newSchema);
        const updateResponse = await UpdateField(siteUrlOrId, listIdOrTitle, currentField.InternalName, {
            SchemaXml: updatedSchema
        });
        return !isNullOrUndefined(updateResponse);
    }

    // If an already existing format was chosen.
    return true;
}

export async function DeleteField(siteUrl: string, listIdOrTitle: string, fieldInternalName: string, options?: { DeleteHiddenField?: boolean; }): Promise<boolean> {
    siteUrl = GetSiteUrl(siteUrl);

    // let finish = async () => {
    //     //we need to clear and reload the list fields cache, so call it and return our field from that collection.
    //     let fields = await GetListFields(siteUrl, listIdOrTitle, { refreshCache: true });
    //     return firstOrNull(fields, f => f.InternalName === fieldInternalName);
    // };

    if (options && options.DeleteHiddenField)
        await UpdateField(siteUrl, listIdOrTitle, fieldInternalName, { Hidden: false });


    return GetJson(GetListRestUrl(siteUrl, listIdOrTitle) + `/fields/getbyinternalnameortitle('${fieldInternalName}')`, null, {
        method: "POST",
        xHttpMethod: "DELETE"
    })
        .then(r => true)
        .catch<boolean>((e) => false);
}

export interface IListViewOptions { includeViewFields?: boolean; }
export function GetListViews(siteUrl: string, listIdOrTitle: string, options?: IListViewOptions, refreshCache = false): Promise<iListView[]> {
    siteUrl = GetSiteUrl(siteUrl);

    return GetJson<{
        value: iListView[];
    }>(GetListRestUrl(siteUrl, listIdOrTitle) + `/views?$select=Title,Id,ServerRelativeUrl,RowLimit,Paged,ViewQuery,ListViewXml,PersonalView,MobileView,MobileDefaultView,Hidden,DefaultView,ReadOnlyView${options && options.includeViewFields ? "&$expand=ViewFields" : ""}`,
        null, { allowCache: refreshCache !== true, jsonMetadata: jsonTypes.nometadata })
        .then(r => {
            let views = r.value;
            if (isNotEmptyArray(views)) {
                views.forEach(v => {
                    v.Id = normalizeGuid(v.Id);
                    if (options && options.includeViewFields) {
                        v.ViewFields = v.ViewFields && v.ViewFields["Items"] && v.ViewFields["Items"] || [];
                    }
                });
            }
            return views;
        })
        .catch<iListView[]>(() => null);
}

export function GetListViewsSync(siteUrl: string, listIdOrTitle: string, refreshCache = false): iListView[] {
    siteUrl = GetSiteUrl(siteUrl);

    let result = GetJsonSync<{
        d: {
            results: iListView[];
        };
    }>(GetListRestUrl(siteUrl, listIdOrTitle) + `/views`,
        null, { allowCache: refreshCache !== true });
    if (result.success) {
        let views = result && result.result && result.result.d && result.result.d.results;
        if (isNotEmptyArray(views)) {
            views.forEach(v => { v.Id = normalizeGuid(v.Id); });
        }
        return views;
    }
    return null;
}

export async function AddViewFieldToListView(siteUrl: string, listIdOrTitle: string, viewId: string, viewField: string) {
    return _addOrRemoveViewField(siteUrl, listIdOrTitle, viewId, viewField, "addviewfield");
}

export async function RemoveViewFieldFromListView(siteUrl: string, listIdOrTitle: string, viewId: string, viewField: string) {
    return _addOrRemoveViewField(siteUrl, listIdOrTitle, viewId, viewField, "removeviewfield");
}

async function _addOrRemoveViewField(siteUrl: string, listIdOrTitle: string, viewId: string, viewField: string, action: "addviewfield" | "removeviewfield") {
    siteUrl = GetSiteUrl(siteUrl);

    if (isNullOrEmptyString(viewField) || !isValidGuid(viewId)) {
        return false;
    }

    let views = await GetListViews(siteUrl, listIdOrTitle, { includeViewFields: true });

    if (isNullOrEmptyArray(views)) {
        return false;
    }

    let view = views.filter((view) => {
        return normalizeGuid(view.Id) === normalizeGuid(viewId);
    })[0];

    if (isNullOrUndefined(view)) {
        return false;
    }

    let hasField = view.ViewFields.includes(viewField);

    if (action === "addviewfield" && hasField === true) {
        return true;
    }

    if (action === "removeviewfield" && hasField === false) {
        return true;
    }

    try {
        let url = GetListRestUrl(siteUrl, listIdOrTitle) + `/views('${normalizeGuid(view.Id)}')/viewfields/${action}('${viewField}')`;

        let result = await GetJson<{ "odata.null": boolean; }>(url, null, { method: "POST" });

        if (result && result["odata.null"] === true) {
            return true;
        }
    } catch { }

    return false;
}

export function GetListContentTypes(siteUrl: string, listIdOrTitle: string,
    options?: Omit<IGetContentTypesOptions, "listIdOrTitle" | "fromRooWeb">, refreshCache = false): Promise<iContentType[]> {
    return GetContentTypes(siteUrl, { ...(options || {}), listIdOrTitle: listIdOrTitle }, refreshCache);
}

export function GetListContentTypesSync(siteUrl: string, listIdOrTitle: string,
    options?: Omit<IGetContentTypesOptions, "listIdOrTitle" | "fromRooWeb">, refreshCache = false): iContentType[] {
    return GetContentTypesSync(siteUrl, { ...(options || {}), listIdOrTitle: listIdOrTitle }, refreshCache);
}

/** generic version. for the KWIZ forms version that supports action id call GetListFormUrlAppsWeb instead */
export function GetListFormUrl(siteUrl: string, listId: string, pageType: PageType, params?: { contentTypeId?: string; itemId?: number | string; rootFolder?: string }) {
    siteUrl = GetSiteUrl(siteUrl);

    if (!isValidGuid(listId)) console.error('GetListFormUrl requires a list id');
    let url = `${normalizeUrl(siteUrl)}/_layouts/15/listform.aspx?PageType=${pageType}&ListId=${encodeURIComponent(listId)}`;
    if (params) {
        if (!isNullOrEmptyString(params.contentTypeId))
            url += `&ContentTypeId=${encodeURIComponent(params.contentTypeId)}`;
        if (!isNullOrEmptyString(params.itemId))
            url += `&ID=${encodeURIComponent(params.itemId as string)}`;
        if (!isNullOrEmptyString(params.rootFolder))
            url += `&RootFolder=${encodeURIComponent(params.rootFolder)}`;
    }
    return url;
}

export function GetFieldSchemaSync(siteUrl: string, listIdOrTitle: string, fieldInternalName: string, refreshCache?: boolean): IFieldJsonSchema {
    siteUrl = GetSiteUrl(siteUrl);

    //ISSUE: 1516 - The get schema request will fail if the field doesn't exist in the list, so we load the fields and ensure the field
    //exists before requesting the schema.
    let fields = GetListFieldsSync(siteUrl, listIdOrTitle, {
        refreshCache: refreshCache,
        fieldNames: [fieldInternalName]
    });

    if (isNullOrEmptyArray(fields)) {
        return null;
    }

    let field = fields[0];
    return SchemaXmlToJson(field.SchemaXml);
    // let url = GetListRestUrl(siteUrl, listIdOrTitle) + `/fields/getByInternalNameOrTitle('${fieldInternalName}')?$select=SchemaXml`;
    // let result = GetJsonSync<{ d: { SchemaXml: string; }; }>(
    //     url,
    //     null,
    //     {
    //         ...shortLocalCache,
    //         forceCacheUpdate: refreshCache === true
    //     });

    // if (result && result.success) {
    //     return SchemaXmlToJson(result.result.d.SchemaXml);
    // }
    // return null;
    //#endregion
}

export async function GetFieldSchema(siteUrl: string, listIdOrTitle: string, fieldInternalName: string, refreshCache?: boolean) {
    siteUrl = GetSiteUrl(siteUrl);

    //ISSUE: 1516 - The get schema request will fail if the field doesn't exist in the list, so we load the fields and ensure the field
    //exists before requesting the schema
    let fields = await GetListFields(siteUrl, listIdOrTitle, {
        refreshCache: refreshCache,
        fieldNames: [fieldInternalName]
    });

    if (isNullOrEmptyArray(fields)) {
        return null;
    }

    let field = fields[0];
    return SchemaXmlToJson(field.SchemaXml);
}

export async function GetListItems(siteUrl: string, listIdOrTitle: string, options: {
    /** Optional, default: 1000. 0: get all items. */
    rowLimit?: number;
    /** Id, Title, Modified, FileLeafRef, FileDirRef, FileRef, FileSystemObjectType */
    columns: (string | IFieldInfoEX)[];
    foldersBehaviour?: GeListItemsFoldersBehaviour;
    /** Optional, request to expand some columns. */
    expand?: string[];
    /** allow to change the jsonMetadata for this request, default: verbose */
    jsonMetadata?: jsonTypes;
    refreshCache?: boolean;
    /** allow to send a filter statement */
    $filter?: string;
}): Promise<IRestItem[]> {
    let info = _GetListItemsInfo(siteUrl, listIdOrTitle, options);

    let items: IRestItem[] = [];

    do {
        let resultItems: IRestItem[] = [];
        let next: string = null;
        if (info.noMetadata) {
            let requestResult = (await GetJson<{
                value: IRestItem[];
                "odata.nextLink": string;
            }>(info.requestUrl, null, {
                allowCache: options.refreshCache !== true,
                jsonMetadata: options.jsonMetadata
            }));
            resultItems = requestResult.value;
            next = requestResult["odata.nextLink"];
        }
        else {
            let requestResult = (await GetJson<{
                d: {
                    results: IRestItem[];
                    __next?: string;
                };
            }>(info.requestUrl, null, {
                allowCache: options.refreshCache !== true
            }));
            resultItems = requestResult.d.results;
            next = requestResult.d.__next;
        }

        if (isNotEmptyArray(resultItems))
            items.push(...resultItems);

        if (info.totalNumberOfItemsToGet > items.length)
            info.requestUrl = next;
        else
            info.requestUrl = null;

    } while (!isNullOrEmptyString(info.requestUrl));

    return __fixGetListItemsResults(siteUrl, listIdOrTitle, items, options.foldersBehaviour, info.expandedLookupFields);
}

export function GetListItemsSync(siteUrl: string, listIdOrTitle: string, options: {
    /** Optional, default: 1000. 0: get all items. */
    rowLimit?: number;
    /** Id, Title, Modified, FileLeafRef, FileDirRef, FileRef, FileSystemObjectType */
    columns: (string | IFieldInfoEX)[];
    foldersBehaviour?: GeListItemsFoldersBehaviour;
    /** Optional, request to expand some columns. */
    expand?: string[];
    /** allow to send a filter statement */
    $filter?: string;
}): IRestItem[] {
    let info = _GetListItemsInfo(siteUrl, listIdOrTitle, options);

    let items: IRestItem[] = [];

    do {
        let resultItems: IRestItem[] = [];
        let next: string = null;
        if (info.noMetadata) {
            let requestResult = GetJsonSync<{
                value: IRestItem[];
                "odata.nextLink": string;
            }>(info.requestUrl, null, { allowCache: true });
            if (requestResult.success) {
                resultItems = requestResult.result.value;
                next = requestResult.result["odata.nextLink"];
            }
        }
        else {
            let requestResult = GetJsonSync<{
                d: { results: IRestItem[]; __next?: string; };
            }>(info.requestUrl, null, { allowCache: true });
            if (requestResult.success) {
                resultItems = requestResult.result.d.results;
                next = requestResult.result.d.__next;
            }
        }

        if (isNotEmptyArray(resultItems))
            items.push(...resultItems);

        if (info.totalNumberOfItemsToGet > items.length)
            info.requestUrl = next;
        else
            info.requestUrl = null;

    } while (!isNullOrEmptyString(info.requestUrl));

    return __fixGetListItemsResults(siteUrl, listIdOrTitle, items, options.foldersBehaviour, info.expandedLookupFields);
}

function _GetListItemsInfo(siteUrl: string, listIdOrTitle: string, options: {
    /** Optional, default: 1000. 0: get all items. */
    rowLimit?: number;
    /** Id, Title, Modified, FileLeafRef, FileDirRef, FileRef, FileSystemObjectType */
    columns: (string | IFieldInfoEX)[];
    /** Optional, request to expand some columns. */
    expand?: string[];
    /** allow to change the jsonMetadata for this request, default: verbose */
    jsonMetadata?: jsonTypes;
    /** allow to send a filter statement */
    $filter?: string;
}) {
    siteUrl = GetSiteUrl(siteUrl);

    let url = GetListRestUrl(siteUrl, listIdOrTitle) + `/items`;
    let queryParams: string[] = [];

    //Issue 8189 expand lookup fields
    let columns: string[] = [];
    let expand: string[] = [];
    let expandedLookupFields: IFieldInfoEX[] = [];
    options.columns.forEach(c => {
        if (isString(c)) columns.push(c);
        else {
            let internalName = c.InternalName;
            //Issue 828, 336
            if (internalName.startsWith("_")) internalName = `OData_${internalName}`;

            let isLookupField = c.TypeAsString === "Lookup" || c.TypeAsString === "LookupMulti";
            let isUserField = c.TypeAsString === "User" || c.TypeAsString === "UserMulti";

            if (isLookupField || isUserField) {
                //ISSUE: 1519 - Added lookupField property to able to retrieve value of the additional lookup field key
                let lookupField = (c as IFieldLookupInfo).LookupField;
                if (!isNullOrEmptyString(lookupField) && isLookupField) {
                    columns.push(`${internalName}/${lookupField}`);
                }
                //we want to expand it
                columns.push(`${internalName}/Title`);
                columns.push(`${internalName}/Id`);
                expand.push(internalName);
                expandedLookupFields.push(c);
            }
            else columns.push(internalName);
        }
    });
    if (isNotEmptyArray(options.expand)) {
        expand.push(...options.expand);
    }

    //add the ones we need
    PushNoDuplicate(columns, "Id");
    PushNoDuplicate(columns, "FileRef");
    PushNoDuplicate(columns, "FileSystemObjectType");

    queryParams.push(`$select=${encodeURIComponent(makeUniqueArray(columns).join(','))}`);

    if (isNotEmptyArray(expand))
        queryParams.push(`$expand=${encodeURIComponent(makeUniqueArray(expand).join(','))}`);

    let batchSize = 2000;
    let limit = options.rowLimit >= 0 && options.rowLimit < batchSize ? options.rowLimit : batchSize;
    let totalNumberOfItemsToGet = !isNumber(options.rowLimit) || options.rowLimit < 1 ? 99999 : options.rowLimit > batchSize ? options.rowLimit : limit;

    if (!isNullOrEmptyString(options.$filter))
        queryParams.push(`$filter=${options.$filter}`);
    queryParams.push(`$top=${limit}`);

    let requestUrl = url + (queryParams.length > 0 ? '?' + queryParams.join('&') : '');
    let noMetadata = options.jsonMetadata === jsonTypes.nometadata;

    return { requestUrl, noMetadata, totalNumberOfItemsToGet, expandedLookupFields };
}

/** Find an item by id, even if it is nested in a sub-folder */
export function FindListItemById(items: IRestItem[], itemId: number): IRestItem {
    for (let i = 0; i < items.length; i++) {
        let current = items[i];
        if (current.Id === itemId) return current;
        else if (isNotEmptyArray(current.__Items))//folder? look inside
        {
            let nestedResult = FindListItemById(current.__Items, itemId);
            if (!isNullOrUndefined(nestedResult)) return nestedResult;
        }
    }
    //not found
    return null;
}

function _getListEventReceiversRequestUrl(siteUrl: string, listIdOrTitle: string) {
    return GetListRestUrl(siteUrl, listIdOrTitle) + `/EventReceivers`
}

export async function GetListEventReceivers(siteUrl: string, listIdOrTitle: string, refreshCache?: boolean): Promise<ISPEventReceiver[]> {
    try {
        let url = _getListEventReceiversRequestUrl(siteUrl, listIdOrTitle);
        let response = await GetJson<{
            value: ISPEventReceiver[];
        }>(url,
            null, {
            allowCache: refreshCache !== true,
            jsonMetadata: jsonTypes.nometadata
        });

        return !isNullOrUndefined(response) ? response.value : null;
    } catch {
    }

    return null;
}

export async function AddListEventReceiver(siteUrl: string, listIdOrTitle: string, eventReceiverDefinition: Pick<ISPEventReceiver, "EventType" | "ReceiverName" | "ReceiverUrl" | "SequenceNumber">): Promise<ISPEventReceiver> {
    let newEventReceiver: Omit<ISPEventReceiver, "ReceiverId" | "Synchronization"> = {
        ReceiverAssembly: "",
        ReceiverClass: "",
        ...eventReceiverDefinition
    };

    try {
        let url = _getListEventReceiversRequestUrl(siteUrl, listIdOrTitle);
        let response = await GetJson<ISPEventReceiver>(url, JSON.stringify(newEventReceiver), {
            method: "POST",
            includeDigestInPost: true,
            jsonMetadata: jsonTypes.nometadata,
            headers: {
                "content-type": contentTypes.json
            }
        });

        return !isNullOrUndefined(response) && isValidGuid(response.ReceiverId) ? response : null;
    } catch {
    }

    return null;
}

export async function DeleteListEventReceiver(siteUrl: string, listIdOrTitle: string, eventReceiverId: string): Promise<boolean> {
    try {
        let url = `${_getListEventReceiversRequestUrl(siteUrl, listIdOrTitle)}('${normalizeGuid(eventReceiverId)}')/deleteObject`;
        let response = await GetJson<{ "odata.null": boolean }>(url, null, {
            method: "POST",
            includeDigestInPost: true,
            jsonMetadata: jsonTypes.nometadata
        });

        return !isNullOrUndefined(response) && response["odata.null"] === true;
    } catch {
    }

    return false;
}

/** timestamp of changes:
 * - item updates
 * - changes to columns
 * - content types
 * - list versioning settings
 * - list title/description
 * - content approval settings
 * does not track:
 * - Changes to views
 * - changing list/items permissions
 */
export function GetListLastItemModifiedDate(siteUrl: string, listIdOrTitle: string, options: {
    sync: true;
    refreshCache?: boolean;
    /** ignore system changes */
    userChangesOnly?: boolean;
}): string;
/** timestamp of changes:
 * - item updates
 * - changes to columns
 * - content types
 * - list versioning settings
 * - list title/description
 * - content approval settings
 * does not track:
 * - Changes to views
 * - changing list/items permissions
 */
export function GetListLastItemModifiedDate(siteUrl: string, listId