@kwiz/common
Version:
KWIZ common utilities and helpers for M365 platform
648 lines • 33.8 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.GetListItemsByCaml = void 0;
const _dependencies_1 = require("../../../_dependencies");
const collections_base_1 = require("../../../helpers/collections.base");
const sharepoint_1 = require("../../../helpers/sharepoint");
const typecheckers_1 = require("../../../helpers/typecheckers");
const url_1 = require("../../../helpers/url");
const rest_types_1 = require("../../../types/rest.types");
const consolelogger_1 = require("../../consolelogger");
const rest_1 = require("../../rest");
const common_1 = require("../common");
const list_1 = require("../list");
const web_1 = require("../web");
const GetListItemsById_1 = require("./GetListItemsById");
const common_2 = require("./common");
const logger = consolelogger_1.ConsoleLogger.get("sharepoint.rest/list/GetListItemsByCaml");
//Lookup threshold limit is 12 but we use a smaller number here to ensure there are no issues
const lookupOrUserFieldLimit = 11;
/** returns the items or NULL, never errors out since it is used in aggregator.
* set throwErrors = true if you want errors to be thrown instead of returning null.
* camlQuery: one of:
* - view.ViewQuery
* - wrapped view.ViewQuery in a <View>: "<View Scope='RecursiveAll'><Query></Query><RowLimit>1000</RowLimit></View>" or send rowLimit in options.
*/
async function GetListItemsByCaml(siteUrl, listIdOrTitle, camlQuery, options) {
siteUrl = (0, common_1.GetSiteUrl)(siteUrl);
if (!camlQuery.toLowerCase().startsWith("<view")) {
camlQuery = `<View Scope='RecursiveAll'><Query>${camlQuery}</Query></View>`;
}
let xmlParser = new DOMParser();
let xmlDoc = xmlParser.parseFromString(camlQuery, "text/xml");
let viewNode = xmlDoc.querySelector("View, view");
let queryNode = viewNode && viewNode.querySelector("Query, query");
let rowLimitNode = viewNode && viewNode.querySelector("RowLimit, rowlimit");
let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
let parseRowLimitNodeResult = _parseRowLimitNode(rowLimitNode, viewNode, options.rowLimit);
if (!(0, typecheckers_1.isNullOrUndefined)(parseRowLimitNodeResult)) {
options.rowLimit = parseRowLimitNodeResult.rowLimit;
camlQuery = parseRowLimitNodeResult.camlQuery;
}
let removeSingleAndConditionsResult = _removeSingleAndConditions(viewNode, queryNode);
if (!(0, typecheckers_1.isNullOrUndefined)(removeSingleAndConditionsResult)) {
camlQuery = removeSingleAndConditionsResult.camlQuery;
}
let getAllItems = (0, typecheckers_1.isNullOrEmptyString)(options.rowLimit) || options.rowLimit < 1;
let totalItemsInList = 99999;
let maxBatchSize = 5000;
let batchSize = options.rowLimit > 0 && options.rowLimit < maxBatchSize ? options.rowLimit : maxBatchSize;
let totalNumberOfItemsToGet = getAllItems ? totalItemsInList : options.rowLimit > maxBatchSize ? options.rowLimit : batchSize;
let requestUrl = `${(0, list_1.GetListRestUrl)(siteUrl, listIdOrTitle)}/GetItems?$expand=FieldValuesAsText`;
let allListFieldsHash = (0, collections_base_1.toHash)(await (0, list_1.GetListFields)(siteUrl, listIdOrTitle), f => f.InternalName);
let expandFields = [];
let orderByStatement = (0, sharepoint_1.GetOrderByFromCaml)(camlQuery);
if ((0, typecheckers_1.isNullOrUndefined)(options.columns)) {
options.columns = [];
}
let columns = options.columns;
columns = _ensureOrderByColumns(orderByStatement, columns);
columns = _ensureViewFields(viewFieldsNode, columns, allListFieldsHash);
columns = _normalizeColumns(columns, allListFieldsHash);
if (columns.length > options.columns.length) {
logger.warn(`added ${columns.length - options.columns.length} to query`);
}
options.columns = columns;
// Store the result of processing the request
let rtrnProcessRequestResult;
let lookupOrUserFieldsInColumns = options.columns.filter((columnName) => {
let field = allListFieldsHash[columnName];
return _isLookupOrUserField(field);
});
let isThrottled = lookupOrUserFieldsInColumns.length >= lookupOrUserFieldLimit;
try {
if (isThrottled) {
//ISSUE: 1565
rtrnProcessRequestResult = await _processLookupThresholdCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize);
}
else {
rtrnProcessRequestResult = await _processNormalCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize);
}
const { items, needContentTypes, postProcessOrderBy } = rtrnProcessRequestResult;
let itemsResult = (0, common_2.__fixGetListItemsResults)(siteUrl, listIdOrTitle, items, options.foldersBehaviour);
let itemsWithOutContentType = [];
if (needContentTypes) {
itemsResult.forEach((item) => {
if ((0, typecheckers_1.isNullOrUndefined)(item["ContentType"])) {
itemsWithOutContentType.push(item.Id);
}
else if (!(0, typecheckers_1.isNullOrUndefined)(item["ContentType"].Name)) {
item["ContentType"] = item["ContentType"].Name;
}
});
}
if (itemsWithOutContentType.length > 0) {
logger.time("Getting content types");
//Issue 1465 content types no longer come back from get items request...
//Make a separate request to get this info
let ctypes = (await (0, GetListItemsById_1.GetItemsById)(siteUrl, listIdOrTitle, itemsWithOutContentType, {
expand: ['ContentType/Name'],
select: ['ContentType/Name', 'Id'],
jsonMetadata: rest_types_1.jsonTypes.nometadata
}));
itemsResult.forEach(item => {
if (!(0, typecheckers_1.isNullOrUndefined)(ctypes[item.Id])) {
item["ContentType"] = ctypes[item.Id].ContentType.Name;
}
});
logger.timeEnd("Getting content types");
}
if (postProcessOrderBy) {
//re-apply sort
if (_dependencies_1.IsLocalDev) {
logger.table(itemsResult.map(i => {
let row = {
Id: i.Id,
Title: i.Title
};
orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
return row;
}), "before sort", true);
}
itemsResult.sort((a, b) => {
for (let i = 0; i < orderByStatement.length; i++) {
let ob = orderByStatement[i];
let v1 = a[ob.Name];
let v2 = b[ob.Name];
if (v1 === v2) {
//these are equal - continue to second sort statement
}
else {
if (ob.IsAscending)
return (v1 > v2) ? 1 : -1;
else
return (v2 > v1) ? 1 : -1;
}
}
return 0;
});
if (_dependencies_1.IsLocalDev) {
logger.table(itemsResult.map(i => {
let row = {
Id: i.Id,
Title: i.Title
};
orderByStatement.forEach(s => { row[`${s.Name} ${s.IsAscending ? 'asc' : 'desc'}`] = i[s.Name]; });
return row;
}), "after sort", true);
}
}
return itemsResult;
}
catch (ex) {
console.log(`isThrottled: ${isThrottled}`);
}
return null;
}
exports.GetListItemsByCaml = GetListItemsByCaml;
// (window as any).runtTest = true;
// window.setTimeout(() => {
// if ((window as any).runtTest == false) return;
// (window as any).runtTest = false;
// GetListItemsByCaml(null, "aec7756b-daa0-4da1-88ba-c66cb572e816", "", {
// columns: [],
// refreshCache: true
// }).then((r) => {
// });
// }, 500);
//ISSUE: 1565
async function _processLookupThresholdCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize) {
let rtrnProcessRequestResult = {
items: [],
postProcessOrderBy: false,
needContentTypes: false
};
let orderByFields = orderByStatement.map((orderByField) => {
let field = allListFieldsHash[orderByField.Name];
return field;
});
let lookupOrUserFieldsInOrderBy = orderByFields.filter((orderByField) => {
return _isLookupOrUserField(orderByField);
});
let lookupOrUserFieldsToChunk = options.columns.filter((columnName) => {
let field = allListFieldsHash[columnName];
if (_isLookupOrUserField(field)) {
let col = (0, collections_base_1.firstOrNull)(lookupOrUserFieldsInOrderBy, (orderByField) => {
return field.InternalName !== orderByField.InternalName;
});
return col === null;
}
return false;
});
let otherFieldNames = options.columns.filter((columnName) => {
let field = allListFieldsHash[columnName];
return !(0, typecheckers_1.isNullOrUndefined)(field) && !_isLookupOrUserField(field);
});
//The number of lookup columns in each request is based on the lookupOrUserFieldLimit.
//Lookup fields in the order by statement must be sent with each request.
//For example, we have 20 lookup columns and 3 of them are in the order by statement.
//The 3 order by lookup columns are sent with each request. The remaining lookups are split into chunks.
//The request will split into 3 requests
//The first request will have 11 lookup columns (8 standard + 3 order by lookup columns)
//The second request will have 11 lookup columns (8 standard + 3 order by lookup columns)
//The third request will have 4 lookup columns (1 standard + 3 order by lookup columns)
let requestChunks = (0, collections_base_1.chunkArray)(lookupOrUserFieldsToChunk, lookupOrUserFieldLimit - lookupOrUserFieldsInOrderBy.length);
let otherFieldsChunkSize = Math.ceil(otherFieldNames.length / requestChunks.length);
let otherFieldsChunks = (0, collections_base_1.chunkArray)(otherFieldNames, otherFieldsChunkSize);
requestChunks.forEach((chunk, index) => {
//Add all order by fields to each request, this will include the lookup
//fields from the order by statement that we left room for previously
requestChunks[index] = chunk.concat(orderByFields.map((orderByField) => {
return orderByField.InternalName;
}));
//Add the other fields but split them across the requests so we don't request duplicate data
if (otherFieldsChunks[index]) {
requestChunks[index] = chunk.concat(otherFieldsChunks[index]);
}
});
//requestChunks should now have about the same number of fields in each chunk and each
//chunk will have all the order by fields.
let queries = requestChunks.map((requestChunk) => {
let camlQueryClone = camlQuery;
let processColumnsResult = _processColumns(requestChunk, expandFields, allListFieldsHash);
//Id field must be included in oder to merge the items correctly
let viewFields = processColumnsResult.viewFields;
if (viewFields.length && !viewFields.includes("Id")) {
viewFields.push("Id");
}
let selectFields = processColumnsResult.selectFields;
if (selectFields.length && !selectFields.includes("Id")) {
selectFields.push("Id");
}
rtrnProcessRequestResult.needContentTypes = rtrnProcessRequestResult.needContentTypes || processColumnsResult.needContentTypes;
expandFields = processColumnsResult.expandFields;
if ((0, typecheckers_1.isNotEmptyArray)(viewFields)) {
camlQueryClone = (0, sharepoint_1.EnsureViewFields)(camlQueryClone, viewFields, true);
}
// if (isDebug()) {
// let lfields = viewFields.filter((fieldName) => {
// let field = allListFieldsHash[fieldName];
// return _isLookupOrUserField(field);
// });
// let xmlDoc = new DOMParser().parseFromString(camlQueryClone, "text/xml");
// let viewNode = xmlDoc.querySelector("View, view");
// let viewFieldsNode = viewNode && viewNode.querySelector("ViewFields, viewfields");
// let viewFields2 = Array.from(viewFieldsNode.children).map((viewFieldNode) => {
// let name = viewFieldNode.getAttribute("Name") || viewFieldNode.getAttribute("name");
// return name;
// });
// let lfields2 = viewFields2.filter((fieldName) => {
// let field = allListFieldsHash[fieldName];
// return _isLookupOrUserField(field);
// });
// if (lfields2.length !== lfields.length) {
// logger.warn("Lookup fields in caml query do not match look up fields in view fields.");
// logger.warn(`Lookup fields in caml query: ${lfields2}`)
// logger.warn(`Lookup fields in in view fields: ${lfields}`)
// }
// }
// Prepare the REST URL with select fields
let restUrlWithSelect = requestUrl;
if (expandFields.length > 0) {
restUrlWithSelect += ',' + expandFields.join(',');
}
// Include the lookup user select fields with the other select fields
restUrlWithSelect += '&$select=' + selectFields.join(',');
return { camlQueryClone, restUrlWithSelect, viewFields };
});
let mergedItems = null;
for (let i = 0; i < queries.length; i++) {
const { camlQueryClone, restUrlWithSelect, viewFields } = queries[i];
let loopResult = await processRequestResult(requestUrl, restUrlWithSelect, camlQueryClone, { ...options, columns: viewFields }, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
if ((0, typecheckers_1.isNullOrUndefined)(mergedItems)) {
mergedItems = loopResult.items;
}
else {
for (let restItemIndex = 0; restItemIndex < loopResult.items.length; restItemIndex++) {
//The item chunks (loopResult) should be in the same order as mergedItems because each request should have the
//same number of items returned in the same order. The only difference between requests is which fields are present.
//So request 1 will have the 1st set of fields, request 2 will have the 2nd set. etc.
//This means that the restItemIndex should be the same as existingItemIndex for each iteration. But, we do these extra
//checks just in case.
let restItem = loopResult.items[restItemIndex];
let existingItem = mergedItems[restItemIndex];
let existingItemIndex = restItemIndex;
if ((0, typecheckers_1.isNullOrUndefined)(existingItem) || existingItem.Id !== restItem.Id) {
existingItemIndex = (0, collections_base_1.firstIndexOf)(mergedItems, (mergedItem) => {
return mergedItem.Id === restItem.Id;
});
}
if (existingItemIndex === -1) {
//We shouldn't get here. Each chunk should have the same items in the same order.
logger.warn("_processLookupThresholdCamlRequest results are out of sync");
mergedItems.push(restItem);
}
else {
let existingItem = mergedItems[existingItemIndex];
let FieldValuesAsText = {
...(existingItem.FieldValuesAsText || {}),
...(restItem.FieldValuesAsText || {})
};
let FieldValuesForEdit = {
...(existingItem.FieldValuesForEdit || {}),
...(restItem.FieldValuesForEdit || {})
};
existingItem = { ...existingItem, ...restItem };
existingItem.FieldValuesAsText = FieldValuesAsText;
existingItem.FieldValuesForEdit = FieldValuesForEdit;
mergedItems[existingItemIndex] = existingItem;
}
}
}
// only need to put this true if it happens once
if (loopResult.postProcessOrderBy) {
rtrnProcessRequestResult.postProcessOrderBy = true;
}
}
rtrnProcessRequestResult.items = mergedItems;
return rtrnProcessRequestResult;
}
async function _processNormalCamlRequest(orderByStatement, allListFieldsHash, options, camlQuery, requestUrl, expandFields, siteUrl, totalNumberOfItemsToGet, batchSize) {
let processColumnsResult = _processColumns(options.columns, expandFields, allListFieldsHash);
let viewFields = processColumnsResult.viewFields;
let selectFields = processColumnsResult.selectFields;
expandFields = processColumnsResult.expandFields;
let needContentTypes = processColumnsResult.needContentTypes;
if ((0, typecheckers_1.isNotEmptyArray)(viewFields)) {
camlQuery = (0, sharepoint_1.EnsureViewFields)(camlQuery, viewFields, true);
}
// Prepare the REST URL with select fields
let restUrlWithSelect = requestUrl;
if (expandFields.length > 0) {
restUrlWithSelect += ',' + expandFields.join(',');
}
// Include the lookup user select fields with the other select fields
restUrlWithSelect += '&$select=' + selectFields.join(',');
// Process the request and get the result
let result = await processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement);
return {
...result,
needContentTypes
};
}
function _ensureViewFields(viewFieldsNode, columns, allListFieldsHash) {
//ISSUE: 1565
//Cases
//1. Empty view fields element so ALL columns will be requested. We must include all list fields in
//columns parsing so that we get an accurate count of how many lookup/user fields there are
//2. View fields element with a some field names that must also be included in the column parsing so we add them here.
if ((0, typecheckers_1.isNullOrUndefined)(viewFieldsNode) && (0, typecheckers_1.isNullOrEmptyArray)(columns)) {
Object.keys(allListFieldsHash).forEach((fieldName) => {
let field = allListFieldsHash[fieldName];
if (!(0, typecheckers_1.isNullOrUndefined)(field)
&& !field.Hidden
&& !common_2.SkipFields.includes(fieldName.toLowerCase())) {
columns.push(field.InternalName);
}
});
}
else if (!(0, typecheckers_1.isNullOrUndefined)(viewFieldsNode)) {
let fieldRefNodes = Array.from(viewFieldsNode.querySelectorAll("FieldRef"));
if ((0, typecheckers_1.isNotEmptyArray)(fieldRefNodes)) {
fieldRefNodes.forEach((fieldRefNode) => {
let name = fieldRefNode.getAttribute("Name") || fieldRefNode.getAttribute("name");
if (!(0, typecheckers_1.isNullOrEmptyString)(name)) {
columns.push(name);
}
});
}
}
return columns;
}
function _ensureOrderByColumns(orderByStatement, columns) {
//Issue 548: Ensure that the order by field is in the view fields and columns collection so
//that paging works correctly when there is an order by clause
if (orderByStatement.length > 0 && !(0, typecheckers_1.isNullOrEmptyString)(orderByStatement[0].Name)) {
// just add them to columns, and we will add them to view fields below.
// camlQuery = EnsureViewFields(camlQuery, orderByStatement.orderBy.map(o => o.Name), false);
orderByStatement.forEach(o => {
if (columns.indexOf(o.Name) === -1) {
columns.push(o.Name);
}
});
}
return columns;
}
function _normalizeColumns(columns, allListFieldsHash) {
return (0, collections_base_1.makeUniqueArray)(columns.map((column) => {
//some columns will come in lower case (from dvp), normalize them so that we can remove duplicates
let name = Object.keys(allListFieldsHash).filter((fieldName) => {
return fieldName.toLowerCase() === column.toLowerCase();
})[0];
if (!(0, typecheckers_1.isNullOrEmptyString)(name)) {
let field = allListFieldsHash[name];
return field;
}
return null;
}).filter((field) => {
return !(0, typecheckers_1.isNullOrUndefined)(field);
}).map((field) => {
return field.InternalName;
}).concat(['Title', 'Id', 'FileLeafRef', 'FileDirRef', 'FileRef', 'FileSystemObjectType']));
}
function _removeSingleAndConditions(viewNode, queryNode) {
if (!(0, typecheckers_1.isNullOrUndefined)(viewNode.querySelector("And, and"))) {
try {
//Issue 8063: calendar list, will wrongly add a wrapping <and> statement for a single condition
//will result in error 500
//ie: <View Scope='RecursiveAll'><Query><Where><And><Eq><FieldRef Name=\"Title\" /><Value Type=\"Text\">Holiday</Value></Eq></And></Where>
//</Query><RowLimit>5000</RowLimit></View>
let whereNode = queryNode && queryNode.querySelector("Where, where");
if (!(0, typecheckers_1.isNullOrUndefined)(whereNode)) {
let firstCondition = whereNode.firstElementChild;
if (firstCondition.tagName.toLowerCase() === "and" && firstCondition.children.length < 2) {
//this is the bug, <and> tag must have 2 conditions - get rid of it!
whereNode.innerHTML = firstCondition.innerHTML;
return {
camlQuery: viewNode.outerHTML
};
}
}
}
catch (e) { }
}
return null;
}
function _parseRowLimitNode(rowLimitNode, viewNode, rowLimit) {
if (!(0, typecheckers_1.isNullOrUndefined)(rowLimitNode)) {
let value = rowLimitNode.textContent;
//if not provided by options - use it
if ((0, typecheckers_1.isNullOrNaN)(rowLimit) && (0, typecheckers_1.isNumeric)(value) && Number(value) > 0) {
rowLimit = Number(value);
}
//remove it
viewNode.removeChild(rowLimitNode);
return {
camlQuery: viewNode.outerHTML,
rowLimit: rowLimit
};
}
return null;
}
function _processColumns(columns, expandFields, allListFieldsHash) {
let thresholdLimitLookupCount = 0;
let thresholdLimitLookupHit = false;
let selectFields = [];
let viewFields = [];
let needContentTypes = false;
//column parsing
columns.forEach(viewField => {
let viewFieldLower = viewField.toLowerCase();
if (viewFieldLower === 'contenttype' || viewFieldLower === 'contenttypeid') {
needContentTypes = true;
}
else if (viewFieldLower === '_moderationstatus') {
selectFields.push(`FieldValuesAsText/${viewField}`);
viewFields.push('_moderationstatus');
}
else if (viewFieldLower === '_moderationcomments') {
selectFields.push('OData_' + viewField);
viewFields.push('_moderationcomments');
}
else if (viewFieldLower === "filesystemobjecttype") {
selectFields.push("FileSystemObjectType");
}
else if (viewFieldLower === "fileref" || viewFieldLower === "filedirref") {
//treat them similar to lookup fields
selectFields.push(viewField);
selectFields.push(`FieldValuesAsText/${viewField}`);
viewFields.push(`${viewField}`);
viewFields.push(`${viewField}Id`);
}
else {
//prefer to get columns not from FieldValuesAsText, unless special data type requires it (date, lookup, boolean, etc...)
//make the select url shorter
let foundField = allListFieldsHash[viewField];
if (foundField) {
let foundFieldInternalName = foundField.InternalName;
viewFields.push(foundFieldInternalName);
let urlField = null;
if (foundField.TypeAsString === "URL" && foundField.InternalName === "URL") {
urlField = foundField;
}
else if ((viewFieldLower === "urlnomenu"
|| viewFieldLower === "urlwmenu2"
|| viewFieldLower === "urlwmenu")
&& foundField.TypeAsString === "Computed"
&& !(0, typecheckers_1.isNullOrUndefined)(allListFieldsHash["URL"])
&& allListFieldsHash["URL"].TypeAsString === "URL") {
urlField = allListFieldsHash["URL"];
}
if (!(0, typecheckers_1.isNullOrUndefined)(urlField) && urlField.TypeAsString === "URL") {
viewFields.push(urlField.InternalName);
selectFields.push(urlField.InternalName);
selectFields.push(`FieldValuesAsText/${urlField.InternalName}`);
}
else {
//Issue 828, 336
if (foundFieldInternalName.startsWith("_")) {
foundFieldInternalName = `OData_${foundFieldInternalName}`;
}
let outputType = (0, sharepoint_1.getFieldOutputType)(foundField);
switch (outputType) {
case "Lookup":
case "LookupMulti":
case "User":
case "UserMulti":
thresholdLimitLookupCount += 1;
if (thresholdLimitLookupCount >= lookupOrUserFieldLimit) {
thresholdLimitLookupHit = true;
}
//lookup raw values comes with Id appended
selectFields.push(`FieldValuesAsText/${foundFieldInternalName}`);
selectFields.push(`${foundFieldInternalName}Id`);
break;
case "Boolean":
case "Attachments":
case "AllDayEvent":
case "Recurrence":
selectFields.push(`FieldValuesAsText/${foundFieldInternalName}`);
selectFields.push(foundFieldInternalName);
break;
default:
selectFields.push(foundFieldInternalName);
break;
}
}
}
else {
selectFields.push(viewField);
}
}
});
if (needContentTypes) {
expandFields.push("ContentType");
selectFields.push("ContentType/Name");
selectFields.push("ContentTypeId");
viewFields.push("ContentTypeId");
}
return {
thresholdLimitLookupHit,
viewFields: (0, collections_base_1.makeUniqueArray)(viewFields),
selectFields: (0, collections_base_1.makeUniqueArray)(selectFields),
expandFields: (0, collections_base_1.makeUniqueArray)(expandFields),
needContentTypes
};
}
function _isLookupOrUserField(field) {
if ((0, typecheckers_1.isNullOrUndefined)(field)) {
return false;
}
let outputType = (0, sharepoint_1.getFieldOutputType)(field);
switch (outputType) {
case "Lookup":
case "LookupMulti":
case "User":
case "UserMulti":
return true;
default:
return false;
}
}
async function processRequestResult(requestUrl, restUrlWithSelect, camlQuery, options, siteUrl, totalNumberOfItemsToGet, batchSize, orderByStatement) {
//issue 6150: if there are too many fields, url will be too long. just get all columns without adding a $select
if (restUrlWithSelect.length < 2000)
requestUrl = restUrlWithSelect;
let query = { ViewXml: camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`) };
if (!(0, typecheckers_1.isNullOrEmptyString)(options.FolderServerRelativeUrl))
query.FolderServerRelativeUrl = options.FolderServerRelativeUrl;
let data = { query: query };
let items = [];
//let triedWithoutViewFields = false;
let postProcessOrderBy = false;
do {
try {
let requestResult = await (0, rest_1.GetJson)(requestUrl, JSON.stringify(data), {
allowCache: options.refreshCache !== true,
postCacheKey: JSON.stringify(data.query),
jsonMetadata: rest_types_1.jsonTypes.nometadata,
spWebUrl: siteUrl
});
if (requestResult && requestResult.value)
items.push(...requestResult.value);
let itemsLeftToGet = totalNumberOfItemsToGet - items.length;
if (itemsLeftToGet > 0 && //we need more items
requestResult.value.length === batchSize //we might have more on server since we got the full batch size we asked for
) {
let lastItem = items[items.length - 1];
let lastItemId = lastItem.Id;
let pagingInfoSort = "";
//Issue 7542 need to add order by value to the paging info if it is in the query
if (!postProcessOrderBy && orderByStatement && orderByStatement.length) {
let orderFieldName = orderByStatement[0].Name;
let orderFieldValue = lastItem[orderFieldName];
//if field is ID - do not do it.
if (orderFieldName.toLowerCase() !== "id" && !(0, typecheckers_1.isNullOrUndefined)(orderFieldValue)) {
//if value is date - we need it in ISO and in this format: yyyyMMdd HH:mm:ss
try {
//Numbers cast to date properly but they are not dates. ignore number values.
let orderFieldValueAsDate = (0, typecheckers_1.isNumeric)(orderFieldValue) ? null : new Date(orderFieldValue);
if ((0, typecheckers_1.isDate)(orderFieldValueAsDate)) {
try {
//issue 7599 date only field on different time zone...
orderFieldValue = (0, web_1.SPServerLocalTimeToUTCSync)(siteUrl, orderFieldValueAsDate).replace(/T/i, " ").replace(/Z/i, " ");
}
catch (e) { }
}
}
catch (e) { }
pagingInfoSort = `&p_${orderFieldName}=${(0, url_1.encodeURIComponentEX)(orderFieldValue)}`;
}
}
data.query.ListItemCollectionPosition = {
"PagingInfo": `Paged=TRUE${pagingInfoSort}&p_ID=${lastItemId}`
};
if (itemsLeftToGet < batchSize) //the last batch should be smaller, update the row limit
data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${itemsLeftToGet}</RowLimit></View>`);
}
else
data = null;
}
catch (e) {
if ((0, common_1.__getSPRestErrorData)(e).code.indexOf('SPQueryThrottledException')) {
//test again - on awaiting score view, this will work but will ONLY return the order by fields...
// if (!triedWithoutViewFields) {
// //Our own issues list had too many lookup fields.
// logger.info("Query throttled, trying again without view fields... some fields might be missing from results.");
// triedWithoutViewFields = true;
// camlQuery = EnsureViewFields(camlQuery, orderByStatement.map(o => o.Name), false, true);
// data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
// }
// else
if (!postProcessOrderBy && orderByStatement.length > 0) {
logger.warn("Query throttled, trying again without order by...");
postProcessOrderBy = true;
camlQuery = (0, sharepoint_1.RemoveOrderByFromCaml)(camlQuery);
camlQuery = (0, sharepoint_1.EnsureViewFields)(camlQuery, [], false, true);
data.query.ViewXml = camlQuery.replace("</View>", `<RowLimit>${batchSize}</RowLimit></View>`);
}
else
throw e; //still throttled? might be due to filter query
}
else
throw e; //different error
}
} while (!(0, typecheckers_1.isNullOrEmptyString)(data));
return { items: items, postProcessOrderBy: postProcessOrderBy };
}
//# sourceMappingURL=GetListItemsByCaml.js.map