@arcgis/core/layers/graphics/sources/support/ParquetSourceWorker.js

ArcGIS Maps SDK for JavaScript: A complete 2D and 3D mapping and data visualization API

/* COPYRIGHT Esri - https://js.arcgis.com/5.0.19/LICENSE.txt */
import{__addDisposableResource as e,__disposeResources as t}from"tslib";import r from"../../../../core/Collection.js";import{toConst as s}from"../../../../core/compilerUtils.js";import i from"../../../../core/Error.js";import has from"../../../../core/has.js";import o from"../../../../core/Logger.js";import{onAbortOrThrow as a,throwIfAborted as n}from"../../../../core/promiseUtils.js";import{QueueProcessor as u}from"../../../../core/QueueProcessor.js";import{parseWhereClause as l}from"../../../../core/sql.js";import p from"../../../../geometry/SpatialReference.js";import{set as c,create as d,expandWithRect as f,expandWithAABB as m,negativeInfinity as y,fromRect as h}from"../../../../geometry/support/aaBoundingBox.js";import{getQueryResultExtent as g,QueryEngine as w}from"../../data/QueryEngine.js";import{QueryEngineResult as _}from"../../data/QueryEngineResult.js";import{normalizeQueryLike as F}from"../../data/queryUtils.js";import{createDrawingInfo as I}from"./clientSideDefaults.js";import{getParquetFileId as x,getParquetRowId as R}from"./parquetIdUtils.js";import S from"../../../support/Field.js";import q from"../../../support/FieldsIndex.js";import{fromParquetGeometryEncodingJSON as C}from"../../../support/parquetEncodingUtils.js";import{completeParquetLayerInfo as v,fromParquetJSONGeometryType as b,toParquetJSONGeometryType as E}from"../../../support/parquetUtils.js";import{loadParquetModule as O}from"../../../../libs/parquet/loadParquetModule.js";import{createParquetFile as Q,readGeoMetadata as j}from"../../../../libs/parquet/parquet.js";import{FeatureStoreQueryAdapter as T}from"../../../../views/2d/layers/features/FeatureStoreQueryAdapter.js";import{FeatureSnapshotSourceChunk as P}from"../../../../views/2d/layers/features/sources/strategies/chunks/FeatureSnapshotSourceChunk.js";import{FeatureSourceChunkStore as A}from"../../../../views/2d/layers/features/sources/strategies/chunks/SourceChunkStore.js";import{FeatureMetadata as M}from"../../../../views/2d/layers/features/support/FeatureMetadata.js";import{FeatureSetReaderParquet as G}from"../../../../views/2d/layers/features/support/FeatureSetReaderParquet.js";const B=new T,D=4,z="__OBJECTID",W=new TextDecoder;class U{constructor(){this._fileInfos=new Map,this._queue=new u({concurrency:D,process:(e,t)=>this._executeQuery(e,t)}),this._indexMap={}}async load(s){const a=s.spatialReference?p.fromJSON(s.spatialReference):void 0;if(a&&!a.isWGS84&&!a.isWebMercator)throw new i("parquet:unsupported-projection","Only WGS84 and Web Mercator are supported");const n=await v({urls:new r(s.urls),fields:s.fields?.map(e=>S.fromJSON(e)),geometryEncoding:s.geometryEncoding?C(s.geometryEncoding):null,geometryType:s.geometryType?b(s.geometryType):null,displayOptimization:s.displayOptimization,spatialReference:a},{customParameters:s.customParameters});if(!n.fields)throw new i("parquet:unsupported","Fields must be defined");let u;if(n.spatialReference&&n.geometryType){if(!n.spatialReference)throw new i("parquet:unsupported","SpatialReference must be defined");if(!n.spatialReference.isGeographic&&!n.spatialReference.isWebMercator)throw new i("parquet:unsupported-projection","Only WGS84 and Web Mercator are supported");n.spatialReference.isGeographic&&!n.spatialReference.isWGS84&&(o.getLogger("parquet:unsupported-projection").warn("Found a geographic projection that is not WGS84. Handling as WGS84.",{spatialReference:n.spatialReference}),n.spatialReference=p.WGS84),u={geometry:n.geometryEncoding?{geometryType:E(n.geometryType),spatialReference:n.spatialReference.toJSON(),encoding:n.geometryEncoding.toJSON()}:null,displayOptimization:n.displayOptimization}}this.setCustomParameters(s.customParameters),this._geometryInfo=u;const l=s.urls;for(const e of l)this._addFile(e);this._capabilities=H(await this.getFileStatistics());const m=this._fileInfos.values().next().value;if(!m)return{layerDefinition:{},capabilities:H(null)};const h=await m,{fields:g}=n;if(null==g)throw new i("parquet-layer:missing-metadata","Unable to create parquet source: cannot infer fields",g);g.push(new S({name:z,type:"oid",alias:z}));for(const e of g){const t=h.file.columnForFieldName(e.name);null!=t&&(this._indexMap[e.name]=t)}const w=new q(g.map(e=>e.toJSON()));this._fieldsIndex=w;const _=E(n.geometryType??"point");if(this._metadata=M.createFeature({fieldsIndex:w.toJSON(),geometryType:_,featureIdInfo:{type:"object-id",fieldName:"rowId"},subtypes:null,subtypeField:null,types:null,typeIdField:null,globalIdField:null,spatialReference:n.spatialReference,outSpatialReference:null,timeInfo:null,timeReferenceUnknownClient:null,dateFieldsTimeZone:null}),this._queryEngineParams={fieldsIndex:this._metadata.fieldsIndex,geometryType:_,featureIdInfo:{type:"object-id",fieldName:"rowId"},hasM:!1,hasZ:!1,spatialReference:n.spatialReference?.toJSON()??{wkid:4326},aggregateAdapter:null,timeInfo:null,definitionExpression:null},n.spatialReference){const e=await this.getFileInfos();this._fullExtent=V(e,n.spatialReference.toJSON())}if(null==this._fullExtent&&"location"===n.geometryEncoding?.type){const{xField:r,yField:s}=n.geometryEncoding,i=c(d(),y);for(const o of await this.getFileInfos())for(const a of o.file.rowGroups()){const o={stack:[],error:void 0,hasError:!1};try{const t=e(o,a.columnDescriptorForAttribute(r),!1),n=e(o,a.columnDescriptorForAttribute(s),!1),u=[t.minValue(),n.minValue(),t.maxValue(),n.maxValue()];f(i,u),a.free()}catch(F){o.error=F,o.hasError=!0}finally{t(o)}}this._fullExtent={xmin:i[0],ymin:i[1],xmax:i[3],ymax:i[4],spatialReference:n.spatialReference?.toJSON()}}return{capabilities:this._capabilities,layerDefinition:{fields:n.fields?.map(e=>e.toJSON()),drawingInfo:I(_),extent:this._fullExtent??void 0,geometryType:_,geometryEncoding:n.geometryEncoding?.toJSON(),displayOptimization:n.displayOptimization}}}destroy(){for(const e of this._fileInfos.values())e.then(e=>e.file.free);this._fileInfos.clear(),this._queue.destroy()}setCustomParameters(e){this._customParameters=e}getIndexMap(){return this._indexMap}async getFileId(e){const t=this._fileInfos.get(e);if(!t)throw new Error(`InternalError: File ${e} does not exist`);return(await t).id}async getFileInfo(e){const t=this._fileInfos.get(e);if(!t)throw new Error(`InternalError: File ${e} does not exist`);return t}async getFileInfos(){return Promise.all(Array.from(this._fileInfos.values()))}async getFileStatistics(){if(!this._fileInfos.size)return null;const e=(await this.getFileInfos()).reduce((e,t)=>e+t.file.byteLength(),0);return{featureCount:await this._getFeatureCount(),byteLength:e}}async updateFiles(e){const t=new Set(e);for(const[r,s]of this._fileInfos.entries())t.has(r)?t.delete(r):(s.then(e=>e.file.free()),this._fileInfos.delete(r));await Promise.all(Array.from(t.values()).map(e=>this._addFile(e)))}async queryFeatures(e,t){this._validateQuery(e),Z(e)||(e.resultRecordCount=e.resultRecordCount?Math.min(e.resultRecordCount,8e3):8e3,e.resultOffset=e.resultOffset??0),(e.outStatistics||e.returnDistinctValues)&&(e.returnGeometry=void 0);return(await this._enqueueQuery(e,t)).createQueryResponse()}async queryFeatureCount(e,t){const r=await F(e,null,this._queryEngineParams.spatialReference);if(this._validateQuery(r),!N(r))return this._getFeatureCount();r.outFields=void 0,r.returnGeometry=void 0;return(await this._enqueueQuery(r,t)).createQueryResponseForCount()}async queryObjectIds(e,t){const r=await F(e,null,this._queryEngineParams.spatialReference);if(this._validateQuery(r),!N(r))return Array.from({length:await this._getFeatureCount()},(e,t)=>t);r.resultRecordCount=r.resultRecordCount?Math.min(r.resultRecordCount,8e3):8e3,r.resultOffset=r.resultOffset??0,r.returnGeometry=void 0,r.outFields=void 0;return(await this._enqueueQuery(r,t)).items.map(e=>e.getObjectId())}async queryExtent(e,t){const r=await F(e,null,this._queryEngineParams.spatialReference);if(this._validateQuery(r),this._fullExtent&&!N(r))return{count:await this._getFeatureCount(),extent:this._fullExtent};const i=s(this._metadata.spatialReference);r.returnGeometry=!0,r.outFields=void 0;const o=c(d(),y),a=d(),n=await this._enqueueQuery(r,t);let u=0;for(const s of n.items)s.getBounds(a)&&(m(o,a),u+=1);return{count:u,extent:g(o,i,r.outSR?s(r.outSR):i,i,!1)}}async queryStream(e,t,r){if(!r.signal)throw new Error("InternalError: AbortSignal must be passed");const s=await this.getFileInfo(e),i=(await O()).Query.new();"tile"===t.type&&(i.setExtent(t.extent),i.setQuantizationTransform(t.transform),i.setScale(t.scale)),i.setOutFields(t.outFields.filter(e=>null!=this._indexMap[e])),i.setOutSpatialReference(t.outSpatialReference.wkid),i.setReturnGeometry(!0),t.where&&await this._setWhereClause(i,s.file,t.where);const o=await s.file.executeQuery(i,r.signal),n=s.streamIdCounter++,u=a(r.signal,e=>{const t=s.streams.get(n);null!=t&&(t.handle.remove(),s.streams.delete(n))});return s.streams.set(n,{stream:o,handle:u}),n}async getStreamNext(e,t,r){if(!r.signal)throw new Error("InternalError: AbortSignal must be passed");const s=await this.getFileInfo(e),i=s.streams.get(t);if(!i)return null;const o=await i.stream.next(r.signal);return null==o&&(i.handle.remove(),s.streams.delete(t)),o?.serialize().buffer}async createPatch(e,t,r,s,i){if(!i.signal)throw new Error("InternalError: AbortSignal must be passed");const o=await this.getFileInfo(e);return(await o.file.createChunkPatch(t,r,s,i.signal)).serialize().buffer}async _getFile(e){for(const t of await this.getFileInfos())if(t.id===e)return t.file;throw new Error(`InternalError: File ${e} does not exist`)}_addFile(e){const t=this._fileInfos.size;this._fileInfos.set(e,this._createFileInfo(e,t))}async _createFileInfo(e,t){return{id:t,file:await Q(e,{geometryInfo:this._geometryInfo,getCustomParameters:()=>this._customParameters}),streamIdCounter:0,streams:new Map}}async _getFeatureCount(){return(await this.getFileInfos()).reduce((e,t)=>e+t.file.numRows(),0)}_validateQuery(e){if(!this._capabilities.query.supportsStatistics&&e.outStatistics)throw new i("parquet:unsupported","Statistics queries are not supported",{query:e});if(!this._capabilities.query.supportsOrderBy&&e.orderByFields?.length)throw new i("parquet:unsupported","Queries using orderBy are not supported",{query:e});if(!this._capabilities.query.supportsDistinct&&e.returnDistinctValues)throw new i("parquet:unsupported","Queries using returnDistinctValues are not supported",{query:e})}async _setWhereClause(e,t,r){const s=this._indexMap,o=this._fieldsIndex,a={getAttribute(e,r){const i=t.readAttribute(e.rowGroup,e.row,s[r]),a=o.get(r);return"esriFieldTypeString"===a.type||"esriFieldTypeDateOnly"===a.type||"esriFieldTypeTimeOnly"===a.type||"esriFieldTypeTimestampOffset"===a.type?W.decode(i):i}},n=await l(r,this._fieldsIndex);if(!n.isStandardized)throw new i("sql-parse-error","expression is not standardized");const u=(e,t)=>n.testFeatureCompiled({rowGroup:e,row:t},a,null);e.setWhere(r),e.setWhereEvaluator(u),e.setWhereFields(n.fieldNames)}async*_fetchChunks(e,t){const r=await O();for(const s of await this.getFileInfos()){const i=r.Query.new();i.setOutFields(e.fields),i.setReturnGeometry(e.returnGeometry),e.where&&await this._setWhereClause(i,s.file,e.where);const o=await s.file.executeQuery(i),a=[],u=o.next(t);for(let e=0;e<D;e++)a.push(o.next(t));let l=await u;for(;null!=l;){n(t);const e=new G(this._metadata,this._indexMap,l,s.id),r=k([new P(e,null,0,!1)],this._queryEngineParams),i=a.shift();a.push(o.next(t)),yield r,l=await i}}}_enqueueQuery(e,t){return this._queue.push(e,t)}async _executeQuery(e,t){const r=await this._getReadParams(e);if(e.where=void 0,e.objectIds?.length){const r=new Map;for(const t of e.objectIds){const e=x(t),s=R(t);let i=r.get(e);i||(i=[],r.set(e,i)),i.push(s)}const s=[];for(const[i,o]of r.entries()){const r=await this._executeFileIdQuery(e,i,o,t);for(const e of r)s.push(e)}return new _(s,e,{fieldsIndex:this._fieldsIndex,geometryType:this._metadata.geometryType,spatialReference:this._queryEngineParams.spatialReference,objectIdField:"rowId",hasM:!1,hasZ:!1,featureAdapter:B})}let s=e.resultRecordCount??await this._getFeatureCount(),i=e.resultOffset??0;e.resultRecordCount=void 0,e.resultOffset=void 0;const o=[];for await(const a of this._fetchChunks(r,t)){const r=await a.executeQueryForOpaqueFeatures(e,t);if(r.length>i){const t=r.slice(i,Math.min(i+s,r.length));for(const e of t)o.push(e);if(i=0,s-=t.length,0===s)return new _(o,e,{fieldsIndex:this._fieldsIndex,geometryType:this._metadata.geometryType,spatialReference:this._queryEngineParams.spatialReference,objectIdField:"rowId",hasM:!1,hasZ:!1,featureAdapter:B})}else i-=r.length}return new _(o,e,{fieldsIndex:this._fieldsIndex,geometryType:this._metadata.geometryType,spatialReference:this._queryEngineParams.spatialReference,objectIdField:"rowId",hasM:!1,hasZ:!1,featureAdapter:B})}async _executeFileIdQuery(e,t,r,s){const i=await this._getReadParams(e),o=(await O()).Query.new();o.setOutFields(i.fields),o.setReturnGeometry(i.returnGeometry),o.setIds(new Uint32Array(r));const a=await this._getFile(t),n=await a.executeQuery(o,s);let u=await n.next(s),l=0;const p=[];for(;null!=u;){const e=new G(this._metadata,this._indexMap,u,t),r=new P(e,null,l++,!1);p.push(r),u=await n.next(s)}return k(p,this._queryEngineParams).executeQueryForOpaqueFeatures(e,s)}async _getReadParams(e){const t=new Set;if(e.outStatistics)for(const r of e.outStatistics)null!=r.onStatisticField&&t.add(r.onStatisticField);if(e.outFields)for(const r of e.outFields)t.add(r);return{fields:(t.has("*")?this._fieldsIndex.fields.map(e=>e.name):Array.from(t)).filter(e=>null!=this._indexMap[e]),returnGeometry:!!e.returnGeometry||!!e.geometry,where:e.where}}}function N(e){for(const t in e){const r=t;switch(r){case"resultOffset":case"resultRecordCount":case"aggregateIds":case"distance":case"gdbVersion":case"geometry":case"having":case"timeExtent":case"objectIds":case"historicMoment":case"where":return null!=e[r]}}return!1}function k(e,t){const r=new A;for(const s of e)r.insert(s);return new w({...t,featureStore:r})}function J(e){switch(e.length){case 4:return h(d(),e);case 6:return e;default:throw new i("parquet:protocol-violation","Invalid Geoparquet file. BoundingBox size must be 4 or 6.",{bbox:e})}}function V(e,t){const r=c(d(),y);for(const s of e){const e=j(s.file);if(!e)return null;const t=e.columns[e.primary_column];if(!t.bbox)return null;const i=J(t.bbox);m(r,i)}return{xmin:r[0],ymin:r[1],xmax:r[3],ymax:r[4],spatialReference:t}}function H(e){const t=e?.featureCount;let r=!1;return null!=t&&t<has("parquetlayer-full-query-feature-count")&&(r=!0),{analytics:{supportsCacheHint:!1},attachment:null,data:{isVersioned:!1,isBranchVersioned:!1,supportedCurveTypes:[],supportsAttachment:!1,supportsM:!1,supportsTrueCurve:!1,supportsZ:!1},metadata:{supportsAdvancedFieldProperties:!1},operations:{supportsCalculate:!1,supportsTruncate:!1,supportsValidateSql:!1,supportsAdd:!1,supportsDelete:!1,supportsEditing:!1,supportsChangeTracking:!1,supportsQuery:!0,supportsQueryBins:!1,supportsQueryPivot:!1,supportsQueryAnalytics:!1,supportsQueryAttachments:!1,supportsQueryTopFeatures:!1,supportsResizeAttachments:!1,supportsSync:!1,supportsUpdate:!1,supportsExceedsLimitStatistics:!1,supportsAsyncConvert3D:!1},query:{maxRecordCount:8e3,maxRecordCountFactor:void 0,maxUniqueIDCount:void 0,relativeTimeBinWindow:void 0,standardMaxRecordCount:void 0,supportsCacheHint:!1,supportsCentroid:!0,supportsCentroidOnDegeneratedQuantizedGeometry:!1,supportsCurrentUser:!1,supportsDegeneratedQuantizedGeometry:!1,supportsDisjointSpatialRelationship:!1,supportsDistance:!1,supportsDistinct:r,supportsExtent:!1,supportsFormatPBF:!1,supportsFormatPBFWithCurves:!1,supportsGeometryProperties:!1,supportsHavingClause:!1,supportsHistoricMoment:!1,supportsMaxRecordCountFactor:!1,supportsOrderBy:r,supportsPagination:!0,supportsPaginationOnAggregatedQueries:!1,supportsPercentileStatistics:!1,supportsQuantization:!0,supportsQuantizationEditMode:!1,supportsQueryByAnonymous:!1,supportsQueryByOthers:!1,supportsQueryGeometry:!1,supportsResultType:!1,supportsReturnMesh:!1,supportsStandardizedQueriesOnly:!1,supportsTopFeaturesQuery:!1,supportsStatistics:r,supportsSpatialAggregationStatistics:!1,supportedSpatialAggregationStatistics:{envelope:!1,centroid:!1,convexHull:!1},supportsDefaultSpatialReference:!1,supportsFullTextSearch:!1,supportsCompactGeometry:!1,supportsSqlExpression:!1,supportsTrueCurve:!1,tileMaxRecordCount:void 0},queryAttributeBins:{supportsDate:!1,supportsFixedInterval:!1,supportsAutoInterval:!1,supportsFixedBoundaries:!1,supportsStackBy:!1,supportsSplitBy:!1,supportsSnapToData:!1,supportsReturnFullIntervalBin:!1,supportsFirstDayOfWeek:!1,supportsNormalization:!1},queryRelated:{supportsCount:!1,supportsOrderBy:!1,supportsPagination:!1,supportsCacheHint:!1},queryTopFeatures:{supportsCacheHint:!1},editing:{supportsDeleteByAnonymous:!1,supportsDeleteByOthers:!1,supportsGeometryUpdate:!1,supportsGlobalId:!1,supportsTrueCurveUpdate:!1,supportsTrueCurveUpdateByTrueCurveClientsOnly:!0,supportsReturnServiceEditsInSourceSpatialReference:!1,supportsRollbackOnFailure:!1,supportsUpdateByAnonymous:!1,supportsUpdateByOthers:!1,supportsUploadWithItemId:!1,supportsUpdateWithoutM:!1,supportsAsyncApplyEdits:!1,zDefault:void 0}}}function Z(e){return!!(e.objectIds?.length||e.outStatistics||e.orderByFields?.length||e.returnDistinctValues)}export{U as default};