@arcgis/core/views/2d/layers/features/sources/strategies/ParquetTileLoadStrategy.js

ArcGIS Maps SDK for JavaScript: A complete 2D and 3D mapping and data visualization API

/* COPYRIGHT Esri - https://js.arcgis.com/5.0.19/LICENSE.txt */
import t from"../../../../../../core/Error.js";import{handlesGroup as e}from"../../../../../../core/handleUtils.js";import"../../../../../../core/has.js";import{getOrCreateMapValue as s}from"../../../../../../core/MapUtils.js";import{notDeepEqual as i}from"../../../../../../core/object.js";import{throwIfAborted as r,onAbort as a}from"../../../../../../core/promiseUtils.js";import{QueueProcessor as o}from"../../../../../../core/QueueProcessor.js";import{difference as l}from"../../../../../../core/SetUtils.js";import{openWithPorts as n}from"../../../../../../core/workers/workers.js";import{loadParquetModule as h}from"../../../../../../libs/parquet/loadParquetModule.js";import{ALoadStrategy as d}from"./ALoadStrategy.js";import{fetchQueueConcurrency as u}from"./constants.js";import{ParquetSourceChunk as c}from"./chunks/ParquetSourceChunk.js";import{FeatureSetReaderParquet as p}from"../../support/FeatureSetReaderParquet.js";const m=2,_=4;class f{constructor(t,s){this.subscription=t,this.chunks=[],this.ids=[],this._controller=new AbortController,this._handles=e([a(t.signal,()=>this._controller.abort()),a(s,()=>this._controller.abort())])}destroy(){this._controller.abort(),this._handles.remove()}get options(){return{signal:this._controller.signal}}addBinaryData(t){this.chunks.push(t)}addSourceChunkId(t){this.ids.push(t)}}class y extends d{constructor(t,e,s,i){if(super(t.metadata,e,s),this._service=t,this._loadStates=new Map,this._files=new Map,this._queue=new o({concurrency:u,process:t=>this._loadTile(t)}),!t.geometryInfo.displayOptimization)throw new Error("InternalError: ParquetTileLoadStrategy only supports XZ-enabled parquet files");this._loaded=this._load(i)}destroy(){super.destroy(),this._loaded.finally(()=>{this._client.close(),this._client=null;for(const t of this._ports)t.close()}).catch(()=>{})}get about(){return{willQueryAllFeatures:!0,willQueryFullResolutionGeometry:!0}}get availableFields(){return this._schema.partial.availableFields}get definitionExpression(){return this._schema.full.definitionExpression}async tryUpdate(e,s){await this._loaded;let r=!0;if(i(this.availableFields,s.availableFields)&&await this._updateFields(s.availableFields),i(this._schema.partial.urls,s.urls)){for(const t of s.urls)this._files.has(t)||(r=!1,await this._insert(t));for(const e of this._files.keys())if(!s.urls.includes(e))throw new t("unsupported","Removing parquet files is not supported",{previous:this._schema.partial.urls,next:s.urls})}return this._schema.partial=s,r}async load(t){return await this._loaded,s(this._loadStates,t.key.id,()=>new f(t,this._options)),this._queue.push(t)}unload(t){const e=this._loadStates.get(t.tile.id);if(null!=e){for(const t of e.ids)this.store.removeById(t);e.destroy(),this._loadStates.delete(t.tile.id)}}async _load(t){const e=this._schema.partial.urls;this._ports=await t.layerView.openMemoryPorts(),this._client=await n(this._ports,{maxNumberOfConcurrentJobs:20}),this._proxy=this._client.createInvokeProxy(),this._indexMap=await this._proxy.getIndexMap(),await Promise.all(e.map(t=>this._insert(t)))}async _insert(t){this._files.set(t,this._files.size)}_onOverride(){}async _updateFields(t){const e=new Set(t),s=l(e,new Set(this.availableFields));if(null!=this._service.metadata.fieldsIndex&&s.size){const t=Array.from(this.store.insertedChunks()).map(t=>t.updateFields(this._proxy,Array.from(s),this._options));await Promise.all(t)}}async _loadTile(t){const e=t.tile,s=t.signal,i=this._loadStates.get(e.id),a=new o({process:async e=>{const s=await this._proxy.getFileId(e);return this._downloadFileData(t,e,s)},concurrency:m});await Promise.allSettled(Array.from(this._files.keys()).map(t=>a.push(t))),r(s);const l=new c(this.metadata,null,e,null,null,0,!0);i.addSourceChunkId(l.chunkId),this.store.insert(l)}async _downloadFileData(t,e,s){const i=await h(),a=t.tile,o=t.signal,l=this._loadStates.get(a.id),n=await this._proxy.queryStream(e,{type:"tile",extent:a.extent,transform:a.normalizedTransform,outFields:Array.from(this.availableFields),returnGeometry:!0,outSpatialReference:this.metadata.outSpatialReference.toJSON(),scale:a.scale,where:this.definitionExpression},{signal:o});let d=0;const u=[],m=async(t=0)=>{const h=await this._proxy.getStreamNext(e,n,{signal:o});if(r(o),null==h)return;const u=i.ParquetChunk.deserialize(new Uint8Array(h)),_=new p(this.metadata,this._indexMap,u,s),f=new c(this.metadata,_,a,s,e,d++,!1);return l.addBinaryData({data:u,fileUrl:e}),l.addSourceChunkId(f.chunkId),this.store.insert(f),m(t+1)};for(let r=0;r<_;r++)u.push(m());await Promise.all(u)}}export{y as ParquetTileLoadStrategy};