@arcgis/core/libs/parquet/parquet.js

ArcGIS Maps SDK for JavaScript: A complete 2D and 3D mapping and data visualization API

/* COPYRIGHT Esri - https://js.arcgis.com/5.0.19/LICENSE.txt */
import e from"../../request.js";import t from"../../core/Error.js";import has from"../../core/has.js";import{QueueProcessor as n}from"../../core/QueueProcessor.js";import{isBlobProtocol as a}from"../../core/urlUtils.js";import{loadParquetModule as r}from"./loadParquetModule.js";const o="parquet.v1",s=new n({concurrency:12,process:async t=>{const{data:n}=await e(t.url,t.options);return n}}),i=e=>async function(t,n,a,r){const i=e?.(),l={url:t,options:{responseType:"array-buffer",query:i??{},cacheMode:"no-store",useQueue:!0,headers:{range:`bytes=${n}-${a}`}}};if(!has("parquetlayer-cache-enabled"))return s.push(l);const c=new URL(t);c.searchParams.append("range",`${n}-${a}`);try{const e=await caches.open(`${o}:${t}`),n=await e.match(c);if(n)return await n.arrayBuffer();const a=await s.push(l);return await e.put(c,new Response(a,{headers:{"Content-Type":"application/octet-stream","Content-Length":a.byteLength.toString()}})),a}catch(u){return s.push(l)}},l=t=>async function(n){if(a(n)){const{data:t}=await e(n,{responseType:"blob"});return t.size}const{data:r}=await e(n,{responseType:"native",method:"head",query:t?.()}),s=r.headers.get("Content-Length");if(null==s)throw new Error("Unable to parse content length");const i=parseInt(s,10);if(!has("parquetlayer-cache-enabled"))return i;try{const e=`${o}:${n}`,t=new URL(n);t.searchParams.append("metadata","true");let a=await caches.open(e);const s=r.headers.get("etag")??r.headers.get("Last-Modified");if(null==s)return await caches.delete(e),i;const l=await a.match(t),c=await(l?.json());c?.tag!==s&&(await caches.delete(e),a=await caches.open(e));const u=JSON.stringify({tag:s});await a.put(t,new Response(u,{headers:{"Content-Type":"application/json","Content-Length":u.length.toString()}}))}catch(l){}return i};function c(e,n){switch(n){case"esriGeometryPoint":return e.GeometryType.Point;case"esriGeometryPolygon":case"polygon":return e.GeometryType.Polygon;case"esriGeometryPolyline":case"polyline":return e.GeometryType.Polyline;case"esriGeometryMultipoint":case"multipoint":return e.GeometryType.Multipoint;default:throw new t("parquet",`Found unexpected GeometryType: ${n}`)}}function u(e,t){const n=t.encoding,a=c(e,t.geometryType),r=t.spatialReference.wkid;if(null==r)throw new Error("InternalError: Wkid must be defined.");switch(n.type){case"wkb":return e.GeometryField.fromWkb(n.field,r,a);case"location":{const{yField:t,xField:o}=n;return e.GeometryField.fromLocation(t,o,r,a)}}}function p(e,t){const n=e.GeometryInfo.new();t.geometry&&n.setGeometry(u(e,t.geometry));const a=t.displayOptimization;if(a){const{index:t,parentColumn:r}=a,o=e=>null!=r?[r,e].join("."):e;if("z"===t.type){const a=e.DisplayOptimizationZBuilder.new();a.setCodeField(o(t.code)),a.setXColumn(o(t.xColumn)),a.setYColumn(o(t.yColumn)),a.setCooordinatePrecision(t.coordinatePrecision),a.setFullExtent(t.fullExtent),a.setSpatialReference(t.wkid,t.wkt),a.setHasZ(!!t.zColumn),a.setHasM(!!t.mColumn),n.setOptmizationZ(a)}if("xz"===t.type){const a=c(e,t.geometryType),r=e.DisplayOptimizationXZBuilder.new();r.setCodeField(o(t.code)),r.setEncoding(t.encoding),r.setFullExtent(t.fullExtent),r.setGeometryType(a),r.setSpatialReference(t.wkid,t.wkt),r.setMaxLevel(t.maxLevel),r.setHasZ(t.hasZ??!1),r.setHasM(t.hasM??!1);for(const n of t.levels??[]){const[t,a]=n.transform.translate,[s,i]=n.transform.scale,l=new Float64Array([t,a]),c=new Float64Array([s,i]),u=o(n.column);r.addLevel(e.MultiscaleGeometryField.new(n.level,n.scale,u,l,c))}n.setOptmizationXZ(r)}}return n}async function d(e,n={}){const a=await r(),o=i(n.getCustomParameters),s=l(n.getCustomParameters),c=n.geometryInfo?p(a,n.geometryInfo):null;try{return await a.ParquetFile.fromUrl(e,o,s,c)}catch(u){throw new t("parquet","Failed to parse file",{error:u})}}function y(e){const t=e.keyValueMetadata("geo");return null!=t?JSON.parse(t):null}function m(e){if(e.keyValueMetadata("esri"))throw new t("parquet:unsupported","File was created using an unsupported experimental display index. Please regenerate the file.");const n=e.keyValueMetadata("geodisplay");if(null!=n)return JSON.parse(n);const a=e.keyValueMetadata("org.apache.spark.sql.parquet.row.metadata");if(null!=a){const e=JSON.parse(a);for(const n of e.fields)if(null!=n.metadata&&"type"in n.metadata&&("xz"===n.metadata.type||"z"===n.metadata.type)){const e={parentColumn:n.name,index:n.metadata};if("xz"===e.index.type&&(e.index.geometryType=e.index.geometryType.toLowerCase(),"esri-pbf"===e.index.encoding&&(e.index.encoding="esriPBF"),"esriPBF"!==e.index.encoding))throw new t("parquet:unsupported",`Encoding for display index must be of type esriPBF, but found ${e.index.encoding}`);return e}}return null}export{d as createParquetFile,y as readGeoMetadata,m as readGeodisplayMetadata};