From ab1d0334103107794309e8c94e4a20e39749aaf1 Mon Sep 17 00:00:00 2001 From: Don McCurdy Date: Fri, 20 Sep 2024 11:23:06 -0400 Subject: [PATCH] feat(carto): Allow configuring max URL length for GET requests (#9159) --- docs/api-reference/carto/data-sources.md | 36 ++++++------ modules/carto/src/api/common.ts | 4 +- modules/carto/src/api/fetch-map.ts | 56 +++++++++++++------ modules/carto/src/api/query.ts | 4 +- .../carto/src/api/request-with-parameters.ts | 8 ++- modules/carto/src/sources/base-source.ts | 16 ++++-- modules/carto/src/sources/types.ts | 7 +++ .../carto/api/request-with-parameters.spec.ts | 33 +++++++++++ 8 files changed, 119 insertions(+), 45 deletions(-) diff --git a/docs/api-reference/carto/data-sources.md b/docs/api-reference/carto/data-sources.md index fb2ccf85f82..2751c6f4a15 100644 --- a/docs/api-reference/carto/data-sources.md +++ b/docs/api-reference/carto/data-sources.md @@ -11,8 +11,8 @@ import {vectorTableSource} from '@deck.gl/carto'; const data = vectorTableSource({ accessToken: 'XXX', connectionName: 'carto_dw', - tableName: 'carto-demo-data.demo_tables.chicago_crime_sample', -}) + tableName: 'carto-demo-data.demo_tables.chicago_crime_sample' +}); ``` ### Promise API @@ -51,7 +51,7 @@ type SourceOptions = { apiBaseUrl?: string; clientId?: string; headers?: Record; - mapsUrl?: string; + maxLengthURL?: number; }; ``` @@ -64,7 +64,7 @@ type VectorTableSourceOptions = { columns?: string[]; spatialDataColumn?: string; tableName: string; -} +}; ``` #### vectorQuerySource @@ -74,7 +74,7 @@ type VectorQuerySourceOptions = { spatialDataColumn?: string; sqlQuery: string; queryParameters: QueryParameters; -} +}; ``` #### vectorTilesetSource @@ -82,7 +82,7 @@ type VectorQuerySourceOptions = { ```ts type VectorTilesetSourceOptions = { tableName: string; -} +}; ``` #### h3TableSource @@ -94,7 +94,7 @@ type H3TableSourceOptions = { columns?: string[]; spatialDataColumn?: string; tableName: string; -} +}; ``` #### h3QuerySource @@ -106,7 +106,7 @@ type H3QuerySourceOptions = { spatialDataColumn?: string; sqlQuery: string; queryParameters: QueryParameters; -} +}; ``` #### h3TilesetSource @@ -114,7 +114,7 @@ type H3QuerySourceOptions = { ```ts type H3TilesetSourceOptions = { tableName: string; -} +}; ``` #### quadbinTableSource @@ -126,7 +126,7 @@ type QuadbinTableSourceOptions = { columns?: string[]; spatialDataColumn?: string; tableName: string; -} +}; ``` #### quadbinQuerySource @@ -138,7 +138,7 @@ type QuadbinQuerySourceOptions = { spatialDataColumn?: string; sqlQuery: string; queryParameters: QueryParameters; -} +}; ``` #### quadbinTilesetSource @@ -146,7 +146,7 @@ type QuadbinQuerySourceOptions = { ```ts type QuadbinTilesetSourceOptions = { tableName: string; -} +}; ``` #### rasterTilesetSource (Experimental) @@ -154,7 +154,7 @@ type QuadbinTilesetSourceOptions = { ```ts type RasterTilesetSourceOptions = { tableName: string; -} +}; ``` Boundary sources are experimental sources where both the tileset and the properties props need a specific schema to work. [Read more about Boundaries in the CARTO documentation](https://docs.carto.com/carto-for-developers/guides/use-boundaries-in-your-application). @@ -166,7 +166,7 @@ type BoundaryTableSourceOptions = { tilesetTableName: string; columns?: string[]; propertiesTableName: string; -} +}; ``` #### boundaryQuerySource (Experimental) @@ -176,7 +176,7 @@ type BoundaryQuerySourceOptions = { tilesetTableName: string; propertiesSqlQuery: string; queryParameters?: QueryParameters; -} +}; ``` ### QueryParameters @@ -184,6 +184,7 @@ type BoundaryQuerySourceOptions = { QueryParameters are used to parametrize SQL queries. The format depends on the source's provider, some examples: [PostgreSQL and Redshift](https://node-postgres.com/features/queries): + ```ts vectorQuerySource({ ..., @@ -193,6 +194,7 @@ vectorQuerySource({ ``` [BigQuery positional](https://cloud.google.com/bigquery/docs/parameterized-queries#node.js): + ```ts vectorQuerySource({ ..., @@ -201,8 +203,8 @@ vectorQuerySource({ }) ``` - [BigQuery named parameters](https://cloud.google.com/bigquery/docs/parameterized-queries#node.js): + ```ts vectorQuerySource({ ..., @@ -212,6 +214,7 @@ vectorQuerySource({ ``` [Snowflake positional](https://docs.snowflake.com/en/user-guide/nodejs-driver-use.html#binding-statement-parameters) : + ```ts vectorQuerySource({ ..., @@ -230,6 +233,7 @@ vectorQuerySource({ ``` [Databricks ODBC](https://github.com/markdirish/node-odbc#bindparameters-callback) + ```ts vectorQuerySource({ ... diff --git a/modules/carto/src/api/common.ts b/modules/carto/src/api/common.ts index 147c0183f5f..120ffa321e0 100644 --- a/modules/carto/src/api/common.ts +++ b/modules/carto/src/api/common.ts @@ -1,4 +1,6 @@ export const DEFAULT_API_BASE_URL = 'https://gcp-us-east1.api.carto.com'; export const DEFAULT_CLIENT = 'deck-gl-carto'; export const V3_MINOR_VERSION = '3.4'; -export const MAX_GET_LENGTH = 8192; + +// Fastly default limit is 8192; leave some padding. +export const DEFAULT_MAX_LENGTH_URL = 7000; diff --git a/modules/carto/src/api/fetch-map.ts b/modules/carto/src/api/fetch-map.ts index 757a2532866..a4950674a57 100644 --- a/modules/carto/src/api/fetch-map.ts +++ b/modules/carto/src/api/fetch-map.ts @@ -1,6 +1,6 @@ /* eslint-disable camelcase */ import {CartoAPIError} from './carto-api-error'; -import {DEFAULT_API_BASE_URL, DEFAULT_CLIENT} from './common'; +import {DEFAULT_API_BASE_URL, DEFAULT_CLIENT, DEFAULT_MAX_LENGTH_URL} from './common'; import {buildPublicMapUrl, buildStatsUrl} from './endpoints'; import { GeojsonResult, @@ -36,13 +36,14 @@ type Dataset = { }; /* global clearInterval, setInterval, URL */ -/* eslint-disable complexity, max-statements */ +/* eslint-disable complexity, max-statements, max-params */ async function _fetchMapDataset( dataset: Dataset, accessToken: string, apiBaseUrl: string, clientId?: string, - headers?: Record + headers?: Record, + maxLengthURL = DEFAULT_MAX_LENGTH_URL ) { const { aggregationExp, @@ -64,7 +65,8 @@ async function _fetchMapDataset( clientId, connectionName, format, - headers + headers, + maxLengthURL }; if (type === 'tileset') { @@ -114,7 +116,8 @@ async function _fetchTilestats( attribute: string, dataset: Dataset, accessToken: string, - apiBaseUrl: string + apiBaseUrl: string, + maxLengthURL = DEFAULT_MAX_LENGTH_URL ) { const {connectionName, data, id, source, type, queryParameters} = dataset; const errorContext: APIErrorContext = { @@ -144,7 +147,8 @@ async function _fetchTilestats( baseUrl, headers, parameters, - errorContext + errorContext, + maxLengthURL }); // Replace tilestats for attribute with value from API @@ -158,17 +162,19 @@ async function fillInMapDatasets( {datasets, token}: {datasets: Dataset[]; token: string}, clientId: string, apiBaseUrl: string, - headers?: Record + headers?: Record, + maxLengthURL = DEFAULT_MAX_LENGTH_URL ) { const promises = datasets.map(dataset => - _fetchMapDataset(dataset, token, apiBaseUrl, clientId, headers) + _fetchMapDataset(dataset, token, apiBaseUrl, clientId, headers, maxLengthURL) ); return await Promise.all(promises); } async function fillInTileStats( {datasets, keplerMapConfig, token}: {datasets: Dataset[]; keplerMapConfig: any; token: string}, - apiBaseUrl: string + apiBaseUrl: string, + maxLengthURL = DEFAULT_MAX_LENGTH_URL ) { const attributes: {attribute: string; dataset: any}[] = []; const {layers} = keplerMapConfig.config.visState; @@ -197,7 +203,7 @@ async function fillInTileStats( } const promises = filteredAttributes.map(({attribute, dataset}) => - _fetchTilestats(attribute, dataset, token, apiBaseUrl) + _fetchTilestats(attribute, dataset, token, apiBaseUrl, maxLengthURL) ); return await Promise.all(promises); } @@ -237,6 +243,13 @@ export type FetchMapOptions = { * Callback function that will be invoked whenever data in layers is changed. If provided, `autoRefresh` must also be provided. */ onNewData?: (map: any) => void; + + /** + * Maximum URL character length. Above this limit, requests use POST. + * Used to avoid browser and CDN limits. + * @default {@link DEFAULT_MAX_LENGTH_URL} + */ + maxLengthURL?: number; }; export type FetchMapResult = ParseMapResult & { @@ -255,7 +268,8 @@ export async function fetchMap({ clientId = DEFAULT_CLIENT, headers = {}, autoRefresh, - onNewData + onNewData, + maxLengthURL = DEFAULT_MAX_LENGTH_URL }: FetchMapOptions): Promise { assert(cartoMapId, 'Must define CARTO map id: fetchMap({cartoMapId: "XXXX-XXXX-XXXX"})'); assert(apiBaseUrl, 'Must define apiBaseUrl'); @@ -275,7 +289,7 @@ export async function fetchMap({ const baseUrl = buildPublicMapUrl({apiBaseUrl, cartoMapId}); const errorContext: APIErrorContext = {requestType: 'Public map', mapId: cartoMapId}; - const map = await requestWithParameters({baseUrl, headers, errorContext}); + const map = await requestWithParameters({baseUrl, headers, errorContext, maxLengthURL}); // Periodically check if the data has changed. Note that this // will not update when a map is published. @@ -283,10 +297,16 @@ export async function fetchMap({ if (autoRefresh) { // eslint-disable-next-line @typescript-eslint/no-misused-promises const intervalId = setInterval(async () => { - const changed = await fillInMapDatasets(map, clientId, apiBaseUrl, { - ...headers, - 'If-Modified-Since': new Date().toUTCString() - }); + const changed = await fillInMapDatasets( + map, + clientId, + apiBaseUrl, + { + ...headers, + 'If-Modified-Since': new Date().toUTCString() + }, + maxLengthURL + ); if (onNewData && changed.some(v => v === true)) { onNewData(parseMap(map)); } @@ -315,11 +335,11 @@ export async function fetchMap({ fetchBasemapProps({config: map.keplerMapConfig.config, errorContext}), // Mutates map.datasets so that dataset.data contains data - fillInMapDatasets(map, clientId, apiBaseUrl, headers) + fillInMapDatasets(map, clientId, apiBaseUrl, headers, maxLengthURL) ]); // Mutates attributes in visualChannels to contain tile stats - await fillInTileStats(map, apiBaseUrl); + await fillInTileStats(map, apiBaseUrl, maxLengthURL); const out = {...parseMap(map), basemap, ...{stopAutoRefresh}}; diff --git a/modules/carto/src/api/query.ts b/modules/carto/src/api/query.ts index 060c23039ae..db91dc0a431 100644 --- a/modules/carto/src/api/query.ts +++ b/modules/carto/src/api/query.ts @@ -11,6 +11,7 @@ export const query = async function (options: QueryOptions): Promise({ baseUrl, parameters = {}, headers: customHeaders = {}, - errorContext + errorContext, + maxLengthURL = DEFAULT_MAX_LENGTH_URL }: { baseUrl: string; parameters?: Record; headers?: Record; errorContext: APIErrorContext; + maxLengthURL?: number; }): Promise { parameters = {...DEFAULT_PARAMETERS, ...parameters}; baseUrl = excludeURLParameters(baseUrl, Object.keys(parameters)); @@ -44,7 +46,7 @@ export async function requestWithParameters({ /* global fetch */ const fetchPromise = - url.length > MAX_GET_LENGTH + url.length > maxLengthURL ? fetch(baseUrl, {method: 'POST', body: JSON.stringify(parameters), headers}) : fetch(url, {headers}); diff --git a/modules/carto/src/sources/base-source.ts b/modules/carto/src/sources/base-source.ts index 45b31fbc0ce..e80f7ad5ab6 100644 --- a/modules/carto/src/sources/base-source.ts +++ b/modules/carto/src/sources/base-source.ts @@ -1,5 +1,5 @@ /* eslint-disable camelcase */ -import {DEFAULT_API_BASE_URL, DEFAULT_CLIENT} from '../api/common'; +import {DEFAULT_API_BASE_URL, DEFAULT_CLIENT, DEFAULT_MAX_LENGTH_URL} from '../api/common'; import {buildSourceUrl} from '../api/endpoints'; import {requestWithParameters} from '../api/request-with-parameters'; import type {APIErrorContext, MapType} from '../api/types'; @@ -16,7 +16,8 @@ export const SOURCE_DEFAULTS: SourceOptionalOptions = { apiBaseUrl: DEFAULT_API_BASE_URL, clientId: DEFAULT_CLIENT, format: 'tilejson', - headers: {} + headers: {}, + maxLengthURL: DEFAULT_MAX_LENGTH_URL }; export async function baseSource>( @@ -32,7 +33,7 @@ export async function baseSource>( } } const baseUrl = buildSourceUrl(mergedOptions); - const {clientId, format} = mergedOptions; + const {clientId, maxLengthURL, format} = mergedOptions; const headers = {Authorization: `Bearer ${options.accessToken}`, ...options.headers}; const parameters = {client: clientId, ...urlParameters}; @@ -46,7 +47,8 @@ export async function baseSource>( baseUrl, parameters, headers, - errorContext + errorContext, + maxLengthURL }); const dataUrl = mapInstantiation[format].url[0]; @@ -59,7 +61,8 @@ export async function baseSource>( const json = await requestWithParameters({ baseUrl: dataUrl, headers, - errorContext + errorContext, + maxLengthURL }); if (accessToken) { json.accessToken = accessToken; @@ -70,6 +73,7 @@ export async function baseSource>( return await requestWithParameters({ baseUrl: dataUrl, headers, - errorContext + errorContext, + maxLengthURL }); } diff --git a/modules/carto/src/sources/types.ts b/modules/carto/src/sources/types.ts index 11786e1c2dd..a43b390298d 100644 --- a/modules/carto/src/sources/types.ts +++ b/modules/carto/src/sources/types.ts @@ -35,6 +35,13 @@ export type SourceOptionalOptions = { clientId: string; /** @deprecated use `query` instead **/ format: Format; + + /** + * Maximum URL character length. Above this limit, requests use POST. + * Used to avoid browser and CDN limits. + * @default {@link DEFAULT_MAX_LENGTH_URL} + */ + maxLengthURL?: number; }; export type SourceOptions = SourceRequiredOptions & Partial; diff --git a/test/modules/carto/api/request-with-parameters.spec.ts b/test/modules/carto/api/request-with-parameters.spec.ts index f17c7303d6a..0624b2a35b3 100644 --- a/test/modules/carto/api/request-with-parameters.spec.ts +++ b/test/modules/carto/api/request-with-parameters.spec.ts @@ -192,3 +192,36 @@ test('requestWithParameters#precedence', async t => { }); t.end(); }); + +test('requestWithParameters#maxLengthURL', async t => { + await withMockFetchMapsV3(async calls => { + t.equals(calls.length, 0, '0 initial calls'); + + await Promise.all([ + requestWithParameters({ + baseUrl: 'https://example.com/v1/item/1' + }), + requestWithParameters({ + baseUrl: 'https://example.com/v1/item/2', + maxLengthURL: 10 + }), + requestWithParameters({ + baseUrl: `https://example.com/v1/item/3`, + parameters: {content: 'long'.padEnd(10_000, 'g')} // > default limit + }), + requestWithParameters({ + baseUrl: `https://example.com/v1/item/4`, + parameters: {content: 'long'.padEnd(10_000, 'g')}, + maxLengthURL: 15_000 + }) + ]); + + t.equals(calls.length, 4, '4 requests'); + t.deepEquals( + calls.map(({method}) => method ?? 'GET'), + ['GET', 'POST', 'POST', 'GET'], + 'request method' + ); + }); + t.end(); +});