diff --git a/src/client.js b/src/client.js index c997b43..be4c423 100644 --- a/src/client.js +++ b/src/client.js @@ -28,6 +28,7 @@ export class Saturn { * @param {number} [opts.connectTimeout=5000] * @param {number} [opts.downloadTimeout=0] * @param {string} [opts.orchURL] + * @param {string} [opts.originURL] * @param {number} [opts.fallbackLimit] * @param {boolean} [opts.experimental] * @param {import('./storage/index.js').Storage} [opts.storage] @@ -64,7 +65,6 @@ export class Saturn { * @param {string} cidPath * @param {object} [opts={}] * @param {Node[]} [opts.nodes] - * @param {Node} [opts.node] * @param {('car'|'raw')} [opts.format] * @param {number} [opts.connectTimeout=5000] * @param {number} [opts.downloadTimeout=0] @@ -87,7 +87,7 @@ export class Saturn { let nodes = options.nodes if (!nodes || nodes.length === 0) { - const replacementNode = options.node ?? { url: this.opts.cdnURL } + const replacementNode = { url: this.opts.cdnURL } nodes = [replacementNode] } const controllers = [] @@ -159,7 +159,7 @@ export class Saturn { * @param {string} cidPath * @param {object} [opts={}] * @param {('car'|'raw')} [opts.format] - * @param {Node} [opts.node] + * @param {Node[]} [opts.nodes] * @param {number} [opts.connectTimeout=5000] * @param {number} [opts.downloadTimeout=0] * @returns {Promise} @@ -171,7 +171,7 @@ export class Saturn { const jwt = await getJWT(this.opts, this.storage) const options = Object.assign({}, this.opts, { format: 'car', jwt }, opts) - const node = options.node + const node = options.nodes && options.nodes[0] const origin = node?.url ?? this.opts.cdnURL const url = this.createRequestURL(cidPath, { ...options, url: origin }) @@ -245,13 +245,14 @@ export class Saturn { * @param {object} [opts={}] * @param {('car'|'raw')} [opts.format] * @param {boolean} [opts.raceNodes] - * @param {string} [opts.url] + * @param {string} [opts.originURL] * @param {number} [opts.connectTimeout=5000] * @param {number} [opts.downloadTimeout=0] * @returns {Promise>} */ async * fetchContentWithFallback (cidPath, opts = {}) { let lastError = null + let skipNodes = false // we use this to checkpoint at which chunk a request failed. // this is temporary until range requests are supported. let byteCountCheckpoint = 0 @@ -260,9 +261,10 @@ export class Saturn { throw new Error(`All attempts to fetch content have failed. Last error: ${lastError.message}`) } - const fetchContent = async function * () { + const fetchContent = async function * (options) { let byteCount = 0 - const byteChunks = await this.fetchContent(cidPath, opts) + const fetchOptions = Object.assign(opts, options) + const byteChunks = await this.fetchContent(cidPath, fetchOptions) for await (const chunk of byteChunks) { // avoid sending duplicate chunks if (byteCount < byteCountCheckpoint) { @@ -280,33 +282,37 @@ export class Saturn { } }.bind(this) + // Use CDN origin if node list is not loaded if (this.nodes.length === 0) { // fetch from origin in the case that no nodes are loaded - opts.url = this.opts.cdnURL + opts.nodes = Array({ url: this.opts.cdnURL }) try { yield * fetchContent() return } catch (err) { lastError = err if (err.res?.status === 410 || isErrorUnavoidable(err)) { - throwError() + skipNodes = true + } else { + await this.loadNodesPromise } - await this.loadNodesPromise } } let fallbackCount = 0 const nodes = this.nodes for (let i = 0; i < nodes.length; i++) { + if (skipNodes) { + break + } if (fallbackCount > this.opts.fallbackLimit) { return } if (opts.raceNodes) { opts.nodes = nodes.slice(i, i + Saturn.defaultRaceCount) } else { - opts.node = nodes[i] + opts.nodes = Array(nodes[i]) } - try { yield * fetchContent() return @@ -320,6 +326,17 @@ export class Saturn { } if (lastError) { + const originUrl = opts.originURL ?? this.opts.originURL + // Use customer origin if cid is not retrievable by lassie. + if (originUrl) { + opts.nodes = Array({ url: originUrl }) + try { + yield * fetchContent() + return + } catch (err) { + lastError = err + } + } throwError() } } diff --git a/src/utils/errors.js b/src/utils/errors.js index f80566b..5833367 100644 --- a/src/utils/errors.js +++ b/src/utils/errors.js @@ -19,7 +19,8 @@ export function isErrorUnavoidable (error) { /file does not exist/, /Cannot read properties of undefined \(reading '([^']+)'\)/, /([a-zA-Z_.]+) is undefined/, - /undefined is not an object \(evaluating '([^']+)'\)/ + /undefined is not an object \(evaluating '([^']+)'\)/, + /all retrievals failed/ ] for (const pattern of errorPatterns) { diff --git a/test/fallback.spec.js b/test/fallback.spec.js index 425393d..6b8a1a5 100644 --- a/test/fallback.spec.js +++ b/test/fallback.spec.js @@ -3,12 +3,13 @@ import assert from 'node:assert/strict' import { describe, mock, test } from 'node:test' import { Saturn } from '#src/index.js' -import { concatChunks, generateNodes, getMockServer, HTTP_STATUS_GONE, mockJWT, mockNodesHandlers, mockOrchHandler, mockSaturnOriginHandler, MSW_SERVER_OPTS } from './test-utils.js' +import { concatChunks, generateNodes, getMockServer, HTTP_STATUS_GONE, HTTP_STATUS_TIMEOUT, mockJWT, mockNodesHandlers, mockOrchHandler, mockOriginHandler, MSW_SERVER_OPTS } from './test-utils.js' const TEST_DEFAULT_ORCH = 'https://orchestrator.strn.pl.test/nodes' const TEST_NODES_LIST_KEY = 'saturn-nodes' const TEST_AUTH = 'https://auth.test/' const TEST_ORIGIN_DOMAIN = 'l1s.saturn.test' +const TEST_CUSTOMER_ORIGIN = 'customer.test' const CLIENT_KEY = 'key' const options = { @@ -120,7 +121,7 @@ describe('Client Fallback', () => { const handlers = [ mockOrchHandler(2, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN), mockJWT(TEST_AUTH), - mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), + mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), ...mockNodesHandlers(2, TEST_ORIGIN_DOMAIN) ] const server = getMockServer(handlers) @@ -153,7 +154,7 @@ describe('Client Fallback', () => { const handlers = [ mockOrchHandler(5, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN), mockJWT(TEST_AUTH), - mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), + mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), ...mockNodesHandlers(5, TEST_ORIGIN_DOMAIN) ] const server = getMockServer(handlers) @@ -186,7 +187,7 @@ describe('Client Fallback', () => { const handlers = [ mockOrchHandler(5, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN), mockJWT(TEST_AUTH), - mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), + mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), ...mockNodesHandlers(5, TEST_ORIGIN_DOMAIN, 2) ] const server = getMockServer(handlers) @@ -273,6 +274,42 @@ describe('Client Fallback', () => { server.close() }) + test('should hit origin if failed to fetch', async (t) => { + const numNodes = 3 + const handlers = [ + mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN), + mockJWT(TEST_AUTH), + mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true), + mockOriginHandler(TEST_CUSTOMER_ORIGIN, 0, false), + ...mockNodesHandlers(numNodes, TEST_ORIGIN_DOMAIN, numNodes, HTTP_STATUS_TIMEOUT) + ] + + const server = getMockServer(handlers) + server.listen(MSW_SERVER_OPTS) + + const expectedNodes = generateNodes(5, TEST_ORIGIN_DOMAIN) + + // Mocking storage object + const mockStorage = { + get: async (key) => expectedNodes, + set: async (key, value) => { return null } + } + t.mock.method(mockStorage, 'get') + t.mock.method(mockStorage, 'set') + + const saturn = new Saturn({ storage: mockStorage, originURL: TEST_CUSTOMER_ORIGIN, ...options }) + + const cid = saturn.fetchContentWithFallback('bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4', { raceNodes: true }) + + const buffer = await concatChunks(cid) + const actualContent = String.fromCharCode(...buffer) + const expectedContent = 'hello world\n' + + assert.strictEqual(actualContent, expectedContent) + mock.reset() + server.close() + }) + test('Should abort fallback on 410s', async () => { const numNodes = 3 const handlers = [ diff --git a/test/test-utils.js b/test/test-utils.js index 7f2b33e..d77ce6f 100644 --- a/test/test-utils.js +++ b/test/test-utils.js @@ -46,15 +46,15 @@ export function generateNodes (count, originDomain) { /** * Generates a mock handler to mimick Saturn's orchestrator /nodes endpoint. * - * @param {string} cdnURL - orchestratorUrl + * @param {string} originUrl - originUrl * @param {number} delay - request delay in ms * @param {boolean} error * @returns {RestHandler} */ -export function mockSaturnOriginHandler (cdnURL, delay = 0, error = false) { - cdnURL = addHttpPrefix(cdnURL) - cdnURL = `${cdnURL}/ipfs/:cid` - return rest.get(cdnURL, (req, res, ctx) => { +export function mockOriginHandler (originUrl, delay = 0, error = false) { + originUrl = addHttpPrefix(originUrl) + originUrl = `${originUrl}/ipfs/:cid` + return rest.get(originUrl, (req, res, ctx) => { if (error) { throw Error('Simulated Error') }