feat: add customer origin url fallback

filecoin-saturn · Oct 31, 2023 · 6f90f07 · 6f90f07
1 parent e571d94
commit 6f90f07
Show file tree

Hide file tree

Showing 4 changed files with 77 additions and 22 deletions.
diff --git a/src/client.js b/src/client.js
@@ -28,6 +28,7 @@ export class Saturn {
    * @param {number} [opts.connectTimeout=5000]
    * @param {number} [opts.downloadTimeout=0]
    * @param {string} [opts.orchURL]
+   * @param {string} [opts.originURL]
    * @param {number} [opts.fallbackLimit]
    * @param {boolean} [opts.experimental]
    * @param {import('./storage/index.js').Storage} [opts.storage]
@@ -64,7 +65,6 @@ export class Saturn {
    * @param {string} cidPath
    * @param {object} [opts={}]
    * @param {Node[]} [opts.nodes]
-   * @param {Node} [opts.node]
    * @param {('car'|'raw')} [opts.format]
    * @param {number} [opts.connectTimeout=5000]
    * @param {number} [opts.downloadTimeout=0]
@@ -87,7 +87,7 @@ export class Saturn {
 
     let nodes = options.nodes
     if (!nodes || nodes.length === 0) {
-      const replacementNode = options.node ?? { url: this.opts.cdnURL }
+      const replacementNode = { url: this.opts.cdnURL }
       nodes = [replacementNode]
     }
     const controllers = []
@@ -159,7 +159,7 @@ export class Saturn {
    * @param {string} cidPath
    * @param {object} [opts={}]
    * @param {('car'|'raw')} [opts.format]
-   * @param {Node} [opts.node]
+   * @param {Node[]} [opts.nodes]
    * @param {number} [opts.connectTimeout=5000]
    * @param {number} [opts.downloadTimeout=0]
    * @returns {Promise<object>}
@@ -171,7 +171,7 @@ export class Saturn {
     const jwt = await getJWT(this.opts, this.storage)
 
     const options = Object.assign({}, this.opts, { format: 'car', jwt }, opts)
-    const node = options.node
+    const node = options.nodes && options.nodes[0]
     const origin = node?.url ?? this.opts.cdnURL
     const url = this.createRequestURL(cidPath, { ...options, url: origin })
 
@@ -245,13 +245,14 @@ export class Saturn {
    * @param {object} [opts={}]
    * @param {('car'|'raw')} [opts.format]
    * @param {boolean} [opts.raceNodes]
-   * @param {string} [opts.url]
+   * @param {string} [opts.originURL]
    * @param {number} [opts.connectTimeout=5000]
    * @param {number} [opts.downloadTimeout=0]
    * @returns {Promise<AsyncIterable<Uint8Array>>}
    */
   async * fetchContentWithFallback (cidPath, opts = {}) {
     let lastError = null
+    let skipNodes = false
     // we use this to checkpoint at which chunk a request failed.
     // this is temporary until range requests are supported.
     let byteCountCheckpoint = 0
@@ -260,9 +261,10 @@ export class Saturn {
       throw new Error(`All attempts to fetch content have failed. Last error: ${lastError.message}`)
     }
 
-    const fetchContent = async function * () {
+    const fetchContent = async function * (options) {
       let byteCount = 0
-      const byteChunks = await this.fetchContent(cidPath, opts)
+      const fetchOptions = Object.assign(opts, options)
+      const byteChunks = await this.fetchContent(cidPath, fetchOptions)
       for await (const chunk of byteChunks) {
         // avoid sending duplicate chunks
         if (byteCount < byteCountCheckpoint) {
@@ -280,33 +282,37 @@ export class Saturn {
       }
     }.bind(this)
 
+    // Use CDN origin if node list is not loaded
     if (this.nodes.length === 0) {
       // fetch from origin in the case that no nodes are loaded
-      opts.url = this.opts.cdnURL
+      opts.nodes = Array({ url: this.opts.cdnURL })
       try {
         yield * fetchContent()
         return
       } catch (err) {
         lastError = err
         if (err.res?.status === 410 || isErrorUnavoidable(err)) {
-          throwError()
+          skipNodes = true
+        } else {
+          await this.loadNodesPromise
         }
-        await this.loadNodesPromise
       }
     }
 
     let fallbackCount = 0
     const nodes = this.nodes
     for (let i = 0; i < nodes.length; i++) {
+      if (skipNodes) {
+        break
+      }
       if (fallbackCount > this.opts.fallbackLimit) {
         return
       }
       if (opts.raceNodes) {
         opts.nodes = nodes.slice(i, i + Saturn.defaultRaceCount)
       } else {
-        opts.node = nodes[i]
+        opts.nodes = Array(nodes[i])
       }
-
       try {
         yield * fetchContent()
         return
@@ -320,6 +326,17 @@ export class Saturn {
     }
 
     if (lastError) {
+      const originUrl = opts.originURL ?? this.opts.originURL
+      // Use customer origin if cid is not retrievable by lassie.
+      if (originUrl) {
+        opts.nodes = Array({ url: originUrl })
+        try {
+          yield * fetchContent()
+          return
+        } catch (err) {
+          lastError = err
+        }
+      }
       throwError()
     }
   }

diff --git a/src/utils/errors.js b/src/utils/errors.js
@@ -19,7 +19,8 @@ export function isErrorUnavoidable (error) {
     /file does not exist/,
     /Cannot read properties of undefined \(reading '([^']+)'\)/,
     /([a-zA-Z_.]+) is undefined/,
-    /undefined is not an object \(evaluating '([^']+)'\)/
+    /undefined is not an object \(evaluating '([^']+)'\)/,
+    /all retrievals failed/
   ]
 
   for (const pattern of errorPatterns) {

diff --git a/test/fallback.spec.js b/test/fallback.spec.js
@@ -3,12 +3,13 @@ import assert from 'node:assert/strict'
 import { describe, mock, test } from 'node:test'
 
 import { Saturn } from '#src/index.js'
-import { concatChunks, generateNodes, getMockServer, HTTP_STATUS_GONE, mockJWT, mockNodesHandlers, mockOrchHandler, mockSaturnOriginHandler, MSW_SERVER_OPTS } from './test-utils.js'
+import { concatChunks, generateNodes, getMockServer, HTTP_STATUS_GONE, HTTP_STATUS_TIMEOUT, mockJWT, mockNodesHandlers, mockOrchHandler, mockOriginHandler, MSW_SERVER_OPTS } from './test-utils.js'
 
 const TEST_DEFAULT_ORCH = 'https://orchestrator.strn.pl.test/nodes'
 const TEST_NODES_LIST_KEY = 'saturn-nodes'
 const TEST_AUTH = 'https://auth.test/'
 const TEST_ORIGIN_DOMAIN = 'l1s.saturn.test'
+const TEST_CUSTOMER_ORIGIN = 'customer.test'
 const CLIENT_KEY = 'key'
 
 const options = {
@@ -120,7 +121,7 @@ describe('Client Fallback', () => {
     const handlers = [
       mockOrchHandler(2, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
       mockJWT(TEST_AUTH),
-      mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true),
+      mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true),
       ...mockNodesHandlers(2, TEST_ORIGIN_DOMAIN)
     ]
     const server = getMockServer(handlers)
@@ -153,7 +154,7 @@ describe('Client Fallback', () => {
     const handlers = [
       mockOrchHandler(5, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
       mockJWT(TEST_AUTH),
-      mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true),
+      mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true),
       ...mockNodesHandlers(5, TEST_ORIGIN_DOMAIN)
     ]
     const server = getMockServer(handlers)
@@ -186,7 +187,7 @@ describe('Client Fallback', () => {
     const handlers = [
       mockOrchHandler(5, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
       mockJWT(TEST_AUTH),
-      mockSaturnOriginHandler(TEST_ORIGIN_DOMAIN, 0, true),
+      mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true),
       ...mockNodesHandlers(5, TEST_ORIGIN_DOMAIN, 2)
     ]
     const server = getMockServer(handlers)
@@ -273,6 +274,42 @@ describe('Client Fallback', () => {
     server.close()
   })
 
+  test('should hit origin if failed to fetch', async (t) => {
+    const numNodes = 3
+    const handlers = [
+      mockOrchHandler(numNodes, TEST_DEFAULT_ORCH, TEST_ORIGIN_DOMAIN),
+      mockJWT(TEST_AUTH),
+      mockOriginHandler(TEST_ORIGIN_DOMAIN, 0, true),
+      mockOriginHandler(TEST_CUSTOMER_ORIGIN, 0, false),
+      ...mockNodesHandlers(numNodes, TEST_ORIGIN_DOMAIN, numNodes, HTTP_STATUS_TIMEOUT)
+    ]
+
+    const server = getMockServer(handlers)
+    server.listen(MSW_SERVER_OPTS)
+
+    const expectedNodes = generateNodes(5, TEST_ORIGIN_DOMAIN)
+
+    // Mocking storage object
+    const mockStorage = {
+      get: async (key) => expectedNodes,
+      set: async (key, value) => { return null }
+    }
+    t.mock.method(mockStorage, 'get')
+    t.mock.method(mockStorage, 'set')
+
+    const saturn = new Saturn({ storage: mockStorage, originURL: TEST_CUSTOMER_ORIGIN, ...options })
+
+    const cid = saturn.fetchContentWithFallback('bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4', { raceNodes: true })
+
+    const buffer = await concatChunks(cid)
+    const actualContent = String.fromCharCode(...buffer)
+    const expectedContent = 'hello world\n'
+
+    assert.strictEqual(actualContent, expectedContent)
+    mock.reset()
+    server.close()
+  })
+
   test('Should abort fallback on 410s', async () => {
     const numNodes = 3
     const handlers = [

diff --git a/test/test-utils.js b/test/test-utils.js
@@ -46,15 +46,15 @@ export function generateNodes (count, originDomain) {
 /**
  * Generates a mock handler to mimick Saturn's orchestrator /nodes endpoint.
  *
- * @param {string} cdnURL - orchestratorUrl
+ * @param {string} originUrl - originUrl
  * @param {number} delay - request delay in ms
  * @param {boolean} error
  * @returns {RestHandler<any>}
  */
-export function mockSaturnOriginHandler (cdnURL, delay = 0, error = false) {
-  cdnURL = addHttpPrefix(cdnURL)
-  cdnURL = `${cdnURL}/ipfs/:cid`
-  return rest.get(cdnURL, (req, res, ctx) => {
+export function mockOriginHandler (originUrl, delay = 0, error = false) {
+  originUrl = addHttpPrefix(originUrl)
+  originUrl = `${originUrl}/ipfs/:cid`
+  return rest.get(originUrl, (req, res, ctx) => {
     if (error) {
       throw Error('Simulated Error')
     }