feat(client): support range requests

- pass entity-bytes param to the backend
- utilize offset/length params on unixfs-exporter to do ranged
  verification

src/client.js

@@ -392,7 +392,7 @@ export class Saturn {
       if (!opts.format) {
         yield * itr
       } else {
-        yield * extractVerifiedContent(cidPath, itr)
+        yield * extractVerifiedContent(cidPath, itr, opts.range || {})
       }
     } catch (err) {
       log.error = err.message
@@ -422,6 +422,7 @@ export class Saturn {
    * @param {string} [opts.format]
    * @param {string} [opts.originFallback]
    * @param {object} [opts.jwt]
+   * @param {import('./types.js').ContentRange} [opts.range]
    * @returns {URL}
    */
   createRequestURL (cidPath, opts = {}) {
@@ -442,6 +443,11 @@ export class Saturn {
       url.searchParams.set('jwt', opts.jwt)
     }
 
+    if (typeof opts.range === 'object' && (opts.range.rangeStart || opts.range.rangeEnd)) {
+      const rangeStart = opts.range.rangeStart?.toString() || '0'
+      const rangeEnd = opts.range.rangeEnd?.toString() || '*'
+      url.searchParams.set('entity-bytes', rangeStart + ':' + rangeEnd)
+    }
     return url
   }
 

src/types.js

@@ -18,6 +18,7 @@
  * @typedef {object} FetchOptions
  * @property {Node[]} [nodes] - An array of nodes.
  * @property {('car'|'raw')} [format] - The format of the fetched content.
+ * 
  * @property {boolean} [originFallback] - Is this a fallback to the customer origin
  * @property {boolean} [raceNodes] - Does the fetch race multiple nodes on requests.
  * @property {string} [customerFallbackURL] - Customer Origin that is a fallback.
@@ -26,6 +27,15 @@
  * @property {number} [downloadTimeout=0] - Download timeout in milliseconds.
  * @property {AbortController} [controller] - Abort controller
  * @property {boolean} [firstHitDNS] - First request hit is always to CDN origin.
+ * @property {ContentRange} [range] - range to fetch, compatible with entity bytes parameter
+ */
+
+/**
+ * Options for a range request
+ *
+ * @typedef {object} ContentRange
+ * @property {number} [rangeStart]
+ * @property {number} [rangeEnd]
  */
 
 export {}

src/utils/car.js

@@ -2,7 +2,7 @@ import { CarBlockIterator } from '@ipld/car'
 import * as dagCbor from '@ipld/dag-cbor'
 import * as dagPb from '@ipld/dag-pb'
 import * as dagJson from '@ipld/dag-json'
-import { exporter } from 'ipfs-unixfs-exporter'
+import * as unixfs from 'ipfs-unixfs-exporter'
 import { bytes } from 'multiformats'
 import * as raw from 'multiformats/codecs/raw'
 import * as json from 'multiformats/codecs/json'
@@ -89,12 +89,45 @@ export async function verifyBlock (cid, bytes) {
  *
  * @param {string} cidPath
  * @param {ReadableStream|AsyncIterable} carStream
+ * @param {import('../types.js').ContentRange} options
  */
-export async function * extractVerifiedContent (cidPath, carStream) {
+export async function * extractVerifiedContent (cidPath, carStream, options = {}) {
   const getter = await CarBlockGetter.fromStream(carStream)
-  const node = await exporter(cidPath, getter)
+  const node = await unixfs.exporter(cidPath, getter)
 
-  for await (const chunk of node.content()) {
+  for await (const chunk of contentGenerator(node, options)) {
     yield chunk
   }
 }
+
+/**
+ * 
+ * @param {unixfs.UnixFSEntry} node
+ * @param {import('../types.js').ContentRange} options
+ */
+function contentGenerator(node, options = {}) {
+
+  let rangeStart = options.rangeStart ?? 0
+  if (rangeStart < 0) {
+    rangeStart = rangeStart + Number(node.size)
+    if (rangeStart < 0) {
+      rangeStart = 0
+    }
+
+  }
+  if (options.rangeEnd === null || options.rangeEnd === undefined) {
+    return node.content({offset: rangeStart})
+  }
+
+    let rangeEnd = options.rangeEnd
+    if (rangeEnd < 0) {
+      rangeEnd = rangeEnd + Number(node.size)
+    } else {
+      rangeEnd = rangeEnd+1
+    }
+    const toRead =  rangeEnd - rangeStart
+    if (toRead < 0) {
+      throw new Error("range end must be greater than range start")
+    }
+    return node.content({offset: rangeStart, length: toRead})
+}

test/car.spec.js

@@ -23,6 +23,62 @@ describe('CAR Verification', () => {
     assert.strictEqual(actualContent, expectedContent)
   })
 
+  it('should extract content from a valid CAR with a range', async () => {
+    const cidPath =
+      'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4'
+    const filepath = getFixturePath('hello.car')
+    const carStream = fs.createReadStream(filepath)
+
+    const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: 1, rangeEnd: 3})
+    const buffer = await concatChunks(contentItr)
+    const actualContent = String.fromCharCode(...buffer)
+    const expectedContent = 'ell'
+
+    assert.strictEqual(actualContent, expectedContent)
+  })
+
+  it('should extract content from a valid CAR with a range with only a start', async () => {
+    const cidPath =
+      'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4'
+    const filepath = getFixturePath('hello.car')
+    const carStream = fs.createReadStream(filepath)
+
+    const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: 1})
+    const buffer = await concatChunks(contentItr)
+    const actualContent = String.fromCharCode(...buffer)
+    const expectedContent = 'ello world\n'
+
+    assert.strictEqual(actualContent, expectedContent)
+  })
+
+  it('should extract content from a valid CAR with a range with a negative end', async () => {
+    const cidPath =
+      'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4'
+    const filepath = getFixturePath('hello.car')
+    const carStream = fs.createReadStream(filepath)
+
+    const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: 1, rangeEnd: -1})
+    const buffer = await concatChunks(contentItr)
+    const actualContent = String.fromCharCode(...buffer)
+    const expectedContent = 'ello world'
+
+    assert.strictEqual(actualContent, expectedContent)
+  })
+
+  it('should extract content from a valid CAR with a range with a negative start and end', async () => {
+    const cidPath =
+      'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4'
+    const filepath = getFixturePath('hello.car')
+    const carStream = fs.createReadStream(filepath)
+
+    const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: -5, rangeEnd: -1})
+    const buffer = await concatChunks(contentItr)
+    const actualContent = String.fromCharCode(...buffer)
+    const expectedContent = 'orld'
+
+    assert.strictEqual(actualContent, expectedContent)
+  })
+
   it('should verify intermediate path segments', async () => {
     const cidPath =
       'bafybeigeqgfwhivuuxgmuvcrrwvs4j3yfzgljssvnuqzokm6uby4fpmwsa/subdir/hello.txt'

test/index.spec.js

@@ -81,6 +81,18 @@ describe('Saturn client', () => {
     })
   })
 
+  describe('Create a request URL', () => {
+    const client = new Saturn({ clientKey })
+    it('should translate entity bytes params', () => {
+      assert.strictEqual(client.createRequestURL('bafy...').searchParams.get('entity-bytes'), null)
+      assert.strictEqual(client.createRequestURL('bafy...', { range: {} }).searchParams.get('entity-bytes'), null)
+      assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeStart: 0 } }).searchParams.get('entity-bytes'), null)
+      assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeStart: 10 } }).searchParams.get('entity-bytes'), '10:*')
+      assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeStart: 10, rangeEnd: 20 } }).searchParams.get('entity-bytes'), '10:20')
+      assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeEnd: 20 } }).searchParams.get('entity-bytes'), '0:20')
+    })
+  })
+
   describe('Logging', () => {
     const handlers = [
       mockJWT(TEST_AUTH)