Merge branch 'main' into feat/storage-updates

package.json

@@ -36,10 +36,12 @@
     "p-limit": "^5.0.0"
   },
   "devDependencies": {
+    "crypto-browserify": "^3.12.0",
     "eslint": "^8.24.0",
     "eslint-config-ipfs": "^3.1.1",
     "eslint-plugin-jsdoc": "^39.3.6",
     "eslint-plugin-promise": "^6.0.1",
+    "stream-browserify": "^3.0.0",
     "webpack": "^5.74.0",
     "webpack-cli": "^4.10.0"
   },
@@ -63,4 +65,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

src/client.js

@@ -406,7 +406,7 @@ export class Saturn {
       if (!opts.format) {
         yield * itr
       } else {
-        yield * extractVerifiedContent(cidPath, itr)
+        yield * extractVerifiedContent(cidPath, itr, opts.range || {})
       }
     } catch (err) {
       log.error = err.message
@@ -436,6 +436,7 @@ export class Saturn {
    * @param {string} [opts.format]
    * @param {string} [opts.originFallback]
    * @param {object} [opts.jwt]
+   * @param {import('./types.js').ContentRange} [opts.range]
    * @returns {URL}
    */
   createRequestURL (cidPath, opts = {}) {
@@ -456,6 +457,11 @@ export class Saturn {
       url.searchParams.set('jwt', opts.jwt)
     }
 
+    if (typeof opts.range === 'object' && (opts.range.rangeStart || opts.range.rangeEnd)) {
+      const rangeStart = opts.range.rangeStart?.toString() || '0'
+      const rangeEnd = opts.range.rangeEnd?.toString() || '*'
+      url.searchParams.set('entity-bytes', rangeStart + ':' + rangeEnd)
+    }
     return url
   }
 

src/types.js

@@ -18,6 +18,7 @@
  * @typedef {object} FetchOptions
  * @property {Node[]} [nodes] - An array of nodes.
  * @property {('car'|'raw')} [format] - The format of the fetched content.
+ * 
  * @property {boolean} [originFallback] - Is this a fallback to the customer origin
  * @property {boolean} [raceNodes] - Does the fetch race multiple nodes on requests.
  * @property {string} [customerFallbackURL] - Customer Origin that is a fallback.
@@ -26,6 +27,15 @@
  * @property {number} [downloadTimeout=0] - Download timeout in milliseconds.
  * @property {AbortController} [controller] - Abort controller
  * @property {boolean} [firstHitDNS] - First request hit is always to CDN origin.
+ * @property {ContentRange} [range] - range to fetch, compatible with entity bytes parameter
+ */
+
+/**
+ * Options for a range request
+ *
+ * @typedef {object} ContentRange
+ * @property {number} [rangeStart]
+ * @property {number} [rangeEnd]
  */
 
 export {}

src/utils/car.js

@@ -2,7 +2,7 @@ import { CarBlockIterator } from '@ipld/car'
 import * as dagCbor from '@ipld/dag-cbor'
 import * as dagPb from '@ipld/dag-pb'
 import * as dagJson from '@ipld/dag-json'
-import { exporter } from 'ipfs-unixfs-exporter'
+import * as unixfs from 'ipfs-unixfs-exporter'
 import { bytes } from 'multiformats'
 import * as raw from 'multiformats/codecs/raw'
 import * as json from 'multiformats/codecs/json'
@@ -89,12 +89,45 @@ export async function verifyBlock (cid, bytes) {
  *
  * @param {string} cidPath
  * @param {ReadableStream|AsyncIterable} carStream
+ * @param {import('../types.js').ContentRange} options
  */
-export async function * extractVerifiedContent (cidPath, carStream) {
+export async function * extractVerifiedContent (cidPath, carStream, options = {}) {
   const getter = await CarBlockGetter.fromStream(carStream)
-  const node = await exporter(cidPath, getter)
+  const node = await unixfs.exporter(cidPath, getter)
 
-  for await (const chunk of node.content()) {
+  for await (const chunk of contentGenerator(node, options)) {
     yield chunk
   }
 }
+
+/**
+ * 
+ * @param {unixfs.UnixFSEntry} node
+ * @param {import('../types.js').ContentRange} options
+ */
+function contentGenerator(node, options = {}) {
+
+  let rangeStart = options.rangeStart ?? 0
+  if (rangeStart < 0) {
+    rangeStart = rangeStart + Number(node.size)
+    if (rangeStart < 0) {
+      rangeStart = 0
+    }
+
+  }
+  if (options.rangeEnd === null || options.rangeEnd === undefined) {
+    return node.content({offset: rangeStart})
+  }
+
+    let rangeEnd = options.rangeEnd
+    if (rangeEnd < 0) {
+      rangeEnd = rangeEnd + Number(node.size)
+    } else {
+      rangeEnd = rangeEnd+1
+    }
+    const toRead =  rangeEnd - rangeStart
+    if (toRead < 0) {
+      throw new Error("range end must be greater than range start")
+    }
+    return node.content({offset: rangeStart, length: toRead})
+}

test/car.spec.js

@@ -23,6 +23,62 @@ describe('CAR Verification', () => {
     assert.strictEqual(actualContent, expectedContent)
   })
 
+  it('should extract content from a valid CAR with a range', async () => {
+    const cidPath =
+      'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4'
+    const filepath = getFixturePath('hello.car')
+    const carStream = fs.createReadStream(filepath)
+
+    const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: 1, rangeEnd: 3})
+    const buffer = await concatChunks(contentItr)
+    const actualContent = String.fromCharCode(...buffer)
+    const expectedContent = 'ell'
+
+    assert.strictEqual(actualContent, expectedContent)
+  })
+
+  it('should extract content from a valid CAR with a range with only a start', async () => {
+    const cidPath =
+      'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4'
+    const filepath = getFixturePath('hello.car')
+    const carStream = fs.createReadStream(filepath)
+
+    const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: 1})
+    const buffer = await concatChunks(contentItr)
+    const actualContent = String.fromCharCode(...buffer)
+    const expectedContent = 'ello world\n'
+
+    assert.strictEqual(actualContent, expectedContent)
+  })
+
+  it('should extract content from a valid CAR with a range with a negative end', async () => {
+    const cidPath =
+      'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4'
+    const filepath = getFixturePath('hello.car')
+    const carStream = fs.createReadStream(filepath)
+
+    const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: 1, rangeEnd: -1})
+    const buffer = await concatChunks(contentItr)
+    const actualContent = String.fromCharCode(...buffer)
+    const expectedContent = 'ello world'
+
+    assert.strictEqual(actualContent, expectedContent)
+  })
+
+  it('should extract content from a valid CAR with a range with a negative start and end', async () => {
+    const cidPath =
+      'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4'
+    const filepath = getFixturePath('hello.car')
+    const carStream = fs.createReadStream(filepath)
+
+    const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: -5, rangeEnd: -1})
+    const buffer = await concatChunks(contentItr)
+    const actualContent = String.fromCharCode(...buffer)
+    const expectedContent = 'orld'
+
+    assert.strictEqual(actualContent, expectedContent)
+  })
+
   it('should verify intermediate path segments', async () => {
     const cidPath =
       'bafybeigeqgfwhivuuxgmuvcrrwvs4j3yfzgljssvnuqzokm6uby4fpmwsa/subdir/hello.txt'

test/index.spec.js

@@ -81,6 +81,18 @@ describe('Saturn client', () => {
     })
   })
 
+  describe('Create a request URL', () => {
+    const client = new Saturn({ clientKey })
+    it('should translate entity bytes params', () => {
+      assert.strictEqual(client.createRequestURL('bafy...').searchParams.get('entity-bytes'), null)
+      assert.strictEqual(client.createRequestURL('bafy...', { range: {} }).searchParams.get('entity-bytes'), null)
+      assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeStart: 0 } }).searchParams.get('entity-bytes'), null)
+      assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeStart: 10 } }).searchParams.get('entity-bytes'), '10:*')
+      assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeStart: 10, rangeEnd: 20 } }).searchParams.get('entity-bytes'), '10:20')
+      assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeEnd: 20 } }).searchParams.get('entity-bytes'), '0:20')
+    })
+  })
+
   describe('Logging', () => {
     const handlers = [
       mockJWT(TEST_AUTH)

webpack.config.cjs

@@ -10,5 +10,11 @@ module.exports = {
       name: 'SaturnModule',
       type: 'var',
     }
+  },
+  resolve: {
+    fallback: {
+      crypto: require.resolve('crypto-browserify'),
+      stream: require.resolve('stream-browserify')
+    }
   }
 };