From 000a26657f3e25c539de926afb10df6e623732e7 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Mon, 8 Jan 2024 20:44:11 -0800 Subject: [PATCH] feat(client): support range requests - pass entity-bytes param to the backend - utilize offset/length params on unixfs-exporter to do ranged verification --- src/client.js | 8 ++++++- src/types.js | 10 +++++++++ src/utils/car.js | 41 +++++++++++++++++++++++++++++---- test/car.spec.js | 56 ++++++++++++++++++++++++++++++++++++++++++++++ test/index.spec.js | 12 ++++++++++ 5 files changed, 122 insertions(+), 5 deletions(-) diff --git a/src/client.js b/src/client.js index 55d3893..554d72e 100644 --- a/src/client.js +++ b/src/client.js @@ -392,7 +392,7 @@ export class Saturn { if (!opts.format) { yield * itr } else { - yield * extractVerifiedContent(cidPath, itr) + yield * extractVerifiedContent(cidPath, itr, opts.range || {}) } } catch (err) { log.error = err.message @@ -422,6 +422,7 @@ export class Saturn { * @param {string} [opts.format] * @param {string} [opts.originFallback] * @param {object} [opts.jwt] + * @param {import('./types.js').ContentRange} [opts.range] * @returns {URL} */ createRequestURL (cidPath, opts = {}) { @@ -442,6 +443,11 @@ export class Saturn { url.searchParams.set('jwt', opts.jwt) } + if (typeof opts.range === 'object' && (opts.range.rangeStart || opts.range.rangeEnd)) { + const rangeStart = opts.range.rangeStart?.toString() || '0' + const rangeEnd = opts.range.rangeEnd?.toString() || '*' + url.searchParams.set('entity-bytes', rangeStart + ':' + rangeEnd) + } return url } diff --git a/src/types.js b/src/types.js index a333ce3..a23e98f 100644 --- a/src/types.js +++ b/src/types.js @@ -18,6 +18,7 @@ * @typedef {object} FetchOptions * @property {Node[]} [nodes] - An array of nodes. * @property {('car'|'raw')} [format] - The format of the fetched content. + * * @property {boolean} [originFallback] - Is this a fallback to the customer origin * @property {boolean} [raceNodes] - Does the fetch race multiple nodes on requests. * @property {string} [customerFallbackURL] - Customer Origin that is a fallback. @@ -26,6 +27,15 @@ * @property {number} [downloadTimeout=0] - Download timeout in milliseconds. * @property {AbortController} [controller] - Abort controller * @property {boolean} [firstHitDNS] - First request hit is always to CDN origin. + * @property {ContentRange} [range] - range to fetch, compatible with entity bytes parameter + */ + +/** + * Options for a range request + * + * @typedef {object} ContentRange + * @property {number} [rangeStart] + * @property {number} [rangeEnd] */ export {} diff --git a/src/utils/car.js b/src/utils/car.js index facd291..60c8ff7 100644 --- a/src/utils/car.js +++ b/src/utils/car.js @@ -2,7 +2,7 @@ import { CarBlockIterator } from '@ipld/car' import * as dagCbor from '@ipld/dag-cbor' import * as dagPb from '@ipld/dag-pb' import * as dagJson from '@ipld/dag-json' -import { exporter } from 'ipfs-unixfs-exporter' +import * as unixfs from 'ipfs-unixfs-exporter' import { bytes } from 'multiformats' import * as raw from 'multiformats/codecs/raw' import * as json from 'multiformats/codecs/json' @@ -89,12 +89,45 @@ export async function verifyBlock (cid, bytes) { * * @param {string} cidPath * @param {ReadableStream|AsyncIterable} carStream + * @param {import('../types.js').ContentRange} options */ -export async function * extractVerifiedContent (cidPath, carStream) { +export async function * extractVerifiedContent (cidPath, carStream, options = {}) { const getter = await CarBlockGetter.fromStream(carStream) - const node = await exporter(cidPath, getter) + const node = await unixfs.exporter(cidPath, getter) - for await (const chunk of node.content()) { + for await (const chunk of contentGenerator(node, options)) { yield chunk } } + +/** + * + * @param {unixfs.UnixFSEntry} node + * @param {import('../types.js').ContentRange} options + */ +function contentGenerator(node, options = {}) { + + let rangeStart = options.rangeStart ?? 0 + if (rangeStart < 0) { + rangeStart = rangeStart + Number(node.size) + if (rangeStart < 0) { + rangeStart = 0 + } + + } + if (options.rangeEnd === null || options.rangeEnd === undefined) { + return node.content({offset: rangeStart}) + } + + let rangeEnd = options.rangeEnd + if (rangeEnd < 0) { + rangeEnd = rangeEnd + Number(node.size) + } else { + rangeEnd = rangeEnd+1 + } + const toRead = rangeEnd - rangeStart + if (toRead < 0) { + throw new Error("range end must be greater than range start") + } + return node.content({offset: rangeStart, length: toRead}) +} \ No newline at end of file diff --git a/test/car.spec.js b/test/car.spec.js index 02e9c1b..b1305c9 100644 --- a/test/car.spec.js +++ b/test/car.spec.js @@ -23,6 +23,62 @@ describe('CAR Verification', () => { assert.strictEqual(actualContent, expectedContent) }) + it('should extract content from a valid CAR with a range', async () => { + const cidPath = + 'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4' + const filepath = getFixturePath('hello.car') + const carStream = fs.createReadStream(filepath) + + const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: 1, rangeEnd: 3}) + const buffer = await concatChunks(contentItr) + const actualContent = String.fromCharCode(...buffer) + const expectedContent = 'ell' + + assert.strictEqual(actualContent, expectedContent) + }) + + it('should extract content from a valid CAR with a range with only a start', async () => { + const cidPath = + 'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4' + const filepath = getFixturePath('hello.car') + const carStream = fs.createReadStream(filepath) + + const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: 1}) + const buffer = await concatChunks(contentItr) + const actualContent = String.fromCharCode(...buffer) + const expectedContent = 'ello world\n' + + assert.strictEqual(actualContent, expectedContent) + }) + + it('should extract content from a valid CAR with a range with a negative end', async () => { + const cidPath = + 'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4' + const filepath = getFixturePath('hello.car') + const carStream = fs.createReadStream(filepath) + + const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: 1, rangeEnd: -1}) + const buffer = await concatChunks(contentItr) + const actualContent = String.fromCharCode(...buffer) + const expectedContent = 'ello world' + + assert.strictEqual(actualContent, expectedContent) + }) + + it('should extract content from a valid CAR with a range with a negative start and end', async () => { + const cidPath = + 'bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4' + const filepath = getFixturePath('hello.car') + const carStream = fs.createReadStream(filepath) + + const contentItr = await extractVerifiedContent(cidPath, carStream, {rangeStart: -5, rangeEnd: -1}) + const buffer = await concatChunks(contentItr) + const actualContent = String.fromCharCode(...buffer) + const expectedContent = 'orld' + + assert.strictEqual(actualContent, expectedContent) + }) + it('should verify intermediate path segments', async () => { const cidPath = 'bafybeigeqgfwhivuuxgmuvcrrwvs4j3yfzgljssvnuqzokm6uby4fpmwsa/subdir/hello.txt' diff --git a/test/index.spec.js b/test/index.spec.js index 96015b9..4460806 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -81,6 +81,18 @@ describe('Saturn client', () => { }) }) + describe('Create a request URL', () => { + const client = new Saturn({ clientKey }) + it('should translate entity bytes params', () => { + assert.strictEqual(client.createRequestURL('bafy...').searchParams.get('entity-bytes'), null) + assert.strictEqual(client.createRequestURL('bafy...', { range: {} }).searchParams.get('entity-bytes'), null) + assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeStart: 0 } }).searchParams.get('entity-bytes'), null) + assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeStart: 10 } }).searchParams.get('entity-bytes'), '10:*') + assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeStart: 10, rangeEnd: 20 } }).searchParams.get('entity-bytes'), '10:20') + assert.strictEqual(client.createRequestURL('bafy...', { range: { rangeEnd: 20 } }).searchParams.get('entity-bytes'), '0:20') + }) + }) + describe('Logging', () => { const handlers = [ mockJWT(TEST_AUTH)