From 211dd595a7653c7db626512fc0be70301d734496 Mon Sep 17 00:00:00 2001 From: Sadiq Khoja Date: Tue, 12 Sep 2023 15:59:03 -0400 Subject: [PATCH] Odata opaque cursor (#934) * added integration tests to capture requirement * Made changes for Submission OData need to add more unit tests * changes for Entiites Odata * refactored a bit and added unit tests * prefix skiptoken with 01 * just use repeatId for subtables * fix entity count * datasets/:name is extendable * added test for extended datasets/:name * polish some of the tests --- lib/data/entity.js | 10 +- lib/data/odata.js | 12 +- lib/formats/odata.js | 93 +++++++++++-- lib/http/endpoint.js | 2 +- lib/model/query/entities.js | 29 +++- lib/model/query/submissions.js | 23 +++- lib/resources/datasets.js | 4 +- lib/resources/odata-entities.js | 4 +- lib/resources/odata.js | 6 +- lib/util/db.js | 22 ++- lib/util/odata.js | 14 +- lib/util/util.js | 14 ++ test/integration/api/datasets.js | 69 ++++++++-- test/integration/api/odata-entities.js | 125 ++++++++++++++++- test/integration/api/odata.js | 178 +++++++++++++++++++++++-- test/unit/data/odata.js | 65 ++++----- test/unit/formats/odata.js | 50 +++++-- test/unit/util/db.js | 9 ++ test/unit/util/util.js | 8 ++ 19 files changed, 623 insertions(+), 114 deletions(-) diff --git a/lib/data/entity.js b/lib/data/entity.js index 411982179..9478de8bd 100644 --- a/lib/data/entity.js +++ b/lib/data/entity.js @@ -263,12 +263,13 @@ const extractSelectedProperties = (query, properties) => { }; // Pagination is done at the database level -const streamEntityOdata = (inStream, properties, domain, originalUrl, query, tableCount) => { +const streamEntityOdata = (inStream, properties, domain, originalUrl, query, tableCount, tableRemaining) => { const serviceRoot = getServiceRoot(originalUrl); - const { limit, offset, shouldCount } = extractPaging(query); + const { limit, offset, shouldCount, skipToken } = extractPaging(query); const selectedProperties = extractSelectedProperties(query, properties); let isFirstEntity = true; + let lastUuid; const rootStream = new Transform({ writableObjectMode: true, // we take a stream of objects from the db, but readableObjectMode: false, // we put out a stream of text. @@ -281,6 +282,8 @@ const streamEntityOdata = (inStream, properties, domain, originalUrl, query, tab this.push(','); } + lastUuid = entity.uuid; + this.push(JSON.stringify(selectFields(entity, properties, selectedProperties))); done(); @@ -290,8 +293,9 @@ const streamEntityOdata = (inStream, properties, domain, originalUrl, query, tab }, flush(done) { this.push((isFirstEntity) ? '{"value":[],' : '],'); // open object or close row array. + const remaining = skipToken ? tableRemaining - limit : tableCount - (limit + offset); // @odata.count and nextUrl. - const nextUrl = nextUrlFor(limit, offset, tableCount, originalUrl); + const nextUrl = nextUrlFor(remaining, originalUrl, { uuid: lastUuid }); this.push(jsonDataFooter({ table: 'Entities', domain, serviceRoot, nextUrl, count: (shouldCount ? tableCount.toString() : null) })); done(); diff --git a/lib/data/odata.js b/lib/data/odata.js index 6aad6f4bc..e7f0906bd 100644 --- a/lib/data/odata.js +++ b/lib/data/odata.js @@ -136,9 +136,7 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi // on option.metadata even though they are not part of the form's own schema. // So rather than try to inject them into the xml transformation below, we just // formulate them herein advance: - if (!options.metadata || options.metadata.__id) { - root.__id = submission.instanceId; - } + root.__id = submission.instanceId; if (table === 'Submissions') { const systemObj = { submissionDate: submission.createdAt, @@ -175,7 +173,7 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi } // bail out without doing any work if we are encrypted. - if (encrypted === true) return resolve(result); + if (encrypted === true) return resolve({ data: result, instanceId: submission.instanceId }); // we keep a dataStack, so we build an appropriate nested structure overall, and // we can select the appropriate layer of that nesting at will. @@ -220,9 +218,7 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi // create our new databag, push into result data, and set it as our result ptr. const bag = generateDataFrame(schemaStack); - if (!options.metadata || options.metadata.__id) { - bag.__id = hashId(schemaStack, submission.instanceId); - } + bag.__id = hashId(schemaStack, submission.instanceId); dataPtr[outname].push(bag); dataStack.push(bag); @@ -342,7 +338,7 @@ const submissionToOData = (fields, table, submission, options = {}) => new Promi if (schemaStack.hasExited()) { parser.reset(); - resolve(result); + resolve({ data: result, instanceId: submission.instanceId }); } } }, { xmlMode: true, decodeEntities: true }); diff --git a/lib/formats/odata.js b/lib/formats/odata.js index 11464e81f..f71011a85 100644 --- a/lib/formats/odata.js +++ b/lib/formats/odata.js @@ -23,6 +23,8 @@ const { sanitizeOdataIdentifier, without } = require('../util/util'); const { jsonDataFooter, extractOptions, nextUrlFor } = require('../util/odata'); const { submissionToOData, systemFields } = require('../data/odata'); const { SchemaStack } = require('../data/schema'); +const { QueryOptions } = require('../util/db'); + //////////////////////////////////////////////////////////////////////////////// // UTIL @@ -52,9 +54,10 @@ const extractPathContext = (subpath) => const extractPaging = (table, query) => { const parsedLimit = parseInt(query.$top, 10); const limit = Number.isNaN(parsedLimit) ? Infinity : parsedLimit; - const offset = parseInt(query.$skip, 10) || 0; + const offset = (!query.$skiptoken && parseInt(query.$skip, 10)) || 0; const shouldCount = isTrue(query.$count); - const result = { limit: max(0, limit), offset: max(0, offset), shouldCount }; + const skipToken = query.$skiptoken ? QueryOptions.parseSkiptoken(query.$skiptoken) : null; + const result = { limit: max(0, limit), offset: max(0, offset), shouldCount, skipToken }; return Object.assign(result, (table === 'Submissions') ? { doLimit: Infinity, doOffset: 0 } @@ -366,11 +369,12 @@ const edmxForEntities = (datasetName, properties) => { // originalUrl: String is the request URL; we need it as well to formulate response URLs. // query: Object is the Express Request query object indicating request querystring parameters. // inStream: Stream[Row] is the postgres Submissions rowstream. -const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, tableCount) => { +const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, tableCount, tableRemaining) => { // cache values we'll need repeatedly. const serviceRoot = getServiceRoot(originalUrl); - const { limit, offset, doLimit, doOffset, shouldCount } = extractPaging(table, query); + const { doLimit, doOffset, shouldCount, skipToken } = extractPaging(table, query); const options = extractOptions(query); + const isSubTable = table !== 'Submissions'; // make sure the target table actually exists. // TODO: now that this doesn't require schema computation, should we move it up @@ -378,26 +382,53 @@ const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, t if (!verifyTablePath(table.split('.'), fields)) throw Problem.user.notFound(); // write the header, then transform and stream each row. + // To count total number of items for subtable (repeats) let counted = 0; + // To count items added to the downstream, required only for subtable + let added = 0; + // To count remaining items in case of subtable + let remainingItems = 0; + + // skipToken is created based on following two variables + let lastInstanceId = null; + let lastRepeatId = null; + + // For Submissions table, it is true because cursor is handled at database level + let cursorPredicate = !isSubTable || !skipToken; + const parserStream = new Transform({ writableObjectMode: true, // we take a stream of objects from the db, but readableObjectMode: false, // we put out a stream of text. transform(row, _, done) { // per row, we do our asynchronous parsing, jam the result onto the // text resultstream, and call done to indicate that the row is processed. - submissionToOData(fields, table, row, options).then((data) => { + submissionToOData(fields, table, row, options).then(({ data, instanceId }) => { const parentIdProperty = data[0] ? Object.keys(data[0]).find(p => /^__.*-id$/.test(p)) : null; + // In case of subtable we are reading all Submissions without pagination because we have to + // count repeat items in each Submission for (const field of data) { // if $select is there and parentId is not requested then remove it - const fieldRefined = options.metadata && !options.metadata[parentIdProperty] ? without([parentIdProperty], field) : field; + let fieldRefined = options.metadata && !options.metadata[parentIdProperty] ? without([parentIdProperty], field) : field; + // if $select is there and __id is not requested then remove it + fieldRefined = options.metadata && !options.metadata.__id ? without(['__id'], fieldRefined) : fieldRefined; + + if (added === doLimit) remainingItems += 1; - if ((counted >= doOffset) && (counted < (doOffset + doLimit))) { - this.push((counted === doOffset) ? '{"value":[' : ','); // header or fencepost. + if (added < doLimit && counted >= doOffset && cursorPredicate) { + this.push((added === 0) ? '{"value":[' : ','); // header or fencepost. this.push(JSON.stringify(fieldRefined)); + lastInstanceId = instanceId; + if (isSubTable) lastRepeatId = field.__id; + added += 1; } + + // Controls the rows to be skipped based on skipToken + // Once set to true remains true + cursorPredicate = cursorPredicate || skipToken.repeatId === field.__id; + counted += 1; } done(); // signifies that this stream element is fully processed. @@ -406,12 +437,20 @@ const rowStreamToOData = (fields, table, domain, originalUrl, query, inStream, t flush(done) { // flush is called just before the transform stream is done and closed; we write // our footer information, close the object, and tell the stream we are done. - this.push((counted <= doOffset) ? '{"value":[],' : '],'); // open object or close row array. + this.push((added === 0) ? '{"value":[],' : '],'); // open object or close row array. // if we were given an explicit count, use it from here out, to create // @odata.count and nextUrl. const totalCount = (tableCount != null) ? tableCount : counted; - const nextUrl = nextUrlFor(limit, offset, totalCount, originalUrl); + + // How many items are remaining for the next page? + // if there aren't any then we don't need nextUrl + const remaining = (tableRemaining != null) ? tableRemaining - added : remainingItems; + + let skipTokenData = { instanceId: lastInstanceId }; + if (isSubTable) skipTokenData = { repeatId: lastRepeatId }; + + const nextUrl = nextUrlFor(remaining, originalUrl, skipTokenData); // we do toString on the totalCount because mustache won't bother rendering // the block if it sees integer zero. @@ -448,8 +487,10 @@ const singleRowToOData = (fields, row, domain, originalUrl, query) => { const table = tableParts.join('.'); if (!verifyTablePath(tableParts, fields)) throw Problem.user.notFound(); + const isSubTable = table !== 'Submissions'; + // extract all our fields first, the field extractor doesn't know about target contexts. - return submissionToOData(fields, table, row, options).then((subrows) => { + return submissionToOData(fields, table, row, options).then(({ data: subrows, instanceId }) => { // now we actually filter to the requested set. we actually only need to compare // the very last specified id, since it is fully unique. const filterContextIdx = targetContext.reduce(((extant, pair, idx) => ((pair[1] != null) ? idx : extant)), -1); @@ -462,12 +503,36 @@ const singleRowToOData = (fields, row, domain, originalUrl, query) => { const count = filtered.length; // now we can process $top/$skip/$count: - const { limit, offset, shouldCount } = extractPaging(table, query); - const nextUrl = nextUrlFor(limit, offset, count, originalUrl); + const paging = extractPaging(table, query); + const { limit, shouldCount, skipToken } = paging; + let { offset } = paging; + + if (skipToken) { + offset = filtered.fiindIndex(s => skipToken.repeatId === s.__id); + } + const pared = filtered.slice(offset, offset + limit); + let nextUrl = null; + + if (pared.length > 0) { + const remaining = count - (offset + limit); + + let skipTokenData = { + instanceId + }; + + if (isSubTable) skipTokenData = { repeatId: pared[pared.length - 1].__id }; + + nextUrl = nextUrlFor(remaining, originalUrl, skipTokenData); + } + + + // if $select is there and parentId is not requested then remove it + let paredRefined = options.metadata && !options.metadata[filterField] ? pared.map(p => without([filterField], p)) : pared; + // if $select is there and parentId is not requested then remove it - const paredRefined = options.metadata && !options.metadata[filterField] ? pared.map(p => without([filterField], p)) : pared; + paredRefined = options.metadata && !options.metadata.__id ? paredRefined.map(p => without(['__id'], p)) : paredRefined; // and finally splice together and return our result: const dataContents = paredRefined.map(JSON.stringify).join(','); diff --git a/lib/http/endpoint.js b/lib/http/endpoint.js index 5c1ebf790..c524bbedd 100644 --- a/lib/http/endpoint.js +++ b/lib/http/endpoint.js @@ -309,7 +309,7 @@ const isJsonType = (x) => /(^|,)(application\/json|json)($|;|,)/i.test(x); const isXmlType = (x) => /(^|,)(application\/(atom(svc)?\+)?xml|atom|xml)($|;|,)/i.test(x); // various supported odata constants: -const supportedParams = [ '$format', '$count', '$skip', '$top', '$filter', '$wkt', '$expand', '$select' ]; +const supportedParams = [ '$format', '$count', '$skip', '$top', '$filter', '$wkt', '$expand', '$select', '$skiptoken' ]; const supportedFormats = { json: [ 'application/json', 'json' ], xml: [ 'application/xml', 'atom' ] diff --git a/lib/model/query/entities.js b/lib/model/query/entities.js index 0a02ae291..10cb0c81a 100644 --- a/lib/model/query/entities.js +++ b/lib/model/query/entities.js @@ -315,6 +315,11 @@ INNER JOIN SELECT "entityId", (COUNT(id) - 1) AS "updates" FROM entity_defs GROUP BY "entityId" ) stats ON stats."entityId"=entity_defs."entityId" LEFT JOIN actors ON entities."creatorId"=actors.id +${options.skiptoken ? sql` + INNER JOIN + ( SELECT id, "createdAt" FROM entities WHERE "uuid" = ${options.skiptoken.uuid}) AS cursor + ON entities."createdAt" <= cursor."createdAt" AND entities.id < cursor.id + `: sql``} WHERE entities."datasetId" = ${datasetId} AND entities."deletedAt" IS NULL @@ -324,10 +329,28 @@ ORDER BY entities."createdAt" DESC, entities.id DESC ${page(options)}`) .then(stream.map(_exportUnjoiner)); -const countByDatasetId = (datasetId, options = QueryOptions.none) => ({ oneFirst }) => oneFirst(sql` -SELECT count(*) FROM entities +const countByDatasetId = (datasetId, options = QueryOptions.none) => ({ one }) => one(sql` +SELECT * FROM + +( + SELECT count(*) count FROM entities + WHERE "datasetId" = ${datasetId} + AND "deletedAt" IS NULL + AND ${odataFilter(options.filter, odataToColumnMap)} +) AS "all" + +CROSS JOIN +( + SELECT COUNT(*) remaining FROM entities + ${options.skiptoken ? sql` + INNER JOIN + ( SELECT id, "createdAt" FROM entities WHERE "uuid" = ${options.skiptoken.uuid}) AS cursor + ON entities."createdAt" <= cursor."createdAt" AND entities.id < cursor.id + `: sql``} WHERE "datasetId" = ${datasetId} - AND ${odataFilter(options.filter, odataToColumnMap)}`); + AND "deletedAt" IS NULL + AND ${odataFilter(options.filter, odataToColumnMap)} +) AS skiptoken`); //////////////////////////////////////////////////////////////////////////////// diff --git a/lib/model/query/submissions.js b/lib/model/query/submissions.js index a11f1c5b8..3faeba804 100644 --- a/lib/model/query/submissions.js +++ b/lib/model/query/submissions.js @@ -199,9 +199,21 @@ const getById = (submissionId) => ({ maybeOne }) => maybeOne(sql`select * from submissions where id=${submissionId} and "deletedAt" is null`) .then(map(construct(Submission))); -const countByFormId = (formId, draft, options = QueryOptions.none) => ({ oneFirst }) => oneFirst(sql` -select count(*) from submissions - where ${equals({ formId, draft })} and "deletedAt" is null and ${odataFilter(options.filter, odataToColumnMap)}`); +const countByFormId = (formId, draft, options = QueryOptions.none) => ({ one }) => one(sql` +SELECT * FROM +( SELECT COUNT(*) count FROM submissions + WHERE ${equals({ formId, draft })} AND "deletedAt" IS NULL AND ${odataFilter(options.filter, odataToColumnMap)}) AS "all" +CROSS JOIN +( SELECT COUNT(*) remaining FROM submissions + ${options.skiptoken ? sql` + INNER JOIN + ( SELECT id, "createdAt" FROM submissions WHERE "instanceId" = ${options.skiptoken.instanceId}) AS cursor + ON submissions."createdAt" <= cursor."createdAt" AND submissions.id < cursor.id + `: sql``} + WHERE ${equals({ formId, draft })} + AND "deletedAt" IS NULL + AND ${odataFilter(options.filter, odataToColumnMap)}) AS skiptoken +`); const verifyVersion = (formId, rootId, instanceId, draft) => ({ maybeOne }) => maybeOne(sql` select true from submissions @@ -337,6 +349,11 @@ inner join (select "submissionId", (count(id) - 1) as count from submission_defs group by "submissionId") as edits on edits."submissionId"=submission_defs."submissionId" +${options.skiptoken && !options.skiptoken.repeatId ? sql` -- in case of subtable we are fetching all Submissions without pagination + inner join + (select id, "createdAt" from submissions where "instanceId" = ${options.skiptoken.instanceId}) as cursor + on submissions."createdAt" <= cursor."createdAt" and submissions.id < cursor.id +`: sql``} where ${encrypted ? sql`(form_defs."encKeyId" is null or form_defs."encKeyId" in (${sql.join(keyIds, sql`,`)})) and` : sql``} ${odataFilter(options.filter, options.isSubmissionsTable ? odataToColumnMap : odataSubTableToColumnMap)} and diff --git a/lib/resources/datasets.js b/lib/resources/datasets.js index c22ffd423..aa99ef8f0 100644 --- a/lib/resources/datasets.js +++ b/lib/resources/datasets.js @@ -22,8 +22,8 @@ module.exports = (service, endpoint) => { .then((project) => auth.canOrReject('dataset.list', project)) .then(() => Datasets.getList(params.id, queryOptions)))); - service.get('/projects/:projectId/datasets/:name', endpoint(({ Datasets }, { params, auth }) => - Datasets.get(params.projectId, params.name) + service.get('/projects/:projectId/datasets/:name', endpoint(({ Datasets }, { params, auth, queryOptions }) => + Datasets.get(params.projectId, params.name, true, queryOptions.extended) .then(getOrNotFound) .then((dataset) => auth.canOrReject('dataset.read', dataset) .then(() => Datasets.getMetadata(dataset))))); diff --git a/lib/resources/odata-entities.js b/lib/resources/odata-entities.js index 74cf54402..1859cd18d 100644 --- a/lib/resources/odata-entities.js +++ b/lib/resources/odata-entities.js @@ -46,9 +46,9 @@ module.exports = (service, endpoint) => { const dataset = await Datasets.get(params.projectId, params.name, true).then(getOrNotFound); const properties = await Datasets.getProperties(dataset.id); const options = QueryOptions.fromODataRequestEntities(query); - const count = await Entities.countByDatasetId(dataset.id, options); + const { count, remaining } = await Entities.countByDatasetId(dataset.id, options); const entities = await Entities.streamForExport(dataset.id, options); - return json(streamEntityOdata(entities, properties, env.domain, originalUrl, query, count)); + return json(streamEntityOdata(entities, properties, env.domain, originalUrl, query, count, remaining)); })); diff --git a/lib/resources/odata.js b/lib/resources/odata.js index b0560c7d0..efdf3b1fc 100644 --- a/lib/resources/odata.js +++ b/lib/resources/odata.js @@ -67,10 +67,10 @@ module.exports = (service, endpoint) => { Forms.getFields(form.def.id).then(selectFields(query, params.table)), Submissions.streamForExport(form.id, draft, undefined, options), ((params.table === 'Submissions') && options.hasPaging()) - ? Submissions.countByFormId(form.id, draft, options) : resolve(null) + ? Submissions.countByFormId(form.id, draft, options) : resolve({}) ]) - .then(([fields, stream, count]) => - json(rowStreamToOData(fields, params.table, env.domain, originalUrl, query, stream, count))); + .then(([fields, stream, { count, remaining }]) => + json(rowStreamToOData(fields, params.table, env.domain, originalUrl, query, stream, count, remaining))); }))); }; diff --git a/lib/util/db.js b/lib/util/db.js index f74d6adc3..2df12907e 100644 --- a/lib/util/db.js +++ b/lib/util/db.js @@ -15,7 +15,7 @@ const { reject } = require('./promise'); const Problem = require('./problem'); const Option = require('./option'); const { PartialPipe, mapStream } = require('./stream'); -const { construct } = require('./util'); +const { construct, base64ToUtf8, utf8ToBase64 } = require('./util'); const { isTrue, isFalse } = require('./http'); const { Transform } = require('stream'); @@ -222,7 +222,7 @@ const equals = (obj) => { const page = (options) => { const parts = []; - if (options.offset != null) parts.push(sql`offset ${options.offset}`); + if (options.offset != null && !options.skiptoken) parts.push(sql`offset ${options.offset}`); if (options.limit != null) parts.push(sql`limit ${options.limit}`); return parts.length ? sql.join(parts, sql` `) : nothing; }; @@ -360,27 +360,41 @@ class QueryOptions { return f(this.args[arg]); } + static parseSkiptoken(token) { + const jsonString = base64ToUtf8(token.substr(2)); + return JSON.parse(jsonString); + } + + static getSkiptoken(data) { + const jsonString = JSON.stringify(data); + return '01' + utf8ToBase64(jsonString); // 01 is the version number of this scheme + } + static fromODataRequest(params, query) { const result = { extended: true }; result.isSubmissionsTable = params.table === 'Submissions'; - if ((params.table === 'Submissions') && (query.$skip != null)) + if ((params.table === 'Submissions') && (!query.$skiptoken) && (query.$skip != null)) result.offset = parseInt(query.$skip, 10); if ((params.table === 'Submissions') && (query.$top != null)) result.limit = parseInt(query.$top, 10); if (query.$filter != null) result.filter = query.$filter; + if ((params.table === 'Submissions') && (query.$skiptoken != null)) + result.skiptoken = QueryOptions.parseSkiptoken(query.$skiptoken); return new QueryOptions(result); } static fromODataRequestEntities(query) { const result = { extended: true }; - if (query.$skip != null) + if (!query.$skiptoken && query.$skip != null) result.offset = parseInt(query.$skip, 10); if (query.$top != null) result.limit = parseInt(query.$top, 10); if (query.$filter != null) result.filter = query.$filter; + if (query.$skiptoken != null) + result.skiptoken = QueryOptions.parseSkiptoken(query.$skiptoken); return new QueryOptions(result); } diff --git a/lib/util/odata.js b/lib/util/odata.js index cc2c93226..417c4abef 100644 --- a/lib/util/odata.js +++ b/lib/util/odata.js @@ -13,6 +13,7 @@ const { max } = Math; const Problem = require('./problem'); const { parse, render } = require('mustache'); const { isTrue, urlWithQueryParams } = require('./http'); +const { QueryOptions } = require('./db'); const template = (body) => { parse(body); // caches template for future perf. @@ -22,9 +23,10 @@ const template = (body) => { const extractPaging = (query) => { const parsedLimit = parseInt(query.$top, 10); const limit = Number.isNaN(parsedLimit) ? Infinity : parsedLimit; - const offset = parseInt(query.$skip, 10) || 0; + const offset = (!query.$skiptoken && parseInt(query.$skip, 10)) || 0; const shouldCount = isTrue(query.$count); - const result = { limit: max(0, limit), offset: max(0, offset), shouldCount }; + const skipToken = query.$skiptoken ? QueryOptions.parseSkiptoken(query.$skiptoken) : null; + const result = { limit: max(0, limit), offset: max(0, offset), shouldCount, skipToken }; return Object.assign(result, { doLimit: Infinity, doOffset: 0 }); }; @@ -49,10 +51,10 @@ const getServiceRoot = (subpath) => { // Given limit: Int, offset: Int, count: Int, originalUrl: String, calculates // what the nextUrl should be to supply server-driven paging (11.2.5.7). Returns // url: String? -const nextUrlFor = (limit, offset, count, originalUrl) => - ((offset + limit >= count) - ? null - : urlWithQueryParams(originalUrl, { $skip: (offset + limit), $top: null })); +const nextUrlFor = (remaining, originalUrl, skipTokenData) => ((!skipTokenData || remaining <= 0) + ? null + : urlWithQueryParams(originalUrl, { $skip: null, $skiptoken: QueryOptions.getSkiptoken(skipTokenData) })); + // Given a querystring object, returns an object of relevant OData options. Right // now that is only { wkt: Bool, expand: String, metadata: Array } diff --git a/lib/util/util.js b/lib/util/util.js index 0bb2de50a..cb3fa893c 100644 --- a/lib/util/util.js +++ b/lib/util/util.js @@ -60,6 +60,19 @@ const pickAll = (keys, obj) => { return result; }; +// source: https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem + +function base64ToUtf8(base64) { + const binString = atob(base64); + const bytes = Uint8Array.from(binString, (m) => m.codePointAt(0)); + return new TextDecoder().decode(bytes); +} + +function utf8ToBase64(string) { + const bytes = new TextEncoder().encode(string); + const binString = Array.from(bytes, (x) => String.fromCodePoint(x)).join(''); + return btoa(binString); +} //////////////////////////////////////// // CLASSES @@ -76,6 +89,7 @@ module.exports = { noop, noargs, isBlank, isPresent, blankStringToNull, sanitizeOdataIdentifier, printPairs, without, pickAll, + base64ToUtf8, utf8ToBase64, construct }; diff --git a/test/integration/api/datasets.js b/test/integration/api/datasets.js index 57402e434..d3b3eb1e2 100644 --- a/test/integration/api/datasets.js +++ b/test/integration/api/datasets.js @@ -554,18 +554,67 @@ describe('datasets and entities', () => { sourceForms.should.be.eql([ { name: 'simpleEntity', xmlFormId: 'simpleEntity' }, - { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' } ]); + { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' }]); properties.map(({ publishedAt, ...p }) => { publishedAt.should.be.isoDate(); return p; }).should.be.eql([ - { name: 'first_name', odataName: 'first_name', forms: [ - { name: 'simpleEntity', xmlFormId: 'simpleEntity' }, - { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' } - ] }, - { name: 'the.age', odataName: 'the_age', forms: [ { name: 'simpleEntity', xmlFormId: 'simpleEntity' }, ] }, - { name: 'address', odataName: 'address', forms: [ { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' }, ] } + { + name: 'first_name', odataName: 'first_name', forms: [ + { name: 'simpleEntity', xmlFormId: 'simpleEntity' }, + { name: 'simpleEntity2', xmlFormId: 'simpleEntity2' } + ] + }, + { name: 'the.age', odataName: 'the_age', forms: [{ name: 'simpleEntity', xmlFormId: 'simpleEntity' },] }, + { name: 'address', odataName: 'address', forms: [{ name: 'simpleEntity2', xmlFormId: 'simpleEntity2' },] } + ]); + + }); + + })); + + it('should return the extended metadata of the dataset', testService(async (service) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .send(testData.forms.simpleEntity) + .set('Content-Type', 'application/xml') + .expect(200); + + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + uuid: '12345678-1234-4123-8234-111111111aaa', + label: 'Johnny Doe' + }) + .expect(200); + + await asAlice.get('/v1/projects/1/datasets/people') + .set('X-Extended-Metadata', 'true') + .expect(200) + .then(({ body }) => { + + const { createdAt, properties, lastEntity, ...ds } = body; + + ds.should.be.eql({ + name: 'people', + projectId: 1, + approvalRequired: false, + entities: 1, + linkedForms: [], + sourceForms: [{ name: 'simpleEntity', xmlFormId: 'simpleEntity' }] + }); + + lastEntity.should.be.recentIsoDate(); + + createdAt.should.be.recentIsoDate(); + + properties.map(({ publishedAt, ...p }) => { + publishedAt.should.be.isoDate(); + return p; + }).should.be.eql([ + { name: 'first_name', odataName: 'first_name', forms: [{ name: 'simpleEntity', xmlFormId: 'simpleEntity' }] }, + { name: 'age', odataName: 'age', forms: [{ name: 'simpleEntity', xmlFormId: 'simpleEntity' },] } ]); }); @@ -902,7 +951,7 @@ describe('datasets and entities', () => { await asAlice.get('/v1/projects/1/datasets/people') .expect(200) .then(({ body }) => { - body.sourceForms.should.be.eql([ { name: 'simpleEntity', xmlFormId: 'simpleEntity' } ]); + body.sourceForms.should.be.eql([{ name: 'simpleEntity', xmlFormId: 'simpleEntity' }]); }); })); @@ -2362,7 +2411,7 @@ describe('datasets and entities', () => { await Audits.getLatestByAction('dataset.update') .then(o => o.get()) - .then(audit => audit.details.should.eql({ properties: ['first_name', 'age', 'color_name', ] })); + .then(audit => audit.details.should.eql({ properties: ['first_name', 'age', 'color_name',] })); })); @@ -2397,7 +2446,7 @@ describe('datasets and entities', () => { container.oneFirst(sql`select count(*) from form_fields as fs join forms as f on fs."formId" = f.id where f."xmlFormId"='simpleEntity'`), container.oneFirst(sql`select count(*) from ds_property_fields`), ]) - .then((counts) => counts.should.eql([ 2, 6, 6 ])); + .then((counts) => counts.should.eql([2, 6, 6])); })); }); diff --git a/test/integration/api/odata-entities.js b/test/integration/api/odata-entities.js index 49c24e225..76211d2b2 100644 --- a/test/integration/api/odata-entities.js +++ b/test/integration/api/odata-entities.js @@ -12,6 +12,8 @@ const testData = require('../../data/xml'); const { exhaust } = require('../../../lib/worker/worker'); const { v4: uuid } = require('uuid'); const { sql } = require('slonik'); +const { QueryOptions } = require('../../../lib/util/db'); +const should = require('should'); describe('api: /datasets/:name.svc', () => { describe('GET /Entities', () => { @@ -21,6 +23,7 @@ describe('api: /datasets/:name.svc', () => { await user.post('/v1/projects/1/forms/simpleEntity/submissions') .send(testData.instances.simpleEntity.one .replace(/one/g, `submission${i+skip}`) + .replace(/88/g, i + skip + 1) .replace('uuid:12345678-1234-4123-8234-123456789abc', uuid())) .set('Content-Type', 'application/xml') .expect(200); @@ -49,9 +52,9 @@ describe('api: /datasets/:name.svc', () => { .then(({ body }) => { body.value.length.should.be.eql(2); - body.value.forEach(r => { + body.value.forEach((r, i) => { r.first_name.should.be.eql('Alice'); - r.age.should.be.eql('88'); + r.age.should.be.eql((2 - i).toString()); }); body.value[0].__id.should.not.be.eql(body.value[1].__id); @@ -126,8 +129,93 @@ describe('api: /datasets/:name.svc', () => { await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=1') .expect(200) .then(({ body }) => { - body['@odata.nextLink'].should.be.equal('http://localhost:8989/v1/projects/1/datasets/people.svc/Entities?%24skip=1'); + const tokenData = { + uuid: body.value[0].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + body['@odata.nextLink'].should.be.equal(`http://localhost:8989/v1/projects/1/datasets/people.svc/Entities?%24top=1&%24skiptoken=${token}`); + }); + })); + + it('should not duplicate or skip entities - opaque cursor', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.simpleEntity) + .expect(200); + + await createSubmissions(asAlice, container, 2); + + const nextlink = await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=1&$count=true') + .expect(200) + .then(({ body }) => { + body.value[0].age.should.be.eql('2'); + const tokenData = { + uuid: body.value[0].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + body['@odata.nextLink'].should.be.equal(`http://localhost:8989/v1/projects/1/datasets/people.svc/Entities?%24top=1&%24count=true&%24skiptoken=${token}`); + body['@odata.count'].should.be.eql(2); + return body['@odata.nextLink']; + }); + + // create of these 2 entities have no impact on the nextlink + await createSubmissions(asAlice, container, 2, 2); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + body.value[0].age.should.be.eql('1'); + body['@odata.count'].should.be.eql(4); + should.not.exist(body['@odata.nextLink']); + }); + })); + + it('should not return deleted entities - opaque cursor', testService(async (service, container) => { + const asAlice = await service.login('alice'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.simpleEntity) + .expect(200); + + await createSubmissions(asAlice, container, 5); + + const uuids = await asAlice.get('/v1/projects/1/datasets/people/entities') + .then(({ body }) => body.map(e => e.uuid)); + + const nextlink = await asAlice.get('/v1/projects/1/datasets/people.svc/Entities?$top=2&$count=true') + .expect(200) + .then(({ body }) => { + body.value[0].age.should.be.eql('5'); + body.value[1].age.should.be.eql('4'); + const tokenData = { + uuid: body.value[1].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + body['@odata.nextLink'].should.be.equal(`http://localhost:8989/v1/projects/1/datasets/people.svc/Entities?%24top=2&%24count=true&%24skiptoken=${token}`); + body['@odata.count'].should.be.eql(5); + return body['@odata.nextLink']; }); + + // let's delete entities + await asAlice.delete(`/v1/projects/1/datasets/people/entities/${uuids[0]}`) + .expect(200); + await asAlice.delete(`/v1/projects/1/datasets/people/entities/${uuids[2]}`) + .expect(200); + await asAlice.delete(`/v1/projects/1/datasets/people/entities/${uuids[4]}`) + .expect(200); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + body.value[0].age.should.be.eql('2'); + body['@odata.count'].should.be.eql(2); + should.not.exist(body['@odata.nextLink']); + }); + + })); it('should return filtered entities', testService(async (service, container) => { @@ -151,6 +239,37 @@ describe('api: /datasets/:name.svc', () => { }); })); + it('should return filtered entities with pagination', testService(async (service, container) => { + const asAlice = await service.login('alice'); + const asBob = await service.login('bob'); + + await asAlice.post('/v1/projects/1/forms?publish=true') + .set('Content-Type', 'application/xml') + .send(testData.forms.simpleEntity) + .expect(200); + + await createSubmissions(asAlice, container, 2); + + await createSubmissions(asBob, container, 2, 2); + + const bobId = await asBob.get('/v1/users/current').then(({ body }) => body.id); + + const nextlink = await asAlice.get(`/v1/projects/1/datasets/people.svc/Entities?$top=1&$filter=__system/creatorId eq ${bobId}`) + .expect(200) + .then(({ body }) => { + body.value.length.should.be.eql(1); + body.value[0].age.should.be.eql('4'); + return body['@odata.nextLink']; + }); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + body.value[0].age.should.be.eql('3'); + should.not.exist(body['@odata.nextLink']); + }); + })); + it('should throw error if filter criterion is invalid', testService(async (service, container) => { const asAlice = await service.login('alice'); diff --git a/test/integration/api/odata.js b/test/integration/api/odata.js index 497292f10..22adf5c67 100644 --- a/test/integration/api/odata.js +++ b/test/integration/api/odata.js @@ -2,6 +2,8 @@ const { testService } = require('../setup'); const { sql } = require('slonik'); const testData = require('../../data/xml'); const { dissocPath, identity } = require('ramda'); +const { QueryOptions } = require('../../../lib/util/db'); +const should = require('should'); // NOTE: for the data output tests, we do not attempt to extensively determine if every // internal case is covered; there are already two layers of tests below these, at @@ -305,7 +307,7 @@ describe('api: /forms/:id.svc', () => { .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/Submissions(%27double%27)/children/child?%24skip=2', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/Submissions(%27double%27)/children/child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6ImI2ZTkzYTgxYTUzZWVkMDU2NmU2NWU0NzJkNGE0YjlhZTM4M2VlNmQifQ%3D%3D', value: [{ __id: 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', '__Submissions-id': 'double', @@ -324,7 +326,6 @@ describe('api: /forms/:id.svc', () => { .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/doubleRepeat.svc/Submissions(%27double%27)/children/child?%24count=true&%24skip=0', '@odata.count': 3, value: [] }); @@ -387,7 +388,7 @@ describe('api: /forms/:id.svc', () => { asAlice.get("/v1/projects/1/forms/double%20repeat.svc/Submissions('uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4')/children/child?$top=1") .expect(200) .then(({ body }) => { - body['@odata.nextLink'].should.equal('http://localhost:8989/v1/projects/1/forms/double%20repeat.svc/Submissions(%27uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4%27)/children/child?%24skip=1'); + body['@odata.nextLink'].should.equal('http://localhost:8989/v1/projects/1/forms/double%20repeat.svc/Submissions(%27uuid%3A17b09e96-4141-43f5-9a70-611eb0e8f6b4%27)/children/child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6IjdhYzVmNGQ0ZmFjYmFhOTY1N2MyMWZmMjIxYjg4NTI0MWMyODRiNmMifQ%3D%3D'); }) ])))))); @@ -573,7 +574,7 @@ describe('api: /forms/:id.svc', () => { asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions') .expect(200) .then(({ body }) => { - for (const idx of [ 0, 1, 2 ]) { + for (const idx of [0, 1, 2]) { body.value[idx].__system.submissionDate.should.be.an.isoDate(); // eslint-disable-next-line no-param-reassign delete body.value[idx].__system.submissionDate; @@ -691,7 +692,7 @@ describe('api: /forms/:id.svc', () => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24skip=2', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnR3byJ9', value: [{ __id: 'rtwo', __system: { @@ -717,6 +718,120 @@ describe('api: /forms/:id.svc', () => { }); })))); + // nb: order of id and createdAt is not guaranteed to be same + // in test env, see submission id 134849 and 134850 + // 50 (at 873 ms) was created before 49 (at 874 ms) + it('should limit Submissions', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1') + .expect(200) + .then(({ body }) => { + const tokenData = { + instanceId: body.value[0].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + body['@odata.nextLink'].should.be.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24skiptoken=${(token)}`); + }); + })); + + it('should ignore $skip when $skipToken is given', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1&$skip=1') + .expect(200) + .then(({ body }) => { + + body.value[0].__id.should.be.eql('rtwo'); + + const tokenData = { + instanceId: body.value[0].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + + const expectedNextLink = `http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24skiptoken=${(token)}`; + body['@odata.nextLink'].should.eql(expectedNextLink); + return body['@odata.nextLink']; + }); + + await asAlice.get(nextlink.replace('http://localhost:8989', '') + '&$skip=1') + .expect(200) + .then(({ body }) => { + + body.value[0].__id.should.be.eql('rone'); + + should.not.exist(body['@odata.nextLink']); + }); + })); + + it('should have no impact on skipToken when a new submission is created', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=2') + .expect(200) + .then(({ body }) => { + + body.value[0].__id.should.be.eql('rthree'); + body.value[1].__id.should.be.eql('rtwo'); + + const tokenData = { + instanceId: body.value[1].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + + const expectedNextLink = `http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=2&%24skiptoken=${(token)}`; + body['@odata.nextLink'].should.eql(expectedNextLink); + return body['@odata.nextLink']; + }); + + await asAlice.post('/v1/projects/1/forms/withrepeat/submissions') + .send(testData.instances.withrepeat.one + .replace('one', 'four') + .replace('Alice', 'John')) + .set('Content-Type', 'text/xml') + .expect(200); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + + body.value[0].__id.should.be.eql('rone'); + + should.not.exist(body['@odata.nextLink']); + }); + })); + + it('should limit and filter Submissions', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + await asAlice.patch('/v1/projects/1/forms/withrepeat/submissions/rtwo') + .send({ reviewState: 'rejected' }) + .expect(200); + + await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1&$filter=not __system/reviewState eq \'rejected\'') + .expect(200) + .then(({ body }) => { + const tokenData = { + instanceId: body.value[0].__id, + }; + const token = encodeURIComponent(QueryOptions.getSkiptoken(tokenData)); + body['@odata.nextLink'].should.eql(`http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24filter=not+__system%2FreviewState+eq+%27rejected%27&%24skiptoken=${token}`); + }); + })); + + it('should limit and return selected fields of Submissions', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1&$select=age') + .expect(200) + .then(({ body }) => { + body.value[0].should.be.eql({ + age: 38, + }); + body['@odata.nextLink'].should.be.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24select=age&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnRocmVlIn0%3D'); + }); + })); + it('should provide toplevel row count if requested', testService((service) => withSubmissions(service, (asAlice) => asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$top=1&$count=true') @@ -728,7 +843,7 @@ describe('api: /forms/:id.svc', () => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24count=true&%24skip=1', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions?%24top=1&%24count=true&%24skiptoken=01eyJpbnN0YW5jZUlkIjoicnRocmVlIn0%3D', '@odata.count': 3, value: [{ __id: 'rthree', @@ -777,7 +892,7 @@ describe('api: /forms/:id.svc', () => { .then(() => asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions?$filter=__system/submitterId eq 5') .expect(200) .then(({ body }) => { - for (const idx of [ 0, 1 ]) { + for (const idx of [0, 1]) { body.value[idx].__system.submissionDate.should.be.an.isoDate(); // eslint-disable-next-line no-param-reassign delete body.value[idx].__system.submissionDate; @@ -1285,7 +1400,7 @@ describe('api: /forms/:id.svc', () => { .then(({ body }) => { body.should.eql({ '@odata.context': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24skip=2', + '@odata.nextLink': 'http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6IjUyZWZmOWVhODI1NTAxODM4ODBiOWQ2NGMyMDQ4NzY0MmZhNmU2MGMifQ%3D%3D', value: [{ __id: '52eff9ea82550183880b9d64c20487642fa6e60c', '__Submissions-id': 'rtwo', @@ -1369,6 +1484,51 @@ describe('api: /forms/:id.svc', () => { }); })); + it('should limit subtable results', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=2') + .expect(200) + .then(({ body }) => { + body.value[0].name.should.be.eql('Candace'); + body.value[1].name.should.be.eql('Billy'); + body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=2&%24skiptoken=01eyJyZXBlYXRJZCI6IjUyZWZmOWVhODI1NTAxODM4ODBiOWQ2NGMyMDQ4NzY0MmZhNmU2MGMifQ%3D%3D'); + return body['@odata.nextLink']; + }); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + body.value[0].name.should.be.eql('Blaine'); + should.not.exist(body['@odata.nextLink']); + }); + })); + + it('should limit and filter subtable', testService(async (service) => { + const asAlice = await withSubmissions(service, identity); + + await asAlice.patch('/v1/projects/1/forms/withrepeat/submissions/rtwo') + .send({ reviewState: 'rejected' }) + .expect(200); + + const nextlink = await asAlice.get('/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?$top=1&$filter=$root/Submissions/__system/reviewState eq \'rejected\'') + .expect(200) + .then(({ body }) => { + body.value[0].name.should.be.eql('Billy'); + body['@odata.nextLink'].should.eql('http://localhost:8989/v1/projects/1/forms/withrepeat.svc/Submissions.children.child?%24top=1&%24filter=%24root%2FSubmissions%2F__system%2FreviewState+eq+%27rejected%27&%24skiptoken=01eyJyZXBlYXRJZCI6IjUyZWZmOWVhODI1NTAxODM4ODBiOWQ2NGMyMDQ4NzY0MmZhNmU2MGMifQ%3D%3D'); + return body['@odata.nextLink']; + }); + + await asAlice.get(nextlink.replace('http://localhost:8989', '')) + .expect(200) + .then(({ body }) => { + body.value[0].name.should.be.eql('Blaine'); + should.not.exist(body['@odata.nextLink']); + }); + + + })); + // we cheat here. see mark1. it('should gracefully degrade on encrypted subtables', testService((service) => service.login('alice', (asAlice) => @@ -1832,7 +1992,7 @@ describe('api: /forms/:id.svc', () => { .then(() => asAlice.get('/v1/projects/1/forms/withrepeat/draft.svc/Submissions') .expect(200) .then(({ body }) => { - for (const idx of [ 0, 1, 2 ]) { + for (const idx of [0, 1, 2]) { body.value[idx].__system.submissionDate.should.be.an.isoDate(); // eslint-disable-next-line no-param-reassign delete body.value[idx].__system.submissionDate; diff --git a/test/unit/data/odata.js b/test/unit/data/odata.js index 59957c63e..0e03e7359 100644 --- a/test/unit/data/odata.js +++ b/test/unit/data/odata.js @@ -35,7 +35,8 @@ describe('submissionToOData', () => { it('should parse and transform a basic submission', () => fieldsFor(testData.forms.simple).then((fields) => { const submission = mockSubmission('one', testData.instances.simple.one); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result, instanceId }) => { + instanceId.should.be.eql('one'); result.should.eql([{ __id: 'one', __system, @@ -56,7 +57,7 @@ describe('submissionToOData', () => { // have one for explicity this purpose in case things change. it('should include submission metadata on the root output', () => { const submission = mockSubmission('test', testData.instances.simple.one); - return submissionToOData([], 'Submissions', submission).then((result) => { + return submissionToOData([], 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'test', __system }]); }); }); @@ -64,7 +65,7 @@ describe('submissionToOData', () => { it('should set the correct review state', () => { const submission = Object.assign(mockSubmission('test', testData.instances.simple.one), { reviewState: 'hasIssues' }); - return submissionToOData([], 'Submissions', submission).then((result) => { + return submissionToOData([], 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'test', __system: Object.assign({}, __system, { reviewState: 'hasIssues' }) }]); }); }); @@ -72,7 +73,7 @@ describe('submissionToOData', () => { it('should set the correct deviceId', () => { const submission = Object.assign(mockSubmission('test', testData.instances.simple.one), { deviceId: 'cool device' }); - return submissionToOData([], 'Submissions', submission).then((result) => { + return submissionToOData([], 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'test', __system: Object.assign({}, __system, { deviceId: 'cool device' }) }]); }); }); @@ -81,7 +82,7 @@ describe('submissionToOData', () => { const submission = mockSubmission('test', testData.instances.simple.one); submission.aux.edit = { count: 42 }; - return submissionToOData([], 'Submissions', submission).then((result) => { + return submissionToOData([], 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'test', __system: Object.assign({}, __system, { edits: 42 }) }]); }); }); @@ -89,7 +90,7 @@ describe('submissionToOData', () => { it('should not crash if no submitter exists', () => { const submission = mockSubmission('test', testData.instances.simple.one); submission.aux.submitter = {}; // wipe it back out. - return submissionToOData([], 'Submissions', submission).then((result) => { + return submissionToOData([], 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'test', __system: { @@ -134,7 +135,7 @@ describe('submissionToOData', () => { hello what could it be? `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'types', __system, @@ -160,7 +161,7 @@ describe('submissionToOData', () => { new MockField({ path: '/uranus', name: 'uranus', type: 'repeat' }) ]; const submission = mockSubmission('nulls', '42'); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'nulls', __system, @@ -184,7 +185,7 @@ describe('submissionToOData', () => { new MockField({ path: '/neptune', name: 'neptune', type: 'structure', order: 6 }) ]; const submission = mockSubmission('nulls', '42'); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'nulls', __system, @@ -207,7 +208,7 @@ describe('submissionToOData', () => { new MockField({ path: '/sun/uranus', name: 'uranus', type: 'repeat', order: 5 }) ]; const submission = mockSubmission('nulls', '42'); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'nulls', __system, @@ -231,7 +232,7 @@ describe('submissionToOData', () => { new MockField({ path: '/sun/uranus', name: 'uranus', type: 'repeat', order: 5 }) ]; const submission = mockSubmission('nulls', '42'); - return submissionToOData(fields, 'Submissions.sun', submission).then((result) => { + return submissionToOData(fields, 'Submissions.sun', submission).then(({ data: result }) => { result.should.eql([{ '__Submissions-id': 'nulls', __id: '68874cc5985b68898fbd0af1156e12b6270820f7', @@ -254,7 +255,7 @@ describe('submissionToOData', () => { new MockField({ path: '/sun/uranus', name: 'uranus', type: 'repeat', order: 6 }) ]; const submission = mockSubmission('nulls', '42'); - return submissionToOData(fields, 'Submissions.sun', submission).then((result) => { + return submissionToOData(fields, 'Submissions.sun', submission).then(({ data: result }) => { result.should.eql([{ '__Submissions-id': 'nulls', __id: '68874cc5985b68898fbd0af1156e12b6270820f7', @@ -275,7 +276,7 @@ describe('submissionToOData', () => { hello <42>108 `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'sanitize', __system, @@ -297,7 +298,7 @@ describe('submissionToOData', () => { dos `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'sanitize2', __system, @@ -311,7 +312,7 @@ describe('submissionToOData', () => { const submission = mockSubmission('entities', ` «hello» `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'entities', __system, @@ -329,7 +330,7 @@ describe('submissionToOData', () => { 100 this is nonsensical `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'geo', __system, @@ -348,7 +349,7 @@ describe('submissionToOData', () => { 4.8 15.16 23.42 11.38 -11.38 `); - return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then((result) => { + return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then(({ data: result }) => { result.should.eql([{ __id: 'wkt', __system, @@ -367,7 +368,7 @@ describe('submissionToOData', () => { 1.1 2.2 3.3 4.4;5.5 6.6 7.7 8.8; 11.1 22.2;33.3 44.4;55.5 66.6; `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'geojson', __system, @@ -395,7 +396,7 @@ describe('submissionToOData', () => { 1.1 2.2 3.3 4.4;5.5 6.6 7.7 8.8; 11.1 22.2;33.3 44.4;55.5 66.6; `); - return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then((result) => { + return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then(({ data: result }) => { result.should.eql([{ __id: 'wkt', __system, @@ -414,7 +415,7 @@ describe('submissionToOData', () => { 1.1 2.2 3.3 4.4;5.5 6.6 7.7 8.8;10.0 20.0 30.0 40.0;1.1 2.2 3.3 4.4; 11.1 22.2;33.3 44.4;55.5 66.6;11.1 22.2; `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'geojson', __system, @@ -442,7 +443,7 @@ describe('submissionToOData', () => { 1.1 2.2 3.3 4.4; 5.5 6.6 7.7 8.8; 10.0 20.0 30.0 40.0;1.1 2.2 3.3 4.4; 11.1 22.2; 33.3 44.4;55.5 66.6; 11.1 22.2; `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'geojson', __system, @@ -470,7 +471,7 @@ describe('submissionToOData', () => { 1.1 2.2 3.3 4.4;5.5 6.6 7.7 8.8;10.0 20.0 30.0 40.0;1.1 2.2 3.3 4.4; 11.1 22.2;33.3 44.4;55.5 66.6;11.1 22.2; `); - return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then((result) => { + return submissionToOData(fields, 'Submissions', submission, { wkt: true }).then(({ data: result }) => { result.should.eql([{ __id: 'wkt', __system, @@ -488,7 +489,7 @@ describe('submissionToOData', () => { new MockField({ path: '/age', name: 'age', type: 'int' }) ]; const submission = mockSubmission('one', testData.instances.simple.one); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'one', __system, @@ -514,7 +515,7 @@ describe('submissionToOData', () => { tres `); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'nesting', __system, @@ -526,7 +527,7 @@ describe('submissionToOData', () => { it('should provide navigation links for repeats', () => fieldsFor(testData.forms.withrepeat).then((fields) => { const submission = mockSubmission('rtwo', testData.instances.withrepeat.two); - return submissionToOData(fields, 'Submissions', submission).then((result) => { + return submissionToOData(fields, 'Submissions', submission).then(({ data: result }) => { result.should.eql([{ __id: 'rtwo', __system, @@ -545,7 +546,7 @@ describe('submissionToOData', () => { .then((fields) => { const submission = mockSubmission('two', testData.instances.doubleRepeat.double); return submissionToOData(fields, 'Submissions', submission, { expand: '*' }) - .then((result) => { + .then(({ data: result }) => { result.should.eql([ { __id: 'two', @@ -631,7 +632,7 @@ describe('submissionToOData', () => { it('should extract subtable rows within repeats', () => fieldsFor(testData.forms.withrepeat).then((fields) => { const row = { instanceId: 'two', xml: testData.instances.withrepeat.two, def: {}, aux: { encryption: {}, attachment: {} } }; - return submissionToOData(fields, 'Submissions.children.child', row).then((result) => { + return submissionToOData(fields, 'Submissions.children.child', row).then(({ data: result }) => { result.should.eql([{ '__Submissions-id': 'two', __id: 'cf9a1b5cc83c6d6270c1eb98860d294eac5d526d', @@ -649,7 +650,7 @@ describe('submissionToOData', () => { it('should return navigation links to repeats within a subtable result set', () => fieldsFor(testData.forms.doubleRepeat).then((fields) => { const row = { instanceId: 'double', xml: testData.instances.doubleRepeat.double, def: {}, aux: { encryption: {}, attachment: {} } }; - return submissionToOData(fields, 'Submissions.children.child', row).then((result) => { + return submissionToOData(fields, 'Submissions.children.child', row).then(({ data: result }) => { result.should.eql([{ __id: '46ebf42ee83ddec5028c42b2c054402d1e700208', '__Submissions-id': 'double', @@ -676,7 +677,7 @@ describe('submissionToOData', () => { it('should return second-order subtable results', () => fieldsFor(testData.forms.doubleRepeat).then((fields) => { const row = { instanceId: 'double', xml: testData.instances.doubleRepeat.double, def: {}, aux: { encryption: {}, attachment: {} } }; - return submissionToOData(fields, 'Submissions.children.child.toys.toy', row).then((result) => { + return submissionToOData(fields, 'Submissions.children.child.toys.toy', row).then(({ data: result }) => { result.should.eql([{ __id: 'a9058d7b2ed9557205ae53f5b1dc4224043eca2a', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', @@ -720,7 +721,7 @@ describe('submissionToOData', () => { new MockField({ path: '/name', name: 'name', type: 'string' }) ]; const submission = mockSubmission('one', testData.instances.simple.one); - return submissionToOData(fields, 'Submissions', submission, { metadata: { __id: true, '__system/status': true } }).then((result) => { + return submissionToOData(fields, 'Submissions', submission, { metadata: { __id: true, '__system/status': true } }).then(({ data: result }) => { result.should.eql([{ __id: 'one', __system: { status: null }, @@ -736,7 +737,7 @@ describe('submissionToOData', () => { new MockField({ path: '/children/child', name: 'child', type: 'repeat' }) ]; const submission = { instanceId: 'double', xml: testData.instances.doubleRepeat.double, def: {}, aux: { encryption: {}, attachment: {} } }; - return submissionToOData(fields, 'Submissions.children.child', submission, { metadata: { __id: true } }).then((result) => { + return submissionToOData(fields, 'Submissions.children.child', submission, { metadata: { __id: true } }).then(({ data: result }) => { result.should.eql([{ __id: '46ebf42ee83ddec5028c42b2c054402d1e700208', '__Submissions-id': 'double' @@ -755,7 +756,7 @@ describe('submissionToOData', () => { .then(selectFields({ $select: 'name' }, 'Submissions.children.child.toys.toy')) .then((fields) => { const row = { instanceId: 'double', xml: testData.instances.doubleRepeat.double, def: {}, aux: { encryption: {}, attachment: {} } }; - return submissionToOData(fields, 'Submissions.children.child.toys.toy', row, { metadata: { __id: true } }).then((result) => { + return submissionToOData(fields, 'Submissions.children.child.toys.toy', row, { metadata: { __id: true } }).then(({ data: result }) => { result.should.eql([{ __id: 'a9058d7b2ed9557205ae53f5b1dc4224043eca2a', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', diff --git a/test/unit/formats/odata.js b/test/unit/formats/odata.js index 353882f67..73a30c966 100644 --- a/test/unit/formats/odata.js +++ b/test/unit/formats/odata.js @@ -5,6 +5,7 @@ const { serviceDocumentFor, edmxFor, rowStreamToOData, singleRowToOData, selectF const { fieldsFor, MockField } = require(appRoot + '/test/util/schema'); const testData = require(appRoot + '/test/data/xml'); const should = require('should'); +const { QueryOptions } = require('../../../lib/util/db'); // Helpers to deal with repeated system metadata generation. const submitter = { id: 5, displayName: 'Alice' }; @@ -574,10 +575,10 @@ describe('odata message composition', () => { const query = { $top: '3', $skip: '2' }; const inRows = streamTest.fromObjects(instances(6)); // make it close to check the off-by-one. fieldsFor(testData.forms.simple) - .then((fields) => rowStreamToOData(fields, 'Submissions', 'http://localhost:8989', '/simple.svc/Submissions?$top=3&$skip=2', query, inRows)) + .then((fields) => rowStreamToOData(fields, 'Submissions', 'http://localhost:8989', '/simple.svc/Submissions?$top=3&$skip=2', query, inRows, 10, 8)) .then((stream) => stream.pipe(streamTest.toText((_, result) => { const resultObj = JSON.parse(result); - resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24skip=5'); + resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24top=3&%24skiptoken=01e30%3D'); done(); }))); }); @@ -586,10 +587,10 @@ describe('odata message composition', () => { const query = { $top: '3', $skip: '2', $wkt: 'true', $count: 'true' }; const inRows = streamTest.fromObjects(instances(6)); // make it close to check the off-by-one. fieldsFor(testData.forms.simple) - .then((fields) => rowStreamToOData(fields, 'Submissions', 'http://localhost:8989', '/simple.svc/Submissions?$top=3&$skip=2&$wkt=true&$count=true', query, inRows)) + .then((fields) => rowStreamToOData(fields, 'Submissions', 'http://localhost:8989', '/simple.svc/Submissions?$top=3&$skip=2&$wkt=true&$count=true', query, inRows, 10, 8)) .then((stream) => stream.pipe(streamTest.toText((_, result) => { const resultObj = JSON.parse(result); - resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24skip=5&%24wkt=true&%24count=true'); + resultObj['@odata.nextLink'].should.equal('http://localhost:8989/simple.svc/Submissions?%24top=3&%24wkt=true&%24count=true&%24skiptoken=01e30%3D'); done(); }))); }); @@ -666,7 +667,6 @@ describe('odata message composition', () => { .then((stream) => stream.pipe(streamTest.toText((_, result) => { JSON.parse(result).should.eql({ '@odata.context': 'http://localhost:8989/simple.svc/$metadata#Submissions', - '@odata.nextLink': 'http://localhost:8989/simple.svc/Submissions?%24skip=2', value: [ { __id: 'one', __system, meta: { instanceID: 'one' }, name: 'Alice', age: 30 }, { __id: 'two', __system, meta: { instanceID: 'two' }, name: 'Bob', age: 34 }, @@ -743,7 +743,7 @@ describe('odata message composition', () => { .then((stream) => stream.pipe(streamTest.toText((_, result) => { JSON.parse(result).should.eql({ '@odata.context': 'http://localhost:8989/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24skip=2', + '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=2&%24skiptoken=01eyJyZXBlYXRJZCI6ImM3NmQwY2NjNmQ1ZGEyMzZiZTdiOTNiOTg1YTgwNDEzZDJlM2UxNzIifQ%3D%3D', value: [{ __id: 'cf9a1b5cc83c6d6270c1eb98860d294eac5d526d', '__Submissions-id': 'two', @@ -800,7 +800,7 @@ describe('odata message composition', () => { .then((stream) => stream.pipe(streamTest.toText((_, result) => { JSON.parse(result).should.eql({ '@odata.context': 'http://localhost:8989/withrepeat.svc/$metadata#Submissions.children.child', - '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24skip=2', + '@odata.nextLink': 'http://localhost:8989/withrepeat.svc/Submissions.children.child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6ImM3NmQwY2NjNmQ1ZGEyMzZiZTdiOTNiOTg1YTgwNDEzZDJlM2UxNzIifQ%3D%3D', value: [{ __id: 'c76d0ccc6d5da236be7b93b985a80413d2e3e172', '__Submissions-id': 'two', @@ -811,6 +811,34 @@ describe('odata message composition', () => { done(); }))); }); + + it('should offset subtable row data by skipToken', (done) => { + const query = { $skiptoken: QueryOptions.getSkiptoken({ instanceId: 'two', repeatId: 'cf9a1b5cc83c6d6270c1eb98860d294eac5d526d' }) }; + const inRows = streamTest.fromObjects([ + mockSubmission('one', testData.instances.withrepeat.one), + mockSubmission('two', testData.instances.withrepeat.two), + mockSubmission('three', testData.instances.withrepeat.three) + ]); + fieldsFor(testData.forms.withrepeat) + .then((fields) => rowStreamToOData(fields, 'Submissions.children.child', 'http://localhost:8989', '/withrepeat.svc/Submissions.children.child?$skip=1&$top=1', query, inRows)) + .then((stream) => stream.pipe(streamTest.toText((_, result) => { + JSON.parse(result).should.eql({ + '@odata.context': 'http://localhost:8989/withrepeat.svc/$metadata#Submissions.children.child', + value: [{ + __id: 'c76d0ccc6d5da236be7b93b985a80413d2e3e172', + '__Submissions-id': 'two', + name: 'Blaine', + age: 6 + }, { + __id: 'beaedcdba519e6e6b8037605c9ae3f6a719984fa', + '__Submissions-id': 'three', + name: 'Candace', + age: 2 + }] + }); + done(); + }))); + }); }); }); @@ -897,7 +925,7 @@ describe('odata message composition', () => { return singleRowToOData(fields, submission, 'http://localhost:8989', "/withrepeat.svc/Submissions('two')/children/child?$top=1", query) .then(JSON.parse) .then((result) => { - result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24skip=1"); + result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24skiptoken=01eyJyZXBlYXRJZCI6ImNmOWExYjVjYzgzYzZkNjI3MGMxZWI5ODg2MGQyOTRlYWM1ZDUyNmQifQ%3D%3D"); }); }); }); @@ -910,7 +938,7 @@ describe('odata message composition', () => { return singleRowToOData(fields, submission, 'http://localhost:8989', "/withrepeat.svc/Submissions('two')/children/child?$top=1&$wkt=true", query) .then(JSON.parse) .then((result) => { - result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24wkt=true&%24skip=1"); + result['@odata.nextLink'].should.equal("http://localhost:8989/withrepeat.svc/Submissions('two')/children/child?%24top=1&%24wkt=true&%24skiptoken=01eyJyZXBlYXRJZCI6ImNmOWExYjVjYzgzYzZkNjI3MGMxZWI5ODg2MGQyOTRlYWM1ZDUyNmQifQ%3D%3D"); }); }); }); @@ -1007,7 +1035,7 @@ describe('odata message composition', () => { .then((result) => { result.should.eql({ '@odata.context': 'http://localhost:8989/doubleRepeat.svc/$metadata#Submissions.children.child.toys.toy', - '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24skip=2", + '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=01eyJyZXBlYXRJZCI6IjhkMmRjN2JkM2U5N2E2OTBjMDgxM2U2NDY2NThlNTEwMzhlYjQxNDQifQ%3D%3D", value: [{ __id: 'a9058d7b2ed9557205ae53f5b1dc4224043eca2a', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', @@ -1060,7 +1088,7 @@ describe('odata message composition', () => { .then((result) => { result.should.eql({ '@odata.context': 'http://localhost:8989/doubleRepeat.svc/$metadata#Submissions.children.child.toys.toy', - '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24skip=3", + '@odata.nextLink': "http://localhost:8989/doubleRepeat.svc/Submissions('double')/children/child('b6e93a81a53eed0566e65e472d4a4b9ae383ee6d')/toys/toy?%24top=2&%24skiptoken=01eyJyZXBlYXRJZCI6ImI3MTZkZDhiNzlhNGM5MzY5ZDZiMWU3YTljOWQ1NWFjMThkYTEzMTkifQ%3D%3D", value: [{ __id: '8d2dc7bd3e97a690c0813e646658e51038eb4144', '__Submissions-children-child-id': 'b6e93a81a53eed0566e65e472d4a4b9ae383ee6d', diff --git a/test/unit/util/db.js b/test/unit/util/db.js index 099d94d88..ad1bb4f68 100644 --- a/test/unit/util/db.js +++ b/test/unit/util/db.js @@ -420,6 +420,15 @@ returning *`); .args.should.eql({ b: 2, c: 3, f: 9 }); }); + it('should create and parse cursor token', () => { + const data = { + someid: '123' + }; + + const token = QueryOptions.getSkiptoken(data); + QueryOptions.parseSkiptoken(token).should.be.eql(data); + }); + describe('related functions', () => { it('should run the handler only if the arg is present', () => { let ran = false; diff --git a/test/unit/util/util.js b/test/unit/util/util.js index 6408f02f1..bd7e44749 100644 --- a/test/unit/util/util.js +++ b/test/unit/util/util.js @@ -63,5 +63,13 @@ describe('util/util', () => { }); }); + describe('UTF-8 and Base64 conversion', () => { + const { utf8ToBase64, base64ToUtf8 } = util; + it('should convert unicode to base64', () => { + const input = 'a Ā 𐀀 文 🦄'; + const base64 = utf8ToBase64(input); + base64ToUtf8(base64).should.be.eql(input); + }); + }); });