-
Notifications
You must be signed in to change notification settings - Fork 19
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: neDB persistent datastore #48
base: develop
Are you sure you want to change the base?
Changes from 17 commits
67ccf2e
e0fc92a
12cc5db
5ac9f8e
b38bdfa
c6f92ef
7e07524
8f5e897
4703666
83a1fe7
8e87ce2
7b13c0b
e4a5652
4695f14
21990d6
e8523bb
232aec5
fd47b90
95e1297
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,3 +3,5 @@ node_modules/ | |
# Prevent test coverage reports from being added | ||
.coverage | ||
.nyc_output | ||
|
||
/lib/data | ||
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
class BatchQueue { | ||
constructor(queueProcessor) { | ||
this.queue = []; | ||
this.processing = false; | ||
this.queueProcessor = queueProcessor; | ||
} | ||
|
||
add(data) { | ||
this.queue.push(data); | ||
this.processQueue(); | ||
} | ||
|
||
async processQueue() { | ||
if (this.queue.length == 0) { | ||
this.processing = false; | ||
return; | ||
} | ||
|
||
if (this.processing) return; | ||
|
||
this.processing = true; | ||
const currentQueue = this.queue; | ||
this.queue = []; | ||
|
||
await currentQueue.reduce(async (prevPromise, item) => { | ||
await prevPromise; | ||
await this.queueProcessor(item); | ||
}, Promise.resolve()); | ||
|
||
return this.processQueue; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Shouldn't this be a function call? |
||
} | ||
} | ||
|
||
module.exports = BatchQueue; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,83 +1,98 @@ | ||
const Run = require('../models/run'); | ||
const db = require('../store'); | ||
const Event = require('../models/event'); | ||
const RunStat = require('../models/runStat'); | ||
const BatchQueue = require('./batchQueue'); | ||
|
||
const api = { | ||
insert: async (data) => { | ||
const runs = await db.getTable('runs'); | ||
await runs.insert(data.id, new Run(data)); | ||
}, | ||
|
||
findOne: async (id) => { | ||
const runs = await db.getTable('runs'); | ||
|
||
if (!id) throw new TypeError('Invalid id'); | ||
return runs.findOne(id); | ||
}, | ||
if (!data.id) throw new TypeError('Invalid run data'); | ||
|
||
find: async () => { | ||
const runs = await db.getTable('runs'); | ||
return runs.find(); | ||
}, | ||
const runDocument = Run.create(data); | ||
const run = await runDocument.save(); | ||
|
||
clear: async () => { | ||
const runs = await db.getTable('runs'); | ||
return runs.clear(); | ||
if (!run) | ||
throw new Error('Unable to save run data. Incoherent schema.'); | ||
}, | ||
|
||
pause: async (id) => { | ||
if (!id) throw new TypeError('Invalid id'); | ||
const run = await api.findOne(id); | ||
const run = await Run.findOne({ _id: id }); | ||
|
||
if (!run) throw new Error('Run not found.'); | ||
run.setPaused(); | ||
|
||
run.status = 'paused'; | ||
await run.save(); | ||
}, | ||
|
||
resume: async (id) => { | ||
if (!id) throw new TypeError('Invalid id'); | ||
const run = await api.findOne(id); | ||
const run = await Run.findOne({ _id: id }); | ||
|
||
if (!run) throw new Error('Run not found.'); | ||
run.setActive(); | ||
|
||
run.status = 'active'; | ||
await run.save(); | ||
}, | ||
|
||
abort: async (id) => { | ||
if (!id) throw new TypeError('Invalid id'); | ||
const run = await api.findOne(id); | ||
const run = await Run.findOne({ _id: id }); | ||
|
||
if (!run) throw new Error('Run not found.'); | ||
run.setAborted(); | ||
|
||
run.status = 'aborted'; | ||
run.endTime = Date.now(); | ||
await run.save(); | ||
}, | ||
|
||
done: async (id) => { | ||
if (!id) throw new TypeError('Invalid id'); | ||
const run = await api.findOne(id); | ||
const run = await Run.findOne({ _id: id }); | ||
|
||
if (!run) throw new Error('Run not found.'); | ||
run.setFinished(); | ||
|
||
if (run.status === 'aborted') return; | ||
|
||
run.status = 'finished'; | ||
run.endTime = Date.now(); | ||
await run.save(); | ||
}, | ||
|
||
interrupt: async (id) => { | ||
if (!id) throw new TypeError('Invalid id'); | ||
const run = await api.findOne(id); | ||
const run = await Run.findOne({ _id: id }); | ||
|
||
if (!run) throw new Error('Run not found.'); | ||
run.setInterrupted(); | ||
|
||
run.status = 'interrupted'; | ||
await run.save(); | ||
}, | ||
|
||
addEvent: async (data) => { | ||
if (!data.id) throw new TypeError('Invalid id'); | ||
const run = await api.findOne(data.id); | ||
saveEvent: async (data) => { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's make these methods ( There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These methods are defined in an object and not a class, can they be private as well? 🤔 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yep, just don't include them in the object. It's not using Sidenote: If it did use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. On it @coditva, just busy with my endsems this week. Will get this done asap |
||
if (!data.runId) throw new TypeError('Invalid parent id'); | ||
|
||
if (!run) throw new Error('Run not found.'); | ||
run.addEvent(data); | ||
const event = Event.create(data); | ||
if (!event) throw new Error('Invalid event type'); | ||
await event.save(); | ||
}, | ||
|
||
addStats: async (data) => { | ||
if (!data.id) throw new TypeError('Invalid id'); | ||
const run = await api.findOne(data.id); | ||
saveStats: async (data) => { | ||
if (!data.runId) throw new TypeError('Invalid parent id'); | ||
|
||
if (!run) throw new Error('Run not found.'); | ||
run.addRunStats(data); | ||
const stat = RunStat.create(data); | ||
if (!stat) throw new Error('Invalid stat type'); | ||
|
||
await stat.save(); | ||
}, | ||
|
||
addEvent: async (data) => { | ||
const batch = new BatchQueue(api.saveEvent); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We are creating a new queue every time this function is called. Is this expected? 🤔 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, my bad. Should I initialize separate queues in this file and use them? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes. The queue should be initialised once. Either when the file is imported or when the first call is made. Also, do note that:
|
||
batch.add(data); | ||
}, | ||
|
||
addStats: async (data) => { | ||
const batch = new BatchQueue(api.saveStats); | ||
batch.add(data); | ||
}, | ||
}; | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
const Document = require('camo').Document; | ||
|
||
class RunStat extends Document { | ||
constructor() { | ||
super(); | ||
|
||
this.cpu = Number; | ||
this.memory = Number; | ||
this.runId = String; | ||
} | ||
} | ||
|
||
module.exports = RunStat; |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,17 +1,16 @@ | ||
const Table = require('./table'); | ||
const connect = require('camo').connect; | ||
const paths = require('env-paths')('newman-dashboard'); | ||
const uri = `nedb://${paths.data}`; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should be in config file. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we create a separate PR for config files? |
||
|
||
const cache = {}; | ||
const init = async () => { | ||
try { | ||
await connect(uri); | ||
|
||
const api = { | ||
_createTable: (tableName) => { | ||
cache[tableName] = new Table({}); | ||
return; | ||
}, | ||
|
||
getTable: async (tableName) => { | ||
if (!cache.hasOwnProperty(tableName)) api._createTable(tableName); | ||
return cache[tableName]; | ||
}, | ||
// cleanup function to terminate db connection | ||
return () => process.exit(0); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The cleanup function would terminate the whole process! Shouldn't we only close the connection? |
||
} catch (e) { | ||
console.log('Error in connecting to database.'); | ||
} | ||
}; | ||
|
||
module.exports = api; | ||
module.exports = { init }; |
This file was deleted.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Are we storing the data in the project directory itself?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah right now we are. I'll move this to a config file.