-
Notifications
You must be signed in to change notification settings - Fork 121
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
e1134d5
commit 7254fc9
Showing
29 changed files
with
2,233 additions
and
1,028 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,6 +5,7 @@ src/generated/ | |
jobs/export/ | ||
public/css/bootstrap | ||
/temp/ | ||
/model/ | ||
|
||
.python-version | ||
|
||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
require('dotenv').config(); | ||
const fs = require('fs'); | ||
|
||
const AWS = require('aws-sdk'); | ||
|
||
const downloadFromS3 = async () => { | ||
const s3 = new AWS.S3({ | ||
accessKeyId: process.env.AWS_ACCESS_KEY, | ||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, | ||
}); | ||
|
||
// list all from s3 under s3://cubecobra/model | ||
const listResult = await s3.listObjectsV2({ Bucket: process.env.DATA_BUCKET, Prefix: 'model/' }).promise(); | ||
|
||
// for each file, download it to the local model directory | ||
for (const file of listResult.Contents) { | ||
// eslint-disable-next-line no-await-in-loop | ||
const res = await s3.getObject({ Bucket: process.env.DATA_BUCKET, Key: file.Key }).promise(); | ||
|
||
// make sure folders exist | ||
const folders = file.Key.split('/'); | ||
folders.pop(); | ||
|
||
let folderPath = ''; | ||
for (const folder of folders) { | ||
folderPath += `${folder}/`; | ||
if (!fs.existsSync(folderPath)) { | ||
fs.mkdirSync(folderPath); | ||
} | ||
} | ||
|
||
fs.writeFileSync(file.Key, res.Body); | ||
console.log(`Downloaded ${file.Key}`); | ||
} | ||
}; | ||
|
||
downloadFromS3(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
/* eslint-disable no-await-in-loop */ | ||
require('dotenv').config(); | ||
|
||
const fs = require('fs'); | ||
const carddb = require('../serverjs/carddb'); | ||
|
||
const Cube = require('../dynamo/models/cube'); | ||
|
||
const processCube = async (cube, oracleToIndex) => { | ||
const cards = await Cube.getCards(cube.id); | ||
|
||
return { | ||
cards: cards.mainboard.map((card) => oracleToIndex[card.details.oracle_id] || -1), | ||
id: cube.id, | ||
name: cube.name, | ||
owner: cube.owner.username, | ||
owner_id: cube.owner.id, | ||
image_uri: cube.image.uri, | ||
iamge_artist: cube.image.artist, | ||
card_count: cards.mainboard.length, | ||
following: cube.following, | ||
}; | ||
}; | ||
|
||
(async () => { | ||
await carddb.initializeCardDb(); | ||
|
||
const allOracles = carddb.allOracleIds(); | ||
const oracleToIndex = Object.fromEntries(allOracles.map((oracle, index) => [oracle, index])); | ||
const indexToOracleMap = Object.fromEntries(allOracles.map((oracle, index) => [index, oracle])); | ||
|
||
let lastKey = null; | ||
let processed = 0; | ||
const cubes = []; | ||
|
||
do { | ||
const result = await Cube.getByVisibility(Cube.VISIBILITY.PUBLIC, lastKey, 100); | ||
lastKey = result.lastKey; | ||
processed += result.items.length; | ||
|
||
const processedCubes = await Promise.all(result.items.map((item) => processCube(item, oracleToIndex))); | ||
|
||
cubes.push(...processedCubes); | ||
|
||
console.log(`Processed ${processed} cubes`); | ||
} while (lastKey); | ||
|
||
// if /temp doesn't exist, create it | ||
if (!fs.existsSync('./temp')) { | ||
fs.mkdirSync('./temp'); | ||
} | ||
|
||
// if /temp/export doesn't exist, create it | ||
if (!fs.existsSync('./temp/export')) { | ||
fs.mkdirSync('./temp/export'); | ||
} | ||
|
||
fs.writeFileSync('./temp/export/cubes.json', JSON.stringify(cubes)); | ||
fs.writeFileSync('./temp/export/indexToOracleMap.json', JSON.stringify(indexToOracleMap)); | ||
})(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,130 @@ | ||
/* eslint-disable no-await-in-loop */ | ||
require('dotenv').config(); | ||
|
||
const fs = require('fs'); | ||
const carddb = require('../serverjs/carddb'); | ||
|
||
const { getDrafterState } = require('../dist/drafting/draftutil'); | ||
const Draft = require('../dynamo/models/draft'); | ||
|
||
const draftCardIndexToOracle = (cardIndex, draftCards, oracleToIndex) => { | ||
const card = draftCards[cardIndex]; | ||
if (!card) { | ||
return -1; | ||
} | ||
|
||
return card.details.oracle_id; | ||
}; | ||
|
||
const draftCardIndexToOracleIndex = (cardIndex, draftCards, oracleToIndex) => { | ||
return oracleToIndex[draftCardIndexToOracle(cardIndex, draftCards, oracleToIndex)] || -1; | ||
}; | ||
|
||
const processDeck = (draft, oracleToIndex) => { | ||
const seats = []; | ||
|
||
if (!draft.seats) { | ||
return []; | ||
} | ||
|
||
draft.seats.forEach((seat) => { | ||
if (seat.owner) { | ||
seats.push({ | ||
cube: draft.cube, | ||
owner: seat.owner.id, | ||
mainboard: seat.mainboard.flat(2).map((pick) => draftCardIndexToOracleIndex(pick, draft.cards, oracleToIndex)), | ||
sideboard: seat.sideboard.flat(2).map((pick) => draftCardIndexToOracleIndex(pick, draft.cards, oracleToIndex)), | ||
basics: (draft.basics || []).map((pick) => draftCardIndexToOracleIndex(pick, draft.cards, oracleToIndex)), | ||
}); | ||
} | ||
}); | ||
|
||
return seats; | ||
}; | ||
|
||
const processPicks = (draft, oracleToIndex) => { | ||
const picks = []; | ||
|
||
if (!draft.seats) { | ||
return []; | ||
} | ||
|
||
draft.seats.forEach((seat) => { | ||
if ( | ||
draft.InitialState && | ||
Object.entries(draft.InitialState).length > 0 && | ||
seat.pickorder && | ||
draft.type === Draft.TYPES.DRAFT && | ||
seat.owner | ||
) { | ||
for (let j = 0; j < draft.seats[0].pickorder.length; j++) { | ||
const drafterState = getDrafterState(draft, 0, j); | ||
|
||
const picked = draftCardIndexToOracleIndex(drafterState.selection, draft.cards, oracleToIndex); | ||
const pack = drafterState.cardsInPack.map((pick) => | ||
draftCardIndexToOracleIndex(pick, draft.cards, oracleToIndex), | ||
); | ||
const pool = drafterState.picked.map((pick) => draftCardIndexToOracleIndex(pick, draft.cards, oracleToIndex)); | ||
|
||
picks.push({ | ||
cube: draft.cube, | ||
owner: seat.owner.id, | ||
pack, | ||
picked, | ||
pool, | ||
}); | ||
} | ||
} | ||
}); | ||
|
||
return picks; | ||
}; | ||
|
||
(async () => { | ||
await carddb.initializeCardDb(); | ||
|
||
const indexToOracleMap = JSON.parse(fs.readFileSync('./temp/export/indexToOracleMap.json', 'utf8')); | ||
const oracleToIndex = Object.fromEntries( | ||
Object.entries(indexToOracleMap).map(([index, oracle]) => [oracle, parseInt(index, 10)]), | ||
); | ||
|
||
// load all draftlogs into memory | ||
let lastKey = null; | ||
let draftLogs = []; | ||
do { | ||
const result = await Draft.scan(1000000, lastKey); | ||
draftLogs = draftLogs.concat(result.items); | ||
lastKey = result.lastKey; | ||
|
||
console.log(`Loaded ${draftLogs.length} draftlogs`); | ||
} while (lastKey); | ||
|
||
console.log('Loaded all draftlogs'); | ||
|
||
const batches = []; | ||
|
||
for (let i = 0; i < draftLogs.length; i += 1000) { | ||
batches.push(draftLogs.slice(i, i + 1000)); | ||
} | ||
|
||
if (!fs.existsSync('./temp/export/decks')) { | ||
fs.mkdirSync('./temp/export/decks'); | ||
} | ||
if (!fs.existsSync('./temp/export/picks')) { | ||
fs.mkdirSync('./temp/export/picks'); | ||
} | ||
|
||
for (let i = 0; i < batches.length; i += 1) { | ||
const batch = batches[i]; | ||
|
||
const drafts = await Draft.batchGet(batch.filter((item) => item.complete).map((row) => row.id)); | ||
|
||
const processedDrafts = drafts.map((draft) => processDeck(draft, oracleToIndex)); | ||
// const processedPicks = drafts.map((draft) => processPicks(draft, oracleToIndex)); | ||
|
||
fs.writeFileSync(`./temp/export/decks/${i}.json`, JSON.stringify(processedDrafts.flat())); | ||
// fs.writeFileSync(`./temp/export/picks/${i}.json`, JSON.stringify(processedPicks.flat())); | ||
|
||
console.log(`Processed ${i + 1} / ${batches.length} batches`); | ||
} | ||
})(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
/* eslint-disable no-await-in-loop */ | ||
require('dotenv').config(); | ||
|
||
const fs = require('fs'); | ||
const carddb = require('../serverjs/carddb'); | ||
|
||
(async () => { | ||
await carddb.initializeCardDb(); | ||
|
||
const allOracleIds = carddb.allOracleIds(); | ||
|
||
const result = Object.fromEntries( | ||
allOracleIds.map((oracleId) => { | ||
const card = carddb.getVersionsByOracleId(oracleId)[0]; | ||
const reasonable = carddb.getMostReasonableById(card); | ||
|
||
return [ | ||
oracleId, | ||
{ | ||
name: reasonable.name, | ||
image: reasonable.image_small, | ||
elo: reasonable.elo, | ||
type: reasonable.type, | ||
cmc: reasonable.cmc, | ||
}, | ||
]; | ||
}), | ||
); | ||
|
||
fs.writeFileSync('./temp/export/simpleCardDict.json', JSON.stringify(result)); | ||
})(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.