add new ml model

.env_EXAMPLE

@@ -11,8 +11,6 @@ DYNAMO_PREFIX="LOCAL"
 EMAIL_CONFIG_PASSWORD=""
 EMAIL_CONFIG_USERNAME=""
 ENV="development"
-FLASKROOT="http://167.172.233.110"
-HOST="https://cubecobra.com"
 NITROPAY_ENABLED="false"
 NODE_ENV="development"
 PATREON_CLIENT_ID=""

.gitignore

@@ -5,6 +5,7 @@ src/generated/
 jobs/export/
 public/css/bootstrap
 /temp/
+/model/
 
 .python-version
 

README.md

@@ -46,9 +46,7 @@ Here is a table on how to fill out the env vars:
 | DYNAMO_PREFIX | The prefix to use for DynamoDB tables. You can leave this as the default value | Yes |
 | EMAIL_CONFIG_PASSWORD | The password for the email account to use for sending emails. |  |
 | EMAIL_CONFIG_USERNAME | The username for the email account to use for sending emails. |  |
-| ENV | The environment to run Cube Cobra in. | Yes |
-| FLASKROOT | The URL of the Flask server. This server is what vends recommendations from our ML model. |  |
-| HOST | The URL of the Cube Cobra server for the ML server to redirect to. |  |
+| ENV | The environment to run Cube Cobra in. | Yes |\
 | NITROPAY_ENABLED | Whether or not to enable NitroPay, our ad provider. |  |
 | NODE_ENV | The environment to run Cube Cobra in. | Yes |
 | PATREON_CLIENT_ID | The client ID for the Patreon OAuth app. |  |

dynamo/models/cube.js

@@ -260,7 +260,7 @@ module.exports = {
       lastKey: result.LastEvaluatedKey,
     };
   },
-  getByVisibility: async (visibility, lastKey) => {
+  getByVisibility: async (visibility, lastKey, limit = 36) => {
     const result = await client.query({
       IndexName: 'ByVisiblity',
       KeyConditionExpression: `#p1 = :visibility`,
@@ -272,6 +272,7 @@ module.exports = {
       },
       ExclusiveStartKey: lastKey,
       ScanIndexForward: false,
+      Limit: limit,
     });
     return {
       items: await batchHydrate(result.Items),

dynamo/models/featuredQueue.js

@@ -74,6 +74,6 @@ module.exports = {
       [FIELDS.STATUS]: STATUS.ACTIVE,
     }));
   },
-  delete: async (id) => client.delete({ id }),
+  delete: async (id) => client.delete({ cube: id }),
   FIELDS,
 };

jobs/download_model.js

@@ -0,0 +1,37 @@
+require('dotenv').config();
+const fs = require('fs');
+
+const AWS = require('aws-sdk');
+
+const downloadFromS3 = async () => {
+  const s3 = new AWS.S3({
+    accessKeyId: process.env.AWS_ACCESS_KEY,
+    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+  });
+
+  // list all from s3 under s3://cubecobra/model
+  const listResult = await s3.listObjectsV2({ Bucket: process.env.DATA_BUCKET, Prefix: 'model/' }).promise();
+
+  // for each file, download it to the local model directory
+  for (const file of listResult.Contents) {
+    // eslint-disable-next-line no-await-in-loop
+    const res = await s3.getObject({ Bucket: process.env.DATA_BUCKET, Key: file.Key }).promise();
+
+    // make sure folders exist
+    const folders = file.Key.split('/');
+    folders.pop();
+
+    let folderPath = '';
+    for (const folder of folders) {
+      folderPath += `${folder}/`;
+      if (!fs.existsSync(folderPath)) {
+        fs.mkdirSync(folderPath);
+      }
+    }
+
+    fs.writeFileSync(file.Key, res.Body);
+    console.log(`Downloaded ${file.Key}`);
+  }
+};
+
+downloadFromS3();

jobs/export_cubes.js

@@ -0,0 +1,60 @@
+/* eslint-disable no-await-in-loop */
+require('dotenv').config();
+
+const fs = require('fs');
+const carddb = require('../serverjs/carddb');
+
+const Cube = require('../dynamo/models/cube');
+
+const processCube = async (cube, oracleToIndex) => {
+  const cards = await Cube.getCards(cube.id);
+
+  return {
+    cards: cards.mainboard.map((card) => oracleToIndex[card.details.oracle_id] || -1),
+    id: cube.id,
+    name: cube.name,
+    owner: cube.owner.username,
+    owner_id: cube.owner.id,
+    image_uri: cube.image.uri,
+    iamge_artist: cube.image.artist,
+    card_count: cards.mainboard.length,
+    following: cube.following,
+  };
+};
+
+(async () => {
+  await carddb.initializeCardDb();
+
+  const allOracles = carddb.allOracleIds();
+  const oracleToIndex = Object.fromEntries(allOracles.map((oracle, index) => [oracle, index]));
+  const indexToOracleMap = Object.fromEntries(allOracles.map((oracle, index) => [index, oracle]));
+
+  let lastKey = null;
+  let processed = 0;
+  const cubes = [];
+
+  do {
+    const result = await Cube.getByVisibility(Cube.VISIBILITY.PUBLIC, lastKey, 100);
+    lastKey = result.lastKey;
+    processed += result.items.length;
+
+    const processedCubes = await Promise.all(result.items.map((item) => processCube(item, oracleToIndex)));
+
+    cubes.push(...processedCubes);
+
+    console.log(`Processed ${processed} cubes`);
+  } while (lastKey);
+
+  // if /temp doesn't exist, create it
+  if (!fs.existsSync('./temp')) {
+    fs.mkdirSync('./temp');
+  }
+
+  // if /temp/export doesn't exist, create it
+  if (!fs.existsSync('./temp/export')) {
+    fs.mkdirSync('./temp/export');
+  }
+
+  fs.writeFileSync('./temp/export/cubes.json', JSON.stringify(cubes));
+  fs.writeFileSync('./temp/export/indexToOracleMap.json', JSON.stringify(indexToOracleMap));
+})();

jobs/export_decks.js

@@ -0,0 +1,130 @@
+/* eslint-disable no-await-in-loop */
+require('dotenv').config();
+
+const fs = require('fs');
+const carddb = require('../serverjs/carddb');
+
+const { getDrafterState } = require('../dist/drafting/draftutil');
+const Draft = require('../dynamo/models/draft');
+
+const draftCardIndexToOracle = (cardIndex, draftCards, oracleToIndex) => {
+  const card = draftCards[cardIndex];
+  if (!card) {
+    return -1;
+  }
+
+  return card.details.oracle_id;
+};
+
+const draftCardIndexToOracleIndex = (cardIndex, draftCards, oracleToIndex) => {
+  return oracleToIndex[draftCardIndexToOracle(cardIndex, draftCards, oracleToIndex)] || -1;
+};
+
+const processDeck = (draft, oracleToIndex) => {
+  const seats = [];
+
+  if (!draft.seats) {
+    return [];
+  }
+
+  draft.seats.forEach((seat) => {
+    if (seat.owner) {
+      seats.push({
+        cube: draft.cube,
+        owner: seat.owner.id,
+        mainboard: seat.mainboard.flat(2).map((pick) => draftCardIndexToOracleIndex(pick, draft.cards, oracleToIndex)),
+        sideboard: seat.sideboard.flat(2).map((pick) => draftCardIndexToOracleIndex(pick, draft.cards, oracleToIndex)),
+        basics: (draft.basics || []).map((pick) => draftCardIndexToOracleIndex(pick, draft.cards, oracleToIndex)),
+      });
+    }
+  });
+
+  return seats;
+};
+
+const processPicks = (draft, oracleToIndex) => {
+  const picks = [];
+
+  if (!draft.seats) {
+    return [];
+  }
+
+  draft.seats.forEach((seat) => {
+    if (
+      draft.InitialState &&
+      Object.entries(draft.InitialState).length > 0 &&
+      seat.pickorder &&
+      draft.type === Draft.TYPES.DRAFT &&
+      seat.owner
+    ) {
+      for (let j = 0; j < draft.seats[0].pickorder.length; j++) {
+        const drafterState = getDrafterState(draft, 0, j);
+
+        const picked = draftCardIndexToOracleIndex(drafterState.selection, draft.cards, oracleToIndex);
+        const pack = drafterState.cardsInPack.map((pick) =>
+          draftCardIndexToOracleIndex(pick, draft.cards, oracleToIndex),
+        );
+        const pool = drafterState.picked.map((pick) => draftCardIndexToOracleIndex(pick, draft.cards, oracleToIndex));
+
+        picks.push({
+          cube: draft.cube,
+          owner: seat.owner.id,
+          pack,
+          picked,
+          pool,
+        });
+      }
+    }
+  });
+
+  return picks;
+};
+
+(async () => {
+  await carddb.initializeCardDb();
+
+  const indexToOracleMap = JSON.parse(fs.readFileSync('./temp/export/indexToOracleMap.json', 'utf8'));
+  const oracleToIndex = Object.fromEntries(
+    Object.entries(indexToOracleMap).map(([index, oracle]) => [oracle, parseInt(index, 10)]),
+  );
+
+  // load all draftlogs into memory
+  let lastKey = null;
+  let draftLogs = [];
+  do {
+    const result = await Draft.scan(1000000, lastKey);
+    draftLogs = draftLogs.concat(result.items);
+    lastKey = result.lastKey;
+
+    console.log(`Loaded ${draftLogs.length} draftlogs`);
+  } while (lastKey);
+
+  console.log('Loaded all draftlogs');
+
+  const batches = [];
+
+  for (let i = 0; i < draftLogs.length; i += 1000) {
+    batches.push(draftLogs.slice(i, i + 1000));
+  }
+
+  if (!fs.existsSync('./temp/export/decks')) {
+    fs.mkdirSync('./temp/export/decks');
+  }
+  if (!fs.existsSync('./temp/export/picks')) {
+    fs.mkdirSync('./temp/export/picks');
+  }
+
+  for (let i = 0; i < batches.length; i += 1) {
+    const batch = batches[i];
+
+    const drafts = await Draft.batchGet(batch.filter((item) => item.complete).map((row) => row.id));
+
+    const processedDrafts = drafts.map((draft) => processDeck(draft, oracleToIndex));
+    // const processedPicks = drafts.map((draft) => processPicks(draft, oracleToIndex));
+
+    fs.writeFileSync(`./temp/export/decks/${i}.json`, JSON.stringify(processedDrafts.flat()));
+    // fs.writeFileSync(`./temp/export/picks/${i}.json`, JSON.stringify(processedPicks.flat()));
+
+    console.log(`Processed ${i + 1} / ${batches.length} batches`);
+  }
+})();

jobs/export_simple_card_dict.js

@@ -0,0 +1,31 @@
+/* eslint-disable no-await-in-loop */
+require('dotenv').config();
+
+const fs = require('fs');
+const carddb = require('../serverjs/carddb');
+
+(async () => {
+  await carddb.initializeCardDb();
+
+  const allOracleIds = carddb.allOracleIds();
+
+  const result = Object.fromEntries(
+    allOracleIds.map((oracleId) => {
+      const card = carddb.getVersionsByOracleId(oracleId)[0];
+      const reasonable = carddb.getMostReasonableById(card);
+
+      return [
+        oracleId,
+        {
+          name: reasonable.name,
+          image: reasonable.image_small,
+          elo: reasonable.elo,
+          type: reasonable.type,
+          cmc: reasonable.cmc,
+        },
+      ];
+    }),
+  );
+
+  fs.writeFileSync('./temp/export/simpleCardDict.json', JSON.stringify(result));
+})();

jobs/rotate_featured.js

@@ -14,15 +14,14 @@ const DAYS_BETWEEN_ROTATIONS = 7;
 
     const { items } = queue;
 
-    console.log(queue);
     const msTimeDiff = Date.now() - items[0].featuredOn;
     const daysTimeDiff = msTimeDiff / MS_PER_DAY;
 
     if (daysTimeDiff >= DAYS_BETWEEN_ROTATIONS) {
       console.log(
         `Rotation period elapsed (period=${DAYS_BETWEEN_ROTATIONS}, elapsed=${daysTimeDiff}). Rotating featured cubes.`,
       );
-      const rotate = await fq.rotateFeatured();
+      const rotate = await fq.rotateFeatured(items);
       for (const message of rotate.messages) {
         console.warn(message);
       }

jobs/update_cards.js

@@ -936,10 +936,11 @@ const uploadCardDb = async () => {
 };
 
 const loadMetadatadict = async () => {
-  if (!fs.existsSync('./temp') && !fs.existsSync('./temp/metadatadict.json')) {
+  if (fs.existsSync('./temp') && fs.existsSync('./temp/metadatadict.json')) {
     return fs.promises.readFile('./temp/metadatadict.json', 'utf8').then((data) => JSON.parse(data));
   }
 
+  console.log("Couldn't find metadatadict.json");
   return {};
 };