Revamp JoinRecords

.github/workflows/playwright.yml

@@ -10,6 +10,22 @@ jobs:
     name: Run Playwright Tests
     timeout-minutes: 60
     runs-on: ubuntu-latest
+    services:
+      minio:
+        image: quay.io/minio/minio
+        ports:
+          - '9000:9000'
+          - '9001:9001'
+        volumes:
+          - 'minio_data:/data'
+        command: |
+          server /data 
+        environment:
+          - MINIO_ROOT_USER=admin
+          - MINIO_ROOT_PASSWORD=admin1234
+          - MINIO_DEFAULT_BUCKETS=meshdb-join-form-log
+          - MINIO_ACCESS_KEY=sampleaccesskey
+          - MINIO_SECRET_KEY=samplesecretkey
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
     - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6
@@ -24,6 +40,12 @@ jobs:
       env:
         NEXT_PUBLIC_MESHDB_URL: https://127.0.0.1:8000 # Throwaway to make the mock work
         MESHDB_URL: https://127.0.0.1:8000 # Throwaway to make the mock work
+        # We now check the JoinRecord stuff, so submit that too.
+        S3_ENDPOINT: http://127.0.0.1:9000
+        S3_BUCKET_NAME: meshdb-join-form-log
+        S3_BASE_NAME: dev-join-form-submissions
+        S3_ACCESS_KEY: testaccesskey 
+        S3_SECRET_KEY: testsecretkey
     - uses: actions/upload-artifact@v4
       if: always()
       with:

app/join_record.ts

@@ -0,0 +1,137 @@
+"use server";
+// import { access, constants, appendFileSync, readFile } from "node:fs";
+import {
+  S3Client,
+  PutObjectCommand,
+  GetObjectCommand,
+} from "@aws-sdk/client-s3";
+import { JoinFormValues } from "@/components/JoinForm/JoinForm";
+import { Readable } from "stream";
+import { JoinRecord } from "./types";
+
+// const JOIN_FORM_LOG = process.env.JOIN_FORM_LOG as string;
+
+class JoinRecordS3 {
+  private s3Client: S3Client;
+
+  private S3_REGION: string;
+  private S3_ENDPOINT: string;
+  private S3_BUCKET_NAME: string;
+  private S3_BASE_NAME: string;
+  private S3_ACCESS_KEY: string;
+  private S3_SECRET_KEY: string;
+
+  constructor() {
+    this.S3_REGION = process.env.S3_REGION as string;
+    this.S3_ENDPOINT = process.env.S3_ENDPOINT as string;
+    this.S3_BUCKET_NAME = process.env.S3_BUCKET_NAME as string;
+    this.S3_BASE_NAME = process.env.S3_BASE_NAME as string;
+    this.S3_ACCESS_KEY = process.env.S3_ACCESS_KEY as string;
+    this.S3_SECRET_KEY = process.env.S3_SECRET_KEY as string;
+
+    // Setup the S3 client
+    this.s3Client = new S3Client({
+      region: this.S3_REGION != undefined ? this.S3_REGION : "us-east-1",
+      endpoint:
+        this.S3_ENDPOINT != undefined
+          ? this.S3_ENDPOINT
+          : "https://s3.us-east-1.amazonaws.com",
+      credentials: {
+        accessKeyId: this.S3_ACCESS_KEY,
+        secretAccessKey: this.S3_SECRET_KEY,
+      },
+    });
+  }
+
+  // Records the submission we just got as a JSON object in an S3 bucket.
+  // submission: A Join Form Submission. We append a few things to this.
+  // key: The S3 path we store the submission at
+  // responseCode: If we have a response code for this submission, add it here.
+  async save(submission: JoinRecord, key: string = "") {
+    // Bail if there's no S3 key
+    if (this.S3_ACCESS_KEY === undefined || this.S3_SECRET_KEY === undefined) {
+      console.error(
+        "S3 credentials not configured. I WILL NOT SAVE THIS SUBMISSION.",
+      );
+      return;
+    }
+
+    // Get the date to store this submission under (this is part of the path)
+    const submissionKey = new Date()
+      .toISOString()
+      .replace(/[-:T]/g, "/")
+      .slice(0, 19);
+
+    // Create the path, or use the one provided.
+    key = key != "" ? key : `${this.S3_BASE_NAME}/${submissionKey}.json`;
+
+    let body = JSON.stringify(
+      submission,
+      /*
+      Object.assign(submission, {
+        code: responseCode, // Code the server returned to us
+        replayed: 0, // Number of times we've replayed it from the backend (obvs we haven't)
+        install_number: NaN, // Install number if we do manage to successfully submit it
+      }),
+      */
+    );
+
+    const command = new PutObjectCommand({
+      Bucket: this.S3_BUCKET_NAME,
+      Key: key,
+      Body: body,
+    });
+
+    try {
+      const response = await this.s3Client.send(command);
+      console.log(response);
+    } catch (err) {
+      // Oof, guess we'll drop this on the floor.
+      console.error(err);
+      throw err;
+    }
+
+    // Return the key later so we can update it.
+    return key;
+  }
+
+  // Gets the contents of a JoinRecord for testing
+  async get(key: string) {
+    const getObjectCommand = new GetObjectCommand({
+      Bucket: this.S3_BUCKET_NAME,
+      Key: key,
+    });
+    const getObjectResponse = await this.s3Client.send(getObjectCommand);
+
+    if (getObjectResponse.Body) {
+      const content = await this.streamToString(
+        getObjectResponse.Body as Readable,
+      );
+      return JSON.parse(content);
+    }
+    throw new Error("Could not get Record from S3");
+  }
+
+  // Decode S3 response
+  async streamToString(stream: Readable): Promise<string> {
+    return await new Promise((resolve, reject) => {
+      const chunks: Buffer[] = [];
+      stream.on("data", (chunk) => chunks.push(chunk));
+      stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf-8")));
+      stream.on("error", reject);
+    });
+  }
+}
+
+const joinRecordS3 = new JoinRecordS3();
+
+export async function saveJoinRecordToS3(
+  submission: JoinRecord,
+  key: string = "",
+) {
+  return joinRecordS3.save(submission, key);
+}
+
+export async function getJoinRecordFromS3(key: string) {
+  return joinRecordS3.get(key);
+}

app/types.ts

@@ -0,0 +1,9 @@
+import { JoinFormValues } from "@/components/JoinForm/JoinForm";
+
+type JoinRecord = JoinFormValues & {
+  code: string;
+  replayed: number;
+  install_number: number | null;
+};
+
+export type { JoinRecord };

components/JoinForm/JoinForm.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import React, { useState } from "react";
+import React, { useEffect, useState } from "react";
 import { useForm, SubmitHandler } from "react-hook-form";
 import styles from "./JoinForm.module.scss";
 import { parsePhoneNumberFromString } from "libphonenumber-js";
@@ -13,9 +13,10 @@ import {
 } from "@mui/material";
 import { ToastContainer, toast } from "react-toastify";
 import "react-toastify/dist/ReactToastify.css";
-import { recordJoinFormSubmissionToS3 } from "@/app/data";
+import { saveJoinRecordToS3 } from "@/app/join_record";
 import { getMeshDBAPIEndpoint } from "@/app/endpoint";
 import InfoConfirmationDialog from "../InfoConfirmation/InfoConfirmation";
+import { JoinRecord } from "@/app/types";
 
 type JoinFormValues = {
   first_name: string;
@@ -86,6 +87,8 @@ export default function App() {
     useState(false);
   const [isSubmitted, setIsSubmitted] = useState(false);
   const [isBadPhoneNumber, setIsBadPhoneNumber] = useState(false);
+  const [joinRecordKey, setJoinRecordKey] = useState("");
+
   const isBeta = true;
 
   // Store the values submitted by the user or returned by the server
@@ -150,20 +153,44 @@ export default function App() {
   };
 
   async function submitJoinFormToMeshDB(joinFormSubmission: JoinFormValues) {
-    console.debug(JSON.stringify(joinFormSubmission));
+    // First up, before we try anything else, submit to S3 for safety.
+    const safetyRecord: JoinRecord = Object.assign(
+      structuredClone(joinFormSubmission),
+      { code: "", replayed: 0, install_number: null },
+    ) as JoinRecord;
+    saveJoinRecordToS3(safetyRecord, joinRecordKey).then((key) => {
+      setJoinRecordKey(key as string);
+    });
 
     return fetch(`${await getMeshDBAPIEndpoint()}/api/v1/join/`, {
       method: "POST",
       body: JSON.stringify(joinFormSubmission),
     })
       .then(async (response) => {
+        // Update the submission in S3 with the status code.
+        const record: JoinRecord = Object.assign(joinFormSubmission, {
+          code: response.status.toString(),
+          replayed: 0,
+          install_number: NaN,
+        }) as JoinRecord;
+
         if (response.ok) {
           console.debug("Join Form submitted successfully");
           setIsLoading(false);
           setIsSubmitted(true);
+          const install_number = (await response.json()).install_number;
+          record.install_number = Number(install_number);
+          saveJoinRecordToS3(record, joinRecordKey).then((key) => {
+            setJoinRecordKey(key as string);
+          });
           return;
         }
 
+        saveJoinRecordToS3(record, joinRecordKey).then((key) => {
+          setJoinRecordKey(key as string);
+        });
+
+        // If the response was not good, then get angry
         throw response;
       })
       .catch(async (error) => {
@@ -174,11 +201,8 @@ export default function App() {
         if (error.status == 409) {
           let needsConfirmation: Array<ConfirmationField> = [];
           const changedInfo = errorJson.changed_info;
-          console.log(joinFormSubmission);
-          console.log(errorJson.changed_info);
 
           for (const key in joinFormSubmission) {
-            console.log(key);
             if (
               joinFormSubmission.hasOwnProperty(key) &&
               changedInfo.hasOwnProperty(key)
@@ -209,7 +233,6 @@ export default function App() {
 
   const onSubmit: SubmitHandler<JoinFormValues> = (data) => {
     setIsLoading(true);
-    recordJoinFormSubmissionToS3(data);
     data.trust_me_bro = false;
     submitJoinFormToMeshDB(data);
   };
@@ -401,6 +424,7 @@ export default function App() {
         handleClickReject={handleClickReject}
         handleClickCancel={handleClickCancel}
       />
+      <div data-testid="test-join-record-key" data-state={joinRecordKey}></div>
     </>
   );
 }