diff --git a/.env.sample b/.env.sample index 8231526..dbec4f4 100644 --- a/.env.sample +++ b/.env.sample @@ -1,8 +1,17 @@ MESHDB_URL=http://127.0.0.1:8000 NEXT_PUBLIC_MESHDB_URL=http://127.0.0.1:8000 +JOIN_RECORD_BUCKET_NAME=meshdb-join-form-log +JOIN_RECORD_PREFIX=dev-join-form-submissions + +S3_ENDPOINT=http://127.0.0.1:9000 +AWS_ACCESS_KEY=sampleaccesskey +AWS_SECRET_KEY=samplesecretkey +AWS_REGION=us-east-1 + # Docker compose environment variables # Set this to true in prod COMPOSE_EXTERNAL_NETWORK=false # Set this to traefik-net in prod COMPOSE_NETWORK_NAME= + diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 6556aa6..ea41afb 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -15,6 +15,20 @@ jobs: - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 with: node-version: lts/* + - name: Setup Minio + run: | + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ACCESS_KEY=sampleaccesskey" \ + -e "MINIO_SECRET_KEY=samplesecretkey" \ + -e "MINIO_DEFAULT_BUCKETS=meshdb-join-form-log" \ + -v /tmp/data:/data \ + -v /tmp/config:/root/.minio \ + minio/minio server /data + + export AWS_ACCESS_KEY_ID=sampleaccesskey + export AWS_SECRET_ACCESS_KEY=samplesecretkey + export AWS_EC2_METADATA_DISABLED=true + aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://meshdb-join-form-log - name: Install dependencies run: npm ci - name: Install Playwright Browsers @@ -24,6 +38,13 @@ jobs: env: NEXT_PUBLIC_MESHDB_URL: https://127.0.0.1:8000 # Throwaway to make the mock work MESHDB_URL: https://127.0.0.1:8000 # Throwaway to make the mock work + # We now check the JoinRecord stuff, so submit that too. + JOIN_RECORD_BUCKET_NAME: meshdb-join-form-log + JOIN_RECORD_PREFIX: dev-join-form-submissions + S3_ENDPOINT: http://127.0.0.1:9000 + AWS_ACCESS_KEY_ID: sampleaccesskey + AWS_SECRET_ACCESS_KEY: samplesecretkey + AWS_REGION: us-east-1 - uses: actions/upload-artifact@v4 if: always() with: diff --git a/app/data.ts b/app/data.ts deleted file mode 100644 index 201e965..0000000 --- a/app/data.ts +++ /dev/null @@ -1,76 +0,0 @@ -"use server"; -import { access, constants, appendFileSync, readFile } from "node:fs"; -import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3"; -import { JoinFormValues } from "@/components/JoinForm/JoinForm"; -const JOIN_FORM_LOG = process.env.JOIN_FORM_LOG as string; - -const S3_REGION = process.env.S3_REGION as string; -const S3_ENDPOINT = process.env.S3_ENDPOINT as string; -const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME as string; -const S3_BASE_NAME = process.env.S3_BASE_NAME as string; -const S3_ACCESS_KEY = process.env.S3_ACCESS_KEY as string; -const S3_SECRET_KEY = process.env.S3_SECRET_KEY as string; - -export async function recordJoinFormSubmissionToCSV( - submission: JoinFormValues, -) { - const keys = Object.keys(submission).join(","); - // Surround each value in quotes to avoid confusion with strings like - // "Brooklyn, NY" - const values = Object.values(submission) - .map((v) => `"${v}"`) - .join(","); - - access(JOIN_FORM_LOG, constants.F_OK, async (err) => { - if (err) { - console.log(err); - // Initialize with headers if it doesn't exist - appendFileSync(JOIN_FORM_LOG, `${keys}\n`); - } - // Write the submission - appendFileSync(JOIN_FORM_LOG, `${values}\n`); - }); -} - -// Records the submission we just got as a JSON object in an S3 bucket. -export async function recordJoinFormSubmissionToS3(submission: JoinFormValues) { - if (S3_ACCESS_KEY === undefined || S3_SECRET_KEY === undefined) { - console.error( - "S3 credentials not configured. I WILL NOT SAVE THIS SUBMISSION.", - ); - return; - } - - const s3Client = new S3Client({ - region: S3_REGION != undefined ? S3_REGION : "us-east-1", - endpoint: - S3_ENDPOINT != undefined - ? S3_ENDPOINT - : "https://s3.us-east-1.amazonaws.com", - credentials: { - accessKeyId: S3_ACCESS_KEY, - secretAccessKey: S3_SECRET_KEY, - }, - }); - - const submissionKey = new Date() - .toISOString() - .replace(/[-:T]/g, "/") - .slice(0, 19); - - const command = new PutObjectCommand({ - Bucket: S3_BUCKET_NAME, - Key: `${S3_BASE_NAME}/${submissionKey}.json`, - Body: JSON.stringify(submission), - }); - - try { - const response = await s3Client.send(command); - console.log(response); - } catch (err) { - console.error(err); - // Record the submission to a local CSV file as a last-ditch effort - recordJoinFormSubmissionToCSV(submission); - throw err; - } -} diff --git a/app/join_record.ts b/app/join_record.ts new file mode 100644 index 0000000..207f85a --- /dev/null +++ b/app/join_record.ts @@ -0,0 +1,126 @@ +"use server"; +// import { access, constants, appendFileSync, readFile } from "node:fs"; +import { + S3Client, + PutObjectCommand, + GetObjectCommand, + ListBucketsCommand, +} from "@aws-sdk/client-s3"; +import { JoinFormValues } from "@/components/JoinForm/JoinForm"; +import { Readable } from "stream"; +import { JoinRecord } from "./types"; + +// const JOIN_FORM_LOG = process.env.JOIN_FORM_LOG as string; + +class JoinRecordS3 { + private s3Client: S3Client; + + private BUCKET_NAME: string; + private PREFIX: string; + + private S3_ENDPOINT: string; + private AWS_ACCESS_KEY_ID: string; + private AWS_SECRET_ACCESS_KEY: string; + + constructor() { + this.BUCKET_NAME = process.env.JOIN_RECORD_BUCKET_NAME as string; + this.PREFIX = process.env.JOIN_RECORD_PREFIX as string; + this.S3_ENDPOINT = process.env.S3_ENDPOINT as string; + this.AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID as string; + this.AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY as string; + + // Setup the S3 client + this.s3Client = new S3Client({ + endpoint: + this.S3_ENDPOINT != undefined + ? this.S3_ENDPOINT + : "https://s3.us-east-1.amazonaws.com", + }); + } + + // Records the submission we just got as a JSON object in an S3 bucket. + // submission: A Join Form Submission. We append a few things to this. + // key: The S3 path we store the submission at + // responseCode: If we have a response code for this submission, add it here. + async save(joinRecord: JoinRecord, key: string = "") { + // Check if the S3 client is working and exit gracefully with a warning if it is not. + try { + const command = new ListBucketsCommand({}); + await this.s3Client.send(command); + } catch (error) { + console.warn( + "S3 Client not configured properly. I WILL NOT SAVE THIS SUBMISSION.", + error, + ); + return; + } + + // Get the date to store this submission under (this is part of the path) + const submissionKey = joinRecord.submission_time + .replace(/[-:T]/g, "/") + .slice(0, 19); + + // Create the path, or use the one provided. + key = key != "" ? key : `${this.PREFIX}/${submissionKey}.json`; + + let body = JSON.stringify(joinRecord); + + const command = new PutObjectCommand({ + Bucket: this.BUCKET_NAME, + Key: key, + Body: body, + }); + + try { + const response = await this.s3Client.send(command); + console.log(response); + } catch (err) { + // Oof, guess we'll drop this on the floor. + console.error(err); + throw err; + } + + // Return the key later so we can update it. + return key; + } + + // Gets the contents of a JoinRecord for testing + async get(key: string) { + const getObjectCommand = new GetObjectCommand({ + Bucket: this.BUCKET_NAME, + Key: key, + }); + const getObjectResponse = await this.s3Client.send(getObjectCommand); + + if (getObjectResponse.Body) { + const content = await this.streamToString( + getObjectResponse.Body as Readable, + ); + return JSON.parse(content); + } + throw new Error("Could not get Record from S3"); + } + + // Decode S3 response + async streamToString(stream: Readable): Promise { + return await new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on("data", (chunk) => chunks.push(chunk)); + stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf-8"))); + stream.on("error", reject); + }); + } +} + +const joinRecordS3 = new JoinRecordS3(); + +export async function saveJoinRecordToS3( + submission: JoinRecord, + key: string = "", +) { + return joinRecordS3.save(submission, key); +} + +export async function getJoinRecordFromS3(key: string) { + return joinRecordS3.get(key); +} diff --git a/app/types.ts b/app/types.ts new file mode 100644 index 0000000..282bfb0 --- /dev/null +++ b/app/types.ts @@ -0,0 +1,10 @@ +import { JoinFormValues } from "@/components/JoinForm/JoinForm"; + +type JoinRecord = JoinFormValues & { + submission_time: string; + code: number | null; + replayed: number; + install_number: number | null; +}; + +export type { JoinRecord }; diff --git a/components/JoinForm/JoinForm.tsx b/components/JoinForm/JoinForm.tsx index bc44418..878be60 100644 --- a/components/JoinForm/JoinForm.tsx +++ b/components/JoinForm/JoinForm.tsx @@ -1,6 +1,6 @@ "use client"; -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; import { useForm, SubmitHandler } from "react-hook-form"; import styles from "./JoinForm.module.scss"; import { parsePhoneNumberFromString } from "libphonenumber-js"; @@ -13,46 +13,40 @@ import { } from "@mui/material"; import { ToastContainer, toast } from "react-toastify"; import "react-toastify/dist/ReactToastify.css"; -import { recordJoinFormSubmissionToS3 } from "@/app/data"; +import { saveJoinRecordToS3 } from "@/app/join_record"; import { getMeshDBAPIEndpoint } from "@/app/endpoint"; import InfoConfirmationDialog from "../InfoConfirmation/InfoConfirmation"; - -type JoinFormValues = { - first_name: string; - last_name: string; - email_address: string; - phone_number: string; - street_address: string; - apartment: string; - city: string; - state: string; - zip_code: string; - roof_access: boolean; - referral: string; - ncl: boolean; - trust_me_bro: boolean; -}; - -// Coding like it's 1997 -export function NewJoinFormValues() { - return { - first_name: "", - last_name: "", - email_address: "", - phone_number: "", - street_address: "", - apartment: "", - city: "", - state: "", - zip_code: "", - roof_access: false, - referral: "", - ncl: false, - trust_me_bro: false, - }; +import { JoinRecord } from "@/app/types"; + +export class JoinFormValues { + constructor( + public first_name: string = "", + public last_name: string = "", + public email_address: string = "", + public phone_number: string = "", + public street_address: string = "", + public apartment: string = "", + public city: string = "", + public state: string = "", + public zip_code: string = "", + public roof_access: boolean = false, + public referral: string = "", + public ncl: boolean = false, + public trust_me_bro: boolean = false, + ) {} } -export type { JoinFormValues }; +export class JoinFormResponse { + constructor( + public detail: string = "", + public building_id: string = "", // UUID + public member_id: string = "", // UUID + public install_id: string = "", // UUID + public install_number: number | null = null, + public member_exists: boolean = false, + public changed_info: { [id: string]: string } = {}, + ) {} +} type ConfirmationField = { key: keyof JoinFormValues; @@ -68,14 +62,14 @@ const selectStateOptions = [ ]; export default function App() { - let defaultFormValues = NewJoinFormValues(); + let defaultFormValues = new JoinFormValues(); defaultFormValues.state = selectStateOptions[0].value; const { register, setValue, getValues, handleSubmit, - formState: { isDirty, isValid }, + formState: { isValid }, } = useForm({ mode: "onChange", defaultValues: defaultFormValues, @@ -85,7 +79,10 @@ export default function App() { const [isInfoConfirmationDialogueOpen, setIsInfoConfirmationDialogueOpen] = useState(false); const [isSubmitted, setIsSubmitted] = useState(false); + const [isMeshDBProbablyDown, setIsMeshDBProbablyDown] = useState(false); const [isBadPhoneNumber, setIsBadPhoneNumber] = useState(false); + const [joinRecordKey, setJoinRecordKey] = useState(""); + const isBeta = true; // Store the values submitted by the user or returned by the server @@ -150,35 +147,68 @@ export default function App() { }; async function submitJoinFormToMeshDB(joinFormSubmission: JoinFormValues) { - console.debug(JSON.stringify(joinFormSubmission)); - - return fetch(`${await getMeshDBAPIEndpoint()}/api/v1/join/`, { - method: "POST", - body: JSON.stringify(joinFormSubmission), - }) - .then(async (response) => { - if (response.ok) { - console.debug("Join Form submitted successfully"); - setIsLoading(false); - setIsSubmitted(true); - return; - } + // Before we try anything else, submit to S3 for safety. + let record: JoinRecord = Object.assign( + structuredClone(joinFormSubmission), + { + submission_time: new Date().toISOString(), + code: null, + replayed: 0, + install_number: null, + }, + ) as JoinRecord; + setJoinRecordKey( + (await saveJoinRecordToS3(record, joinRecordKey)) as string, + ); - throw response; - }) - .catch(async (error) => { - const errorJson = await error.json(); - const detail = await errorJson.detail; + try { + const response = await fetch( + `${await getMeshDBAPIEndpoint()}/api/v1/join/`, + { + method: "POST", + body: JSON.stringify(joinFormSubmission), + }, + ); + const j = await response.json(); + const responseData = new JoinFormResponse( + j.detail, + j.building_id, + j.member_id, + j.install_id, + j.install_number, + j.member_exists, + j.changed_info, + ); + + // Grab the HTTP code and the install_number (if we have it) for the joinRecord + record.code = response.status; + record.install_number = responseData.install_number; + // Update the join record with our data if we have it. + setJoinRecordKey( + (await saveJoinRecordToS3(record, joinRecordKey)) as string, + ); + + if (response.ok) { + console.debug("Join Form submitted successfully"); + setIsLoading(false); + setIsSubmitted(true); + return; + } + + // If the response was not good, then get angry. + throw responseData; + } catch (error: unknown) { + // The error should always be a JoinFormResponse. TS insists that + // errors be handled as type Unknown, but this is (and should always be) + // the happy path of error handling + if (error instanceof JoinFormResponse) { // We just need to confirm some information - if (error.status == 409) { + if (record.code == 409) { let needsConfirmation: Array = []; - const changedInfo = errorJson.changed_info; - console.log(joinFormSubmission); - console.log(errorJson.changed_info); + const changedInfo = error.changed_info; for (const key in joinFormSubmission) { - console.log(key); if ( joinFormSubmission.hasOwnProperty(key) && changedInfo.hasOwnProperty(key) @@ -200,16 +230,48 @@ export default function App() { return; } + // If it was the server's fault, then just accept the record and move + // on. + if (record.code !== null && (500 <= record.code && record.code <= 599)) { + setIsMeshDBProbablyDown(true); + setIsLoading(false); + setIsSubmitted(true); + // Log the error to the console + console.error(error.detail); + return; + } + + // If it was another kind of 4xx, the member did something wrong and needs + // to fix their information (i.e. move out of nj) + const detail = error.detail; // This looks disgusting when Debug is on in MeshDB because it replies with HTML. // There's probably a way to coax the exception out of the response somewhere toast.error(`Could not submit Join Form: ${detail}`); + console.error(`An error occurred: ${detail}`); setIsLoading(false); - }); + return; + } + + // If we didn't get a JoinFormResponse, chances are that MeshDB is hard down. + // Tell the user we recorded their submission, but change the message. + setIsMeshDBProbablyDown(true); + setIsLoading(false); + setIsSubmitted(true); + + + // Log the message to the console. + if (error instanceof Error) { + console.error(`An error occurred: ${error.message}`); + return; + } + + console.error(`An unknown error occurred: ${JSON.stringify(error)}`); + return; + } } const onSubmit: SubmitHandler = (data) => { setIsLoading(true); - recordJoinFormSubmissionToS3(data); data.trust_me_bro = false; submitJoinFormToMeshDB(data); }; @@ -346,15 +408,20 @@ export default function App() { Network Commons License + {/* +
+

State Debugger

+ isLoading: {isLoading ? "true" : "false"}
+ isSubmitted: {isSubmitted ? "true" : "false"}
+ isBadPhoneNumber: {isBadPhoneNumber ? "true" : "false"}
+ !isValid: {!isValid ? "true" : "false"}
+
+ */}