From 2dbf316c8eead0367d8ed99bb56c75246f46d29c Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 28 Oct 2024 09:04:17 -0400 Subject: [PATCH 1/5] checkpoint --- app/data.ts | 71 +++++++++++++++---- app/join-explorer/layout.tsx | 15 ++++ app/join-explorer/page.tsx | 11 +++ app/query/page.tsx | 6 +- .../JoinExplorer/JoinExplorer.module.scss | 0 components/JoinExplorer/JoinExplorer.tsx | 20 ++++++ components/Landing/Landing.tsx | 1 + tests/join_form.spec.ts | 1 - 8 files changed, 106 insertions(+), 19 deletions(-) create mode 100644 app/join-explorer/layout.tsx create mode 100644 app/join-explorer/page.tsx create mode 100644 components/JoinExplorer/JoinExplorer.module.scss create mode 100644 components/JoinExplorer/JoinExplorer.tsx diff --git a/app/data.ts b/app/data.ts index 201e965..ada9385 100644 --- a/app/data.ts +++ b/app/data.ts @@ -1,6 +1,6 @@ "use server"; import { access, constants, appendFileSync, readFile } from "node:fs"; -import { S3Client, PutObjectCommand } from "@aws-sdk/client-s3"; +import { S3Client, PutObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3"; import { JoinFormValues } from "@/components/JoinForm/JoinForm"; const JOIN_FORM_LOG = process.env.JOIN_FORM_LOG as string; @@ -11,6 +11,24 @@ const S3_BASE_NAME = process.env.S3_BASE_NAME as string; const S3_ACCESS_KEY = process.env.S3_ACCESS_KEY as string; const S3_SECRET_KEY = process.env.S3_SECRET_KEY as string; +if (S3_ACCESS_KEY === undefined || S3_SECRET_KEY === undefined) { + console.error( + "S3 credentials not configured. I WILL NOT SAVE THIS SUBMISSION.", + ); +} + +const s3Client = new S3Client({ + region: S3_REGION != undefined ? S3_REGION : "us-east-1", + endpoint: + S3_ENDPOINT != undefined + ? S3_ENDPOINT + : "https://s3.us-east-1.amazonaws.com", + credentials: { + accessKeyId: S3_ACCESS_KEY, + secretAccessKey: S3_SECRET_KEY, + }, +}); + export async function recordJoinFormSubmissionToCSV( submission: JoinFormValues, ) { @@ -41,18 +59,6 @@ export async function recordJoinFormSubmissionToS3(submission: JoinFormValues) { return; } - const s3Client = new S3Client({ - region: S3_REGION != undefined ? S3_REGION : "us-east-1", - endpoint: - S3_ENDPOINT != undefined - ? S3_ENDPOINT - : "https://s3.us-east-1.amazonaws.com", - credentials: { - accessKeyId: S3_ACCESS_KEY, - secretAccessKey: S3_SECRET_KEY, - }, - }); - const submissionKey = new Date() .toISOString() .replace(/[-:T]/g, "/") @@ -74,3 +80,42 @@ export async function recordJoinFormSubmissionToS3(submission: JoinFormValues) { throw err; } } + + +async function listAllObjects(bucketName: string) { + let continuationToken: string | undefined = undefined; + const allObjects = []; + + try { + do { + const command = new ListObjectsV2Command({ + Bucket: bucketName, + ContinuationToken: continuationToken, + }); + + const response = await s3Client.send(command); + if (response.Contents) { + allObjects.push(...response.Contents); + } + + // Update the continuation token to get the next page of results + continuationToken = response.NextContinuationToken; + + } while (continuationToken); + + return allObjects; // Contains all objects in the bucket + } catch (error) { + console.error("Error listing S3 objects:", error); + throw error; + } +} + + +export async function fetchSubmissions() { + listAllObjects(S3_BUCKET_NAME).then(objects => { + objects.forEach(obj => { + console.log("Object Key:", obj.Key, " | Object Value: ", obj); + }); + return objects; + }); +} diff --git a/app/join-explorer/layout.tsx b/app/join-explorer/layout.tsx new file mode 100644 index 0000000..b924e33 --- /dev/null +++ b/app/join-explorer/layout.tsx @@ -0,0 +1,15 @@ +export default function JoinExplorerLayout({ + children, +}: { + children: React.ReactNode; +}) { + return ( + <> +
+
+ {children} +
+
+ + ); +} diff --git a/app/join-explorer/page.tsx b/app/join-explorer/page.tsx new file mode 100644 index 0000000..aab73b1 --- /dev/null +++ b/app/join-explorer/page.tsx @@ -0,0 +1,11 @@ +import { JoinExplorer } from "@/components/JoinExplorer/JoinExplorer"; + +export default async function Query() { + return ( + <> +
+ +
+ + ); +} diff --git a/app/query/page.tsx b/app/query/page.tsx index 2e7f4d4..370d815 100644 --- a/app/query/page.tsx +++ b/app/query/page.tsx @@ -2,11 +2,7 @@ import { Footer } from "@/components/Footer/Footer"; import { Header } from "@/components/Header/Header"; import { QueryForm } from "@/components/QueryForm/QueryForm"; -// TODO: -// https://www.npmjs.com/package/react-phone-number-input -// https://www.npmjs.com/package/react-error-boundary - -export default async function Join() { +export default async function Query() { return ( <>
diff --git a/components/JoinExplorer/JoinExplorer.module.scss b/components/JoinExplorer/JoinExplorer.module.scss new file mode 100644 index 0000000..e69de29 diff --git a/components/JoinExplorer/JoinExplorer.tsx b/components/JoinExplorer/JoinExplorer.tsx new file mode 100644 index 0000000..02eae1e --- /dev/null +++ b/components/JoinExplorer/JoinExplorer.tsx @@ -0,0 +1,20 @@ +"use client" +import React, { useState } from "react"; +import { fetchSubmissions } from "@/app/data"; + +export async function JoinExplorer() { + const [submissionList, setSubmissionList] = useState({}); + const submissions = await fetchSubmissions(); + setSubmissionList(submissions); + + + return (<> + + ); +} diff --git a/components/Landing/Landing.tsx b/components/Landing/Landing.tsx index 77c0c5c..fd4f8d9 100644 --- a/components/Landing/Landing.tsx +++ b/components/Landing/Landing.tsx @@ -13,6 +13,7 @@ const Landing = () => { text: "MeshDB Admin", link: process.env.NEXT_PUBLIC_MESHDB_URL + "/admin/", }, + { text: "View Submissions", link: "/join-explorer" }, ]; return ( diff --git a/tests/join_form.spec.ts b/tests/join_form.spec.ts index 3f76ae4..236e471 100644 --- a/tests/join_form.spec.ts +++ b/tests/join_form.spec.ts @@ -1,4 +1,3 @@ -import { JoinFormInput, JoinFormResponse } from "@/app/io"; import { JoinFormValues } from "@/components/JoinForm/JoinForm"; import { test, expect } from "@/tests/mock/test"; From 22414eb288a13d7167586cbf6458805b1f15a845 Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 28 Oct 2024 09:06:16 -0400 Subject: [PATCH 2/5] I can't stop listing objects --- app/data.ts | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/app/data.ts b/app/data.ts index ada9385..84a6801 100644 --- a/app/data.ts +++ b/app/data.ts @@ -1,7 +1,8 @@ "use server"; import { access, constants, appendFileSync, readFile } from "node:fs"; -import { S3Client, PutObjectCommand, ListObjectsV2Command } from "@aws-sdk/client-s3"; +import { S3Client, PutObjectCommand, ListObjectsV2Command, GetObjectCommand } from "@aws-sdk/client-s3"; import { JoinFormValues } from "@/components/JoinForm/JoinForm"; +import { Readable } from "node:stream"; const JOIN_FORM_LOG = process.env.JOIN_FORM_LOG as string; const S3_REGION = process.env.S3_REGION as string; @@ -82,24 +83,45 @@ export async function recordJoinFormSubmissionToS3(submission: JoinFormValues) { } +async function streamToString(stream: Readable): Promise { + return await new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on("data", (chunk) => chunks.push(chunk)); + stream.on("end", () => resolve(Buffer.concat(chunks).toString("utf-8"))); + stream.on("error", reject); + }); +} + + async function listAllObjects(bucketName: string) { let continuationToken: string | undefined = undefined; const allObjects = []; try { do { - const command = new ListObjectsV2Command({ + const listCommand = new ListObjectsV2Command({ Bucket: bucketName, ContinuationToken: continuationToken, }); - const response = await s3Client.send(command); - if (response.Contents) { - allObjects.push(...response.Contents); + const listResponse = await s3Client.send(listCommand); + + for (const obj of listResponse.Contents || []) { + const getObjectCommand = new GetObjectCommand({ + Bucket: bucketName, + Key: obj.Key, + }); + const getObjectResponse = await s3Client.send(getObjectCommand); + + if (getObjectResponse.Body) { + const content = await streamToString(getObjectResponse.Body as Readable); + console.log(`Content of ${obj.Key}:`, content); + } } + // Update the continuation token to get the next page of results - continuationToken = response.NextContinuationToken; + continuationToken = listResponse.NextContinuationToken; } while (continuationToken); From 85f936af3561697ef3a5959bdea606aec78ca8e5 Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 28 Oct 2024 19:23:24 -0400 Subject: [PATCH 3/5] nevermind --- app/data.ts | 18 ++++++++++-------- components/JoinExplorer/JoinExplorer.tsx | 16 ++++++++++------ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/app/data.ts b/app/data.ts index 84a6801..d3a5135 100644 --- a/app/data.ts +++ b/app/data.ts @@ -115,7 +115,10 @@ async function listAllObjects(bucketName: string) { if (getObjectResponse.Body) { const content = await streamToString(getObjectResponse.Body as Readable); - console.log(`Content of ${obj.Key}:`, content); + allObjects.push({ + key: obj.Key, + content: content + }); } } @@ -132,12 +135,11 @@ async function listAllObjects(bucketName: string) { } } +type JoinLogLine = { + key: string; + submission: JoinFormValues; +}; -export async function fetchSubmissions() { - listAllObjects(S3_BUCKET_NAME).then(objects => { - objects.forEach(obj => { - console.log("Object Key:", obj.Key, " | Object Value: ", obj); - }); - return objects; - }); +export async function fetchSubmissions(): Array { + return listAllObjects(S3_BUCKET_NAME); } diff --git a/components/JoinExplorer/JoinExplorer.tsx b/components/JoinExplorer/JoinExplorer.tsx index 02eae1e..dc53595 100644 --- a/components/JoinExplorer/JoinExplorer.tsx +++ b/components/JoinExplorer/JoinExplorer.tsx @@ -1,18 +1,22 @@ "use client" import React, { useState } from "react"; import { fetchSubmissions } from "@/app/data"; +import { Button } from "@mui/material"; -export async function JoinExplorer() { - const [submissionList, setSubmissionList] = useState({}); - const submissions = await fetchSubmissions(); - setSubmissionList(submissions); +export function JoinExplorer() { + const [joinList, setJoinList] = useState([]); + async function loadSubmissions() { + console.log("Chom"); + console.log(await fetchSubmissions()); + } return (<> +
    - {submissionList.map((s) => ( + {joinList.map((s) => (
  • - {s.json()} + {s}
    • ))}
From e570aa9b7ac3d665ed2db8c0f3e2c0ca7c836ce5 Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Mon, 28 Oct 2024 23:53:14 -0400 Subject: [PATCH 4/5] checkpoint --- app/data.ts | 23 +++++++++++++++-------- app/types.ts | 0 components/JoinExplorer/JoinExplorer.tsx | 8 +++++--- 3 files changed, 20 insertions(+), 11 deletions(-) create mode 100644 app/types.ts diff --git a/app/data.ts b/app/data.ts index d3a5135..82a8419 100644 --- a/app/data.ts +++ b/app/data.ts @@ -1,8 +1,18 @@ "use server"; import { access, constants, appendFileSync, readFile } from "node:fs"; import { S3Client, PutObjectCommand, ListObjectsV2Command, GetObjectCommand } from "@aws-sdk/client-s3"; -import { JoinFormValues } from "@/components/JoinForm/JoinForm"; +import { JoinFormValues, NewJoinFormValues } from "@/components/JoinForm/JoinForm"; import { Readable } from "node:stream"; + +type JoinLogLine = { + key: string; + submission: JoinFormValues; +}; + +export async function NewJoinLogLine() { + return {key: "", submission: NewJoinFormValues()} +} + const JOIN_FORM_LOG = process.env.JOIN_FORM_LOG as string; const S3_REGION = process.env.S3_REGION as string; @@ -95,7 +105,7 @@ async function streamToString(stream: Readable): Promise { async function listAllObjects(bucketName: string) { let continuationToken: string | undefined = undefined; - const allObjects = []; + const allObjects: Array = []; try { do { @@ -117,12 +127,11 @@ async function listAllObjects(bucketName: string) { const content = await streamToString(getObjectResponse.Body as Readable); allObjects.push({ key: obj.Key, - content: content + submission: JSON.parse(content) }); } } - // Update the continuation token to get the next page of results continuationToken = listResponse.NextContinuationToken; @@ -135,10 +144,8 @@ async function listAllObjects(bucketName: string) { } } -type JoinLogLine = { - key: string; - submission: JoinFormValues; -}; + +export type { JoinLogLine }; export async function fetchSubmissions(): Array { return listAllObjects(S3_BUCKET_NAME); diff --git a/app/types.ts b/app/types.ts new file mode 100644 index 0000000..e69de29 diff --git a/components/JoinExplorer/JoinExplorer.tsx b/components/JoinExplorer/JoinExplorer.tsx index dc53595..d197e0b 100644 --- a/components/JoinExplorer/JoinExplorer.tsx +++ b/components/JoinExplorer/JoinExplorer.tsx @@ -1,14 +1,16 @@ "use client" import React, { useState } from "react"; -import { fetchSubmissions } from "@/app/data"; +import { JoinLogLine, NewJoinLogLine, fetchSubmissions } from "@/app/data"; import { Button } from "@mui/material"; +import { NewJoinFormValues } from "../JoinForm/JoinForm"; export function JoinExplorer() { - const [joinList, setJoinList] = useState([]); + const [joinList, setJoinList]: Array = useState([NewJoinLogLine()]); async function loadSubmissions() { console.log("Chom"); - console.log(await fetchSubmissions()); + setJoinList(await fetchSubmissions()); + } return (<> From b75e0a3890aae8fae52e70c0207a7e060f708e6d Mon Sep 17 00:00:00 2001 From: Willard Nilges Date: Tue, 29 Oct 2024 00:29:33 -0400 Subject: [PATCH 5/5] chom --- app/data.ts | 11 +----- app/types.ts | 47 ++++++++++++++++++++++++ components/JoinExplorer/JoinExplorer.tsx | 4 +- components/JoinForm/JoinForm.tsx | 37 ------------------- tests/join_form.spec.ts | 2 +- 5 files changed, 51 insertions(+), 50 deletions(-) diff --git a/app/data.ts b/app/data.ts index 82a8419..ac88288 100644 --- a/app/data.ts +++ b/app/data.ts @@ -1,17 +1,8 @@ "use server"; import { access, constants, appendFileSync, readFile } from "node:fs"; import { S3Client, PutObjectCommand, ListObjectsV2Command, GetObjectCommand } from "@aws-sdk/client-s3"; -import { JoinFormValues, NewJoinFormValues } from "@/components/JoinForm/JoinForm"; import { Readable } from "node:stream"; - -type JoinLogLine = { - key: string; - submission: JoinFormValues; -}; - -export async function NewJoinLogLine() { - return {key: "", submission: NewJoinFormValues()} -} +import { JoinFormValues } from "./types"; const JOIN_FORM_LOG = process.env.JOIN_FORM_LOG as string; diff --git a/app/types.ts b/app/types.ts index e69de29..44f8725 100644 --- a/app/types.ts +++ b/app/types.ts @@ -0,0 +1,47 @@ +type JoinFormValues = { + first_name: string; + last_name: string; + email_address: string; + phone_number: string; + street_address: string; + apartment: string; + city: string; + state: string; + zip_code: string; + roof_access: boolean; + referral: string; + ncl: boolean; + trust_me_bro: boolean; +}; + +// Coding like it's 1997 +export function NewJoinFormValues() { + return { + first_name: "", + last_name: "", + email_address: "", + phone_number: "", + street_address: "", + apartment: "", + city: "", + state: "", + zip_code: "", + roof_access: false, + referral: "", + ncl: false, + trust_me_bro: false, + }; +} + +export type { JoinFormValues }; + +type JoinLogLine = { + key: string; + submission: JoinFormValues; +}; + +export async function NewJoinLogLine() { + return {key: "", submission: NewJoinFormValues()} +} + +export type { JoinLogLine }; diff --git a/components/JoinExplorer/JoinExplorer.tsx b/components/JoinExplorer/JoinExplorer.tsx index d197e0b..e17b447 100644 --- a/components/JoinExplorer/JoinExplorer.tsx +++ b/components/JoinExplorer/JoinExplorer.tsx @@ -1,8 +1,8 @@ "use client" import React, { useState } from "react"; -import { JoinLogLine, NewJoinLogLine, fetchSubmissions } from "@/app/data"; +import { fetchSubmissions } from "@/app/data"; import { Button } from "@mui/material"; -import { NewJoinFormValues } from "../JoinForm/JoinForm"; +import { JoinLogLine, NewJoinLogLine } from "@/app/types"; export function JoinExplorer() { const [joinList, setJoinList]: Array = useState([NewJoinLogLine()]); diff --git a/components/JoinForm/JoinForm.tsx b/components/JoinForm/JoinForm.tsx index bc44418..3ec4830 100644 --- a/components/JoinForm/JoinForm.tsx +++ b/components/JoinForm/JoinForm.tsx @@ -17,43 +17,6 @@ import { recordJoinFormSubmissionToS3 } from "@/app/data"; import { getMeshDBAPIEndpoint } from "@/app/endpoint"; import InfoConfirmationDialog from "../InfoConfirmation/InfoConfirmation"; -type JoinFormValues = { - first_name: string; - last_name: string; - email_address: string; - phone_number: string; - street_address: string; - apartment: string; - city: string; - state: string; - zip_code: string; - roof_access: boolean; - referral: string; - ncl: boolean; - trust_me_bro: boolean; -}; - -// Coding like it's 1997 -export function NewJoinFormValues() { - return { - first_name: "", - last_name: "", - email_address: "", - phone_number: "", - street_address: "", - apartment: "", - city: "", - state: "", - zip_code: "", - roof_access: false, - referral: "", - ncl: false, - trust_me_bro: false, - }; -} - -export type { JoinFormValues }; - type ConfirmationField = { key: keyof JoinFormValues; original: string; diff --git a/tests/join_form.spec.ts b/tests/join_form.spec.ts index 236e471..7a8792d 100644 --- a/tests/join_form.spec.ts +++ b/tests/join_form.spec.ts @@ -1,4 +1,4 @@ -import { JoinFormValues } from "@/components/JoinForm/JoinForm"; +import { JoinFormValues } from "@/app/types"; import { test, expect } from "@/tests/mock/test"; import {