From 878e854be8a34d09c1e2db4e299712da68427f66 Mon Sep 17 00:00:00 2001 From: Wind-Explorer <66894537+Wind-Explorer@users.noreply.github.com> Date: Mon, 12 Aug 2024 22:42:34 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=A4=B9=20AI=20IMG=20RECOGNITION=20=D0=91?= =?UTF-8?q?=D0=9B=D0=AF=D0=A2=D0=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/src/components/InsertBillImage.tsx | 128 +++++++++ client/src/pages/HBFormPage.tsx | 237 +++++++++------- client/src/security/http.ts | 2 + server/connections/openai.js | 35 ++- server/index.js | 5 +- server/routes/connections.js | 41 ++- server/routes/hbcform.js | 318 ++++++++++++---------- 7 files changed, 527 insertions(+), 239 deletions(-) create mode 100644 client/src/components/InsertBillImage.tsx diff --git a/client/src/components/InsertBillImage.tsx b/client/src/components/InsertBillImage.tsx new file mode 100644 index 0000000..dcfeeef --- /dev/null +++ b/client/src/components/InsertBillImage.tsx @@ -0,0 +1,128 @@ +import React, { useState } from "react"; +import config from "../config"; +import instance from "../security/http"; +import { AxiosResponse } from "axios"; +import { Card } from "@nextui-org/react"; + +interface InsertImageProps { + label: string; + onImageSelected: (file: File | null) => void; + onAmountResolved: (amount: number) => void; + onAiProcessingChange: React.Dispatch>; +} + +const InsertBillImage: React.FC = ({ + label, + onImageSelected, + onAmountResolved, + onAiProcessingChange, +}) => { + const [selectedFile, setSelectedFile] = useState(null); + const [previewImage, setPreviewImage] = useState(""); + + const base64StringToChunks = (base64String: string): string[] => { + const chunks: string[] = []; + const chunkSize = 4096; + let offset = 0; + + while (offset < base64String.length) { + const chunk = base64String.slice(offset, offset + chunkSize); + chunks.push(chunk); + offset += chunkSize; + } + + return chunks; + }; + + const getAmountPayableFromBase64 = async ( + base64String: string + ): Promise => { + const chunks = base64StringToChunks(base64String); + let result: AxiosResponse; + for (let i = 0; i < chunks.length; i++) { + const chunkData = { + chunk: chunks[i], + chunkIndex: i, + totalChunks: chunks.length, + }; + + let e = await instance.post( + `${config.serverAddress}/connections/resolve-home-bill-payable-amount`, + chunkData, + { + headers: { + "Content-Type": "application/json", + }, + } + ); + result = e; + } + return parseFloat(result!.data.response); + }; + + const handleImageSelect = async ( + event: React.ChangeEvent + ) => { + const selectedFiles = event.target.files as FileList; + const file = selectedFiles?.[0] || null; + setSelectedFile(file); + setPreviewImage(file ? URL.createObjectURL(file) : ""); + onImageSelected(file); + + if (file) { + const formData = new FormData(); + formData.append("image", file); + + try { + onAiProcessingChange(true); + instance + .post(`${config.serverAddress}/hbcform/stringify-image`, formData, { + headers: { + "Content-Type": "multipart/form-data", + }, + }) + .then((response) => { + const base64String = response.data.base64Image; + console.log("stringified!"); + + getAmountPayableFromBase64(base64String) + .then((response) => { + console.log( + "Base64 string uploaded successfully! result: " + response + ); + onAmountResolved(response); // 目前暂时设置为 0 + }) + .catch((error) => { + console.error("Error uploading base64 string:", error); + }) + .finally(() => { + onAiProcessingChange(false); + }); + }); + } catch (error) { + console.error("Error stringifying image:", error); + } + } + }; + + return ( + +
+ +
+ {selectedFile && ( + Selected Image + )} + + ); +}; + +export default InsertBillImage; diff --git a/client/src/pages/HBFormPage.tsx b/client/src/pages/HBFormPage.tsx index 74273a8..d92192b 100644 --- a/client/src/pages/HBFormPage.tsx +++ b/client/src/pages/HBFormPage.tsx @@ -1,5 +1,14 @@ import { useEffect, useState } from "react"; -import { Button, Modal, ModalBody, ModalContent, ModalHeader } from "@nextui-org/react"; +import { + Button, + Card, + CircularProgress, + Divider, + Modal, + ModalBody, + ModalContent, + ModalHeader, +} from "@nextui-org/react"; import { ArrowUTurnLeftIcon } from "../icons"; import { useNavigate } from "react-router-dom"; import { Formik, Form } from "formik"; @@ -7,8 +16,8 @@ import * as Yup from "yup"; import config from "../config"; import NextUIFormikInput from "../components/NextUIFormikInput"; import instance from "../security/http"; -import InsertImage from "../components/InsertImage"; import { retrieveUserInformation } from "../security/users"; +import InsertBillImage from "../components/InsertBillImage"; const validationSchema = Yup.object({ electricalBill: Yup.number() @@ -41,12 +50,13 @@ const validationSchema = Yup.object({ export default function HBFormPage() { const [userId, setUserId] = useState(null); + const [aiProcessing, isAiProcessing] = useState(false); const [initialValues, setInitialValues] = useState({ id: "", - electricalBill: "", - waterBill: "", - totalBill: "", - noOfDependents: "", + electricalBill: "69", + waterBill: "69", + totalBill: "0.00", + noOfDependents: 1, avgBill: "", billPicture: null, userId: "", @@ -81,14 +91,14 @@ export default function HBFormPage() { const [hasHandedInForm, setHasHandedInForm] = useState(false); useEffect(() => { - instance.get(`${config.serverAddress}/hbcform/has-handed-in-form/${userId}`) - .then(response => { + instance + .get(`${config.serverAddress}/hbcform/has-handed-in-form/${userId}`) + .then((response) => { const hasHandedInForm = response.data.hasHandedInForm; setHasHandedInForm(hasHandedInForm); }) - .catch(error => { + .catch((error) => { console.error("Error checking if user has handed in form:", error); - }); }, [userId]); @@ -136,7 +146,11 @@ export default function HBFormPage() { console.error("Error creating form:", response.statusText); } } catch (error: any) { - if (error.response && error.response.data && error.response.data.errors) { + if ( + error.response && + error.response.data && + error.response.data.errors + ) { const errors = error.response.data.errors; Object.keys(errors).forEach((key) => { setFieldError(key, errors[key]); @@ -152,7 +166,7 @@ export default function HBFormPage() { // Handler for image selection const handleImageSelection = (name: string, file: File | null) => { - setImagesSelected(prevState => ({ + setImagesSelected((prevState) => ({ ...prevState, [name]: !!file, })); @@ -162,11 +176,13 @@ export default function HBFormPage() { values, resetForm, setFieldError, - setFieldValue + setFieldValue, }: any) => { try { // Fetch the current form ID associated with the userId - const responses = await instance.get(`${config.serverAddress}/hbcform/has-handed-in-form/${userId}`); + const responses = await instance.get( + `${config.serverAddress}/hbcform/has-handed-in-form/${userId}` + ); const formId = responses.data.formId; // Make sure your API response includes the formId if (formId) { @@ -224,18 +240,18 @@ export default function HBFormPage() { }; const handleModalCancel = () => { - navigate(-1) + navigate(-1); }; return ( -
-
-
-
-
+
{ // Calculate the total bill useEffect(() => { - const totalBill = Number(values.electricalBill) + Number(values.waterBill); - setFieldValue("totalBill", totalBill.toFixed(2)); - - const avgBill = Number(values.noOfDependents) > 0 - ? totalBill / Number(values.noOfDependents) - : 0; + const avgBill = + Number(values.noOfDependents) > 0 + ? Number(values.totalBill) / Number(values.noOfDependents) + : 0; setFieldValue("avgBill", avgBill.toFixed(2)); - - }, [values.electricalBill, values.waterBill, values.noOfDependents, setFieldValue]); + }, [values.totalBill, values.noOfDependents, setFieldValue]); // Disabled the submit button because the images field are not selected const isSubmitDisabled = !imagesSelected.billPicture; return (
-
-
-
- - - - - +
+
+
+
+ + + + +
+ +

+ How many people lives in your unit? +

+ + + +
+

Total amount payable:

+ {aiProcessing && ( +
+ +
+ )} + {!aiProcessing && ( +

+ S${values.totalBill} +

+ )} +
+ +
+

Cost per dependent:

+

+ S${values.avgBill} +

+
+
-
- + { setFieldValue("billPicture", file); handleImageSelection("billPicture", file); }} + onAmountResolved={(totalAmount) => { + setFieldValue("totalBill", totalAmount); + }} + onAiProcessingChange={isAiProcessing} />
-
- -
+
-

This form has been submitted before. If you submit again, the previous entry will be deleted. Are you sure you want to resubmit?

+

+ This form has been submitted before. If you submit + again, the previous entry will be deleted. Are you + sure you want to resubmit? +

-
+
); } diff --git a/client/src/security/http.ts b/client/src/security/http.ts index ef9a818..7d29d3a 100644 --- a/client/src/security/http.ts +++ b/client/src/security/http.ts @@ -3,6 +3,8 @@ import config from "../config"; const instance = axios.create({ baseURL: config.serverAddress, + maxContentLength: Infinity, + maxBodyLength: Infinity, }); // Add a request interceptor diff --git a/server/connections/openai.js b/server/connections/openai.js index 3f16e2e..e160bc2 100644 --- a/server/connections/openai.js +++ b/server/connections/openai.js @@ -17,4 +17,37 @@ async function openAiChatCompletion(query, systemPrompt) { return response; } -module.exports = { openAiChatCompletion }; +async function openAiHomeBillVerification(base64Data) { + console.log("hi"); + const openai = new OpenAI({ apiKey: await getApiKey("openai_api_key") }); + const completion = await openai.chat.completions.create({ + messages: [ + { + role: "system", + content: ` + User should upload an image of a bill. + Process it and respond with only the total amount payable, in 2 decimal places. + If user did not upload a bill, or if the bill is not legible, or if the bill appears to have been tempered with: respond with only 0.00 + `, + // , + }, + { + role: "user", + content: [ + { + type: "image_url", + image_url: { url: `data:image/jpeg;base64,${base64Data}` }, + }, + ], + }, + ], + model: "gpt-4o-mini", + }); + + let response = completion.choices[0].message.content; + console.log(response); + + return response; +} + +module.exports = { openAiChatCompletion, openAiHomeBillVerification }; diff --git a/server/index.js b/server/index.js index 36ee883..85f3350 100644 --- a/server/index.js +++ b/server/index.js @@ -1,4 +1,5 @@ const express = require("express"); +const bodyParser = require("body-parser"); const dotenv = require("dotenv"); const cors = require("cors"); const db = require("./models"); @@ -18,6 +19,8 @@ app.use( ); app.use(express.json()); +app.use(bodyParser.json({ limit: "1000mb" })); +app.use(bodyParser.urlencoded({ limit: "1000mb", extended: true })); app.get("/", (req, res) => { res.send("Welcome to ecoconnect."); @@ -44,7 +47,7 @@ const vouchers = require("./routes/vouchers"); app.use("/vouchers", vouchers); const feedback = require("./routes/feedback.js"); -app.use("/feedback", feedback) +app.use("/feedback", feedback); const uservoucher = require("./routes/uservoucher.js"); app.use("/user-vouchers", uservoucher); diff --git a/server/routes/connections.js b/server/routes/connections.js index 94b5b79..4425771 100644 --- a/server/routes/connections.js +++ b/server/routes/connections.js @@ -1,5 +1,8 @@ const express = require("express"); -const { openAiChatCompletion } = require("../connections/openai"); +const { + openAiChatCompletion, + openAiHomeBillVerification, +} = require("../connections/openai"); const { validateToken } = require("../middlewares/auth"); const router = express.Router(); @@ -37,4 +40,40 @@ router.get("/nls/:query", validateToken, async (req, res) => { } }); +async function resolveBillPayableAmount(base64Data) { + return await openAiHomeBillVerification(base64Data); +} + +let base64Chunks = []; + +router.post( + "/resolve-home-bill-payable-amount", + validateToken, + async (req, res) => { + const { chunk, chunkIndex, totalChunks } = req.body; + + // 存储接收到的块 + base64Chunks[chunkIndex] = chunk; + + // 检查是否接收到所有块 + if (base64Chunks.length === parseInt(totalChunks)) { + const completeBase64String = base64Chunks.join(""); + base64Chunks = []; // 清空数组以便下次上传使用 + + try { + console.log("starting actual resolve"); + let verificationResponse = await resolveBillPayableAmount( + completeBase64String + ); + res.json({ response: verificationResponse }); + } catch (error) { + console.error("Error with AI:", error); + res.status(500).json({ message: "Internal Server Error: " + error }); + } + } else { + res.status(200).json({ message: "Chunk received" }); + } + } +); + module.exports = router; diff --git a/server/routes/hbcform.js b/server/routes/hbcform.js index d98058d..b1f4a2d 100644 --- a/server/routes/hbcform.js +++ b/server/routes/hbcform.js @@ -9,185 +9,215 @@ const { sendThankYouEmail } = require("../connections/mailersend"); const upload = multer({ storage: multer.memoryStorage() }); async function processFile(file) { - try { - const { buffer, mimetype } = file; - const maxSize = 5 * 1024 * 1024; // 5MB limit for compressed image size - let processedBuffer; + try { + const { buffer, mimetype } = file; + const maxSize = 5 * 1024 * 1024; // 5MB limit for compressed image size + let processedBuffer; - if (mimetype.startsWith("image/")) { - // Handle image files - const metadata = await sharp(buffer).metadata(); + if (mimetype.startsWith("image/")) { + // Handle image files + const metadata = await sharp(buffer).metadata(); - // Compress the image based on its format - if (metadata.format === "jpeg") { - processedBuffer = await sharp(buffer) - .jpeg({ quality: 80 }) // Compress to JPEG - .toBuffer(); - } else if (metadata.format === "png") { - processedBuffer = await sharp(buffer) - .png({ quality: 80 }) // Compress to PNG - .toBuffer(); - } else if (metadata.format === "webp") { - processedBuffer = await sharp(buffer) - .webp({ quality: 80 }) // Compress to WebP - .toBuffer(); - } else { - // For other image formats (e.g., TIFF), convert to JPEG - processedBuffer = await sharp(buffer) - .toFormat("jpeg") - .jpeg({ quality: 80 }) - .toBuffer(); - } + // Compress the image based on its format + if (metadata.format === "jpeg") { + processedBuffer = await sharp(buffer) + .jpeg({ quality: 80 }) // Compress to JPEG + .toBuffer(); + } else if (metadata.format === "png") { + processedBuffer = await sharp(buffer) + .png({ quality: 80 }) // Compress to PNG + .toBuffer(); + } else if (metadata.format === "webp") { + processedBuffer = await sharp(buffer) + .webp({ quality: 80 }) // Compress to WebP + .toBuffer(); + } else { + // For other image formats (e.g., TIFF), convert to JPEG + processedBuffer = await sharp(buffer) + .toFormat("jpeg") + .jpeg({ quality: 80 }) + .toBuffer(); + } - // Check the size of the compressed image - if (processedBuffer.length > maxSize) { - throw new Error(`Compressed file too large: ${(processedBuffer.length / 1000000).toFixed(2)}MB`); - } - } else if (mimetype === "application/pdf") { - // Handle PDF files - console.log("Processing PDF"); - processedBuffer = buffer; // Store the PDF as is - // Optionally, process PDF using pdf-lib or other libraries - } else { - throw new Error(`Unsupported file type: ${mimetype}`); - } - - return processedBuffer; - } catch (err) { - console.error("Error processing file:", err); - throw err; + // Check the size of the compressed image + if (processedBuffer.length > maxSize) { + throw new Error( + `Compressed file too large: ${( + processedBuffer.length / 1000000 + ).toFixed(2)}MB` + ); + } + } else if (mimetype === "application/pdf") { + // Handle PDF files + console.log("Processing PDF"); + processedBuffer = buffer; // Store the PDF as is + // Optionally, process PDF using pdf-lib or other libraries + } else { + throw new Error(`Unsupported file type: ${mimetype}`); } + + return processedBuffer; + } catch (err) { + console.error("Error processing file:", err); + throw err; + } } router.post( - "/", - upload.fields([ - { name: "billPicture", maxCount: 1 }, - ]), - async (req, res) => { - let data = req.body; - let files = req.files; + "/", + upload.fields([{ name: "billPicture", maxCount: 1 }]), + async (req, res) => { + let data = req.body; + let files = req.files; - // Validate request body - let validationSchema = yup.object({ - electricalBill: yup.number().positive().required(), - waterBill: yup.number().positive().required(), - totalBill: yup.number().positive().required(), - noOfDependents: yup.number().integer().positive().required(), - avgBill: yup.number().positive().required(), - }); + // Validate request body + let validationSchema = yup.object({ + electricalBill: yup.number().positive().required(), + waterBill: yup.number().positive().required(), + totalBill: yup.number().positive().required(), + noOfDependents: yup.number().integer().positive().required(), + avgBill: yup.number().positive().required(), + }); - try { + try { + data = await validationSchema.validate(data, { abortEarly: false }); - data = await validationSchema.validate(data, { abortEarly: false }); + // Process the files + const processedbillPicture = await processFile(files.billPicture[0]); - // Process the files - const processedbillPicture = await processFile(files.billPicture[0]); + // Save the form with processed files + let result = await HBCform.create({ + ...data, + billPicture: processedbillPicture, + }); - // Save the form with processed files - let result = await HBCform.create({ - ...data, - billPicture: processedbillPicture, - }); - - res.json(result); - } catch (err) { - console.error('Error processing request:', err); - res.status(400).json({ message: 'Bad request', errors: err.errors || err.message }); - } + res.json(result); + } catch (err) { + console.error("Error processing request:", err); + res + .status(400) + .json({ message: "Bad request", errors: err.errors || err.message }); } + } ); router.get("/", async (req, res) => { - let condition = {}; - let search = req.query.search; - if (search) { - condition[Op.or] = [ - { electricalBill: { [Op.like]: `%${search}%` } }, - { waterBill: { [Op.like]: `%${search}%` } }, - { totalBill: { [Op.like]: `%${search}%` } }, - { noOfDependents: { [Op.like]: `%${search}%` } }, - { avgBill: { [Op.like]: `%${search}%` } }, - ]; - } - let list = await HBCform.findAll({ - where: condition, - order: [["createdAt", "ASC"]] - }); - res.json(list); + let condition = {}; + let search = req.query.search; + if (search) { + condition[Op.or] = [ + { electricalBill: { [Op.like]: `%${search}%` } }, + { waterBill: { [Op.like]: `%${search}%` } }, + { totalBill: { [Op.like]: `%${search}%` } }, + { noOfDependents: { [Op.like]: `%${search}%` } }, + { avgBill: { [Op.like]: `%${search}%` } }, + ]; + } + let list = await HBCform.findAll({ + where: condition, + order: [["createdAt", "ASC"]], + }); + res.json(list); }); router.get("/:id", async (req, res) => { - let id = req.params.id; - let hbcform = await HBCform.findByPk(id); - if (!hbcform) { - res.sendStatus(404); - return; - } - res.json(hbcform); + let id = req.params.id; + let hbcform = await HBCform.findByPk(id); + if (!hbcform) { + res.sendStatus(404); + return; + } + res.json(hbcform); }); router.get("/billPicture/:id", async (req, res) => { - let id = req.params.id; - let hbcform = await HBCform.findByPk(id); + let id = req.params.id; + let hbcform = await HBCform.findByPk(id); - if (!hbcform || !hbcform.billPicture) { - res.sendStatus(404); - return; - } + if (!hbcform || !hbcform.billPicture) { + res.sendStatus(404); + return; + } - try { - res.set("Content-Type", "image/jpeg"); // Adjust the content type as necessary - res.send(hbcform.billPicture); - } catch (err) { - res - .status(500) - .json({ message: "Error retrieving electrical bill", error: err }); - } + try { + res.set("Content-Type", "image/jpeg"); // Adjust the content type as necessary + res.send(hbcform.billPicture); + } catch (err) { + res + .status(500) + .json({ message: "Error retrieving electrical bill", error: err }); + } }); router.delete("/:id", async (req, res) => { - let id = req.params.id; - try { - const result = await HBCform.destroy({ where: { id } }); - if (result === 0) { - // No rows were deleted - res.sendStatus(404); - } else { - // Successfully deleted - res.sendStatus(204); - } - } catch (err) { - console.error("Error deleting form entry:", err); - res.status(500).json({ message: "Failed to delete form entry", error: err }); + let id = req.params.id; + try { + const result = await HBCform.destroy({ where: { id } }); + if (result === 0) { + // No rows were deleted + res.sendStatus(404); + } else { + // Successfully deleted + res.sendStatus(204); } + } catch (err) { + console.error("Error deleting form entry:", err); + res + .status(500) + .json({ message: "Failed to delete form entry", error: err }); + } }); // Endpoint for sending emails related to home bill contest router.post("/send-homebill-contest-email", async (req, res) => { - const { email, name } = req.body; - try { - await sendThankYouEmail(email, name); - res.status(200).send({ message: "Email sent successfully" }); - } catch (error) { - console.error("Failed to send email:", error); - res.status(500).send({ error: "Failed to send email" }); - } + const { email, name } = req.body; + try { + await sendThankYouEmail(email, name); + res.status(200).send({ message: "Email sent successfully" }); + } catch (error) { + console.error("Failed to send email:", error); + res.status(500).send({ error: "Failed to send email" }); + } }); router.get("/has-handed-in-form/:userId", async (req, res) => { - const userId = req.params.userId; - try { - const form = await HBCform.findOne({ where: { userId } }); - if (form) { - res.json({ hasHandedInForm: true, formId: form.id }); - } else { - res.json({ hasHandedInForm: false }); - } - } catch (err) { - console.error("Error checking if user has handed in form:", err); - res.status(500).json({ message: "Failed to check if user has handed in form", error: err }); + const userId = req.params.userId; + try { + const form = await HBCform.findOne({ where: { userId } }); + if (form) { + res.json({ hasHandedInForm: true, formId: form.id }); + } else { + res.json({ hasHandedInForm: false }); } + } catch (err) { + console.error("Error checking if user has handed in form:", err); + res.status(500).json({ + message: "Failed to check if user has handed in form", + error: err, + }); + } }); -module.exports = router; \ No newline at end of file +router.post("/stringify-image", upload.single("image"), async (req, res) => { + try { + console.log("stringifying image.."); + if (!req.file) { + return res.status(400).send({ error: "No file uploaded" }); + } + + const compressedImageBuffer = await sharp(req.file.buffer) + .resize(720, 1280) + .toBuffer(); + + const base64Image = compressedImageBuffer.toString("base64"); + + console.log("buffered!"); + + res.status(200).send({ base64Image }); + } catch (error) { + console.error("Failed to process image:", error); + res.status(500).send({ error: "Failed to process image" }); + } +}); + +module.exports = router;