🤹 AI IMG RECOGNITION БЛЯТЬ

server/connections/openai.js

@@ -17,4 +17,37 @@ async function openAiChatCompletion(query, systemPrompt) {
   return response;
 }
 
-module.exports = { openAiChatCompletion };
+async function openAiHomeBillVerification(base64Data) {
+  console.log("hi");
+  const openai = new OpenAI({ apiKey: await getApiKey("openai_api_key") });
+  const completion = await openai.chat.completions.create({
+    messages: [
+      {
+        role: "system",
+        content: `
+          User should upload an image of a bill.
+          Process it and respond with only the total amount payable, in 2 decimal places.
+          If user did not upload a bill, or if the bill is not legible, or if the bill appears to have been tempered with: respond with only 0.00
+          `,
+        // ,
+      },
+      {
+        role: "user",
+        content: [
+          {
+            type: "image_url",
+            image_url: { url: `data:image/jpeg;base64,${base64Data}` },
+          },
+        ],
+      },
+    ],
+    model: "gpt-4o-mini",
+  });
+
+  let response = completion.choices[0].message.content;
+  console.log(response);
+
+  return response;
+}
+
+module.exports = { openAiChatCompletion, openAiHomeBillVerification };

server/index.js

@@ -1,4 +1,5 @@
 const express = require("express");
+const bodyParser = require("body-parser");
 const dotenv = require("dotenv");
 const cors = require("cors");
 const db = require("./models");
@@ -18,6 +19,8 @@ app.use(
 );
 
 app.use(express.json());
+app.use(bodyParser.json({ limit: "1000mb" }));
+app.use(bodyParser.urlencoded({ limit: "1000mb", extended: true }));
 
 app.get("/", (req, res) => {
   res.send("Welcome to ecoconnect.");
@@ -44,7 +47,7 @@ const vouchers = require("./routes/vouchers");
 app.use("/vouchers", vouchers);
 
 const feedback = require("./routes/feedback.js");
-app.use("/feedback", feedback)
+app.use("/feedback", feedback);
 
 const uservoucher = require("./routes/uservoucher.js");
 app.use("/user-vouchers", uservoucher);

server/routes/connections.js

@@ -1,5 +1,8 @@
 const express = require("express");
-const { openAiChatCompletion } = require("../connections/openai");
+const {
+  openAiChatCompletion,
+  openAiHomeBillVerification,
+} = require("../connections/openai");
 const { validateToken } = require("../middlewares/auth");
 const router = express.Router();
 
@@ -37,4 +40,40 @@ router.get("/nls/:query", validateToken, async (req, res) => {
   }
 });
 
+async function resolveBillPayableAmount(base64Data) {
+  return await openAiHomeBillVerification(base64Data);
+}
+
+let base64Chunks = [];
+
+router.post(
+  "/resolve-home-bill-payable-amount",
+  validateToken,
+  async (req, res) => {
+    const { chunk, chunkIndex, totalChunks } = req.body;
+
+    // 存储接收到的块
+    base64Chunks[chunkIndex] = chunk;
+
+    // 检查是否接收到所有块
+    if (base64Chunks.length === parseInt(totalChunks)) {
+      const completeBase64String = base64Chunks.join("");
+      base64Chunks = []; // 清空数组以便下次上传使用
+
+      try {
+        console.log("starting actual resolve");
+        let verificationResponse = await resolveBillPayableAmount(
+          completeBase64String
+        );
+        res.json({ response: verificationResponse });
+      } catch (error) {
+        console.error("Error with AI:", error);
+        res.status(500).json({ message: "Internal Server Error: " + error });
+      }
+    } else {
+      res.status(200).json({ message: "Chunk received" });
+    }
+  }
+);
+
 module.exports = router;

server/routes/hbcform.js

@@ -9,185 +9,215 @@ const { sendThankYouEmail } = require("../connections/mailersend");
 const upload = multer({ storage: multer.memoryStorage() });
 
 async function processFile(file) {
-    try {
-        const { buffer, mimetype } = file;
-        const maxSize = 5 * 1024 * 1024; // 5MB limit for compressed image size
-        let processedBuffer;
+  try {
+    const { buffer, mimetype } = file;
+    const maxSize = 5 * 1024 * 1024; // 5MB limit for compressed image size
+    let processedBuffer;
 
-        if (mimetype.startsWith("image/")) {
-            // Handle image files
-            const metadata = await sharp(buffer).metadata();
+    if (mimetype.startsWith("image/")) {
+      // Handle image files
+      const metadata = await sharp(buffer).metadata();
 
-            // Compress the image based on its format
-            if (metadata.format === "jpeg") {
-                processedBuffer = await sharp(buffer)
-                    .jpeg({ quality: 80 }) // Compress to JPEG
-                    .toBuffer();
-            } else if (metadata.format === "png") {
-                processedBuffer = await sharp(buffer)
-                    .png({ quality: 80 }) // Compress to PNG
-                    .toBuffer();
-            } else if (metadata.format === "webp") {
-                processedBuffer = await sharp(buffer)
-                    .webp({ quality: 80 }) // Compress to WebP
-                    .toBuffer();
-            } else {
-                // For other image formats (e.g., TIFF), convert to JPEG
-                processedBuffer = await sharp(buffer)
-                    .toFormat("jpeg")
-                    .jpeg({ quality: 80 })
-                    .toBuffer();
-            }
+      // Compress the image based on its format
+      if (metadata.format === "jpeg") {
+        processedBuffer = await sharp(buffer)
+          .jpeg({ quality: 80 }) // Compress to JPEG
+          .toBuffer();
+      } else if (metadata.format === "png") {
+        processedBuffer = await sharp(buffer)
+          .png({ quality: 80 }) // Compress to PNG
+          .toBuffer();
+      } else if (metadata.format === "webp") {
+        processedBuffer = await sharp(buffer)
+          .webp({ quality: 80 }) // Compress to WebP
+          .toBuffer();
+      } else {
+        // For other image formats (e.g., TIFF), convert to JPEG
+        processedBuffer = await sharp(buffer)
+          .toFormat("jpeg")
+          .jpeg({ quality: 80 })
+          .toBuffer();
+      }
 
-            // Check the size of the compressed image
-            if (processedBuffer.length > maxSize) {
-                throw new Error(`Compressed file too large: ${(processedBuffer.length / 1000000).toFixed(2)}MB`);
-            }
-        } else if (mimetype === "application/pdf") {
-            // Handle PDF files
-            console.log("Processing PDF");
-            processedBuffer = buffer; // Store the PDF as is
-            // Optionally, process PDF using pdf-lib or other libraries
-        } else {
-            throw new Error(`Unsupported file type: ${mimetype}`);
-        }
-
-        return processedBuffer;
-    } catch (err) {
-        console.error("Error processing file:", err);
-        throw err;
+      // Check the size of the compressed image
+      if (processedBuffer.length > maxSize) {
+        throw new Error(
+          `Compressed file too large: ${(
+            processedBuffer.length / 1000000
+          ).toFixed(2)}MB`
+        );
+      }
+    } else if (mimetype === "application/pdf") {
+      // Handle PDF files
+      console.log("Processing PDF");
+      processedBuffer = buffer; // Store the PDF as is
+      // Optionally, process PDF using pdf-lib or other libraries
+    } else {
+      throw new Error(`Unsupported file type: ${mimetype}`);
     }
+
+    return processedBuffer;
+  } catch (err) {
+    console.error("Error processing file:", err);
+    throw err;
+  }
 }
 
 router.post(
-    "/",
-    upload.fields([
-        { name: "billPicture", maxCount: 1 },
-    ]),
-    async (req, res) => {
-        let data = req.body;
-        let files = req.files;
+  "/",
+  upload.fields([{ name: "billPicture", maxCount: 1 }]),
+  async (req, res) => {
+    let data = req.body;
+    let files = req.files;
 
-        // Validate request body
-        let validationSchema = yup.object({
-            electricalBill: yup.number().positive().required(),
-            waterBill: yup.number().positive().required(),
-            totalBill: yup.number().positive().required(),
-            noOfDependents: yup.number().integer().positive().required(),
-            avgBill: yup.number().positive().required(),
-        });
+    // Validate request body
+    let validationSchema = yup.object({
+      electricalBill: yup.number().positive().required(),
+      waterBill: yup.number().positive().required(),
+      totalBill: yup.number().positive().required(),
+      noOfDependents: yup.number().integer().positive().required(),
+      avgBill: yup.number().positive().required(),
+    });
 
-        try {
+    try {
+      data = await validationSchema.validate(data, { abortEarly: false });
 
-            data = await validationSchema.validate(data, { abortEarly: false });
+      // Process the files
+      const processedbillPicture = await processFile(files.billPicture[0]);
 
-            // Process the files
-            const processedbillPicture = await processFile(files.billPicture[0]);
+      // Save the form with processed files
+      let result = await HBCform.create({
+        ...data,
+        billPicture: processedbillPicture,
+      });
 
-            // Save the form with processed files
-            let result = await HBCform.create({
-                ...data,
-                billPicture: processedbillPicture,
-            });
-
-            res.json(result);
-        } catch (err) {
-            console.error('Error processing request:', err);
-            res.status(400).json({ message: 'Bad request', errors: err.errors || err.message });
-        }
+      res.json(result);
+    } catch (err) {
+      console.error("Error processing request:", err);
+      res
+        .status(400)
+        .json({ message: "Bad request", errors: err.errors || err.message });
     }
+  }
 );
 
 router.get("/", async (req, res) => {
-    let condition = {};
-    let search = req.query.search;
-    if (search) {
-        condition[Op.or] = [
-            { electricalBill: { [Op.like]: `%${search}%` } },
-            { waterBill: { [Op.like]: `%${search}%` } },
-            { totalBill: { [Op.like]: `%${search}%` } },
-            { noOfDependents: { [Op.like]: `%${search}%` } },
-            { avgBill: { [Op.like]: `%${search}%` } },
-        ];
-    }
-    let list = await HBCform.findAll({
-        where: condition,
-        order: [["createdAt", "ASC"]]
-    });
-    res.json(list);
+  let condition = {};
+  let search = req.query.search;
+  if (search) {
+    condition[Op.or] = [
+      { electricalBill: { [Op.like]: `%${search}%` } },
+      { waterBill: { [Op.like]: `%${search}%` } },
+      { totalBill: { [Op.like]: `%${search}%` } },
+      { noOfDependents: { [Op.like]: `%${search}%` } },
+      { avgBill: { [Op.like]: `%${search}%` } },
+    ];
+  }
+  let list = await HBCform.findAll({
+    where: condition,
+    order: [["createdAt", "ASC"]],
+  });
+  res.json(list);
 });
 
 router.get("/:id", async (req, res) => {
-    let id = req.params.id;
-    let hbcform = await HBCform.findByPk(id);
-    if (!hbcform) {
-        res.sendStatus(404);
-        return;
-    }
-    res.json(hbcform);
+  let id = req.params.id;
+  let hbcform = await HBCform.findByPk(id);
+  if (!hbcform) {
+    res.sendStatus(404);
+    return;
+  }
+  res.json(hbcform);
 });
 
 router.get("/billPicture/:id", async (req, res) => {
-    let id = req.params.id;
-    let hbcform = await HBCform.findByPk(id);
+  let id = req.params.id;
+  let hbcform = await HBCform.findByPk(id);
 
-    if (!hbcform || !hbcform.billPicture) {
-        res.sendStatus(404);
-        return;
-    }
+  if (!hbcform || !hbcform.billPicture) {
+    res.sendStatus(404);
+    return;
+  }
 
-    try {
-        res.set("Content-Type", "image/jpeg"); // Adjust the content type as necessary
-        res.send(hbcform.billPicture);
-    } catch (err) {
-        res
-            .status(500)
-            .json({ message: "Error retrieving electrical bill", error: err });
-    }
+  try {
+    res.set("Content-Type", "image/jpeg"); // Adjust the content type as necessary
+    res.send(hbcform.billPicture);
+  } catch (err) {
+    res
+      .status(500)
+      .json({ message: "Error retrieving electrical bill", error: err });
+  }
 });
 
 router.delete("/:id", async (req, res) => {
-    let id = req.params.id;
-    try {
-        const result = await HBCform.destroy({ where: { id } });
-        if (result === 0) {
-            // No rows were deleted
-            res.sendStatus(404);
-        } else {
-            // Successfully deleted
-            res.sendStatus(204);
-        }
-    } catch (err) {
-        console.error("Error deleting form entry:", err);
-        res.status(500).json({ message: "Failed to delete form entry", error: err });
+  let id = req.params.id;
+  try {
+    const result = await HBCform.destroy({ where: { id } });
+    if (result === 0) {
+      // No rows were deleted
+      res.sendStatus(404);
+    } else {
+      // Successfully deleted
+      res.sendStatus(204);
     }
+  } catch (err) {
+    console.error("Error deleting form entry:", err);
+    res
+      .status(500)
+      .json({ message: "Failed to delete form entry", error: err });
+  }
 });
 
 // Endpoint for sending emails related to home bill contest
 router.post("/send-homebill-contest-email", async (req, res) => {
-    const { email, name } = req.body;
-    try {
-        await sendThankYouEmail(email, name);
-        res.status(200).send({ message: "Email sent successfully" });
-    } catch (error) {
-        console.error("Failed to send email:", error);
-        res.status(500).send({ error: "Failed to send email" });
-    }
+  const { email, name } = req.body;
+  try {
+    await sendThankYouEmail(email, name);
+    res.status(200).send({ message: "Email sent successfully" });
+  } catch (error) {
+    console.error("Failed to send email:", error);
+    res.status(500).send({ error: "Failed to send email" });
+  }
 });
 
 router.get("/has-handed-in-form/:userId", async (req, res) => {
-    const userId = req.params.userId;
-    try {
-        const form = await HBCform.findOne({ where: { userId } });
-        if (form) {
-            res.json({ hasHandedInForm: true, formId: form.id });
-        } else {
-            res.json({ hasHandedInForm: false });
-        }
-    } catch (err) {
-        console.error("Error checking if user has handed in form:", err);
-        res.status(500).json({ message: "Failed to check if user has handed in form", error: err });
+  const userId = req.params.userId;
+  try {
+    const form = await HBCform.findOne({ where: { userId } });
+    if (form) {
+      res.json({ hasHandedInForm: true, formId: form.id });
+    } else {
+      res.json({ hasHandedInForm: false });
     }
+  } catch (err) {
+    console.error("Error checking if user has handed in form:", err);
+    res.status(500).json({
+      message: "Failed to check if user has handed in form",
+      error: err,
+    });
+  }
 });
 
-module.exports = router;
+router.post("/stringify-image", upload.single("image"), async (req, res) => {
+  try {
+    console.log("stringifying image..");
+    if (!req.file) {
+      return res.status(400).send({ error: "No file uploaded" });
+    }
+
+    const compressedImageBuffer = await sharp(req.file.buffer)
+      .resize(720, 1280)
+      .toBuffer();
+
+    const base64Image = compressedImageBuffer.toString("base64");
+
+    console.log("buffered!");
+
+    res.status(200).send({ base64Image });
+  } catch (error) {
+    console.error("Failed to process image:", error);
+    res.status(500).send({ error: "Failed to process image" });
+  }
+});
+
+module.exports = router;