Smart Grocery Cart Using Computer Vision - OpenMV Cam H7

// Connections
// Beetle ESP32-C3 :
//                                MFRC522
// D7   --------------------------- RST
// D2   --------------------------- SDA
// D6   --------------------------- MOSI
// D5   --------------------------- MISO
// D4   --------------------------- SCK
//                                OpenMV Cam H7
// D0   --------------------------- P4
// D1   --------------------------- P5
//                                5mm Common Anode RGB LED
// D21  --------------------------- R
// D8   --------------------------- G
// D9   --------------------------- B
//                                Buzzer
// D20  --------------------------- +
//
//
// OpenMV Cam H7 :
//                                ST7735 1.8" Color TFT Display
// 3.3V --------------------------- LED
// P2   --------------------------- SCK
// P0   --------------------------- SDA
// P8   --------------------------- AO
// P7   --------------------------- RESET
// P3   --------------------------- CS
// GND  --------------------------- GND
// 3.3V --------------------------- VCC
//                                JoyStick
// P6   --------------------------- VRX
// P9   --------------------------- SW

require("sendgrid-php/sendgrid-php.php");

	public $conn;

	private $sendgrid_API_Key = "<_API_KEY_>";
	private $from_email = "<_FROM_EMAIL_>";

	public function __init__($conn){
		$this->conn = $conn;
	}

	public function database_create_customer_table($table, $firstname, $lastname, $card_info, $email){
		// Create a new database table.
		$sql_create = "CREATE TABLE `$table`(
							id int AUTO_INCREMENT PRIMARY KEY NOT NULL,
							firstname varchar(255) NOT NULL,
							lastname varchar(255) NOT NULL,
							card_info varchar(255) NOT NULL,
							email varchar(255) NOT NULL,
							product_id varchar(255) NOT NULL,
							product_name varchar(255) NOT NULL,
							product_price varchar(255) NOT NULL
					   );";
		if(mysqli_query($this->conn, $sql_create)){
			echo("<br>Database Table Created Successfully!");
			// Add the customer information to the recently created database table.
			$sql_insert = "INSERT INTO `$table`(`firstname`, `lastname`, `card_info`, `email`, `product_id`, `product_name`, `product_price`)
						   VALUES ('$firstname', '$lastname', '$card_info', '$email', 'X', 'X', 'X')"
					      ;
			if(mysqli_query($this->conn, $sql_insert)) echo("<br><br>Customer information added to the database table successfully!");
			// If the customer information is added to the database table successfully, redirect the customer to the product list page.
            header("Location: product_list.php");
			exit();
		}else{
			// Redirect the customer to the home page if there is an error.
			header("Location: ./?databaseTableAlreadyCreated");
			exit();
		}
	}

	public function insert_new_data($table, $product_id, $product_name, $product_price){
		$sql_insert = "INSERT INTO `$table`(`firstname`, `lastname`, `card_info`, `email`, `product_id`, `product_name`, `product_price`)
		               SELECT `firstname`, `lastname`, `card_info`, `email`, '$product_id', '$product_name', '$product_price'
				       FROM `$table` WHERE id=1"
			          ;
		if(mysqli_query($this->conn, $sql_insert)){ return true; } else{ return false; }
	}

	public function remove_data($table, $product_id){
		$sql_delete = "DELETE FROM `$table` WHERE `product_id`='$product_id' limit 1";
		if(mysqli_query($this->conn, $sql_delete)){ return true; } else{ return false; }
	}

	public function get_purchased_product_list($table){
		$product_names = []; $product_ids = []; $product_prices = [];
		$sql_list = "SELECT * FROM `$table` WHERE id!=1 ORDER BY `id` ASC";
		$result = mysqli_query($this->conn, $sql_list);
		$check = mysqli_num_rows($result);
		if($check > 0){
			while($row = mysqli_fetch_assoc($result)){
				array_push($product_names, $row["product_name"]);
				array_push($product_ids, $row["product_id"]);
				array_push($product_prices, $row["product_price"]);
			}
			return array($product_names, $product_ids, $product_prices);
		}else{
			return array(["Not Found!"], ["Not Found!"], ["Not Found!"]);
		}
	}

	public function get_table_name($return){
		$sql_get = "SELECT `table_name`, `create_time` FROM `information_schema`.`TABLES` WHERE `table_schema` = 'smart_grocery_cart' ORDER BY `CREATE_TIME` DESC limit 1";
		$result = mysqli_query($this->conn, $sql_get);
		$check = mysqli_num_rows($result);
		if($check > 0){
			while($row = mysqli_fetch_assoc($result)){
				if(!$return) echo("%".$row["table_name"]."%".$row["create_time"]."%");
				else return $row["table_name"];
			}
		}
	}

    private function get_email($table){
		$sql_email = "SELECT * FROM `$table` WHERE id=1";
		$result = mysqli_query($this->conn, $sql_email);
		$check = mysqli_num_rows($result);
		if($check > 0){
			if($row = mysqli_fetch_assoc($result)){ return $row["email"]; }
			else{ return "Not Found!"; }
		}
	}

	public function send_product_list_email($table, $tag){
		// Get the customer's email address.
		$to_email = $this->get_email($table);
		// Obtain the list of the products added to the cart from the customer's database table.
	    $product_names = []; $product_ids = []; $product_prices = [];
	    list($product_names, $product_ids, $product_prices) = $this->get_purchased_product_list($_GET['table']);
		$list = "";
		for($i=0; $i<count($product_names); $i++){
			$list .= '<tr>
						<td>'.$product_names[$i].'</td>
						<td>'.$product_ids[$i].'</td>
						<td>'.$product_prices[$i].'</td>
					  </tr>
					 ';
		}
		// Send an HTML email via the SendGrid Email PHP API.
		$email = new \SendGrid\Mail\Mail();
		$email->setFrom($this->from_email, "Smart Grocery Cart");
		$email->setSubject("Cart Product List");
		$email->addTo($to_email, "Customer");
		$email->addContent("text/html",
		                   '<!DOCTYPE html>
							<html>
							<head>
							<style>
								h1{text-align:center;font-weight:bold;color:#25282A;}
								table{background-color:#043458;width:100%;border:10px solid #25282A;}
								th{background-color:#D1CDDA;color:#25282A;border:5px solid #25282A;font-size:25px;font-weight:bold;}
								td{color:#AEE1CD;border:5px solid #25282A;text-align:left;font-size:20px;font-weight:bold;}
                                a{text-decoration:none;background-color:#5EB0E5;}
							</style>
							</head>
							<body>
						    <h1>Thanks for shopping at our store :)</h1>
						    <h1>Your Customer Tag: '.$tag.'</h1>
						    <table>
						     <tr>
						      <th>Product Name</th>
						      <th>Product ID</th>
						      <th>Product Price</th>
						     </tr>
							 '.$list.'
							 </table>
							 <a href="http://localhost/smart_grocery_cart/product_list.php"><h1>🛒 Checkout 🛒</h1></a>
							 </body>
							 </html>
						   '
                          );

		$sendgrid = new \SendGrid($this->sendgrid_API_Key);
		try{
			$response = $sendgrid->send($email);
			print $response->statusCode() . "\n";
			print_r($response->headers());
			print $response->body() . "\n";
		}catch(Exception $e){
			echo 'Caught exception: '. $e->getMessage() ."\n";
		}
	}
}

$server = array(
	"name" => "localhost",
	"username" => "root",
	"password" => "",
	"database" => "smart_grocery_cart"
);

$conn = mysqli_connect($server["name"], $server["username"], $server["password"], $server["database"]);

include_once "assets/class.php";

// Define the new 'customer' object:
$customer = new _main();
$customer->__init__($conn);

if(isset($_GET['table']) && isset($_GET['firstname']) && isset($_GET['lastname']) && isset($_GET['card_info']) && isset($_GET['email'])){
	$customer->database_create_customer_table($_GET['table'], $_GET['firstname'], $_GET['lastname'], $_GET['card_info'], $_GET['email']);
}

if(isset($_GET['add']) && isset($_GET['table']) && isset($_GET['product_id']) && isset($_GET['product_name']) && isset($_GET['product_price'])){
	if($customer->insert_new_data($_GET['table'], $_GET['product_id'], $_GET['product_name'], $_GET['product_price'])){
		echo("Product information saved successfully to the customer's database table!");
	}else{
		echo("Database error!");
	}
}

if(isset($_GET['remove']) && isset($_GET['table']) && isset($_GET['product_id'])){
	if($customer->remove_data($_GET['table'], $_GET['product_id'])){
		echo("Product information removed successfully from the customer's database table!");
	}else{
		echo("Database error!");
	}
}

if(isset($_GET['table']) && isset($_GET['send_email']) && isset($_GET['tag'])){
	$customer->send_product_list_email($_GET['table'], $_GET['tag']);
}

if(isset($_GET["deviceChangeTable"])){
	$customer->get_table_name(false);
}

include_once "class.php";

// Define the new 'customer' object:
$customer = new _main();
$customer->__init__($conn);

$table = $customer->get_table_name(true);

$product_names = []; $product_ids = []; $product_prices = [];
list($product_names, $product_ids, $product_prices) = $customer->get_purchased_product_list($table);
$list = "<tr><th>Product Name</th><th>Product ID</th><th>Product Price</th></tr>";
for($i=0; $i<count($product_names); $i++){
	$list .= '<tr>
				<td>'.$product_names[$i].'</td>
				<td>'.$product_ids[$i].'</td>
				<td>'.$product_prices[$i].'</td>
			  </tr>
			 ';
}

echo($list);

setInterval(function(){
	$.ajax({
		url: "./assets/update_list.php",
		type: "GET",
		success: (response) => {
			$(".container table").html(response);
		}
	});
}, 3000);

import sensor, image, math, lcd
from time import sleep
from pyb import RTC, Pin, ADC, LED

sensor.reset()
sensor.set_pixformat(sensor.RGB565) # or sensor.GRAYSCALE
sensor.set_framesize(sensor.QQVGA2) # Special 128x160 framesize for LCD Shield.

lcd.init()

# Set the built-in RTC (real-time clock).
rtc = RTC()
rtc.datetime((2022, 12, 6, 5, 15, 35, 52, 0))

red = LED(1)
green = LED(2)
blue = LED(3)

# Joystick:
J_VRX = ADC(Pin('P6'))
J_SW = Pin("P9", Pin.IN, Pin.PULL_UP)

def adjustColor(leds):
    if leds[0]: red.on()
    else: red.off()
    if leds[1]: green.on()
    else: green.off()
    if leds[2]: blue.on()
    else: blue.off()

def save_sample(name, color, leds):
    # Get the current date and time.
    date = rtc.datetime()
    date = ".{}_{}_{}_{}-{}-{}".format(date[0], date[1], date[2], date[4], date[5], date[6])
    # Take a picture with the given frame settings (160X160).
    sensor.set_framesize(sensor.B160X160)
    sample = sensor.snapshot()
    sleep(1)
    # Save the captured image.
    file_name = "/samples/" + name + date + ".jpg"
    sample.save(file_name, quality=20)
    adjustColor(leds)
    print("\nSample Saved: " + file_name + "\n")
    # Show a glimpse of the captured image on the ST7735 1.8" color TFT screen.
    sensor.set_framesize(sensor.QQVGA2)
    lcd_img = sensor.snapshot()
    lcd_img.draw_rectangle(0, 0, 128, 30, fill=1, color =(0,0,0))
    lcd_img.draw_string(int((128-16*len(name))/2), 3, name, color=color, scale=2)
    lcd_img.draw_rectangle(0, 130, 128, 160, fill=1, color =(0,0,0))
    lcd_img.draw_string(int((128-16*len("Saved!"))/2), 132, "Saved!", color=color, scale=2)
    lcd_img.draw_cross(64, 80, color=color, size=8, thickness=2)
    lcd.display(lcd_img)
    sleep(5)
    adjustColor((0,0,0))

def readJoystick():
    j_x = ((J_VRX.read() * 3.3) + 2047.5) / 4095
    j_x = math.ceil(j_x)
    sw = J_SW.value()
    return (j_x, sw)

while(True):
    # Display a real-time video stream on the ST7735 1.8" color TFT screen.
    sensor.set_framesize(sensor.QQVGA2)
    lcd_img = sensor.snapshot()
    lcd.display(lcd_img)

    # Read controls from the joystick.
    j_x, sw = readJoystick()

    # Save samples (images) distinguished by 'food' and 'drink' labels.
    if(j_x > 3):
        save_sample("Food", (255,0,255), (1,0,1))
    if(j_x < 2):
        save_sample("Drink", (0,255,255), (0,1,1))

#include <WiFi.h>
#include <SPI.h>
#include <MFRC522.h>

char ssid[] = "<_SSID_>";        // your network SSID (name)
char pass[] = "<_PASSWORD_>";    // your network password (use for WPA, or use as key for WEP)
int keyIndex = 0;                // your network key Index number (needed only for WEP)

// Define the server on LattePanda 3 Delta 864.
char server[] = "192.168.1.22";
// Define the web application path.
String application = "/smart_grocery_cart/shopping.php";

// Initialize the WiFiClient object.
WiFiClient client; /* WiFiSSLClient client; */

#define RST_PIN   7
#define SS_PIN    2
MFRC522 mfrc522(SS_PIN, RST_PIN);

// Define the MFRC522 module key input.
MFRC522::MIFARE_Key key;

  Serial1.begin(SERIAL_BAUDRATE ,SERIAL_8N1,/*RX =*/0,/*TX =*/1);

  ledcSetup(3, pwm_frequency, resolution); ledcSetup(4, pwm_frequency, resolution); ledcSetup(5, pwm_frequency, resolution);
  ledcAttachPin(red_p, 3); ledcAttachPin(green_p, 4); ledcAttachPin(blue_p, 5);

  SPI.begin();
  mfrc522.PCD_Init();
  Serial.println("\n----------------------------------\nApproximate a New Card or Key Tag : \n----------------------------------\n");
  delay(3000);

  WiFi.begin(ssid, pass);
  // Attempt to connect to the WiFi network:
  while(WiFi.status() != WL_CONNECTED){
    // Wait for the connection:
    delay(500);
    Serial.print(".");
  }
  // If connected to the network successfully:
  Serial.println("Connected to the WiFi network successfully!");
  adjustColor(0,0,255);

void adjustColor(int r, int g, int b){
  ledcWrite(3, 255-r);
  delay(100);
  ledcWrite(4, 255-g);
  delay(100);
  ledcWrite(5, 255-b);
  delay(100);
}

int read_UID(){
  // Detect the new card or tag UID.
  if ( ! mfrc522.PICC_IsNewCardPresent()) {
    return 0;
  }
  if ( ! mfrc522.PICC_ReadCardSerial()) {
    return 0;
  }

  // Display the detected UID on the serial monitor.
  Serial.print("\n----------------------------------\nNew Card or Key Tag UID : ");
  for (int i = 0; i < mfrc522.uid.size; i++) {
    lastRead += mfrc522.uid.uidByte[i] < 0x10 ? " 0" : " ";
    lastRead += String(mfrc522.uid.uidByte[i], HEX);
  }
  lastRead.trim();
  lastRead.toUpperCase();
  Serial.print(lastRead);
  Serial.print("\n----------------------------------\n\n");
  adjustColor(255,0,255);
  mfrc522.PICC_HaltA();
  delay(2000);
  return 1;
}

void make_a_get_request(String request){
  // Connect to the web application named smart_grocery_cart. Change '80' with '443' if you are using SSL connection.
  if (client.connect(server, 80)){
    // If successful:
    Serial.println("\nConnected to the web application successfully!\n");
    // Create the query string:
    String query = application + request;
    // Make an HTTP GET request:
    client.println("GET " + query + " HTTP/1.1");
    client.println("Host: 192.168.1.22");
    client.println("Connection: close");
    client.println();
  }else{
    Serial.println("\nConnection failed to the web application!\n");
    adjustColor(255,0,0);
    delay(2000);
  }
  adjustColor(255,255,0);
  delay(2000); // Wait 2 seconds after connecting...
  // If there are incoming bytes available, get the response from the web application.
  response = "";
  while(client.available()) { char c = client.read(); response += c; }
  if(response != ""){ Serial.println(response); adjustColor(0,255,0); }
}

  read_UID();
  delay(500);
  if(lastRead != ""){
    // Remove spaces from the detected UID.
    lastRead.replace(" ", "_");
    // Send an HTML email to the customer's registered email address via the web application, including the list of the products added to the cart and the link to the payment page.
    make_a_get_request("?table="+table_name+"&send_email=OK&tag="+lastRead);
    // Clear the latest detected UID.
    lastRead = "";
    adjustColor(0,0,0);
  }

void get_data_from_OpenMV(){
  // Obtain the transferred data packet from the OpenMV Cam H7 via serial communication.
  if(Serial1.available() > 0){
    Serial.println("\nTransferred query from the OpenMV Cam H7:");
    OpenMV_data = "";
    OpenMV_data = Serial1.readString();
    Serial.println(OpenMV_data);
    adjustColor(0,51,0);
    delay(2000);
    tone(buzzer_pin, 500);
    delay(1000);
    noTone(buzzer_pin);
    delay(1000);
  }
}

  get_data_from_OpenMV();
  if(OpenMV_data != ""){
    // If the customer requested to change the table name to the latest registered table name in the database:
    if(OpenMV_data == "get_table"){
      // Make an HTTP Get request to obtain the latest registered table name:
      make_a_get_request("?deviceChangeTable");
      if(response != ""){
        // Elicit and format the received table name:
        int delimiter_1, delimiter_2;
        delimiter_1 = response.indexOf("%");
        delimiter_2 = response.indexOf("%", delimiter_1 + 1);
        // Glean information as substrings.
        table_name = response.substring(delimiter_1 + 1, delimiter_2);
        Serial.println("\nLatest Registered Table Name Obtained: " + table_name);
      }
    }else{
      // Make an HTTP Get request directly with the received command:
      make_a_get_request("?table="+table_name+OpenMV_data);
    }
    // Clear the latest received command:
    OpenMV_data = "";
    adjustColor(0,0,0);
  }

import sensor, image, math, lcd, tf
from time import sleep
from pyb import Pin, ADC, LED, UART
from product_list import products

sensor.reset()
sensor.set_pixformat(sensor.RGB565) # or sensor.GRAYSCALE
sensor.set_framesize(sensor.QQVGA2) # Special 128x160 framesize for LCD Shield.

net = None
labels = None
min_confidence = 0.7

try:
    labels, net = tf.load_builtin_model('trained')
except Exception as e:
    raise Exception(e)

uart = UART(3, 115200, timeout_char=1000)

lcd.init()

red = LED(1)
green = LED(2)
blue = LED(3)

# Joystick:
J_VRX = ADC(Pin('P6'))
J_SW = Pin("P9", Pin.IN, Pin.PULL_UP)

while(True):
    menu_config = 0

    # Read controls from the joystick.
    j_x, sw = readJoystick()

    # Take a picture with the given frame settings.
    img = sensor.snapshot()
    lcd.display(img)

    # Change the table name on Beetle ESP32-C3 with the latest registered customer's table name in the database.
    if(sw == 0):
        query = "get_table"
        print("\nQuery: " + query + "\n")
        img.draw_rectangle(0, int(160/4), 128, int(160/2), fill=1, color =(0,0,255))
        img.draw_string(int((128-16*len("Query"))/2), int((160/2)-28), "Query", color=(255,255,255), scale=2)
        img.draw_string(int((128-16*len("Sent!"))/2), int((160/2)+8), "Sent!", color=(255,255,255), scale=2)
        lcd.display(img)
        uart.write(query)
        adjustColor((0,0,1))
        sleep(5)
        adjustColor((0,0,0))

    # Run an inference with the FOMO model to detect products.
    # Via the detect function, obtain all detected objects found in the recently captured image, split out per label (class).
    for i, detection_list in enumerate(net.detect(img, thresholds=[(math.ceil(min_confidence * 255), 255)])):
        # Exclude the class index 0 since it is the background class.
        if (i == 0): continue

        # If the Edge Impulse FOMO model predicted a label (class) successfully:
        if (len(detection_list) == 0): continue

        # Activate the selection (options) menu for adding or removing products to/from the database table.
        menu_config = 1

        # Obtain the detected product's label to retrieve its details saved in the given product list.
        detected_product = i

        # Get the prediction (detection) results and print them on the serial terminal.
        print("\n********** %s **********" % labels[i])
        for d in detection_list:
            # Draw a circle in the center of the detected objects (products).
            [x, y, w, h] = d.rect()
            center_x = math.floor(x + (w / 2))
            center_y = math.floor(y + (h / 2))
            img.draw_circle((center_x, center_y, 20), color=(255,0,255), thickness=3)
            print('\nDetected Label: %d | p: (%d, %d)' % (i, center_x, center_y))

    if(menu_config == 1):
        # Display the selection (options) menu.
        img.draw_rectangle(0, 0, 128, 30, fill=1, color =(0,0,0))
        img.draw_string(int((128-16*len("Add Cart"))/2), 3, "Add Cart", color=(0,255,0), scale=2)
        img.draw_rectangle(0, 130, 128, 160, fill=1, color =(0,0,0))
        img.draw_string(int((128-16*len("Remove"))/2), 135, "Remove", color=(255,0,0), scale=2)
        lcd.display(img)
        print("Selection Menu Activated!")
        while(menu_config == 1):
            j_x, sw = readJoystick()
            # Add the detected product to the database table.
            if(j_x > 3):
                cart_choice = "add"
                print("Selected Cart Option: " + cart_choice)
                img.draw_rectangle(0, 0, 128, 30, fill=1, color =(0,255,0))
                img.draw_string(int((128-16*len("Add Cart"))/2), 3, "Add Cart", color=(255,255,255), scale=2)
                img.draw_rectangle(0, 130, 128, 160, fill=1, color =(0,0,0))
                img.draw_string(int((128-16*len("Remove"))/2), 135, "Remove", color=(255,0,0), scale=2)
                lcd.display(img)
                adjustColor((0,1,0))
                sleep(1)
            # Remove the detected product from the database table.
            if(j_x < 2):
                cart_choice = "remove"
                print("Selected Cart Option: " + cart_choice)
                img.draw_rectangle(0, 0, 128, 30, fill=1, color =(0,0,0))
                img.draw_string(int((128-16*len("Add Cart"))/2), 3, "Add Cart", color=(0,255,0), scale=2)
                img.draw_rectangle(0, 130, 128, 160, fill=1, color =(255,0,0))
                img.draw_string(int((128-16*len("Remove"))/2), 135, "Remove", color=(255,255,255), scale=2)
                adjustColor((1,0,0))
                lcd.display(img)
                sleep(1)
            # Send the generated query (command), including the selected option (cart choice) and
            # the detected product's details, to the web application via Beetle ESP32-C3.
            if(sw == 0):
                query = "&"+cart_choice+"=OK&product_id="+products[detected_product]["id"]+"&product_name="+products[detected_product]["name"]+"&product_price="+products[detected_product]["price"]
                print("\nQuery: " + query + "\n")
                img.draw_rectangle(0, int(160/4), 128, int(160/2), fill=1, color =(0,0,255))
                img.draw_string(int((128-16*len("Query"))/2), int((160/2)-28), "Query", color=(255,255,255), scale=2)
                img.draw_string(int((128-16*len("Sent!"))/2), int((160/2)+8), "Sent!", color=(255,255,255), scale=2)
                lcd.display(img)
                uart.write(query)
                adjustColor((0,0,1))
                sleep(5)
                adjustColor((0,0,0))
                # Close the selection menu after transferring the product information and the selected option to the web application via Beetle ESP32-C3.
                menu_config = 0
                # Clear the detected label and the selected option (cart choice).
                detected_product = 0
                cart_choice = "EMPTY"