Water Pollution Detection - Arduino Nano ESP32 + Ultrasonic Scan

// Connections
// Arduino Nano ESP32 :
//                                URM15 - 75KHZ Ultrasonic Sensor via RS485-to-UART Signal Adapter Module
// D3      ------------------------ TX
// D2      ------------------------ RX
// 3.3V    ------------------------ +
// GND     ------------------------ -
//                                Serial 6-Axis Accelerometer
// 3.3V    ------------------------ VCC
// D5      ------------------------ RXD
// D4      ------------------------ TXD
// GND     ------------------------ GND
//                                DS18B20 Waterproof Temperature Sensor
// A1      ------------------------ Data
//                                SSD1306 OLED Display (128x64)
// A4      ------------------------ SDA
// A5      ------------------------ SCL
//                                Control Button (A)
// D6      ------------------------ +
//                                Control Button (B)
// D7      ------------------------ +
//                                Control Button (C)
// D8      ------------------------ +
//                                Control Button (D)
// D9      ------------------------ +
//                                5mm Common Anode RGB LED
// D10     ------------------------ R
// D11     ------------------------ G
// D12     ------------------------ B
&
&
&
// Connections
// UNIHIKER :
//                                5mm Common Anode RGB LED
// P4      ------------------------ R
// P5      ------------------------ G
// P6      ------------------------ B

$date = date("Y_m_d_H_i_s");

# Define the text file name of the received ultrasonic scan data.
$txt_file = "%s_%s__".$date;
$save_folder = "";

if(isset($_GET["scan"]) && isset($_GET["type"]) && isset($_GET["class"])){
	$txt_file = sprintf($txt_file, $_GET["type"], $_GET["class"]);
	$save_folder = $_GET["type"];
}

if(!empty($_FILES["ultrasonic_scan"]['name'])){
	// Text File:
	$received_scan_properties = array(
	    "name" => $_FILES["ultrasonic_scan"]["name"],
	    "tmp_name" => $_FILES["ultrasonic_scan"]["tmp_name"],
		"size" => $_FILES["ultrasonic_scan"]["size"],
		"extension" => pathinfo($_FILES["ultrasonic_scan"]["name"], PATHINFO_EXTENSION)
	);
	
    // Check whether the uploaded file's extension is in the allowed file formats.
	$allowed_formats = array('jpg', 'png', 'bmp', 'txt');
	if(!in_array($received_scan_properties["extension"], $allowed_formats)){
		echo 'FILE => File Format Not Allowed!';
	}else{
		// Check whether the uploaded file size exceeds the 5 MB data limit.
		if($received_scan_properties["size"] > 5000000){
			echo "FILE => File size cannot exceed 5MB!";
		}else{
			// Save the uploaded file (TXT).
			move_uploaded_file($received_scan_properties["tmp_name"], "./".$save_folder."/".$txt_file.".".$received_scan_properties["extension"]);
			echo "FILE => Saved Successfully!";
		}
	}
}

function read_scans(){
	$information = [];
	// Get all text file paths under the sample folder.
	$files = glob("./sample/*.txt");
	// Read each text file to obtain the ultrasonic scan information — data items.
	foreach($files as $scan){
		$line = [];
		// Derive the provided air bubble label from the given text file name.
		$label = explode("_", $scan)[1];
		array_push($line, $label);
		// Read the ultrasonic scan information.
		$record = fopen($scan, "r"); 
		$data_items = fread($record, filesize($scan));
		// Remove the redundant comma from the data record (scan).
		$data_items = substr($data_items, 0, -1);
		// Append the retrieved data items.
		$data_items = explode(",", $data_items);
		$line = array_merge($line, $data_items);
        array_push($information, $line);
        // Close the text file.
		fclose($record);
	}
	// Return the fetched data items.
	return $information;
}

function create_CSV(){
	// Obtain the generated data items array from ultrasonic scans — data records.
	$information = read_scans();
	// Create the scan_data_items.csv file.
	$filename = "scan_data_items.csv";
	$fp = fopen($filename, 'w');
	// Create and add the header to the CSV file.
	$header = [];
	array_push($header, "air_bubble_label");
	for($i=0;$i<400;$i++){ array_push($header, "p_".strval($i)); }
	fputcsv($fp, $header);
	// Append the retrieved data items as rows for each ultrasonic scan to the CSV file.
	foreach($information as $row){
		fputcsv($fp, $row);
	}
	// Close the CSV file.
	fclose($fp);
}

function get_latest_detection($folder){
	$scan = scandir($folder, 1);
	// Label (model result).
	$model_result = explode("_", $scan[0])[1];
	// Data record.
	$file = $folder.$scan[0];
	$record = fopen($file, "r");
	$data_items = fread($record, filesize($file));
	// Remove the redundant comma from the data record (scan).
	$data_items = substr($data_items, 0, -1);
	// Append the model result to the data record.
	$data_packet = $model_result."_".$data_items;
	// Pass the generated data packet.
	echo $data_packet;
    // Close the text file.
    fclose($record);
}

if(isset($_GET["create"]) && $_GET["create"] == "csv"){
	create_CSV();
	echo "Server => CSV file created successfully!";
}

if(isset($_GET["model_result"]) && $_GET["model_result"] == "OK"){
	get_latest_detection("./detection/");
}

// Define the assigned interface logo information as arrays.
PROGMEM static const unsigned char *interface_logos[] = {home_bits, data_bits, sensor_bits, save_bits, run_bits};
int interface_widths[] = {home_width, data_width, sensor_width, save_width, run_width};
int interface_heights[] = {home_height, data_height, sensor_height, save_height, run_height};

// Define the assigned air bubble class icon information as arrays.
PROGMEM static const unsigned char *class_logos[] = {bubble_bits, normal_bits};
int class_widths[] = {bubble_width, normal_width};
int class_heights[] = {bubble_height, normal_height};

...

display.drawBitmap(SCREEN_WIDTH-l_w, SCREEN_HEIGHT-l_h, interface_logos[menu_option], l_w, l_h, SSD1306_WHITE); 

#include <WiFi.h>
#include "DFRobot_RTU.h"
#include <DFRobot_WT61PC.h>
#include <OneWire.h>
#include <DallasTemperature.h>
#include <Adafruit_GFX.h>
#include <Adafruit_SSD1306.h>

#include "logo.h"

char server[] = "192.168.1.22";
// Define the web application path.
String application = "/Aquatic_Ultrasonic_Imaging/";

// Initialize the WiFiClient object.
WiFiClient client; /* WiFiSSLClient client; */

#define scan_buffer_size  400
float ultrasonic_scan[scan_buffer_size] = {0};

#define SLAVE_ADDR  ((uint16_t)0x0F)

typedef enum{ 
  ePid,
  eVid,
  eAddr,
  eComBaudrate,
  eComParityStop,
  eDistance,
  eInternalTempreture,
  eExternTempreture,
  eControl
}eRegIndex_t;

DFRobot_RTU modbus(/*s =*/&Serial1);

DFRobot_WT61PC accelerometer(&Serial2);

#define ONE_WIRE_BUS A1
OneWire oneWire(ONE_WIRE_BUS);
DallasTemperature DS18B20(&oneWire);

#define SCREEN_WIDTH 128 // OLED display width, in pixels
#define SCREEN_HEIGHT 64 // OLED display height, in pixels
#define OLED_RESET    -1 // Reset pin # (or -1 if sharing Arduino reset pin)

Adafruit_SSD1306 display(SCREEN_WIDTH, SCREEN_HEIGHT, &Wire, OLED_RESET);

struct _data {
  float acc_x;
  float acc_y;
  float acc_z;
  float gyro_x;
  float gyro_y;
  float gyro_z;
  float ang_x;
  float ang_y;
  float ang_z;
};

  Serial1.begin(19200, SERIAL_8N1, RX_1_PIN, TX_1_PIN);

  Serial2.begin(9600, SERIAL_8N1, RX_2_PIN, TX_2_PIN);

  /*
     bit0:
      0 - select onboard temperature
      1 - select external temperature
     bit1:
      0 - enable temperature compensation function
      1 - disable temperature compensation function
     bit2:
      0 - activate auto detection
      1 - activate passive detection
     bit3:
      1 - read distance every 65 ms (in passive detection mode) 
  */
  modbus.writeHoldingRegister(/*id =*/SLAVE_ADDR, /*reg =*/ eControl, /*val =*/0b00000001);

  accelerometer.modifyFrequency(FREQUENCY_200HZ); /* FREQUENCY_0_1HZ, FREQUENCY_0_5HZ, FREQUENCY_1HZ, FREQUENCY_2HZ, FREQUENCY_5HZ, FREQUENCY_10HZ, FREQUENCY_20HZ, FREQUENCY_50HZ, FREQUENCY_100HZ, FREQUENCY_125HZ, FREQUENCY_200HZ */

  DS18B20.begin();

  WiFi.mode(WIFI_STA);
  WiFi.begin(ssid, pass);
  // Attempt to connect to the given Wi-Fi network.
  while(WiFi.status() != WL_CONNECTED){
    // Wait for the network connection.
    delay(500);
    Serial.print(".");
  }
  // If connected to the network successfully:
  Serial.println("Connected to the Wi-Fi network successfully!");

boolean make_a_post_request(String request){
  // Connect to the web application named Aquatic_Ultrasonic_Imaging. Change '80' with '443' if you are using SSL connection.
  if (client.connect(server, 80)){
    // If successful:
    Serial.println("\nConnected to the web application successfully!\n");
    // Create the query string:
    String query = application + request;
    // Make an HTTP POST request:
    String head = "--UltrasonicScan\r\nContent-Disposition: form-data; name=\"ultrasonic_scan\"; filename=\"new_scan.txt\"\r\nContent-Type: text/plain\r\n\r\n";
    String tail = "\r\n--UltrasonicScan--\r\n";
    // Get the total message length.
    uint32_t totalLen = head.length() + sizeof(ultrasonic_scan) + (scan_buffer_size*sizeof(char)) + tail.length();
    // Start the request:
    client.println("POST " + query + " HTTP/1.1");
    client.println("Host: 192.168.1.22");
    client.println("Content-Length: " + String(totalLen));
    client.println("Connection: Keep-Alive");
    client.println("Content-Type: multipart/form-data; boundary=UltrasonicScan");
    client.println();
    client.print(head);
    for(int i=0; i<scan_buffer_size; i++){ client.print(ultrasonic_scan[i]); client.print(",");}
    client.print(tail);
    // Wait until transferring the ultrasonic scan (text) buffer (20x20).
    delay(2000);
    // If successful:
    Serial.println("HTTP POST => Data transfer completed!\n");
    return true;
  }else{
    Serial.println("\nConnection failed to the web application!\n");
    delay(2000);
    return false;
  }
}

void read_ultrasonic_sensor(float water_temp){
  // Configure the external temperature value by utilizing the evaluated water temperature to generate precise distance measurements.
  water_temp = water_temp*10;
  modbus.writeHoldingRegister(/*id =*/SLAVE_ADDR, /*reg =*/eExternTempreture, /*val =*/water_temp);
  delay(50);
  // Obtain the temperature-compensated distance measurement produced by the URM15 ultrasonic sensor.
  distance = modbus.readHoldingRegister(SLAVE_ADDR, eDistance);
  delay(50);
  // If the sensor is out of range, set the distance to -1.
  if(distance == 65535){
    distance = -1;
    Serial.println("Ultrasonic sensor is out of range!");
  }else{
    distance = distance/10;
  }
  delay(50);
}

void read_accelerometer(){
  // Obtain the X, Y, and Z-axis measurements generated by the 6-axis accelerometer — acceleration, angular velocity, angle.
  if(accelerometer.available()){
    _acc.acc_x = accelerometer.Acc.X; _acc.acc_y = accelerometer.Acc.Y; _acc.acc_z = accelerometer.Acc.Z;
    _acc.gyro_x = accelerometer.Gyro.X; _acc.gyro_y = accelerometer.Gyro.Y; _acc.gyro_z = accelerometer.Gyro.Z;
    _acc.ang_x = accelerometer.Angle.X; _acc.ang_y = accelerometer.Angle.Y; _acc.ang_z = accelerometer.Angle.Z;
  }
}

float get_temperature(){
  // Obtain the temperature measurement in Celsius, estimated by the DS18B20 temperature sensor.
  DS18B20.requestTemperatures(); 
  float t = DS18B20.getTempCByIndex(0);
  delay(50);
  return t;
}

void ultrasonic_imaging(){
  // Define underwater device movements by utilizing the axis measurements generated by the 6-axis accelerometer — acceleration and angular velocity.
  if(_acc.acc_x > 0 && _acc.gyro_x > 0 && _acc.acc_y > 0 && _acc.gyro_y > 0){
    // If the device is moving underwater inside an arbitrary square, collect the temperature-compensated distance measurements produced by the URM15 ultrasonic sensor
    // and save them as data points to the scan data buffer — 20 x 20 (400 points).
    if(scanned_points < 399){
      scanned_points+=1;
      ultrasonic_scan[scanned_points] = distance/100;
      delay(50);
    }else{
      adjustColor(0,255,0);
      Serial.println("Scan Completed!");
      delay(50);
    }
  }
}

  if(!digitalRead(control_button_A)){
    menu_option-=1;
    if(menu_option < 0) menu_option = 4;
    delay(500);
  }
  if(!digitalRead(control_button_C)){
    menu_option+=1;
    if(menu_option > 4) menu_option = 0;
    delay(500);
  }

  // Show the interface (home) screen.
  show_interface("home", menu_option);

  if(!digitalRead(control_button_B) && menu_option == 1){
    selected_interface[menu_option-1] = true;
    adjustColor(255,255,0);
    while(selected_interface[menu_option-1]){
      // Read multiple sensor data packets.
      read_ultrasonic_sensor(get_temperature());
      read_accelerometer();
      // Display the retrieved sensor information on the SSD1306 screen.
      show_interface("sensor", menu_option);
      // If the control button D is pressed, redirect the user to the home screen.
      if(!digitalRead(control_button_D)){
        selected_interface[menu_option-1] = false;
        adjustColor(0,0,0);
      }
    }
  }

  if(!digitalRead(control_button_B) && menu_option == 2){
    selected_interface[menu_option-1] = true;
    adjustColor(0,255,255);
    // Clear the data buffer.
    scanned_points = -1;
    while(selected_interface[menu_option-1]){
      // Read multiple sensor data packets.
      read_ultrasonic_sensor(get_temperature());
      read_accelerometer();
      // Initiate the ultrasonic image scanning procedure.
      ultrasonic_imaging();
      // Display the ultrasonic scanning progress on the SSD1306 screen.
      show_interface("scan", menu_option);
      // If the control button D is pressed, redirect the user to the home screen.
      if(!digitalRead(control_button_D)){
        selected_interface[menu_option-1] = false;
        adjustColor(0,0,0);
      }
    }
  }

  if(!digitalRead(control_button_B) && menu_option == 3){
    selected_interface[menu_option-1] = true;
    adjustColor(255,0,255);
    while(selected_interface[menu_option-1]){
      // Display the retrieved sensor information on the SSD1306 screen.
      show_interface("save", menu_option);
      // Depending on the passed air bubble class via the control buttons (A and C), transfer the collected ultrasonic scan data (buffer) to the web application via an HTTP POST request.
      if(!digitalRead(control_button_A)){
        if(make_a_post_request("?scan=OK&type=sample&class=normal")){
          // If successful:
          display.clearDisplay();
          display.drawBitmap((SCREEN_WIDTH-connected_width)/2, (SCREEN_HEIGHT-connected_height)/2, connected_bits, connected_width, connected_height, SSD1306_WHITE);
          display.display();
          adjustColor(0,255,0);
          delay(2000);
          adjustColor(255,0,255);
        }else{
          display.clearDisplay();
          display.drawBitmap((SCREEN_WIDTH-error_width)/2, (SCREEN_HEIGHT-error_height)/2, error_bits, error_width, error_height, SSD1306_WHITE);
          display.display();
          adjustColor(255,0,0);
          delay(2000);
          adjustColor(255,0,255);
        }
      }
      if(!digitalRead(control_button_C)){
        if(make_a_post_request("?scan=OK&type=sample&class=bubble")){
          // If successful:
          display.clearDisplay();
          display.drawBitmap((SCREEN_WIDTH-connected_width)/2, (SCREEN_HEIGHT-connected_height)/2, connected_bits, connected_width, connected_height, SSD1306_WHITE);
          display.display();
          adjustColor(0,255,0);
          delay(2000);
          adjustColor(255,0,255);
        }else{
          display.clearDisplay();
          display.drawBitmap((SCREEN_WIDTH-error_width)/2, (SCREEN_HEIGHT-error_height)/2, error_bits, error_width, error_height, SSD1306_WHITE);
          display.display();
          adjustColor(255,0,0);
          delay(2000);
          adjustColor(255,0,255);
        }
      }
      // If the control button D is pressed, redirect the user to the home screen.
      if(!digitalRead(control_button_D)){
        selected_interface[menu_option-1] = false;
        adjustColor(0,0,0);
      }
    }
  }

    def display_camera_feed(self):
        # Display the real-time video stream generated by the USB webcam.
        ret, img = self.camera.read()
        # Resize the captured frame depending on the given object detection model.
        self.latest_frame_m = cv2.resize(img, self.frame_size_m)
        # Resize the same frame to display it on the UNIHIKER screen (snapshot).
        self.latest_frame_s = cv2.resize(img, self.frame_size_s)
        # Stop the camera feed if requested.
        if cv2.waitKey(1) & 0xFF == ord('q'):
            self.camera.release()
            cv2.destroyAllWindows()
            print("\nCamera Feed Stopped!")

    def take_snapshot(self, filename="assets/snapshot.jpg"):
        # Show the latest camera frame (snapshot) on UNIHIKER to inform the user.
        cv2.imwrite("./"+filename, self.latest_frame_s)
        self.cam_snapshot_img.config(image=filename)
        # Store the latest modified image sample on the memory.
        self.modified_image = self.latest_frame_m

    def save_img_sample(self, given_class):
        if(given_class > -1):
            # Create the file name for the image sample.
            date = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
            filename = "IMG_{}_{}.jpg".format(self.class_names[given_class], date)
            # Save the modified image sample.
            cv2.imwrite("./samples/"+filename, self.modified_image)
            print("\nSample Saved Successfully: ./samples/" + filename)
            # Notify the user.
            self.cam_info_text.config(text="Saved: "+filename)
        else:
            self.cam_info_text.config(text="Please select a class.")

    def create_user_interface(self, _x=120, _y=10, offset=15, origin="top_left"):
        # Design the user interface (GUI) via the built-in unihiker module.
        # Camera interface for AI-based chemical water quality test.
        self.cam_backg = self.interface.fill_rect(x=0, y=0, w=240, h=320, color="#9BB5CE")
        self.cam_snapshot_img = self.interface.draw_image(x=60, y=5, image="assets/cam_wait.jpg", origin=origin, onclick=lambda:self.interface_config("clear_class"))
        self.cam_section = self.interface.fill_round_rect(x=5, y=130, r=10, w=230, h=185, color="#215E7C")
        self.cam_run_button = self.interface.fill_round_rect(x=45, y=250, r=5, w=150, h=45, color="#FAE0D8", onclick=self.run_inference)
        self.cam_run_text = self.interface.draw_text(x=120, y=272, text="Run Inference", origin="center", color="#5C5B57", font_size=12, onclick=self.run_inference)
        self.cam_save_button = self.interface.fill_round_rect(x=45, y=195, r=5, w=150, h=45, color="#FAE0D8", onclick=lambda:self.save_img_sample(self.selected_class))
        self.cam_save_text = self.interface.draw_text(x=120, y=217, text="Capture Sample", origin="center", color="#5C5B57", font_size=12, onclick=lambda:self.save_img_sample(self.selected_class))
        self.cam_snap_button = self.interface.fill_round_rect(x=45, y=140, r=5, w=150, h=45, color="#FAE0D8", onclick=self.take_snapshot)
        self.cam_snap_text = self.interface.draw_text(x=120, y=162, text="Snapshot", origin="center", color="#5C5B57", font_size=12)
        self.cam_info_text = self.interface.draw_text(x=120, y=305, text="Pending...", origin="center", color="white", font_size=8)
        # Elements and coordinates — Camera. 
        self.cam_int_vars = [self.cam_backg, self.cam_snapshot_img, self.cam_section, self.cam_run_button, self.cam_run_text, self.cam_save_button, self.cam_save_text, self.cam_snap_button, self.cam_snap_text, self.cam_info_text]
        self.cam_int_vals = [0, 60, 5, 45, 120, 45, 120, 45, 120, 120]
        # Ultrasonic sensor interface for AI-based ultrasonic imaging.
        self.ultra_backg = self.interface.fill_rect(x=0, y=0, w=240, h=320, color="#5C5B57")
        self.ultrasonic_img = self.interface.draw_image(x=20, y=0, image="assets/ultrasonic_temp.jpg", origin=origin, onclick=lambda:self.telegram_send_data("ultrasonic", "6465514194"))
        self.ultra_section = self.interface.fill_round_rect(x=5, y=205, r=10, w=230, h=110, color="#F9E5C9")
        self.ultra_ins_button = self.interface.fill_round_rect(x=45, y=260, r=5, w=150, h=35, color="#F5F5F0", onclick=lambda:self.make_a_get_request("get_model_result"))
        self.ultra_ins_text = self.interface.draw_text(x=120, y=277, text="Generate Image", origin="center", color="#5C5B57", font_size=12, onclick=lambda:self.make_a_get_request("get_model_result"))
        self.ultra_gen_button = self.interface.fill_round_rect(x=45, y=215, r=5, w=150, h=35, color="#F5F5F0", onclick=lambda:self.make_a_get_request("csv"))
        self.ultra_gen_text = self.interface.draw_text(x=120, y=232, text="Generate CSV", origin="center", color="#5C5B57", font_size=12, onclick=lambda:self.make_a_get_request("csv"))
        self.ultra_info_text = self.interface.draw_text(x=120, y=305, text="Pending...", origin="center", color="#5C5B57", font_size=8)
        # Elements and coordinates — Ultrasonic Sensor.
        self.ultra_int_vars = [self.ultra_backg, self.ultrasonic_img, self.ultra_section, self.ultra_ins_button, self.ultra_ins_text, self.ultra_gen_button, self.ultra_gen_text, self.ultra_info_text]
        self.ultra_int_vals = [0, 20, 5, 45, 120, 45, 120, 120]
        # Home screen.
        self.main_backg = self.interface.draw_image(x=0, y=0, image="assets/background.jpg", origin=origin, onclick=lambda:self.adjust_color([0,0,0]))
        self.main_ultra_button = self.interface.fill_round_rect(x=20, y=10, r=5, w=200, h=45, color="#5C5B57", onclick=lambda:self.interface_config("ultra"))
        self.main_ultra_text = self.interface.draw_text(x=120, y=32, text="Aquatic Ultrasonic Scan", origin="center", color="white", font_size=12, onclick=lambda:self.interface_config("ultra"))
        self.main_cam_button = self.interface.fill_round_rect(x=20, y=265, r=5, w=200, h=45, color="#9BB5CE", onclick=lambda:self.interface_config("cam"))
        self.main_cam_text = self.interface.draw_text(x=120, y=287, text="Water Quality Test", origin="center", color="white", font_size=12, onclick=lambda:self.interface_config("cam"))
        # Elements and coordinates — Home Screen.
        self.home_int_vars = [self.main_backg, self.main_ultra_button, self.main_ultra_text, self.main_cam_button, self.main_cam_text]
        self.home_int_vals = [0, 20, 120, 20, 120]

    def board_configuration(self):
        # Utilize the integrated sensors on UNIHIKER to provide a feature-rich user experience.
        while True:
            # If the control button A is pressed, return to the home screen.
            if button_a.is_pressed() == True:
                self.interface_config("home")
                sleep(1)
            # If the control button B is pressed, change the selected class.
            if button_b.is_pressed() == True:
                self.selected_class+=1
                if self.selected_class == 3:
                    self.selected_class = 0
                self.cam_save_button.config(color=self.class_colors[self.selected_class])
                if(self.selected_class == 0): self.adjust_color([0,1,0])
                if(self.selected_class == 1): self.adjust_color([1,1,0])
                if(self.selected_class == 2): self.adjust_color([1,0,0])
                sleep(1)

    def interface_config(self, con, _hide=350):
        if(con == "home"):
            for i in range(len(self.home_int_vals)):
                self.home_int_vars[i].config(x=self.home_int_vals[i])
            for i in range(len(self.cam_int_vals)):
                self.cam_int_vars[i].config(x=_hide)
            for i in range(len(self.ultra_int_vals)):
                self.ultra_int_vars[i].config(x=_hide)
            self.adjust_color([0,0,0])
        elif(con == "cam"):
            for i in range(len(self.home_int_vals)):
                self.home_int_vars[i].config(x=_hide)
            for i in range(len(self.cam_int_vals)):
                self.cam_int_vars[i].config(x=self.cam_int_vals[i])
            for i in range(len(self.ultra_int_vals)):
                self.ultra_int_vars[i].config(x=_hide)
            self.adjust_color([0,1,1])
        elif(con == "ultra"):
            for i in range(len(self.home_int_vals)):
                self.home_int_vars[i].config(x=_hide)
            for i in range(len(self.cam_int_vals)):
                self.cam_int_vars[i].config(x=_hide)
            for i in range(len(self.ultra_int_vals)):
                self.ultra_int_vars[i].config(x=self.ultra_int_vals[i])
            self.adjust_color([1,0,1])
        elif(con == "clear_class"):
            self.selected_class = -1
            self.cam_save_button.config(color="#FAE0D8")
            self.cam_info_text.config(text="Pending...")
            self.adjust_color([0,0,0])

#include <Aquatic_Air_Bubble_Detection_inferencing.h>

#define sample_buffer_size 400

float threshold = 0.60;

// Define the air bubble class names:
String classes[] = {"bubble", "normal"};

void run_inference_to_make_predictions(bool _r){
  // Summarize the Edge Impulse neural network model inference settings (from model_metadata.h):
  Serial.print("\nInference settings:\n");
  Serial.print("\tInterval: "); Serial.print((float)EI_CLASSIFIER_INTERVAL_MS); Serial.print(" ms.\n");
  Serial.printf("\tFrame size: %d\n", EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE);
  Serial.printf("\tSample length: %d ms.\n", EI_CLASSIFIER_RAW_SAMPLE_COUNT / 16);
  Serial.printf("\tNo. of classes: %d\n", sizeof(ei_classifier_inferencing_categories) / sizeof(ei_classifier_inferencing_categories[0]));

  // If the URM15 ultrasonic sensor generates an ultrasonic scan buffer (20 x 20 — 400 points) successfully:
  if(ultrasonic_scan[scan_buffer_size-1] > 0){
    // Run inference:
    ei::signal_t signal;
    // Create a signal object from the resized (scaled) raw data buffer — ultrasonic scan buffer.
    numpy::signal_from_buffer(ultrasonic_scan, EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE, &signal);
    // Run the classifier:
    ei_impulse_result_t result = { 0 };
    EI_IMPULSE_ERROR _err = run_classifier(&signal, &result, false);
    if(_err != EI_IMPULSE_OK){
      Serial.printf("ERR: Failed to run classifier (%d)\n", _err);
      return;
    }

    // Print the inference timings on the serial monitor.
    Serial.printf("\nPredictions (DSP: %d ms., Classification: %d ms., Anomaly: %d ms.): \n",
        result.timing.dsp, result.timing.classification, result.timing.anomaly);

    // Obtain the prediction results for each label (class).
    for(size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++){
      // Print the prediction results on the serial monitor.
      Serial.printf("\t%s:\t%.5f\n", result.classification[ix].label, result.classification[ix].value);
      // Get the imperative predicted label (class).
      if(result.classification[ix].value >= threshold) predicted_class = ix;
    }
    Serial.printf("\nPredicted Class: %d [%s]\n", predicted_class, classes[predicted_class]);  

    // Detect anomalies, if any:
    #if EI_CLASSIFIER_HAS_ANOMALY == 1
      Serial.printf("Anomaly: %d \n", result.anomaly);
    #endif 

    // Release the ultrasonic scan buffer if requested.
    if(!_r){ for(int i=0; i<scan_buffer_size; i++){ ultrasonic_scan[i] = 0; } }
    
  }else{
    Serial.println("\nUltrasonic scan data buffer => Empty!");
  }
}

void show_interface(String com, int menu_option){
  // Get the assigned interface logo information.
  int l_w = interface_widths[menu_option];
  int l_h = interface_heights[menu_option];
  if(com == "home"){
    display.clearDisplay();
    display.drawBitmap(0, (SCREEN_HEIGHT-l_h)/2, interface_logos[menu_option], l_w, l_h, SSD1306_WHITE);   
    display.setTextSize(1); 
    (menu_option == 1) ? display.setTextColor(SSD1306_BLACK, SSD1306_WHITE) : display.setTextColor(SSD1306_WHITE);
    display.setCursor(l_w+5, 5); 
    display.println("1.Show Readings");
    (menu_option == 2) ? display.setTextColor(SSD1306_BLACK, SSD1306_WHITE) : display.setTextColor(SSD1306_WHITE);
    display.setCursor(l_w+5, 20);
    display.println("2.Ultrasonic+++");
    (menu_option == 3) ? display.setTextColor(SSD1306_BLACK, SSD1306_WHITE) : display.setTextColor(SSD1306_WHITE);
    display.setCursor(l_w+5, 35);
    display.println("3.Save Samples");
    (menu_option == 4) ? display.setTextColor(SSD1306_BLACK, SSD1306_WHITE) : display.setTextColor(SSD1306_WHITE);
    display.setCursor(l_w+5, 50);
    display.println("4.Run Inference");
    display.display();
    delay(500);
  }
  else if(com == "sensor"){
    display.clearDisplay();
    display.drawBitmap(SCREEN_WIDTH-l_w, SCREEN_HEIGHT-l_h, interface_logos[menu_option], l_w, l_h, SSD1306_WHITE);   
    display.setTextSize(1);
    display.setCursor(0, 0); 
    display.print("Distance: "); display.print(distance); display.println("cm");
    display.setCursor(0, 20); 
    display.print("X: "); display.print(_acc.acc_x); display.print(" / "); display.print(_acc.gyro_x); 
    display.setCursor(0, 30); 
    display.print("Y: "); display.print(_acc.acc_y); display.print(" / "); display.print(_acc.gyro_y); 
    display.setCursor(0, 40); 
    display.print("Z: "); display.print(_acc.acc_z); display.print(" / "); display.print(_acc.gyro_z);
    display.display();
  }
  else if(com == "scan"){
    display.clearDisplay();
    display.drawBitmap(SCREEN_WIDTH-l_w, SCREEN_HEIGHT-l_h, interface_logos[menu_option], l_w, l_h, SSD1306_WHITE); 
    display.setTextSize(2);
    display.setCursor(0, 0); 
    display.print(scanned_points+1); display.println(" / 400");
    display.setTextSize(1);
    display.setCursor(0, 25); 
    (scanned_points < 399) ? display.print("Scanning...") : display.print("Scan Completed!");
    display.display();
  }
  else if(com == "save"){
    display.clearDisplay();
    display.drawBitmap((SCREEN_WIDTH-l_w)/2, 0, interface_logos[menu_option], l_w, l_h, SSD1306_WHITE);   
    display.setTextSize(1);
    display.setCursor(0, l_h+10); 
    display.print("A) Class => normal");
    display.setCursor(0, l_h+25); 
    display.print("C) Class => bubble");
    display.display();
  }
  else if(com == "run"){
    display.clearDisplay();   
    display.setTextSize(1);
    display.setTextColor(SSD1306_WHITE);
    display.setCursor(0, l_h+5); 
    display.print("A) Run Inference");
    display.setCursor(0, l_h+20);
    // Show the latest model detection result and the assigned class icon if the model yields a label successfully.
    String r = (predicted_class > -1) ? classes[predicted_class] : "Pending"; 
    display.print("C) Send: "+ r);
    (predicted_class > -1) ? display.drawBitmap((SCREEN_WIDTH-class_widths[predicted_class])/2, 0, class_logos[predicted_class], class_widths[predicted_class], class_heights[predicted_class], SSD1306_WHITE) : display.drawBitmap((SCREEN_WIDTH-l_w)/2, 0, interface_logos[menu_option], l_w, l_h, SSD1306_WHITE);
    display.display();
  }
}

  if(!digitalRead(control_button_B) && menu_option == 4){
    selected_interface[menu_option-1] = true;
    adjustColor(255,255,255);
    while(selected_interface[menu_option-1]){
      // Display the running inference progress on the SSD1306 screen.
      show_interface("run", menu_option);
      // If the control button A is pressed, run the Edge Impulse neural network model to detect aquatic air bubbles by applying the ultrasonic scan data points collected via the URM15 ultrasonic sensor.
      if(!digitalRead(control_button_A)){
        // Run inference.
        run_inference_to_make_predictions(true);
        delay(2000);
      }
      // After running the neural network model successfully, if the control button C is pressed, transfer the applied data record (ultrasonic scan buffer) and the detected air bubble class to the web application via an HTTP POST request.
      if(!digitalRead(control_button_C) && predicted_class > -1){
        if(make_a_post_request("?scan=OK&type=detection&class=" + classes[predicted_class])){
          // If successful:
          display.clearDisplay();
          display.drawBitmap((SCREEN_WIDTH-connected_width)/2, (SCREEN_HEIGHT-connected_height)/2, connected_bits, connected_width, connected_height, SSD1306_WHITE);
          display.display();
          adjustColor(0,255,0);
          delay(2000);
          adjustColor(255,255,255);
        }else{
          display.clearDisplay();
          display.drawBitmap((SCREEN_WIDTH-error_width)/2, (SCREEN_HEIGHT-error_height)/2, error_bits, error_width, error_height, SSD1306_WHITE);
          display.display();
          adjustColor(255,0,0);
          delay(2000);
          adjustColor(255,255,255);
        }
      }
      // If the control button D is pressed, redirect the user to the home screen.
      if(!digitalRead(control_button_D)){
        selected_interface[menu_option-1] = false;
        adjustColor(0,0,0);
        // Clear the predicted class (label).
        predicted_class = -1;
      }
    }
  }

import cv2
import numpy
from edge_impulse_linux.image import ImageImpulseRunner
from unihiker import GUI
from pinpong.board import *
from pinpong.extension.unihiker import *
import os
import requests
import datetime
from time import sleep

    def __init__(self, model_file):
        # Initialize the USB high-quality camera feed.
        self.camera = cv2.VideoCapture(0)
        sleep(2)
        # Define the required variables to establish the connection with the web application — Aquatic_Ultrasonic_Imaging.
        self.web_app = "http://192.168.1.22/Aquatic_Ultrasonic_Imaging/"
        # Define the required variables to configure camera settings.
        self.frame_size_m = (320,320)
        self.frame_size_s = (120,120)
        # Define the required configurations to run the Edge Impulse RetinaNet (NVIDIA TAO) object detection model.
        dir_path = os.path.dirname(os.path.realpath(__file__))
        self.model_file = os.path.join(dir_path, model_file)
        self.class_names = ["sterile", "dangerous", "polluted"]
        self.class_colors = ["green", "yellow", "red"]
        self.bb_colors = {"sterile": (0,255,0), "dangerous": (0,255,255), "polluted": (0,0,255)}
        self.selected_class = -1
        self.detected_class = "Pending"
        # Define the required variables to generate an ultrasonic (radar) image.
        self.u_im = {"w": 20, "h": 20, "offset": 20, "temp_path": "./assets/ultrasonic_temp.jpg"}
        # Define the required parameters to transfer information to the given Telegram bot — @aquatic_ultrasonic_bot.
        telegram_bot_token = "<____________>" # e.g., 123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11
        self.telegram_webhook = "https://api.telegram.org/bot{}".format(telegram_bot_token)
        self.latest_air_label = "..."
        # Initiate the user interface (GUI) on UNIHIKER.
        self.interface = GUI()
        # Initiate the built-in sensor features on UNIHIKER.
        Board().begin()
        # Define the RGB LED pins.
        self.rgb = {"r": Pin(Pin.P4, Pin.OUT), "g": Pin(Pin.P5, Pin.OUT), "b": Pin(Pin.P6, Pin.OUT)}

    def run_inference(self, notify="Telegram", bb_offset=40):
        # Run inference to detect water quality levels based on chemical water tests via object detection.
        with ImageImpulseRunner(self.model_file) as runner:
            try:
                resulting_image = ""
                # Print the information of the Edge Impulse model converted to a Linux (AARCH64) application (.eim).
                model_info = runner.init()
                print('\nLoaded runner for "' + model_info['project']['owner'] + ' / ' + model_info['project']['name'] + '"')
                labels = model_info['model_parameters']['labels']
                # Get the currently captured and modified image via the high-quality USB webcam.
                test_img = self.modified_image
                # After obtaining the test frame, resize (if necessary) and generate features from the retrieved frame depending on the provided model so as to run an inference.
                features, cropped = runner.get_features_from_image(test_img)
                res = runner.classify(features)
                # Obtain the prediction (detection) results for each label (class).
                if "bounding_boxes" in res["result"].keys():
                    print('Found %d bounding boxes (%d ms.)' % (len(res["result"]["bounding_boxes"]), res['timing']['dsp'] + res['timing']['classification']))
                    # If the Edge Impulse model predicts a class successfully:
                    if(len(res["result"]["bounding_boxes"]) == 0):
                        self.detected_class = "empty"
                    else:
                        for bb in res["result"]["bounding_boxes"]:
                            # Get the latest detected labels:
                            self.detected_class = bb['label']
                            print('\t%s (%.2f): x=%d y=%d w=%d h=%d' % (bb['label'], bb['value'], bb['x'], bb['y'], bb['width'], bb['height']))
                            cv2.rectangle(cropped, (bb['x']-bb_offset, bb['y']-bb_offset), (bb['x']+bb['width']+bb_offset, bb['y']+bb['height']+bb_offset), self.bb_colors[self.detected_class], 2)
                # Save the generated model resulting image with the passed bounding boxes (if any) to the detections folder.
                if self.detected_class != "empty":
                    date = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
                    resulting_image = "/detections/detection_{}_{}.jpg".format(self.detected_class, date)
                    cv2.imwrite("."+resulting_image, cropped)
                # Notify the user of the model detection results on UNIHIKER.
                self.cam_info_text.config(text="Detection: " + self.detected_class)
                print("\n\nLatest Detected Label => " + self.detected_class)
                if(self.detected_class == "sterile"): self.adjust_color([0,1,0])
                if(self.detected_class == "dangerous"): self.adjust_color([1,1,0])
                if(self.detected_class == "polluted"): self.adjust_color([1,0,0])
                sleep(2)
                self.adjust_color([0,1,1])
                # If requested, also inform the user via Telegram by transferring the modified model resulting image and the latest detected water quality class.
                if(notify == "Telegram" and self.detected_class != "empty"):
                    self.telegram_send_data("water_test", "6465514194", resulting_image)       
            # Stop the running inference.    
            finally:
                if(runner):
                    runner.stop()

    def make_a_get_request(self, com):
        # Depending on the given command, make an HTTP GET request to communicate with the web application.
        if(com == "csv"):
            # If requested, generate a CSV file from the ultrasonic scan information sent by Nano ESP32 — data records.
            req = requests.get(self.web_app + "generate.php?create=csv")
            if(req.status_code == 200):
                if(req.text.find("Server => ") > -1):
                    self.ultra_info_text.config(text="CSV file generated successfully!")
                    self.adjust_color([0,1,1])
                print("\n"+req.text)
            else:
                print("Server => Connection Error: " + str(req.status_code))
        elif(com == "get_model_result"):
            # If requested, get the latest neural network model detection result.
            # Then, convert the retrieved resulting data record to an ultrasonic (radar) image.
            req = requests.get(self.web_app + "generate.php?model_result=OK")
            if(req.status_code == 200):
                data_packet = req.text.split("_")
                self.latest_air_label = data_packet[0]
                data_record = data_packet[1]
                # Generate ultrasonic image.
                self.adjust_color([1,1,0])
                self.generate_ultrasonic_image(data_record)
                # Display the latest generated ultrasonic image with the detected air bubble class (label) for further inspection.
                self.ultrasonic_img.config(image="scans/latest_ultrasonic_image.jpg")
                self.ultra_info_text.config(text="Detected Class: " + self.latest_air_label)
            else:
                print("Server => Connection Error: " + str(req.status_code))

    def generate_ultrasonic_image(self, data_record, scanned_image_path="./scans/latest_ultrasonic_image.jpg"):
        x = 0
        y = 0
        # Get template image.
        template = cv2.imread(self.u_im["temp_path"])
        # Obtain the individual data points by decoding the passed data record.
        data_points = data_record.split(",")
        for point in data_points:
            # Draw depth indicators on the image template according to the given data point.
            p = float(point)*100
            if(p >= 15 and p < 20): cv2.rectangle(template, (x,y), (x+self.u_im["w"],y+self.u_im["h"]), (255,255,255), -1)
            if(p >= 20 and p < 25): cv2.rectangle(template, (x,y), (x+self.u_im["w"],y+self.u_im["h"]), (255,255,0), -1)
            if(p >= 25 and p < 30): cv2.rectangle(template, (x,y), (x+self.u_im["w"],y+self.u_im["h"]), (255,0,0), -1)
            if(p >= 30 and p < 35): cv2.rectangle(template, (x,y), (x+self.u_im["w"],y+self.u_im["h"]), (0,255,255), -1)
            if(p >= 35): cv2.rectangle(template, (x,y), (x + self.u_im["w"], y + self.u_im["h"]), (0,255,0), -1)
            # Configure coordinates.
            x += self.u_im["offset"]
            if(x == 400):
                x = 0
                y += self.u_im["offset"]
            print(str(x) + ", " + str(y))
        # Save the generated ultrasonic image.
        cv2.imwrite(scanned_image_path, template)
        print("\nUltrasonic image generated and saved successfully!")

    def telegram_send_data(self, com, chat_id, file_path="/scans/latest_ultrasonic_image.jpg"):
        # Get the file directory.
        _dir = os.path.abspath(os.getcwd())
        if(com == "ultrasonic"):
            path = self.telegram_webhook + "/sendPhoto"
            image_path = _dir + file_path
            # Make an HTTP POST request to transfer the generated ultrasonic image to the given Telegram bot via the Telegram Bot API.
            req = requests.post(path, data={"chat_id": chat_id, "caption": "🖼 Ultrasonic Image Received!\n\n📡 Detected Class: "+self.latest_air_label}, files={"photo": open(image_path, 'rb')})
            if(req.status_code == 200):
                self.adjust_color([0,1,0])
                self.ultra_info_text.config(text="Image transferred to the Telegram bot!")
                print("\nImage transferred to the Telegram bot!")
            else:
                print("Server => Connection Error: " + str(req.status_code))
        if(com == "water_test"):
            path = self.telegram_webhook + "/sendPhoto"
            image_path = _dir + file_path
            # Make an HTTP POST request to transfer the model resulting image modified with the passed bounding boxes to the given Telegram bot via the Telegram Bot API.
            req = requests.post(path, data={"chat_id": chat_id, "caption": "🤖 Inference running successfully!\n\n💧 Detected Class: " + self.detected_class}, files={"photo": open(image_path, 'rb')})
            if(req.status_code == 200):
                self.adjust_color([0,1,0])
                self.cam_info_text.config(text="Image[{}] sent to Telegram!".format(self.detected_class))
                print("\nModel resulting image transferred to the Telegram bot!")
                sleep(2)
                self.adjust_color([0,1,1])
            else:
                print("Server => Connection Error: " + str(req.status_code))

# Define the aquarium object.
aquarium = aquarium_func("model/ai-based-aquatic-chemical-water-quality-testing-linux-aarch64.eim")

# Define and initialize threads.
Thread(target=aquarium.camera_feed).start()
Thread(target=aquarium.board_configuration).start()

# Show the user interface (GUI) designed with the built-in UNIHIKER modules.
aquarium.create_user_interface()