diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 00000000..cde11470 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1 @@ +ml/ \ No newline at end of file diff --git a/backend/.gitignore b/backend/.gitignore index dfccbf94..36334ccc 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -156,4 +156,5 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -*.db \ No newline at end of file +*.db +ml/ \ No newline at end of file diff --git a/backend/TaskScheduler.py b/backend/TaskScheduler.py index 143d1ab4..cb082dcb 100644 --- a/backend/TaskScheduler.py +++ b/backend/TaskScheduler.py @@ -23,6 +23,12 @@ def add_Daily_Task(self, task, *args) -> None: else: schedule.every().day.at("00:01").do(task) + def add_Weekly_Task(self, task, *args) -> None: + if len(args) > 0: + schedule.every(7).days.do(task).do(task, args) + else: + schedule.every(7).days.do(task) + def add_5min_Task(self, task, *args) -> None: if len(args) > 0: schedule.every(5).minutes.do(task, args) diff --git a/backend/ai.py b/backend/ai.py new file mode 100644 index 00000000..6efb7ef5 --- /dev/null +++ b/backend/ai.py @@ -0,0 +1,167 @@ +import tensorflow as tf +from tensorflow import keras +from tensorflow.keras import layers, regularizers +import matplotlib.pyplot as plt +import numpy as np +import os +from tensorflow.keras.preprocessing.image import ImageDataGenerator + + +# Set parameters +image_size = (224, 224) +dataset_path = "./ml/drinks-combined" +model_path = "./ml/drinks_bottle_classifier-m6.keras" +batch_size = 32 + +# Training generator with augmentation (no rescaling) +train_datagen = ImageDataGenerator( + validation_split=0.2, + rotation_range=20, + width_shift_range=0.2, + height_shift_range=0.2, + shear_range=0.2, + zoom_range=0.2, + horizontal_flip=True, + brightness_range=[0.8, 1.2] +) + +# Validation generator without augmentation (no rescaling) +val_datagen = ImageDataGenerator( + validation_split=0.2 +) + +train_generator = train_datagen.flow_from_directory( + dataset_path, + target_size=image_size, + batch_size=batch_size, + class_mode='categorical', + subset='training' +) + +val_generator = val_datagen.flow_from_directory( + dataset_path, + target_size=image_size, + batch_size=batch_size, + class_mode='categorical', + subset='validation' +) + +# Custom callback to plot training and validation accuracy after each epoch + + +class AccuracyPlotCallback(keras.callbacks.Callback): + def __init__(self, output_file="accuracy_plot.png"): + super().__init__() + self.history_acc = [] + self.history_val_acc = [] + self.output_file = output_file + + def on_epoch_end(self, epoch, logs=None): + self.history_acc.append(logs.get('accuracy')) + self.history_val_acc.append(logs.get('val_accuracy')) + print( + f"Epoch {epoch+1} - Accuracy: {self.history_acc[-1]:.4f}, Val Accuracy: {self.history_val_acc[-1]:.4f}") + plt.figure() + plt.plot(self.history_acc, label='accuracy', color='blue') + plt.plot(self.history_val_acc, label='val_accuracy', color='orange') + plt.xlabel('Epoch') + plt.ylabel('Accuracy') + plt.legend() + plt.savefig(self.output_file) + plt.close() + + def on_train_end(self, logs=None): + plt.figure() + plt.plot(self.history_acc, label='accuracy', color='blue') + plt.plot(self.history_val_acc, label='val_accuracy', color='orange') + plt.xlabel('Epoch') + plt.ylabel('Accuracy') + plt.legend() + plt.savefig(self.output_file) + plt.close() + +# Define a model using pre-trained MobileNetV2 + + +def create_model(): + # Load the MobileNetV2 model with pre-trained ImageNet weights + base_model = keras.applications.MobileNetV2( + input_shape=(image_size[0], image_size[1], 3), + include_top=False, + weights='imagenet' + ) + base_model.trainable = False # Freeze the pre-trained layers + + inputs = keras.Input(shape=(image_size[0], image_size[1], 3)) + # Apply MobileNetV2 preprocessing to convert [0,255] to [-1,1] + x = keras.applications.mobilenet_v2.preprocess_input(inputs) + x = base_model(x, training=False) + x = keras.layers.GlobalAveragePooling2D()(x) + x = keras.layers.Dropout(0.5)(x) + outputs = keras.layers.Dense( + len(train_generator.class_indices), activation='softmax')(x) + model = keras.Model(inputs, outputs) + return model + + +def learn(epochs=30,path=None): + model = create_model() + model.compile(optimizer='adam', + loss='categorical_crossentropy', + metrics=['accuracy']) + + lr_scheduler = keras.callbacks.ReduceLROnPlateau( + monitor='val_loss', patience=3, factor=0.5) + early_stopping = keras.callbacks.EarlyStopping( + monitor='val_loss', patience=5, restore_best_weights=True) + acc_plot_callback = AccuracyPlotCallback( + output_file="./plots/accuracy_plot.png") + os.makedirs("./plots", exist_ok=True) + + history = model.fit( + train_generator, + validation_data=val_generator, + epochs=epochs, + callbacks=[lr_scheduler, early_stopping, acc_plot_callback] + ) + model.save(model_path if path is None else path) + return model + + +def predict_image(image_path, model): + from tensorflow.keras.preprocessing import image + img = image.load_img(image_path, target_size=image_size) + img_array = image.img_to_array(img) + # No rescaling here: MobileNetV2 preprocesses input in the model + img_array = np.expand_dims(img_array, axis=0) + predictions = model.predict(img_array)[0] + + top_4_indices = np.argsort(predictions)[-4:][::-1] + + top_4_labels = [list(train_generator.class_indices.keys())[i] + for i in top_4_indices] + top_4_probs = [predictions[i] for i in top_4_indices] + return list(zip(top_4_labels, top_4_probs)) + + +model = None +# model = learn(30) + + +def learn_and_set(epochs=30): + global model + new_model = learn(epochs=epochs) + model = new_model + +learn_ai = os.environ.get("AI_LEARN") == "true" if os.environ.get( + "AI_LEARN") else False + +if learn_ai: + learn(path=input("Model output path:")+"/drinks_bottle_classifier-m6.keras") + +if model is None: + model = keras.models.load_model(model_path) + + +def predict_jpg(filename): + return predict_image(filename, model) diff --git a/backend/database/Queries.py b/backend/database/Queries.py index f2d5338c..5d97e2ac 100644 --- a/backend/database/Queries.py +++ b/backend/database/Queries.py @@ -770,6 +770,22 @@ def enable_disable_pretix_user(self): # Add user to database self.add_user(name, 0, util.standard_user_password, name, hidden=not checked_in) + + def get_drink_id_by_closest_name(self, name): + drinks = self.session.query(Drink).all() + best_match = None + best_ratio = 0 + for drink in drinks: + ratio = SequenceMatcher(None, name, drink.name).ratio() + if ratio > best_ratio: + best_match = drink + best_ratio = ratio + + return best_match.id + + def get_drink_from_id(self, drink_id): + drink: Drink = self.session.query(Drink).filter_by(id=drink_id).first() + return drink def is_admin(self, member_id): member: Member = self.session.query( diff --git a/backend/main.py b/backend/main.py index 83ad6a97..729dab26 100644 --- a/backend/main.py +++ b/backend/main.py @@ -19,6 +19,11 @@ import flask import mail import secrets +import base64 +import secrets + +if util.ai_enabled: + import ai api_bp = flask.Blueprint("api", __name__, url_prefix="/api/") api = Api(api_bp, doc='/docu/', base_url='/api') @@ -38,6 +43,8 @@ if util.pretix_url is not None: db.enable_disable_pretix_user() taskScheduler.add_5min_Task(db.enable_disable_pretix_user) + # if util.ai_enabled: + # taskScheduler.add_Weekly_Task(ai.learn_and_set) taskScheduler.start() @@ -452,6 +459,110 @@ def get(self): return util.build_response(db.get_drink_categories()) +@api.route('/drinks/ai/train') +class drink_training(Resource): + @authenticated + def post(self): + """ + Receive base64 encoded image and store the image on the server + """ + image = request.json["image"] + image = image.split(",")[1] + image = image.encode() + + # If directory does not exist, create it + if not os.path.exists("ml/training"): + os.makedirs("ml/training") + + # current time with milliseconds as filename + time = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f") + + with open("ml/training/"+time+".jpg", "wb") as fh: + fh.write(base64.decodebytes(image)) + return util.build_response("Image received") + + return util.build_response(code=500) + + +@api.route('/drinks/ai/recognition') +class drink_training(Resource): + @authenticated + def post(self): + """ + Receive base64 encoded image and store the image on the server + """ + image = request.json["image"] + image = image.split(",")[1] + image = image.encode() + + # If directory does not exist, create it + if not os.path.exists("ml/recognition"): + os.makedirs("ml/recognition") + + image_path = secrets.token_urlsafe(24) + ".jpg" + + with open("ml/recognition/"+image_path, "wb") as fh: + fh.write(base64.decodebytes(image)) + + predicted_labels = ai.predict_jpg("ml/recognition/"+image_path) + + # Delete the image after prediction + os.remove("ml/recognition/"+image_path) + + output = [] + + print(predicted_labels) + + if predicted_labels[0][0] == "nothing": + return util.build_response([], code=404) + + if predicted_labels[0][1] > 0.9: + # Search for all labels the best matching drink + for label in predicted_labels: + drink_id_to_add = db.get_drink_id_by_closest_name(label[0]) + if drink_id_to_add is not None: + output.append(drink_id_to_add) + + return util.build_response(output) + + return util.build_response([], code=404) + + +@api.route('/drinks/ai/training/user') +class user_identified_data(Resource): + + @authenticated + def post(self): + """ + Receive base64 encoded image and store the image on the server + """ + image = request.json["image"] + image = image.split(",")[1] + image = image.encode() + + # If directory does not exist, create it + if not os.path.exists("ml/training-user"): + os.makedirs("ml/training-user") + + # current time with milliseconds as filename + time = datetime.now().strftime("%Y-%m-%d-%H-%M-%S-%f") + + drink_id = request.json["drinkID"] + drink: Drink = db.get_drink_from_id(drink_id) + + drink_directory = "ml/training-user/"+drink.name + + # check if drink directory exists + if not os.path.exists(drink_directory): + os.makedirs(drink_directory) + + with open(drink_directory+"/"+time+".jpg", "wb") as fh: + fh.write(base64.decodebytes(image)) + return util.build_response("Image received") + + return util.build_response(code=500) + + @api.route('/drinks//price') class set_drink_price(Resource): @admin @@ -460,6 +571,9 @@ def post(self, drink_id): """ Set the price of a drink """ + if request.json["amount"] is None: + return util.build_response("Price cannot be empty", code=406) + db.change_drink_price(drink_id, request.json["amount"]) return util.build_response("Price changed") diff --git a/backend/requirements.txt b/backend/requirements.txt index 48844c7e..2fc5c471 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -8,3 +8,9 @@ psycopg2 waitress requests werkzeug +tensorflow +numpy +matplotlib +pillow +scipy +autokeras \ No newline at end of file diff --git a/backend/util.py b/backend/util.py index f8c13ad7..1c46197c 100644 --- a/backend/util.py +++ b/backend/util.py @@ -4,6 +4,7 @@ import datetime import time import requests +import random cookie_expire = int(os.environ.get("COOKIE_EXPIRE_TIME")) * \ 60*60 if os.environ.get("COOKIE_EXPIRE_TIME") else 60**3 @@ -83,6 +84,10 @@ pretix_api_token = os.environ.get("PRETIX_API_TOKEN") if os.environ.get( "PRETIX_API_TOKEN") else None +ai_enabled = os.environ.get("AI_ENABLED") == "true" if os.environ.get( + "AI_ENABLED") else False +ai_model_path = os.environ.get("AI_MODEL_PATH") if os.environ.get( + "AI_MODEL_PATH") else "drinks_bottle_classifier-m6.keras" tempfile_path = "tempfiles" backup_file_name = "backup.json" diff --git a/frontend/package-lock.json b/frontend/package-lock.json index d5213993..21120c58 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -31,6 +31,7 @@ "react-redux": "^8.1.3", "react-router-dom": "^7.1.0", "react-string-format": "^1.0.1", + "react-webcam": "^7.2.0", "recharts": "^2.9.0", "redux": "^4.2.1", "rollup": "^4.29.1", @@ -6976,6 +6977,16 @@ "react-dom": ">=16.6.0" } }, + "node_modules/react-webcam": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/react-webcam/-/react-webcam-7.2.0.tgz", + "integrity": "sha512-xkrzYPqa1ag2DP+2Q/kLKBmCIfEx49bVdgCCCcZf88oF+0NPEbkwYk3/s/C7Zy0mhM8k+hpdNkBLzxg8H0aWcg==", + "license": "MIT", + "peerDependencies": { + "react": ">=16.2.0", + "react-dom": ">=16.2.0" + } + }, "node_modules/readdirp": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index ae0e5041..b37e7fe2 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -26,6 +26,7 @@ "react-redux": "^8.1.3", "react-router-dom": "^7.1.0", "react-string-format": "^1.0.1", + "react-webcam": "^7.2.0", "recharts": "^2.9.0", "redux": "^4.2.1", "rollup": "^4.29.1", diff --git a/frontend/public/environment/env.js b/frontend/public/environment/env.js index 59178271..fe87b542 100644 --- a/frontend/public/environment/env.js +++ b/frontend/public/environment/env.js @@ -18,4 +18,5 @@ window.globalTS = { "OIDC_BUTTON_TEXT": null, "DEFAULT_THEME": 3, "SHOW_THEME_SWITCH": false, + "AI_BOTTLE_DETECTION": true, }; \ No newline at end of file diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index ee6962e9..747e7831 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -35,6 +35,7 @@ declare global { AUTH_COOKIE_PREFIX: string, SHOW_THEME_SWITCH: boolean, DEFAULT_THEME: number, + AI_BOTTLE_DETECTION: boolean, }; } } diff --git a/frontend/src/Components/User/Details/AIDrinkDialog.tsx b/frontend/src/Components/User/Details/AIDrinkDialog.tsx new file mode 100644 index 00000000..af8c80bd --- /dev/null +++ b/frontend/src/Components/User/Details/AIDrinkDialog.tsx @@ -0,0 +1,197 @@ +import { Box, Button, ButtonGroup, Dialog, DialogContent, DialogTitle, Paper, Stack, Typography } from '@mui/material' +import React, { useEffect } from 'react' +import style from './details.module.scss' +import Spacer from '../../Common/Spacer' +import { Drink } from '../../../types/ResponseTypes' + +type Props = { + drinks: Array | null + buyDrink: (drink: Drink | null) => void + open: boolean + image: string +} + +const drinkBuytimeout = 5500 + + + +interface RectangularPaperWithLabelProps { + value: number; // 0 to 100 + drinkName: string; + drinkPrice: number; + width?: number | string; + height?: number | string; + buyDrink: () => void; +} + +const RectangularPaperWithLabel: React.FC = ({ + value, + drinkName, + drinkPrice, + width = 200, + height = 100, + buyDrink, +}) => { + const progress = value; // progress from 0 to 100 + + // Ensure numeric dimensions (if passed as numbers or parse strings) + const numericWidth = typeof width === 'number' ? width : parseInt(width, 10) || 200; + const numericHeight = typeof height === 'number' ? height : parseInt(height, 10) || 100; + + const strokeWidth = 3; + // Adjust for stroke so the border fits inside + const adjustedWidth = numericWidth - strokeWidth; + const adjustedHeight = numericHeight - strokeWidth; + const perimeter = 2 * (adjustedWidth + adjustedHeight); + // Compute strokeDashoffset based on progress + const offset = perimeter - (progress / 100) * perimeter; + + return ( + + {/* SVG overlay to render the evolving border */} + + + + {/* Content with the centered label */} + + + + ); +}; + + + + +const AIDrinkDialog = (props: Props) => { + + const [progress, setProgress] = React.useState(100); + + const tickTime = 250 + + useEffect(() => { + if (!props.open) { + return + } + const timer = setInterval(() => { + setProgress((prevProgress) => { + const newProgress = prevProgress - 100 / (drinkBuytimeout / tickTime); + + if (newProgress <= 0) { + props.buyDrink(props.drinks ? props.drinks[0] : null); + return 100; // Reset progress + } + + return newProgress; + }); + }, tickTime); + + return () => clearInterval(timer); + }, [tickTime, props, props.buyDrink, props.drinks, props.open]); // Avoid `progress` in dependencies + + + if (props.drinks && props.drinks.length < 4) { + return + } + + const closeDialog = (drinkIndex: number | null) => { + if (drinkIndex === null) { + props.buyDrink(null); + } else { + props.buyDrink(props.drinks ? props.drinks[drinkIndex] : null); + } + setProgress(100) + } + + return ( + { }} sx={{ zIndex: 20000000}} > +
+ Getränk erkannt + + + { closeDialog(0) }} drinkName={props.drinks ? props.drinks[0].name : ""} drinkPrice={props.drinks ? props.drinks[0].price : 0} /> + Alternative Vorschläge + + + + + + + + + +
+ ) +} + +export default AIDrinkDialog \ No newline at end of file diff --git a/frontend/src/Components/User/Details/AvailableDrinkCard.tsx b/frontend/src/Components/User/Details/AvailableDrinkCard.tsx index c1eb4922..241a89df 100644 --- a/frontend/src/Components/User/Details/AvailableDrinkCard.tsx +++ b/frontend/src/Components/User/Details/AvailableDrinkCard.tsx @@ -14,7 +14,8 @@ import style from './availableDrinkBox.module.scss' type Props = { category: string, drinks: Drink[] | undefined, - memberID: string + memberID: string, + image:string|null } const AvailableDrinkCard = (props: Props) => { @@ -27,6 +28,9 @@ const AvailableDrinkCard = (props: Props) => { {props.drinks?.map(drink => { return <>