Confirm sandwich ingredient added, to avoid button drops. (PokemonAut…

…omation#460)

* confirm sandwich ingredient added, to avoid button drops. also, loop A button press when multiple of same quantity required.

* refactor IngredientSession and SandwichIngredientDetector

SerialPrograms/Source/CommonFramework/ImageMatch/ImageMatchResult.h

@@ -31,7 +31,7 @@ struct ImageMatchResult{
     // Assume there is at least one result in `results`:
     // Remove other weaker match results that have a score that's larger than the best match score + `max_alpha_spread`.
     void clear_beyond_spread(double max_alpha_spread);
-    // Remove match results with scores < `max_alpha`.
+    // Remove match results with scores > `max_alpha`.
     void clear_beyond_alpha(double max_alpha);
 };
 

SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp

@@ -314,6 +314,27 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
     cout << reader.read_quantity(image) << endl;
 #endif
 
+#if 0
+    // ImageRGB32 image("screenshot-20240701-165012250266.png");
+
+    // BattleBallReader reader(console, Language::English);
+    // cout << reader.read_quantity(image) << endl;
+
+    VideoSnapshot image = feed.snapshot();
+    // IngredientSession session(env.inference_dispatcher(), console, context, Language::English, SandwichIngredientType::CONDIMENT);
+    // session.read_ingredient_quantity(console, context, 8);
+
+    SandwichIngredientReader reader(SandwichIngredientType::FILLING);
+    // ImageMatch::ImageMatchResult image_result = reader.read_with_icon_matcher(image, ImageFloatBox(0.508, 0.820, 0.032, 0.057));
+    for (int i = 0; i < 6; i++){
+        ImageMatch::ImageMatchResult image_result = reader.read_with_icon_matcher(image, ImageFloatBox(0.508781 + 0.0468*i, 0.820, 0.032, 0.057));
+        image_result.clear_beyond_spread(SandwichIngredientReader::ALPHA_SPREAD);
+        image_result.log(console, SandwichIngredientReader::MAX_ALPHA);
+        image_result.clear_beyond_alpha(SandwichIngredientReader::MAX_ALPHA);
+    }
+#endif
+
+
 #if 0
     ImageRGB32 image("screenshot-20240630-183016042676.png");
 
@@ -331,8 +352,8 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope&
 #if 0
     ItemPrinterMaterialDetector detector(COLOR_RED, LANGUAGE);
     detector.make_overlays(overlays);
-    cout << (int)detector.find_happiny_dust_row_index(env.inference_dispatcher(), console, context) << endl;
-    // cout << (int)detector.detect_material_quantity(env.inference_dispatcher(), console, context, 2) << endl;
+    // cout << (int)detector.find_happiny_dust_row_index(env.inference_dispatcher(), console, context) << endl;
+    cout << (int)detector.detect_material_quantity(env.inference_dispatcher(), console, context, 2) << endl;
 #endif
 
 #if 0

SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichIngredientDetector.cpp

@@ -270,22 +270,81 @@ OCR::StringMatchResult SandwichCondimentOCR::read_substring(
     );
 }
 
-SandwichIngredientReader::SandwichIngredientReader(SandwichIngredientType ingredient_type, size_t index, Color color)
+SandwichIngredientReader::SandwichIngredientReader(SandwichIngredientType ingredient_type, Color color)
     : m_color(color)
-    , m_icon_box(0.064, 0.179 + 0.074 * index, 0.032, 0.057)
-    , m_text_box(0.100, 0.179 + 0.074 * index, 0.273, 0.057)
     , m_ingredient_type(ingredient_type)
+    , m_box_ingred_text(ingredient_list_boxes(ImageFloatBox(0.100, 0.179, 0.273, 0.057)))
+    , m_box_ingred_icon(ingredient_list_boxes(ImageFloatBox(0.064, 0.179, 0.032, 0.057)))
+    , m_box_confirmed(confirmed_ingredient_boxes(ingredient_type))
 {}
 
 void SandwichIngredientReader::make_overlays(VideoOverlaySet& items) const{
-    items.add(m_color, m_icon_box);
-    items.add(m_color, m_text_box);
+    for (size_t c = 0; c < INGREDIENT_PAGE_LINES; c++){
+        items.add(m_color, m_box_ingred_text[c]);
+        items.add(m_color, m_box_ingred_icon[c]);
+    }
+
+    for (size_t i = 0; i < m_box_confirmed.size(); i++){
+        items.add(m_color, m_box_confirmed[i]);
+    }
+}
+
+
+std::array<ImageFloatBox, 6> SandwichIngredientReader::confirmed_ingredient_boxes(SandwichIngredientType type){
+    std::array<ImageFloatBox, 6> boxes;
+    ImageFloatBox initial_box;
+    size_t total_count = 0;
+    switch (type){
+    case SandwichIngredientType::FILLING:
+        initial_box = ImageFloatBox(0.508781, 0.820, 0.032, 0.057);
+        total_count = 6;
+        break;
+    case SandwichIngredientType::CONDIMENT:
+        initial_box = ImageFloatBox(0.797474, 0.820, 0.032, 0.057);
+        total_count = 4;
+        break;
+    }
+
+    double initial_x = initial_box.x;
+    double width = initial_box.width;
+    double height = initial_box.height;
+    double y = initial_box.y;
+    double x_spacing = 0.0468;
+    for (size_t i = 0; i < total_count; i++){
+        double x = initial_x + i*x_spacing;
+        boxes[i] = ImageFloatBox(x, y, width, height);
+    }
+    return boxes;
 }
 
-ImageMatch::ImageMatchResult SandwichIngredientReader::read_with_icon_matcher(const ImageViewRGB32& screen) const{
+
+std::array<ImageFloatBox, 10> SandwichIngredientReader::ingredient_list_boxes(ImageFloatBox initial_box){
+    std::array<ImageFloatBox, 10> material_boxes;
+    double x = initial_box.x;
+    double width = initial_box.width;
+    double height = initial_box.height;
+    double initial_y = initial_box.y;
+    double y_spacing = 0.074;
+    for (size_t i = 0; i < 10; i++){
+        double y = initial_y + i*y_spacing;
+        material_boxes[i] = ImageFloatBox(x, y, width, height);
+    }
+    return material_boxes;
+}
+
+
+ImageMatch::ImageMatchResult SandwichIngredientReader::read_ingredient_page_with_icon_matcher(const ImageViewRGB32& screen, size_t index) const{
+    return read_with_icon_matcher(screen, m_box_ingred_icon[index]);
+}
+
+ImageMatch::ImageMatchResult SandwichIngredientReader::read_confirmed_list_with_icon_matcher(const ImageViewRGB32& screen, size_t index) const{
+    return read_with_icon_matcher(screen, m_box_confirmed[index]);
+}
+
+ImageMatch::ImageMatchResult SandwichIngredientReader::read_with_icon_matcher(const ImageViewRGB32& screen, const ImageFloatBox icon_box) const{
     // Get a crop of the sandwich ingredient icon
-    ImageViewRGB32 image = extract_box_reference(screen, m_icon_box);
-//    image.save("image.png");
+    ImageViewRGB32 image = extract_box_reference(screen, icon_box);
+//    image.save("image" + std::to_string(icon_box.x) + ".png");
 
 //    // Remove the orange / yellow background when the ingredient is selected
 //    ImageRGB32 filtered_image = filter_rgb32_range(image, 0xffdfaf00, 0xffffef20, Color(0x00000000), true);
@@ -307,9 +366,24 @@ ImageMatch::ImageMatchResult SandwichIngredientReader::read_with_icon_matcher(co
     return results;
 }
 
-OCR::StringMatchResult SandwichIngredientReader::read_with_ocr(const ImageViewRGB32& screen, Logger& logger, Language language) const{
+OCR::StringMatchResult SandwichIngredientReader::read_ingredient_page_with_ocr(
+    const ImageViewRGB32& screen, 
+    Logger& logger, 
+    Language language, 
+    size_t index
+) const{
+    return read_with_ocr(screen, logger, language, m_box_ingred_text[index]);
+}
+
+OCR::StringMatchResult SandwichIngredientReader::read_with_ocr(
+    const ImageViewRGB32& screen, 
+    Logger& logger, 
+    Language language, 
+    const ImageFloatBox icon_box
+) const{
+
     // Get a crop of the sandwich ingredient text
-    ImageViewRGB32 image = extract_box_reference(screen, m_text_box);
+    ImageViewRGB32 image = extract_box_reference(screen, icon_box);
     //image.save("image.png");
 
     OCR::StringMatchResult results;

SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichIngredientDetector.h

@@ -177,24 +177,45 @@ class SandwichIngredientReader{
 public:
     static constexpr double MAX_ALPHA = 180;
     static constexpr double ALPHA_SPREAD = 10;
-
+    static constexpr size_t INGREDIENT_PAGE_LINES = 10;
 
 public:
-    SandwichIngredientReader(SandwichIngredientType ingredient_type, size_t index, Color color = COLOR_RED);
+    SandwichIngredientReader(SandwichIngredientType ingredient_type, Color color = COLOR_RED);
 
     void make_overlays(VideoOverlaySet& items) const;
 
+    std::array<ImageFloatBox, 6> confirmed_ingredient_boxes(SandwichIngredientType type);
+
+    std::array<ImageFloatBox, 10> ingredient_list_boxes(ImageFloatBox initial_box);
+
+    ImageMatch::ImageMatchResult read_ingredient_page_with_icon_matcher(const ImageViewRGB32& screen, size_t index) const;
+
+    ImageMatch::ImageMatchResult read_confirmed_list_with_icon_matcher(const ImageViewRGB32& screen, size_t index) const;
+
     // The icon matcher only works on the selected item, because we want to remove the yellow / orange background
-    ImageMatch::ImageMatchResult read_with_icon_matcher(const ImageViewRGB32& screen) const;
+    ImageMatch::ImageMatchResult read_with_icon_matcher(const ImageViewRGB32& screen, const ImageFloatBox icon_box) const;
+
+    OCR::StringMatchResult read_ingredient_page_with_ocr(
+        const ImageViewRGB32& screen, 
+        Logger& logger, 
+        Language language, 
+        size_t index
+    ) const;
 
     // The OCR works on any ingredient, selected or not
-    OCR::StringMatchResult read_with_ocr(const ImageViewRGB32& screen, Logger& logger, Language language) const;
+    OCR::StringMatchResult read_with_ocr(
+        const ImageViewRGB32& screen, 
+        Logger& logger, 
+        Language language, 
+        const ImageFloatBox icon_box
+    ) const;
 
-private:
+// private:
     Color m_color;
-    ImageFloatBox m_icon_box;
-    ImageFloatBox m_text_box;
     SandwichIngredientType m_ingredient_type;
+    std::array<ImageFloatBox, INGREDIENT_PAGE_LINES> m_box_ingred_text;
+    std::array<ImageFloatBox, INGREDIENT_PAGE_LINES> m_box_ingred_icon;
+    std::array<ImageFloatBox, 6> m_box_confirmed;
 };
 
 }

SerialPrograms/Source/PokemonSV/Programs/Sandwiches/PokemonSV_IngredientSession.cpp

@@ -14,6 +14,9 @@
 #include "PokemonSV/Resources/PokemonSV_Ingredients.h"
 #include "PokemonSV_IngredientSession.h"
 #include "Common/Cpp/PrettyPrint.h"
+#include <iostream>
+using std::cout;
+using std::endl;
 
 namespace PokemonAutomation{
 namespace NintendoSwitch{
@@ -31,16 +34,16 @@ IngredientSession::IngredientSession(
     , m_context(context)
     , m_language(language)
     , m_overlays(console.overlay())
-    , m_ingredients(10)
+    , m_type(type)
+    , m_num_confirmed(0)
     , m_arrow(COLOR_CYAN, GradientArrowType::RIGHT, {0.02, 0.15, 0.05, 0.80})
 {
-    for (size_t c = 0; c < INGREDIENT_PAGE_LINES; c++){
-        m_ingredients.emplace_back(type, c, COLOR_CYAN);
-        m_ingredients.back().make_overlays(m_overlays);
-    }
+    SandwichIngredientReader reader(m_type, COLOR_CYAN);
+    reader.make_overlays(m_overlays);
 }
 
 
+
 PageIngredients IngredientSession::read_screen(std::shared_ptr<const ImageRGB32> screen) const{
     PageIngredients ret;
     ImageFloatBox box;
@@ -65,18 +68,19 @@ PageIngredients IngredientSession::read_screen(std::shared_ptr<const ImageRGB32>
 
     //  Read the names of every line and the sprite of the selected line.
     ImageMatch::ImageMatchResult image_result;
-    m_dispatcher.run_in_parallel(0, INGREDIENT_PAGE_LINES + 1, [&](size_t index){
-        if (index < INGREDIENT_PAGE_LINES){
+    SandwichIngredientReader reader(m_type);
+    m_dispatcher.run_in_parallel(0, SandwichIngredientReader::INGREDIENT_PAGE_LINES + 1, [&](size_t index){
+        if (index < SandwichIngredientReader::INGREDIENT_PAGE_LINES){
             // Read text at line `index`
-            OCR::StringMatchResult result = m_ingredients[index].read_with_ocr(*screen, m_console, m_language);
+            OCR::StringMatchResult result = reader.read_ingredient_page_with_ocr(*screen, m_console, m_language, index);
             result.clear_beyond_log10p(SandwichFillingOCR::MAX_LOG10P);
             result.clear_beyond_spread(SandwichFillingOCR::MAX_LOG10P_SPREAD);
             for (auto& item : result.results){
                 ret.item[index].insert(item.second.token);
             }
         }else{
             // Read current selected icon
-            image_result = m_ingredients[ret.selected].read_with_icon_matcher(*screen);
+            image_result = reader.read_ingredient_page_with_icon_matcher(*screen, ret.selected);
             image_result.clear_beyond_spread(SandwichIngredientReader::ALPHA_SPREAD);
             image_result.log(m_console, SandwichIngredientReader::MAX_ALPHA);
             image_result.clear_beyond_alpha(SandwichIngredientReader::MAX_ALPHA);
@@ -146,7 +150,7 @@ bool IngredientSession::run_move_iteration(
 ) const{
     size_t current_index = page.selected;
     std::map<size_t, std::string> found_ingredients;
-    for (size_t c = 0; c < INGREDIENT_PAGE_LINES; c++){
+    for (size_t c = 0; c < SandwichIngredientReader::INGREDIENT_PAGE_LINES; c++){
         for (const std::string& item : page.item[c]){
             auto iter = ingredients.find(item);
             if (iter != ingredients.end()){
@@ -190,7 +194,7 @@ bool IngredientSession::run_move_iteration(
     }
     m_context.wait_for_all_requests();
     m_context.wait_for(std::chrono::seconds(1));
-
+    // slug = item;
     return true;
 }
 
@@ -205,7 +209,6 @@ std::string IngredientSession::move_to_ingredient(const std::set<std::string>& i
     while (true){
         m_context.wait_for_all_requests();
         PageIngredients page = read_current_page();
-
         std::string found_ingredient;
         if (run_move_iteration(found_ingredient, ingredients, page)){
             if (found_ingredient.empty()){
@@ -216,7 +219,7 @@ std::string IngredientSession::move_to_ingredient(const std::set<std::string>& i
         }
 
         size_t current = page.selected;
-        if (current == INGREDIENT_PAGE_LINES - 1){
+        if (current == SandwichIngredientReader::INGREDIENT_PAGE_LINES - 1){
             not_found_count++;
             if (not_found_count >= 2){
                 m_console.log("Ingredient not found anywhere.", COLOR_RED);
@@ -247,7 +250,7 @@ std::string IngredientSession::move_to_ingredient(const std::set<std::string>& i
 void IngredientSession::add_ingredients(
     ConsoleHandle& console, BotBaseContext& context,
     std::map<std::string, uint8_t>&& ingredients
-) const{
+){
     //  "ingredients" will be what we still need.
     //  Each time we add an ingredient, it will be removed from the map.
     //  Loop until there's nothing left.
@@ -269,19 +272,39 @@ void IngredientSession::add_ingredients(
         const SandwichIngredientNames& name = get_ingredient_name(found);
         console.log("Add " + name.display_name() + " as ingredient", COLOR_BLUE);
 
-        //  Add the item. But don't loop the quantity. Instead, we add one and
-        //  loop again in case we run out.
         //  If you don't have enough ingredient, it errors out instead of proceeding 
         //  with less than the desired quantity.
-        pbf_press_button(context, BUTTON_A, 20, 105);
-        context.wait_for_all_requests();
-        console.overlay().add_log("Added " + name.display_name());
-        // TODO add visual confirmation of ingredients added to avoid button drops
-
         auto iter = ingredients.find(found);
-        if (--iter->second == 0){
-            ingredients.erase(iter);
+        SandwichIngredientReader reader(m_type);
+        while (iter->second > 0){
+            bool ingredient_added = false;
+            for (int attempt = 0; attempt < 5; attempt++){
+                pbf_press_button(context, BUTTON_A, 20, 105);
+                context.wait_for_all_requests();
+                VideoSnapshot image = console.video().snapshot();
+                ImageMatch::ImageMatchResult image_result = 
+                    reader.read_confirmed_list_with_icon_matcher(image, m_num_confirmed);
+                image_result.clear_beyond_spread(SandwichIngredientReader::ALPHA_SPREAD);
+                image_result.clear_beyond_alpha(SandwichIngredientReader::MAX_ALPHA);
+                image_result.log(console, SandwichIngredientReader::MAX_ALPHA);                
+                if (image_result.results.size() > 0){ // confirmed that the ingredient was added
+                    console.overlay().add_log("Added " + name.display_name());
+                    m_num_confirmed++;
+                    ingredient_added = true;
+                    iter->second--;
+                    break;
+                }
+            }
+
+            if (!ingredient_added){
+                    throw OperationFailedException(
+                    ErrorReport::SEND_ERROR_REPORT,
+                    console,
+                    "Unable to add ingredient: \"" + name.display_name() + "\" - Did you run out?"
+                );
+            }
         }
+        ingredients.erase(iter);
     }
 }