From 583600c24e5f107ade327e10986c8f3495358dbb Mon Sep 17 00:00:00 2001 From: jw098 Date: Tue, 23 Jul 2024 21:27:43 -0700 Subject: [PATCH] Confirm sandwich ingredient added, to avoid button drops. (#460) * confirm sandwich ingredient added, to avoid button drops. also, loop A button press when multiple of same quantity required. * refactor IngredientSession and SandwichIngredientDetector --- .../ImageMatch/ImageMatchResult.h | 2 +- .../DevPrograms/TestProgramSwitch.cpp | 25 ++++- .../PokemonSV_SandwichIngredientDetector.cpp | 94 +++++++++++++++++-- .../PokemonSV_SandwichIngredientDetector.h | 35 +++++-- .../PokemonSV_IngredientSession.cpp | 69 +++++++++----- .../Sandwiches/PokemonSV_IngredientSession.h | 9 +- .../Source/Tests/PokemonSV_Tests.cpp | 6 +- 7 files changed, 190 insertions(+), 50 deletions(-) diff --git a/SerialPrograms/Source/CommonFramework/ImageMatch/ImageMatchResult.h b/SerialPrograms/Source/CommonFramework/ImageMatch/ImageMatchResult.h index b4a7cce2f..4b659c12d 100644 --- a/SerialPrograms/Source/CommonFramework/ImageMatch/ImageMatchResult.h +++ b/SerialPrograms/Source/CommonFramework/ImageMatch/ImageMatchResult.h @@ -31,7 +31,7 @@ struct ImageMatchResult{ // Assume there is at least one result in `results`: // Remove other weaker match results that have a score that's larger than the best match score + `max_alpha_spread`. void clear_beyond_spread(double max_alpha_spread); - // Remove match results with scores < `max_alpha`. + // Remove match results with scores > `max_alpha`. void clear_beyond_alpha(double max_alpha); }; diff --git a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp index 4680d2e55..f577e5fd7 100644 --- a/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp +++ b/SerialPrograms/Source/NintendoSwitch/DevPrograms/TestProgramSwitch.cpp @@ -314,6 +314,27 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& cout << reader.read_quantity(image) << endl; #endif +#if 0 + // ImageRGB32 image("screenshot-20240701-165012250266.png"); + + // BattleBallReader reader(console, Language::English); + // cout << reader.read_quantity(image) << endl; + + VideoSnapshot image = feed.snapshot(); + // IngredientSession session(env.inference_dispatcher(), console, context, Language::English, SandwichIngredientType::CONDIMENT); + // session.read_ingredient_quantity(console, context, 8); + + SandwichIngredientReader reader(SandwichIngredientType::FILLING); + // ImageMatch::ImageMatchResult image_result = reader.read_with_icon_matcher(image, ImageFloatBox(0.508, 0.820, 0.032, 0.057)); + for (int i = 0; i < 6; i++){ + ImageMatch::ImageMatchResult image_result = reader.read_with_icon_matcher(image, ImageFloatBox(0.508781 + 0.0468*i, 0.820, 0.032, 0.057)); + image_result.clear_beyond_spread(SandwichIngredientReader::ALPHA_SPREAD); + image_result.log(console, SandwichIngredientReader::MAX_ALPHA); + image_result.clear_beyond_alpha(SandwichIngredientReader::MAX_ALPHA); + } +#endif + + #if 0 ImageRGB32 image("screenshot-20240630-183016042676.png"); @@ -331,8 +352,8 @@ void TestProgram::program(MultiSwitchProgramEnvironment& env, CancellableScope& #if 0 ItemPrinterMaterialDetector detector(COLOR_RED, LANGUAGE); detector.make_overlays(overlays); - cout << (int)detector.find_happiny_dust_row_index(env.inference_dispatcher(), console, context) << endl; - // cout << (int)detector.detect_material_quantity(env.inference_dispatcher(), console, context, 2) << endl; + // cout << (int)detector.find_happiny_dust_row_index(env.inference_dispatcher(), console, context) << endl; + cout << (int)detector.detect_material_quantity(env.inference_dispatcher(), console, context, 2) << endl; #endif #if 0 diff --git a/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichIngredientDetector.cpp b/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichIngredientDetector.cpp index d4eecbe92..ce85a95a0 100644 --- a/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichIngredientDetector.cpp +++ b/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichIngredientDetector.cpp @@ -270,22 +270,81 @@ OCR::StringMatchResult SandwichCondimentOCR::read_substring( ); } -SandwichIngredientReader::SandwichIngredientReader(SandwichIngredientType ingredient_type, size_t index, Color color) +SandwichIngredientReader::SandwichIngredientReader(SandwichIngredientType ingredient_type, Color color) : m_color(color) - , m_icon_box(0.064, 0.179 + 0.074 * index, 0.032, 0.057) - , m_text_box(0.100, 0.179 + 0.074 * index, 0.273, 0.057) , m_ingredient_type(ingredient_type) + , m_box_ingred_text(ingredient_list_boxes(ImageFloatBox(0.100, 0.179, 0.273, 0.057))) + , m_box_ingred_icon(ingredient_list_boxes(ImageFloatBox(0.064, 0.179, 0.032, 0.057))) + , m_box_confirmed(confirmed_ingredient_boxes(ingredient_type)) {} void SandwichIngredientReader::make_overlays(VideoOverlaySet& items) const{ - items.add(m_color, m_icon_box); - items.add(m_color, m_text_box); + for (size_t c = 0; c < INGREDIENT_PAGE_LINES; c++){ + items.add(m_color, m_box_ingred_text[c]); + items.add(m_color, m_box_ingred_icon[c]); + } + + for (size_t i = 0; i < m_box_confirmed.size(); i++){ + items.add(m_color, m_box_confirmed[i]); + } +} + + +std::array SandwichIngredientReader::confirmed_ingredient_boxes(SandwichIngredientType type){ + std::array boxes; + ImageFloatBox initial_box; + size_t total_count = 0; + switch (type){ + case SandwichIngredientType::FILLING: + initial_box = ImageFloatBox(0.508781, 0.820, 0.032, 0.057); + total_count = 6; + break; + case SandwichIngredientType::CONDIMENT: + initial_box = ImageFloatBox(0.797474, 0.820, 0.032, 0.057); + total_count = 4; + break; + } + + double initial_x = initial_box.x; + double width = initial_box.width; + double height = initial_box.height; + double y = initial_box.y; + double x_spacing = 0.0468; + for (size_t i = 0; i < total_count; i++){ + double x = initial_x + i*x_spacing; + boxes[i] = ImageFloatBox(x, y, width, height); + } + return boxes; } -ImageMatch::ImageMatchResult SandwichIngredientReader::read_with_icon_matcher(const ImageViewRGB32& screen) const{ + +std::array SandwichIngredientReader::ingredient_list_boxes(ImageFloatBox initial_box){ + std::array material_boxes; + double x = initial_box.x; + double width = initial_box.width; + double height = initial_box.height; + double initial_y = initial_box.y; + double y_spacing = 0.074; + for (size_t i = 0; i < 10; i++){ + double y = initial_y + i*y_spacing; + material_boxes[i] = ImageFloatBox(x, y, width, height); + } + return material_boxes; +} + + +ImageMatch::ImageMatchResult SandwichIngredientReader::read_ingredient_page_with_icon_matcher(const ImageViewRGB32& screen, size_t index) const{ + return read_with_icon_matcher(screen, m_box_ingred_icon[index]); +} + +ImageMatch::ImageMatchResult SandwichIngredientReader::read_confirmed_list_with_icon_matcher(const ImageViewRGB32& screen, size_t index) const{ + return read_with_icon_matcher(screen, m_box_confirmed[index]); +} + +ImageMatch::ImageMatchResult SandwichIngredientReader::read_with_icon_matcher(const ImageViewRGB32& screen, const ImageFloatBox icon_box) const{ // Get a crop of the sandwich ingredient icon - ImageViewRGB32 image = extract_box_reference(screen, m_icon_box); -// image.save("image.png"); + ImageViewRGB32 image = extract_box_reference(screen, icon_box); +// image.save("image" + std::to_string(icon_box.x) + ".png"); // // Remove the orange / yellow background when the ingredient is selected // ImageRGB32 filtered_image = filter_rgb32_range(image, 0xffdfaf00, 0xffffef20, Color(0x00000000), true); @@ -307,9 +366,24 @@ ImageMatch::ImageMatchResult SandwichIngredientReader::read_with_icon_matcher(co return results; } -OCR::StringMatchResult SandwichIngredientReader::read_with_ocr(const ImageViewRGB32& screen, Logger& logger, Language language) const{ +OCR::StringMatchResult SandwichIngredientReader::read_ingredient_page_with_ocr( + const ImageViewRGB32& screen, + Logger& logger, + Language language, + size_t index +) const{ + return read_with_ocr(screen, logger, language, m_box_ingred_text[index]); +} + +OCR::StringMatchResult SandwichIngredientReader::read_with_ocr( + const ImageViewRGB32& screen, + Logger& logger, + Language language, + const ImageFloatBox icon_box +) const{ + // Get a crop of the sandwich ingredient text - ImageViewRGB32 image = extract_box_reference(screen, m_text_box); + ImageViewRGB32 image = extract_box_reference(screen, icon_box); //image.save("image.png"); OCR::StringMatchResult results; diff --git a/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichIngredientDetector.h b/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichIngredientDetector.h index 200087f23..8cb424496 100644 --- a/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichIngredientDetector.h +++ b/SerialPrograms/Source/PokemonSV/Inference/Picnics/PokemonSV_SandwichIngredientDetector.h @@ -177,24 +177,45 @@ class SandwichIngredientReader{ public: static constexpr double MAX_ALPHA = 180; static constexpr double ALPHA_SPREAD = 10; - + static constexpr size_t INGREDIENT_PAGE_LINES = 10; public: - SandwichIngredientReader(SandwichIngredientType ingredient_type, size_t index, Color color = COLOR_RED); + SandwichIngredientReader(SandwichIngredientType ingredient_type, Color color = COLOR_RED); void make_overlays(VideoOverlaySet& items) const; + std::array confirmed_ingredient_boxes(SandwichIngredientType type); + + std::array ingredient_list_boxes(ImageFloatBox initial_box); + + ImageMatch::ImageMatchResult read_ingredient_page_with_icon_matcher(const ImageViewRGB32& screen, size_t index) const; + + ImageMatch::ImageMatchResult read_confirmed_list_with_icon_matcher(const ImageViewRGB32& screen, size_t index) const; + // The icon matcher only works on the selected item, because we want to remove the yellow / orange background - ImageMatch::ImageMatchResult read_with_icon_matcher(const ImageViewRGB32& screen) const; + ImageMatch::ImageMatchResult read_with_icon_matcher(const ImageViewRGB32& screen, const ImageFloatBox icon_box) const; + + OCR::StringMatchResult read_ingredient_page_with_ocr( + const ImageViewRGB32& screen, + Logger& logger, + Language language, + size_t index + ) const; // The OCR works on any ingredient, selected or not - OCR::StringMatchResult read_with_ocr(const ImageViewRGB32& screen, Logger& logger, Language language) const; + OCR::StringMatchResult read_with_ocr( + const ImageViewRGB32& screen, + Logger& logger, + Language language, + const ImageFloatBox icon_box + ) const; -private: +// private: Color m_color; - ImageFloatBox m_icon_box; - ImageFloatBox m_text_box; SandwichIngredientType m_ingredient_type; + std::array m_box_ingred_text; + std::array m_box_ingred_icon; + std::array m_box_confirmed; }; } diff --git a/SerialPrograms/Source/PokemonSV/Programs/Sandwiches/PokemonSV_IngredientSession.cpp b/SerialPrograms/Source/PokemonSV/Programs/Sandwiches/PokemonSV_IngredientSession.cpp index 0ba9fe980..01a18dc79 100644 --- a/SerialPrograms/Source/PokemonSV/Programs/Sandwiches/PokemonSV_IngredientSession.cpp +++ b/SerialPrograms/Source/PokemonSV/Programs/Sandwiches/PokemonSV_IngredientSession.cpp @@ -14,6 +14,9 @@ #include "PokemonSV/Resources/PokemonSV_Ingredients.h" #include "PokemonSV_IngredientSession.h" #include "Common/Cpp/PrettyPrint.h" +#include +using std::cout; +using std::endl; namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -31,16 +34,16 @@ IngredientSession::IngredientSession( , m_context(context) , m_language(language) , m_overlays(console.overlay()) - , m_ingredients(10) + , m_type(type) + , m_num_confirmed(0) , m_arrow(COLOR_CYAN, GradientArrowType::RIGHT, {0.02, 0.15, 0.05, 0.80}) { - for (size_t c = 0; c < INGREDIENT_PAGE_LINES; c++){ - m_ingredients.emplace_back(type, c, COLOR_CYAN); - m_ingredients.back().make_overlays(m_overlays); - } + SandwichIngredientReader reader(m_type, COLOR_CYAN); + reader.make_overlays(m_overlays); } + PageIngredients IngredientSession::read_screen(std::shared_ptr screen) const{ PageIngredients ret; ImageFloatBox box; @@ -65,10 +68,11 @@ PageIngredients IngredientSession::read_screen(std::shared_ptr // Read the names of every line and the sprite of the selected line. ImageMatch::ImageMatchResult image_result; - m_dispatcher.run_in_parallel(0, INGREDIENT_PAGE_LINES + 1, [&](size_t index){ - if (index < INGREDIENT_PAGE_LINES){ + SandwichIngredientReader reader(m_type); + m_dispatcher.run_in_parallel(0, SandwichIngredientReader::INGREDIENT_PAGE_LINES + 1, [&](size_t index){ + if (index < SandwichIngredientReader::INGREDIENT_PAGE_LINES){ // Read text at line `index` - OCR::StringMatchResult result = m_ingredients[index].read_with_ocr(*screen, m_console, m_language); + OCR::StringMatchResult result = reader.read_ingredient_page_with_ocr(*screen, m_console, m_language, index); result.clear_beyond_log10p(SandwichFillingOCR::MAX_LOG10P); result.clear_beyond_spread(SandwichFillingOCR::MAX_LOG10P_SPREAD); for (auto& item : result.results){ @@ -76,7 +80,7 @@ PageIngredients IngredientSession::read_screen(std::shared_ptr } }else{ // Read current selected icon - image_result = m_ingredients[ret.selected].read_with_icon_matcher(*screen); + image_result = reader.read_ingredient_page_with_icon_matcher(*screen, ret.selected); image_result.clear_beyond_spread(SandwichIngredientReader::ALPHA_SPREAD); image_result.log(m_console, SandwichIngredientReader::MAX_ALPHA); image_result.clear_beyond_alpha(SandwichIngredientReader::MAX_ALPHA); @@ -146,7 +150,7 @@ bool IngredientSession::run_move_iteration( ) const{ size_t current_index = page.selected; std::map found_ingredients; - for (size_t c = 0; c < INGREDIENT_PAGE_LINES; c++){ + for (size_t c = 0; c < SandwichIngredientReader::INGREDIENT_PAGE_LINES; c++){ for (const std::string& item : page.item[c]){ auto iter = ingredients.find(item); if (iter != ingredients.end()){ @@ -190,7 +194,7 @@ bool IngredientSession::run_move_iteration( } m_context.wait_for_all_requests(); m_context.wait_for(std::chrono::seconds(1)); - + // slug = item; return true; } @@ -205,7 +209,6 @@ std::string IngredientSession::move_to_ingredient(const std::set& i while (true){ m_context.wait_for_all_requests(); PageIngredients page = read_current_page(); - std::string found_ingredient; if (run_move_iteration(found_ingredient, ingredients, page)){ if (found_ingredient.empty()){ @@ -216,7 +219,7 @@ std::string IngredientSession::move_to_ingredient(const std::set& i } size_t current = page.selected; - if (current == INGREDIENT_PAGE_LINES - 1){ + if (current == SandwichIngredientReader::INGREDIENT_PAGE_LINES - 1){ not_found_count++; if (not_found_count >= 2){ m_console.log("Ingredient not found anywhere.", COLOR_RED); @@ -247,7 +250,7 @@ std::string IngredientSession::move_to_ingredient(const std::set& i void IngredientSession::add_ingredients( ConsoleHandle& console, BotBaseContext& context, std::map&& ingredients -) const{ +){ // "ingredients" will be what we still need. // Each time we add an ingredient, it will be removed from the map. // Loop until there's nothing left. @@ -269,19 +272,39 @@ void IngredientSession::add_ingredients( const SandwichIngredientNames& name = get_ingredient_name(found); console.log("Add " + name.display_name() + " as ingredient", COLOR_BLUE); - // Add the item. But don't loop the quantity. Instead, we add one and - // loop again in case we run out. // If you don't have enough ingredient, it errors out instead of proceeding // with less than the desired quantity. - pbf_press_button(context, BUTTON_A, 20, 105); - context.wait_for_all_requests(); - console.overlay().add_log("Added " + name.display_name()); - // TODO add visual confirmation of ingredients added to avoid button drops - auto iter = ingredients.find(found); - if (--iter->second == 0){ - ingredients.erase(iter); + SandwichIngredientReader reader(m_type); + while (iter->second > 0){ + bool ingredient_added = false; + for (int attempt = 0; attempt < 5; attempt++){ + pbf_press_button(context, BUTTON_A, 20, 105); + context.wait_for_all_requests(); + VideoSnapshot image = console.video().snapshot(); + ImageMatch::ImageMatchResult image_result = + reader.read_confirmed_list_with_icon_matcher(image, m_num_confirmed); + image_result.clear_beyond_spread(SandwichIngredientReader::ALPHA_SPREAD); + image_result.clear_beyond_alpha(SandwichIngredientReader::MAX_ALPHA); + image_result.log(console, SandwichIngredientReader::MAX_ALPHA); + if (image_result.results.size() > 0){ // confirmed that the ingredient was added + console.overlay().add_log("Added " + name.display_name()); + m_num_confirmed++; + ingredient_added = true; + iter->second--; + break; + } + } + + if (!ingredient_added){ + throw OperationFailedException( + ErrorReport::SEND_ERROR_REPORT, + console, + "Unable to add ingredient: \"" + name.display_name() + "\" - Did you run out?" + ); + } } + ingredients.erase(iter); } } diff --git a/SerialPrograms/Source/PokemonSV/Programs/Sandwiches/PokemonSV_IngredientSession.h b/SerialPrograms/Source/PokemonSV/Programs/Sandwiches/PokemonSV_IngredientSession.h index dd739717e..f6fa643fa 100644 --- a/SerialPrograms/Source/PokemonSV/Programs/Sandwiches/PokemonSV_IngredientSession.h +++ b/SerialPrograms/Source/PokemonSV/Programs/Sandwiches/PokemonSV_IngredientSession.h @@ -24,12 +24,12 @@ namespace PokemonSV{ class SandwichIngredientReader; -static constexpr size_t INGREDIENT_PAGE_LINES = 10; + struct PageIngredients{ // the line index of the current selected ingredient, < INGREDIENT_PAGE_LINES int8_t selected = -1; - std::set item[INGREDIENT_PAGE_LINES]; + std::set item[SandwichIngredientReader::INGREDIENT_PAGE_LINES]; }; @@ -49,7 +49,7 @@ class IngredientSession{ void add_ingredients( ConsoleHandle& console, BotBaseContext& context, std::map&& ingredients - ) const; + ); public: @@ -67,7 +67,8 @@ class IngredientSession{ BotBaseContext& m_context; Language m_language; VideoOverlaySet m_overlays; - FixedLimitVector m_ingredients; + SandwichIngredientType m_type; + int8_t m_num_confirmed; GradientArrowDetector m_arrow; }; diff --git a/SerialPrograms/Source/Tests/PokemonSV_Tests.cpp b/SerialPrograms/Source/Tests/PokemonSV_Tests.cpp index d0d0cf5ac..e6284b60c 100644 --- a/SerialPrograms/Source/Tests/PokemonSV_Tests.cpp +++ b/SerialPrograms/Source/Tests/PokemonSV_Tests.cpp @@ -418,11 +418,11 @@ int test_pokemonSV_SandwichIngredientReader(const ImageViewRGB32& image, const s return 1; } + SandwichIngredientReader reader(sandwich_type); for (size_t i = 0; i < 10; ++i){ - SandwichIngredientReader reader(sandwich_type, i); if (selected_ingredient == i){ // The icon matcher only works on the selected item, because we want to remove the yellow / orange background - ImageMatch::ImageMatchResult results = reader.read_with_icon_matcher(image); + ImageMatch::ImageMatchResult results = reader.read_ingredient_page_with_icon_matcher(image, i); if (results.results.empty()){ cerr << "No ingredient detected via icon matcher" << endl; @@ -432,7 +432,7 @@ int test_pokemonSV_SandwichIngredientReader(const ImageViewRGB32& image, const s TEST_RESULT_COMPONENT_EQUAL(best_match_icon_matcher, words[words.size() - 10 + i], "image matcher : ingredient slot " + std::to_string(i)); } { - OCR::StringMatchResult results = reader.read_with_ocr(image, global_logger_command_line(), language); + OCR::StringMatchResult results = reader.read_ingredient_page_with_ocr(image, global_logger_command_line(), language, i); if (results.results.empty()){ cerr << "No ingredient detected via text" << endl;