fix: use normalized ids for untaxonomized ingredients and specific in…

…gredients (#7131) * fix: use normalized ids for untaxonomized ingredients and specific ingredients #7063 * fix issue with specific ingredients * fix tests Co-authored-by: Alex Garel <[email protected]>
openfoodfacts · Jul 26, 2022 · d57cd90 · d57cd90
1 parent ad78f9e
commit d57cd90
Show file tree

Hide file tree

Showing 19 changed files with 286 additions and 161 deletions.
diff --git a/lib/ProductOpener/Ingredients.pm b/lib/ProductOpener/Ingredients.pm
@@ -1137,7 +1137,7 @@ sub parse_specific_ingredients_from_text($$$) {
 
 		# If we found an ingredient, save it in specific_ingredients
 		if (defined $ingredient) {
-			my $ingredient_id = canonicalize_taxonomy_tag($product_lc, "ingredients", $ingredient);
+			my $ingredient_id = get_taxonomyid($product_lc, canonicalize_taxonomy_tag($product_lc, "ingredients", $ingredient));
 
 			$matched_text =~ s/^\s+//;
 
@@ -1271,7 +1271,7 @@ sub parse_origins_from_text($$) {
 
 				my $matched_text = $matched_ingredient_ref->{matched_text};
 				my $ingredient = $matched_ingredient_ref->{ingredient};
-				my $ingredient_id = canonicalize_taxonomy_tag($product_lc, "ingredients", $ingredient);
+				my $ingredient_id = get_taxonomyid($product_lc, canonicalize_taxonomy_tag($product_lc, "ingredients", $ingredient));
 
 				# Remove extra spaces
 				$ingredient =~ s/\s+$//;
@@ -2075,7 +2075,7 @@ sub parse_ingredients_text($) {
 			if (not $skip_ingredient) {
 
 				my %ingredient = (
-					id => $ingredient_id,
+					id => get_taxonomyid($product_ref->{lc},$ingredient_id),
 					text => $ingredient
 				);
 
@@ -5749,14 +5749,10 @@ sub extract_ingredients_classes_from_text($) {
 		delete $product_ref->{$field . "_next_tags" };
 	}
 
-
-
-
 	if ((defined $product_ref->{ingredients_that_may_be_from_palm_oil_n}) or (defined $product_ref->{ingredients_from_palm_oil_n})) {
 		$product_ref->{ingredients_from_or_that_may_be_from_palm_oil_n} = $product_ref->{ingredients_that_may_be_from_palm_oil_n} + $product_ref->{ingredients_from_palm_oil_n};
 	}
 
-
 	delete $product_ref->{with_sweeteners};
 	if (defined $product_ref->{'additives_tags'}) {
 		foreach my $additive (@{$product_ref->{'additives_tags'}}) {

diff --git a/...s/unit/expected_test_results/ingredients/en-ingredients-analysis-unknown-ingredients.json b/...s/unit/expected_test_results/ingredients/en-ingredients-analysis-unknown-ingredients.json
@@ -10,14 +10,14 @@
          "vegetarian" : "yes"
       },
       {
-         "id" : "en:some unknown ingredient",
+         "id" : "en:some-unknown-ingredient",
          "percent_estimate" : 21.875,
          "percent_max" : 50,
          "percent_min" : 0,
          "text" : "some unknown ingredient"
       },
       {
-         "id" : "en:another unknown ingredient",
+         "id" : "en:another-unknown-ingredient",
          "percent_estimate" : 10.9375,
          "percent_max" : 33.3333333333333,
          "percent_min" : 0,
@@ -74,16 +74,16 @@
          "en:milk"
       ],
       "en:palm-oil-content-unknown" : [
-         "en:some unknown ingredient",
-         "en:another unknown ingredient"
+         "en:some-unknown-ingredient",
+         "en:another-unknown-ingredient"
       ],
       "en:vegan-status-unknown" : [
-         "en:some unknown ingredient",
-         "en:another unknown ingredient"
+         "en:some-unknown-ingredient",
+         "en:another-unknown-ingredient"
       ],
       "en:vegetarian-status-unknown" : [
-         "en:some unknown ingredient",
-         "en:another unknown ingredient"
+         "en:some-unknown-ingredient",
+         "en:another-unknown-ingredient"
       ]
    },
    "ingredients_analysis_tags" : [
@@ -94,8 +94,8 @@
    "ingredients_hierarchy" : [
       "en:milk",
       "en:dairy",
-      "en:some unknown ingredient",
-      "en:another unknown ingredient",
+      "en:some-unknown-ingredient",
+      "en:another-unknown-ingredient",
       "en:salt",
       "en:sugar",
       "en:added-sugar",
@@ -113,8 +113,8 @@
    ],
    "ingredients_original_tags" : [
       "en:milk",
-      "en:some unknown ingredient",
-      "en:another unknown ingredient",
+      "en:some-unknown-ingredient",
+      "en:another-unknown-ingredient",
       "en:salt",
       "en:sugar",
       "en:pepper",

diff --git a/.../unit/expected_test_results/ingredients/en-origin-field-with-not-taxonomized-entries.json b/.../unit/expected_test_results/ingredients/en-origin-field-with-not-taxonomized-entries.json
@@ -0,0 +1,104 @@
+{
+   "ingredients" : [
+      {
+         "id" : "en:peach",
+         "origins" : "en:spain",
+         "percent_estimate" : 66.6666666666667,
+         "percent_max" : 100,
+         "percent_min" : 33.3333333333333,
+         "text" : "Peaches",
+         "vegan" : "yes",
+         "vegetarian" : "yes"
+      },
+      {
+         "id" : "en:some-unknown-ingredient",
+         "origins" : "en:france",
+         "percent_estimate" : 16.6666666666667,
+         "percent_max" : 50,
+         "percent_min" : 0,
+         "text" : "Some unknown ingredient"
+      },
+      {
+         "id" : "en:another-unknown-ingredient",
+         "origins" : "en:malta",
+         "percent_estimate" : 16.6666666666667,
+         "percent_max" : 33.3333333333333,
+         "percent_min" : 0,
+         "text" : "another unknown ingredient"
+      }
+   ],
+   "ingredients_analysis" : {
+      "en:palm-oil-content-unknown" : [
+         "en:some-unknown-ingredient",
+         "en:another-unknown-ingredient"
+      ],
+      "en:vegan-status-unknown" : [
+         "en:some-unknown-ingredient",
+         "en:another-unknown-ingredient"
+      ],
+      "en:vegetarian-status-unknown" : [
+         "en:some-unknown-ingredient",
+         "en:another-unknown-ingredient"
+      ]
+   },
+   "ingredients_analysis_tags" : [
+      "en:palm-oil-content-unknown",
+      "en:vegan-status-unknown",
+      "en:vegetarian-status-unknown"
+   ],
+   "ingredients_hierarchy" : [
+      "en:peach",
+      "en:fruit",
+      "en:some-unknown-ingredient",
+      "en:another-unknown-ingredient"
+   ],
+   "ingredients_n" : 3,
+   "ingredients_n_tags" : [
+      "3",
+      "1-10"
+   ],
+   "ingredients_original_tags" : [
+      "en:peach",
+      "en:some-unknown-ingredient",
+      "en:another-unknown-ingredient"
+   ],
+   "ingredients_percent_analysis" : 1,
+   "ingredients_tags" : [
+      "en:peach",
+      "en:fruit",
+      "en:some-unknown-ingredient",
+      "en:another-unknown-ingredient"
+   ],
+   "ingredients_text" : "Peaches. Some unknown ingredient, another unknown ingredient.\nOrigin of peaches: Spain. Origin of some unknown ingredient: France. origin of Another Unknown Ingredient: Malta",
+   "ingredients_with_specified_percent_n" : 0,
+   "ingredients_with_specified_percent_sum" : 0,
+   "ingredients_with_unspecified_percent_n" : 3,
+   "ingredients_with_unspecified_percent_sum" : 100,
+   "known_ingredients_n" : 2,
+   "lc" : "en",
+   "nutriments" : {
+      "fruits-vegetables-nuts-estimate-from-ingredients_100g" : 33.3333333333333,
+      "fruits-vegetables-nuts-estimate-from-ingredients_serving" : 33.3333333333333
+   },
+   "specific_ingredients" : [
+      {
+         "id" : "en:peach",
+         "ingredient" : "peaches",
+         "origins" : "en:spain",
+         "text" : "Origin of peaches: Spain."
+      },
+      {
+         "id" : "en:some-unknown-ingredient",
+         "ingredient" : "some unknown ingredient",
+         "origins" : "en:france",
+         "text" : "Origin of some unknown ingredient: France."
+      },
+      {
+         "id" : "en:another-unknown-ingredient",
+         "ingredient" : "Another Unknown Ingredient",
+         "origins" : "en:malta",
+         "text" : "origin of Another Unknown Ingredient: Malta"
+      }
+   ],
+   "unknown_ingredients_n" : 2
+}
diff --git a/tests/unit/expected_test_results/ingredients/en-origins-u.json b/tests/unit/expected_test_results/ingredients/en-origins-u.json
@@ -1,10 +1,10 @@
 {
    "ingredients" : [
       {
-         "id" : "en:Something",
+         "id" : "en:something",
          "ingredients" : [
             {
-               "id" : "en:U",
+               "id" : "en:u",
                "percent_estimate" : 100,
                "percent_max" : 100,
                "percent_min" : 100,
@@ -19,16 +19,16 @@
    ],
    "ingredients_analysis" : {
       "en:palm-oil-content-unknown" : [
-         "en:Something",
-         "en:U"
+         "en:something",
+         "en:u"
       ],
       "en:vegan-status-unknown" : [
-         "en:Something",
-         "en:U"
+         "en:something",
+         "en:u"
       ],
       "en:vegetarian-status-unknown" : [
-         "en:Something",
-         "en:U"
+         "en:something",
+         "en:u"
       ]
    },
    "ingredients_analysis_tags" : [
@@ -37,17 +37,17 @@
       "en:vegetarian-status-unknown"
    ],
    "ingredients_hierarchy" : [
-      "en:Something",
-      "en:U"
+      "en:something",
+      "en:u"
    ],
    "ingredients_n" : 2,
    "ingredients_n_tags" : [
       "2",
       "1-10"
    ],
    "ingredients_original_tags" : [
-      "en:Something",
-      "en:U"
+      "en:something",
+      "en:u"
    ],
    "ingredients_percent_analysis" : 1,
    "ingredients_tags" : [

diff --git a/tests/unit/expected_test_results/ingredients/en-vegetal-ingredients.json b/tests/unit/expected_test_results/ingredients/en-vegetal-ingredients.json
@@ -11,7 +11,7 @@
          "vegetarian" : "en:yes"
       },
       {
-         "id" : "en:Charcoal",
+         "id" : "en:charcoal",
          "labels" : "en:vegan",
          "percent_estimate" : 20,
          "percent_max" : 50,
@@ -53,7 +53,7 @@
    ],
    "ingredients_analysis" : {
       "en:palm-oil-content-unknown" : [
-         "en:Charcoal"
+         "en:charcoal"
       ]
    },
    "ingredients_analysis_tags" : [
@@ -63,7 +63,7 @@
    ],
    "ingredients_hierarchy" : [
       "en:e428",
-      "en:Charcoal",
+      "en:charcoal",
       "en:ferment",
       "en:rennet",
       "en:enzyme",
@@ -77,7 +77,7 @@
    ],
    "ingredients_original_tags" : [
       "en:e428",
-      "en:Charcoal",
+      "en:charcoal",
       "en:ferment",
       "en:rennet",
       "en:flavouring"

diff --git a/tests/unit/expected_test_results/ingredients/fr-label-and-multiple-origins.json b/tests/unit/expected_test_results/ingredients/fr-label-and-multiple-origins.json
@@ -4,21 +4,21 @@
          "id" : "en:egg",
          "ingredients" : [
             {
-               "id" : "fr:d'élevage au sol",
+               "id" : "fr:d-elevage-au-sol",
                "percent_estimate" : 66.6666666666667,
                "percent_max" : 100,
                "percent_min" : 33.3333333333333,
                "text" : "d'élevage au sol"
             },
             {
-               "id" : "fr:Suisse",
+               "id" : "fr:suisse",
                "percent_estimate" : 16.6666666666667,
                "percent_max" : 50,
                "percent_min" : 0,
                "text" : "Suisse"
             },
             {
-               "id" : "fr:France",
+               "id" : "fr:france",
                "percent_estimate" : 16.6666666666667,
                "percent_max" : 33.3333333333333,
                "percent_min" : 0,
@@ -38,19 +38,19 @@
          "en:egg"
       ],
       "en:palm-oil-content-unknown" : [
-         "fr:d'élevage au sol",
-         "fr:Suisse",
-         "fr:France"
+         "fr:d-elevage-au-sol",
+         "fr:suisse",
+         "fr:france"
       ],
       "en:vegan-status-unknown" : [
-         "fr:d'élevage au sol",
-         "fr:Suisse",
-         "fr:France"
+         "fr:d-elevage-au-sol",
+         "fr:suisse",
+         "fr:france"
       ],
       "en:vegetarian-status-unknown" : [
-         "fr:d'élevage au sol",
-         "fr:Suisse",
-         "fr:France"
+         "fr:d-elevage-au-sol",
+         "fr:suisse",
+         "fr:france"
       ]
    },
    "ingredients_analysis_tags" : [
@@ -60,9 +60,9 @@
    ],
    "ingredients_hierarchy" : [
       "en:egg",
-      "fr:d'élevage au sol",
-      "fr:Suisse",
-      "fr:France"
+      "fr:d-elevage-au-sol",
+      "fr:suisse",
+      "fr:france"
    ],
    "ingredients_n" : 4,
    "ingredients_n_tags" : [
@@ -71,9 +71,9 @@
    ],
    "ingredients_original_tags" : [
       "en:egg",
-      "fr:d'élevage au sol",
-      "fr:Suisse",
-      "fr:France"
+      "fr:d-elevage-au-sol",
+      "fr:suisse",
+      "fr:france"
    ],
    "ingredients_percent_analysis" : 1,
    "ingredients_tags" : [