[OVC] Import openvino-tokenizers extensions from ovc tool (openvinoto…

…olkit#24681)

**Details:** Previously, we were able to convert models requiring
openvino-tokenizers extensions (like with string operations, ragged
tensors) only in Python by `convert_model`. Now we add option to convert
using `ovc` tool. The tool internally imports extensions if they are
installed.

**Ticket:** TBD

---------

Signed-off-by: Kazantsev, Roman <[email protected]>

src/frontends/tensorflow/src/frontend.cpp

@@ -469,7 +469,7 @@ std::shared_ptr<ov::Model> FrontEnd::convert(const ov::frontend::InputModel::Ptr
                 << "\nEncountered unconverted operation(s) for which openvino-tokenizers package "
                    "provides conversion extension(s): "
                 << unsupported_ops_from_tokenizers
-                << ". Refer to OpenVINO Tokenizers documentation: "
+                << ". Install OpenVINO Tokenizers, refer to the documentation: "
                    "https://docs.openvino.ai/2024/learn-openvino/llm_inference_guide/ov-tokenizers.html \n";
         }
     }

src/frontends/tensorflow/tests/convert_unsupported.cpp

@@ -196,7 +196,7 @@ TEST(FrontEndConvertModelTest, test_unsupported_resource_gather_translator) {
         std::string no_ref_message = "[TensorFlow Frontend] Internal error: No translator found for";
         ASSERT_TRUE(error_message.find(ref_message) != std::string::npos);
         ASSERT_TRUE(error_message.find(no_ref_message) == std::string::npos);
-        std::string ref_message2 = "Refer to OpenVINO Tokenizers documentation";
+        std::string ref_message2 = "Install OpenVINO Tokenizers, refer to the documentation";
         ASSERT_TRUE(error_message.find(ref_message2) == std::string::npos);
         ASSERT_EQ(model, nullptr);
     } catch (...) {
@@ -234,7 +234,7 @@ TEST(FrontEndConvertModelTest, conversion_with_tokenizer_operation) {
         FAIL() << "TensorFlow 1 model with StringLower operation is not supported without openvino-tokenizers.";
     } catch (const OpConversionFailure& error) {
         std::string error_message = error.what();
-        std::string ref_message = "Refer to OpenVINO Tokenizers documentation";
+        std::string ref_message = "Install OpenVINO Tokenizers, refer to the documentation";
         ASSERT_TRUE(error_message.find(ref_message) != std::string::npos);
         ASSERT_EQ(model, nullptr);
     } catch (...) {

tests/layer_tests/common/utils/common_utils.py

@@ -2,15 +2,14 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+import numpy as np
 import os
 import platform
 import shutil
 import subprocess
 import sys
 from pathlib import Path
 
-import numpy as np
-
 logger = logging.getLogger(__name__)
 
 
@@ -75,6 +74,16 @@ def generate_ir_python_api(coverage=False, **kwargs):
     return 0, ""
 
 
+def generate_ir_ovc(input_model, opts):
+    params = ['ovc', input_model]
+    for key, value in opts.items():
+        # handle optional arguments
+        # both key and values are of string type
+        params.extend(("--{}".format(key), value))
+    exit_code, stdout, stderr = shell(params)
+    return exit_code, stdout, stderr
+
+
 def shell(cmd, env=None, cwd=None, out_format="plain"):
     """
     Run command execution in specified environment

tests/layer_tests/tensorflow_tests/test_tf_StringLower.py

@@ -1,12 +1,13 @@
 # Copyright (C) 2018-2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-import platform
-
 import numpy as np
+import os
+import platform
 import pytest
 import tensorflow as tf
 from common.tf_layer_test_class import CommonTFLayerTest
+from common.utils.common_utils import generate_ir_ovc
 from common.utils.tf_utils import run_in_jenkins
 
 rng = np.random.default_rng()
@@ -57,3 +58,45 @@ def test_string_lower(self, input_shape, encoding, strings_dictionary, ie_device
                                                  strings_dictionary=strings_dictionary),
                    ie_device, precision, ir_version, temp_dir=temp_dir,
                    use_legacy_frontend=use_legacy_frontend)
+
+
+class TestStringLowerOVC:
+    def prepare_data(self):
+        inputs_data = {}
+        inputs_data['input:0'] = np.array(['Some sentence', 'ANOTHER sentenCE'], dtype=str)
+        ref_data = np.array(['some sentence', 'another sentence'], dtype=str)
+        return inputs_data, ref_data
+
+    def create_string_lower_model(self, output_dir):
+        tf.compat.v1.reset_default_graph()
+        with tf.compat.v1.Session() as sess:
+            input = tf.compat.v1.placeholder(tf.string, [2], 'input')
+            tf.raw_ops.StringLower(input=input, name='StringLower')
+            tf.compat.v1.global_variables_initializer()
+            tf.compat.v1.io.write_graph(sess.graph, output_dir, 'model_string_lower.pb', as_text=False)
+        return os.path.join(output_dir, 'model_string_lower.pb')
+
+    @pytest.mark.precommit
+    @pytest.mark.nightly
+    @pytest.mark.xfail(condition=platform.system() in ('Darwin', 'Linux') and platform.machine() in ['arm', 'armv7l',
+                                                                                                     'aarch64',
+                                                                                                     'arm64', 'ARM64'],
+                       reason='Ticket - 126314, 132699')
+    def test_string_lower_with_ovc(self, ie_device, temp_dir, precision):
+        if ie_device == 'GPU' or run_in_jenkins():
+            pytest.skip("operation extension is not supported on GPU")
+        input_model_path = self.create_string_lower_model(temp_dir)
+        output_model_path = os.path.join(temp_dir, 'model_string_lower.xml')
+        return_code, _, _ = generate_ir_ovc(input_model_path, {'output_model': output_model_path})
+        assert return_code == 0, "OVC tool is failed for conversion model {}".format(input_model_path)
+
+        import openvino_tokenizers
+        import openvino as ov
+        core = ov.Core()
+        compiled_model = core.compile_model(output_model_path, ie_device)
+        input_data, ref_data = self.prepare_data()
+        ov_result = compiled_model(input_data)['StringLower:0']
+
+        assert np.array_equal(ov_result, ref_data), 'OpenVINO result does not match the reference:' \
+                                                    'OpenVINO result - {},' \
+                                                    'Reference - {}'.format(ov_result, ref_data)

tools/ovc/openvino/tools/ovc/main.py

@@ -10,6 +10,11 @@
     import openvino.tools.ovc.telemetry_stub as tm
 from openvino.tools.ovc.convert_impl import _convert
 from openvino.tools.ovc.cli_parser import get_model_name_from_args
+from openvino.tools.ovc.utils import import_openvino_tokenizers
+
+# TODO 131000: temporal workaround to patch OpenVINO Core and frontends with tokenizers extensions
+# make OVC tool to convert models requiring openvino-tokenizers extensions
+import_openvino_tokenizers()
 
 # pylint: disable=no-name-in-module,import-error
 from openvino.runtime import save_model

tools/ovc/openvino/tools/ovc/utils.py

@@ -1,18 +1,25 @@
 # Copyright (C) 2018-2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-import os
-from typing import Iterable, Union
-
+import importlib.util
+import logging as log
 import numpy as np
+import os
+import sys
 from openvino.tools.ovc.error import Error
+from typing import Iterable, Union
 
 try:
     import openvino_telemetry as tm
     from openvino_telemetry.backend import backend_ga4
 except ImportError:
     import openvino.tools.ovc.telemetry_stub as tm
 
+if sys.version_info < (3, 8):
+    import importlib_metadata
+else:
+    import importlib.metadata as importlib_metadata
+
 dynamic_dimension = np.ma.masked
 
 
@@ -107,3 +114,83 @@ def get_ir_version():
     :return: the IR version
     """
     return 11
+
+
+def import_openvino_tokenizers():
+    # extract openvino version
+    if importlib.util.find_spec("openvino") is None:
+        return False
+    try:
+        from openvino import get_version
+        openvino_version = get_version()
+        openvino_available = True
+    except ImportError:
+        openvino_available = False
+    if not openvino_available:
+        return False
+
+    if importlib.util.find_spec("openvino_tokenizers") is None:
+        return False
+
+    try:
+        pip_metadata_version = importlib_metadata.version("openvino")
+    except importlib_metadata.PackageNotFoundError:
+        pip_metadata_version = False
+    try:
+        pip_metadata_version = importlib_metadata.version("openvino-nightly")
+        is_nightly = True
+    except importlib_metadata.PackageNotFoundError:
+        is_nightly = False
+
+    try:
+        import openvino_tokenizers  # pylint: disable=no-name-in-module,import-error
+
+        openvino_tokenizers._get_factory()
+    except RuntimeError:
+        tokenizers_version = openvino_tokenizers.__version__
+
+        if tokenizers_version == "0.0.0.0":
+            try:
+                tokenizers_version = importlib_metadata.version("openvino_tokenizers") or tokenizers_version
+            except importlib_metadata.PackageNotFoundError:
+                pass
+        message = (
+            "OpenVINO and OpenVINO Tokenizers versions are not binary compatible.\n"
+            f"OpenVINO version:            {openvino_version}\n"
+            f"OpenVINO Tokenizers version: {tokenizers_version}\n"
+            "First 3 numbers should be the same. Update OpenVINO Tokenizers to compatible version. "
+        )
+        if not pip_metadata_version:
+            message += (
+                "For archive installation of OpenVINO try to build OpenVINO Tokenizers from source: "
+                "https://github.com/openvinotoolkit/openvino_tokenizers/tree/master?tab=readme-ov-file"
+                "#build-and-install-from-source"
+            )
+            if sys.platform == "linux":
+                message += (
+                    "\nThe PyPI version of OpenVINO Tokenizers is built on CentOS and may not be compatible with other "
+                    "Linux distributions; rebuild OpenVINO Tokenizers from source."
+                )
+        else:
+            message += (
+                "It is recommended to use the same day builds for pre-release version. "
+                "To install both OpenVINO and OpenVINO Tokenizers release version perform:\n"
+            )
+            if is_nightly:
+                message += "pip uninstall -y openvino-nightly && "
+            message += "pip install --force-reinstall openvino openvino-tokenizers\n"
+            if is_nightly:
+                message += (
+                    "openvino-nightly package will be deprecated in the future - use pre-release drops instead. "
+                )
+            message += "To update both OpenVINO and OpenVINO Tokenizers to the latest pre-release version perform:\n"
+            if is_nightly:
+                message += "pip uninstall -y openvino-nightly && "
+            message += (
+                "pip install --pre -U openvino openvino-tokenizers "
+                "--extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly"
+            )
+        log.warning(message)
+        return False
+
+    return True