diff --git a/notebooks/keras_fashion_mnist.ipynb b/notebooks/keras_fashion_mnist.ipynb index a1e2b36..6e751ec 100644 --- a/notebooks/keras_fashion_mnist.ipynb +++ b/notebooks/keras_fashion_mnist.ipynb @@ -16,7 +16,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "modelscan, version 0.5.0\n" + "modelscan, version 0.5.1\n" ] } ], @@ -48,6 +48,37 @@ "from utils.tensorflow_fashion_mnist_model import train_model, get_predictions" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Optional Settings File for ModelScan\n", + "\n", + "ModelScan scan settings can be configured using a settings file. \n", + "\n", + "- To create a configurable settings file use: `modelscan create-settings-file` (creates a `modelscan-settings.toml` file in current directory). \n", + "\n", + "- The location and name of a settings file can also be specified with `modelscan create-settings-file -l ../path-to/settings-file.toml` \n", + "\n", + "Configurations:\n", + "- A settings file would allow enabling/disabling particular scans such as H5LambdaDetectScan, and PickleUnsafeOpScan. \n", + "\n", + "- A settings file can also be configured to set particular severity level (CRITICAL, HIGH, MEDIUM, or LOW) for unsafe operators. \n", + "\n", + "- Reporting format for ModelScan results can also be specified in a settings file.\n", + "\n", + "If you would like to create a configurable settings file, uncomment the code in the next cell and run it. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "#!modelscan create-settings-file -l my-modelscan-settings.toml" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -59,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -67,35 +98,35 @@ "output_type": "stream", "text": [ "Epoch 1/10\n", - "1875/1875 [==============================] - 2s 1ms/step - loss: 0.4986 - accuracy: 0.8258\n", + "1875/1875 [==============================] - 1s 726us/step - loss: 0.4961 - accuracy: 0.8248\n", "Epoch 2/10\n", - "1875/1875 [==============================] - 2s 800us/step - loss: 0.3741 - accuracy: 0.8649\n", + "1875/1875 [==============================] - 1s 689us/step - loss: 0.3736 - accuracy: 0.8641\n", "Epoch 3/10\n", - "1875/1875 [==============================] - 1s 772us/step - loss: 0.3391 - accuracy: 0.8766\n", + "1875/1875 [==============================] - 1s 718us/step - loss: 0.3351 - accuracy: 0.8782\n", "Epoch 4/10\n", - "1875/1875 [==============================] - 1s 755us/step - loss: 0.3125 - accuracy: 0.8852\n", + "1875/1875 [==============================] - 1s 716us/step - loss: 0.3147 - accuracy: 0.8829\n", "Epoch 5/10\n", - "1875/1875 [==============================] - 2s 965us/step - loss: 0.2955 - accuracy: 0.8908\n", + "1875/1875 [==============================] - 1s 713us/step - loss: 0.2960 - accuracy: 0.8908\n", "Epoch 6/10\n", - "1875/1875 [==============================] - 2s 905us/step - loss: 0.2808 - accuracy: 0.8963\n", + "1875/1875 [==============================] - 1s 715us/step - loss: 0.2797 - accuracy: 0.8963\n", "Epoch 7/10\n", - "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2696 - accuracy: 0.9004\n", + "1875/1875 [==============================] - 1s 715us/step - loss: 0.2672 - accuracy: 0.9008\n", "Epoch 8/10\n", - "1875/1875 [==============================] - 2s 895us/step - loss: 0.2587 - accuracy: 0.9039\n", + "1875/1875 [==============================] - 1s 723us/step - loss: 0.2580 - accuracy: 0.9044\n", "Epoch 9/10\n", - "1875/1875 [==============================] - 2s 925us/step - loss: 0.2479 - accuracy: 0.9068\n", + "1875/1875 [==============================] - 1s 733us/step - loss: 0.2470 - accuracy: 0.9067\n", "Epoch 10/10\n", - "1875/1875 [==============================] - 2s 893us/step - loss: 0.2406 - accuracy: 0.9112\n", - "313/313 [==============================] - 0s 555us/step - loss: 0.3496 - accuracy: 0.8790\n", + "1875/1875 [==============================] - 1s 700us/step - loss: 0.2406 - accuracy: 0.9097\n", + "313/313 [==============================] - 0s 480us/step - loss: 0.3465 - accuracy: 0.8782\n", "\n", - "Model trained with test accuracy: 0.8790000081062317\n" + "Model trained with test accuracy: 0.8781999945640564\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/Users/mehrinkiani/mambaforge/envs/py310/lib/python3.10/site-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n", + "/Users/mehrinkiani/mambaforge/envs/modelscan/lib/python3.10/site-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n", " saving_api.save_model(\n" ] } @@ -119,16 +150,16 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "1/1 [==============================] - 0s 42ms/step\n", + "1/1 [==============================] - 0s 34ms/step\n", "\n", - "The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 98.948 99.926 100. ]\n", + "The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 95.851 99.163 100. ]\n", "\n", "The true labels are ['Ankle boot', 'Pullover', 'Trouser']\n" ] @@ -153,15 +184,22 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Scan the Safe Model\n", + "# Scan Safe Model\n", + "\n", + "\n", + "The scan results include information on the files scanned, and any issues if found. For the safe model scanned, modelscan finds no model serialization attacks, as expected.\n", + "\n", + "### ModelScan Settings File\n", + "- If you have created a settings file with default name and location (`modelscan-settings.toml`) it would automatically be used when scanning a model. \n", "\n", + "- If you want to use a specific name and/or location of settings file, please specify it with `--settings-file` as outlined in the next cell. \n", "\n", - "The scan results include information on the files scanned, and any issues if found. For the safe model scanned, modelscan finds no model serialization attacks, as expected." + "- If you have not created any settings file, ModelScan will scan using default settings. " ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -170,15 +208,19 @@ "text": [ "No settings file detected at /Users/mehrinkiani/Documents/modelscan/notebooks/modelscan-settings.toml. Using defaults. \n", "\n", - "Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/safe_model.h5 using modelscan.scanners.H5LambdaDetectScan model scan\n", "\n", "\u001b[34m--- Summary ---\u001b[0m\n", "\n", - "\u001b[32m No issues found! 🎉\u001b[0m\n" + "\u001b[32m No issues found! 🎉\u001b[0m\n", + "\n", + "\u001b[34m--- Skipped --- \u001b[0m\n", + "\n", + "Total skipped: \u001b[1;36m1\u001b[0m - run with --show-skipped to see the full list.\n" ] } ], "source": [ + "#!modelscan -p ./KerasModels/safe_model.h5 --settings-file my-modelscan-settings.toml\n", "!modelscan -p ./KerasModels/safe_model.h5" ] }, @@ -193,7 +235,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -222,7 +264,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -244,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -255,9 +297,9 @@ "aws_secret_access_key=\n", "aws_access_key_id=\n", "aws_secret_access_key=\n", - "1/1 [==============================] - 0s 35ms/step\n", + "1/1 [==============================] - 0s 26ms/step\n", "\n", - "The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 98.948 99.926 100. ]\n", + "The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 95.851 99.163 100. ]\n", "\n", "The true labels are ['Ankle boot', 'Pullover', 'Trouser']\n" ] @@ -284,16 +326,23 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Scan the Unsafe Model\n", + "# Scan Unsafe Model\n", "\n", "The scan results include information on the files scanned, and any issues if found. In this case, a critical severity level issue is found in the unsafe model scanned. \n", "\n", - "modelscan also outlines the found operator(s) and module(s) deemed unsafe. " + "ModelScan also outlines the found operator(s) and module(s) deemed unsafe. \n", + "\n", + "### ModelScan Settings File\n", + "- If you have created a settings file with default name and location (`modelscan-settings.toml`) it would automatically be used when scanning a model. \n", + "\n", + "- If you want to use a specific name and/or location of settings file, please specify it with `--settings-file` as outlined in the next cell. \n", + "\n", + "- If you have not created any settings file, ModelScan will scan using default settings." ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -327,6 +376,7 @@ } ], "source": [ + "#!modelscan -p KerasModels/unsafe_model.h5 --settings-file my-modelscan-settings.toml\n", "!modelscan -p KerasModels/unsafe_model.h5" ] }, @@ -349,7 +399,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -359,7 +409,7 @@ "No settings file detected at /Users/mehrinkiani/Documents/modelscan/notebooks/modelscan-settings.toml. Using defaults. \n", "\n", "Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/unsafe_model.h5 using modelscan.scanners.H5LambdaDetectScan model scan\n", - "\u001b[1m{\u001b[0m\u001b[32m\"modelscan_version\"\u001b[0m: \u001b[32m\"0.5.0\"\u001b[0m, \u001b[32m\"timestamp\"\u001b[0m: \u001b[32m\"2024-01-25T17:56:36.776966\"\u001b[0m, \n", + "\u001b[1m{\u001b[0m\u001b[32m\"modelscan_version\"\u001b[0m: \u001b[32m\"0.5.1\"\u001b[0m, \u001b[32m\"timestamp\"\u001b[0m: \u001b[32m\"2024-02-06T11:00:43.503049\"\u001b[0m, \n", "\u001b[32m\"input_path\"\u001b[0m: \n", "\u001b[32m\"/Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/unsafe_model.h5\"\u001b[0m, \n", "\u001b[32m\"total_issues\"\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m\"summary\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"total_issues_by_severity\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"LOW\"\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m\"MEDIUM\"\u001b[0m: \n", @@ -389,7 +439,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10.13 ('py310')", + "display_name": "Python 3.10.13 ('modelscan')", "language": "python", "name": "python3" }, @@ -408,7 +458,7 @@ "orig_nbformat": 4, "vscode": { "interpreter": { - "hash": "bd638e2064d9001d4ca93bc8e56e039dad230900dd235e8a6196f1614960903a" + "hash": "4dc76e75b8580a33bdc5a5a988b23a4143f70625528e602e356063408d11834c" } } }, diff --git a/notebooks/pytorch_sentiment_analysis.ipynb b/notebooks/pytorch_sentiment_analysis.ipynb index 282439b..fb813a0 100644 --- a/notebooks/pytorch_sentiment_analysis.ipynb +++ b/notebooks/pytorch_sentiment_analysis.ipynb @@ -7,13 +7,6 @@ "# Setup " ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Installing ModelScan" - ] - }, { "cell_type": "code", "execution_count": 1, @@ -23,7 +16,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "modelscan, version 0.5.0\n" + "modelscan, version 0.5.1\n" ] } ], @@ -52,7 +45,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/mehrinkiani/mambaforge/envs/py310/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + "/Users/mehrinkiani/mambaforge/envs/modelscan/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", " from .autonotebook import tqdm as notebook_tqdm\n" ] }, @@ -72,6 +65,37 @@ "%env TOKENIZERS_PARALLELISM=false\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Optional Settings File for ModelScan\n", + "\n", + "ModelScan scan settings can be configured using a settings file. \n", + "\n", + "- To create a configurable settings file use: `modelscan create-settings-file` (creates a `modelscan-settings.toml` file in current directory). \n", + "\n", + "- The location and name of a settings file can also be specified with `modelscan create-settings-file -l ../path-to/settings-file.toml` \n", + "\n", + "Configurations:\n", + "- A settings file would allow enabling/disabling particular scans such as H5LambdaDetectScan, and PickleUnsafeOpScan. \n", + "\n", + "- A settings file can also be configured to set particular severity level (CRITICAL, HIGH, MEDIUM, or LOW) for unsafe operators. \n", + "\n", + "- Reporting format for ModelScan results can also be specified in a settings file.\n", + "\n", + "If you would like to create a configurable settings file, uncomment the code in the next cell and run it. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "#!modelscan create-settings-file -l my-modelscan-settings.toml" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -84,7 +108,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -107,7 +131,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -128,12 +152,19 @@ "source": [ "# Scan Safe Model\n", "\n", - "The scan results include information on the files scanned, and any issues if found. For the safe model scanned, modelscan finds no model serialization attacks, as expected." + "The scan results include information on the files scanned, and any issues if found. For the safe model scanned, modelscan finds no model serialization attacks, as expected.\n", + "\n", + "### ModelScan Settings File\n", + "- If you have created a settings file with default name and location (`modelscan-settings.toml`) it would automatically be used when scanning a model. \n", + "\n", + "- If you want to use a specific name and/or location of settings file, please specify it with `--settings-file` as outlined in the next cell. \n", + "\n", + "- If you have not created any settings file, ModelScan will scan using default settings. " ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -151,13 +182,15 @@ } ], "source": [ - "!modelscan --path PyTorchModels/safe_model.pt" + "#!modelscan --path PyTorchModels/safe_model.pt --settings-file my-modelscan-settings.toml\n", + "!modelscan --path PyTorchModels/safe_model.pt " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ + "\n", "# Model Serialization Attack\n", "\n", "Here malicious code is injected in the safe model to read aws secret keys. The unsafe model is saved at `./PyTorchModels/unsafe_model.pt`" @@ -165,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -197,7 +230,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -222,12 +255,19 @@ "\n", "The scan results include information on the files scanned, and any issues if found. In this case, a critical severity level issue is found in the unsafe model scanned. \n", "\n", - "modelscan also outlines the found operator(s) and module(s) deemed unsafe. " + "ModelScan also outlines the found operator(s) and module(s) deemed unsafe. \n", + "\n", + "### ModelScan Settings File\n", + "- If you have created a settings file with default name and location (`modelscan-settings.toml`) it would automatically be used when scanning a model. \n", + "\n", + "- If you want to use a specific name and/or location of settings file, please specify it with `--settings-file` as outlined in the next cell. \n", + "\n", + "- If you have not created any settings file, ModelScan will scan using default settings." ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -261,6 +301,7 @@ } ], "source": [ + "#!modelscan --path PyTorchModels/unsafe_model.pt --settings-file my-modelscan-settings.toml\n", "!modelscan --path ./PyTorchModels/unsafe_model.pt " ] }, @@ -283,7 +324,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -293,7 +334,7 @@ "No settings file detected at /Users/mehrinkiani/Documents/modelscan/notebooks/modelscan-settings.toml. Using defaults. \n", "\n", "Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/PyTorchModels/unsafe_model.pt:unsafe_model/data.pkl using modelscan.scanners.PickleUnsafeOpScan model scan\n", - "\u001b[1m{\u001b[0m\u001b[32m\"modelscan_version\"\u001b[0m: \u001b[32m\"0.5.0\"\u001b[0m, \u001b[32m\"timestamp\"\u001b[0m: \u001b[32m\"2024-01-25T17:10:54.306065\"\u001b[0m, \n", + "\u001b[1m{\u001b[0m\u001b[32m\"modelscan_version\"\u001b[0m: \u001b[32m\"0.5.1\"\u001b[0m, \u001b[32m\"timestamp\"\u001b[0m: \u001b[32m\"2024-02-06T10:51:35.006602\"\u001b[0m, \n", "\u001b[32m\"input_path\"\u001b[0m: \n", "\u001b[32m\"/Users/mehrinkiani/Documents/modelscan/notebooks/PyTorchModels/unsafe_model.pt\"\u001b[0m\n", ", \u001b[32m\"total_issues\"\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m\"summary\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"total_issues_by_severity\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"LOW\"\u001b[0m: \u001b[1;36m0\u001b[0m, \n", @@ -312,18 +353,11 @@ "# This will save the scan results in file: pytorch-model-scan-results.json\n", "!modelscan --path ./PyTorchModels/unsafe_model.pt -r json -o pytorch-model-scan-results.json" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3.10.13 ('py310')", + "display_name": "Python 3.10.13 ('modelscan')", "language": "python", "name": "python3" }, @@ -341,7 +375,7 @@ }, "vscode": { "interpreter": { - "hash": "bd638e2064d9001d4ca93bc8e56e039dad230900dd235e8a6196f1614960903a" + "hash": "4dc76e75b8580a33bdc5a5a988b23a4143f70625528e602e356063408d11834c" } } }, diff --git a/notebooks/tensorflow_fashion_mnist.ipynb b/notebooks/tensorflow_fashion_mnist.ipynb index bdd48e6..a208ca3 100644 --- a/notebooks/tensorflow_fashion_mnist.ipynb +++ b/notebooks/tensorflow_fashion_mnist.ipynb @@ -16,7 +16,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "modelscan, version 0.5.0\n" + "modelscan, version 0.5.1\n" ] } ], @@ -49,6 +49,37 @@ "from utils.tensorflow_codeinjection import MaliciousModule" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Optional Settings File for ModelScan\n", + "\n", + "ModelScan scan settings can be configured using a settings file. \n", + "\n", + "- To create a configurable settings file use: `modelscan create-settings-file` (creates a `modelscan-settings.toml` file in current directory). \n", + "\n", + "- The location and name of a settings file can also be specified with `modelscan create-settings-file -l ../path-to/settings-file.toml` \n", + "\n", + "Configurations:\n", + "- A settings file would allow enabling/disabling particular scans such as H5LambdaDetectScan, and PickleUnsafeOpScan. \n", + "\n", + "- A settings file can also be configured to set particular severity level (CRITICAL, HIGH, MEDIUM, or LOW) for unsafe operators. \n", + "\n", + "- Reporting format for ModelScan results can also be specified in a settings file.\n", + "\n", + "If you would like to create a configurable settings file, uncomment the code in the next cell and run it. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "#!modelscan create-settings-file -l my-modelscan-settings.toml" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -60,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -68,28 +99,28 @@ "output_type": "stream", "text": [ "Epoch 1/10\n", - "1875/1875 [==============================] - 2s 1ms/step - loss: 0.4993 - accuracy: 0.8256\n", + "1875/1875 [==============================] - 1s 726us/step - loss: 0.5043 - accuracy: 0.8241\n", "Epoch 2/10\n", - "1875/1875 [==============================] - 2s 915us/step - loss: 0.3759 - accuracy: 0.8643\n", + "1875/1875 [==============================] - 1s 723us/step - loss: 0.3777 - accuracy: 0.8636\n", "Epoch 3/10\n", - "1875/1875 [==============================] - 2s 925us/step - loss: 0.3380 - accuracy: 0.8761\n", + "1875/1875 [==============================] - 1s 697us/step - loss: 0.3380 - accuracy: 0.8758\n", "Epoch 4/10\n", - "1875/1875 [==============================] - 2s 919us/step - loss: 0.3125 - accuracy: 0.8852\n", + "1875/1875 [==============================] - 1s 695us/step - loss: 0.3134 - accuracy: 0.8849\n", "Epoch 5/10\n", - "1875/1875 [==============================] - 2s 947us/step - loss: 0.2955 - accuracy: 0.8910\n", + "1875/1875 [==============================] - 1s 710us/step - loss: 0.2945 - accuracy: 0.8917\n", "Epoch 6/10\n", - "1875/1875 [==============================] - 2s 839us/step - loss: 0.2819 - accuracy: 0.8951\n", + "1875/1875 [==============================] - 1s 718us/step - loss: 0.2813 - accuracy: 0.8960\n", "Epoch 7/10\n", - "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2708 - accuracy: 0.9000\n", + "1875/1875 [==============================] - 1s 704us/step - loss: 0.2675 - accuracy: 0.9005\n", "Epoch 8/10\n", - "1875/1875 [==============================] - 2s 983us/step - loss: 0.2577 - accuracy: 0.9047\n", + "1875/1875 [==============================] - 1s 714us/step - loss: 0.2569 - accuracy: 0.9038\n", "Epoch 9/10\n", - "1875/1875 [==============================] - 2s 983us/step - loss: 0.2496 - accuracy: 0.9065\n", + "1875/1875 [==============================] - 1s 702us/step - loss: 0.2467 - accuracy: 0.9072\n", "Epoch 10/10\n", - "1875/1875 [==============================] - 2s 818us/step - loss: 0.2387 - accuracy: 0.9109\n", - "313/313 [==============================] - 0s 564us/step - loss: 0.4170 - accuracy: 0.8603\n", + "1875/1875 [==============================] - 1s 698us/step - loss: 0.2388 - accuracy: 0.9102\n", + "313/313 [==============================] - 0s 500us/step - loss: 0.3476 - accuracy: 0.8765\n", "\n", - "Model trained with test accuracy: 0.8603000044822693\n" + "Model trained with test accuracy: 0.8765000104904175\n" ] } ], @@ -113,16 +144,16 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "1/1 [==============================] - 0s 36ms/step\n", + "1/1 [==============================] - 0s 34ms/step\n", "\n", - "The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 94.768 65.806 100. ]\n", + "The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 94.972 99.955 100. ]\n", "\n", "The true labels are ['Ankle boot', 'Pullover', 'Trouser']\n" ] @@ -147,14 +178,21 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Scan the Safe Model \n", + "# Scan Safe Model \n", "\n", - "The scan results include information on the files scanned, and any issues if found. For the safe model scanned, modelscan finds no model serialization attacks, as expected." + "The scan results include information on the files scanned, and any issues if found. For the safe model scanned, modelscan finds no model serialization attacks, as expected.\n", + "\n", + "### ModelScan Settings File\n", + "- If you have created a settings file with default name and location (`modelscan-settings.toml`) it would automatically be used when scanning a model. \n", + "\n", + "- If you want to use a specific name and/or location of settings file, please specify it with `--settings-file` as outlined in the next cell. \n", + "\n", + "- If you have not created any settings file, ModelScan will scan using default settings. " ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -178,6 +216,7 @@ } ], "source": [ + "#!modelscan -p ./TensorFlowModels/safe_model --settings-file my-modelscan-settings.toml\n", "!modelscan -p ./TensorFlowModels/safe_model" ] }, @@ -192,7 +231,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -217,7 +256,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -227,9 +266,9 @@ "File found: /tmp/aws_secret.txt\n", "aws_access_key_id=\n", "aws_secret_access_key=\n", - "1/1 [==============================] - 0s 48ms/step\n", + "1/1 [==============================] - 0s 42ms/step\n", "\n", - "The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 94.768 65.806 100. ]\n", + "The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 94.972 99.955 100. ]\n", "\n", "The true labels are ['Ankle boot', 'Pullover', 'Trouser']\n" ] @@ -256,16 +295,23 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Scan the Unsafe Model\n", + "# Scan Unsafe Model\n", "\n", "The scan results include information on the files scanned, and any issues if found. In this case, a critical severity level issue is found in the unsafe model scanned. \n", "\n", - "modelscan also outlines the found operator(s) and module(s) deemed unsafe. " + "ModelScan also outlines the found operator(s) and module(s) deemed unsafe. \n", + "\n", + "### ModelScan Settings File\n", + "- If you have created a settings file with default name and location (`modelscan-settings.toml`) it would automatically be used when scanning a model. \n", + "\n", + "- If you want to use a specific name and/or location of settings file, please specify it with `--settings-file` as outlined in the next cell. \n", + "\n", + "- If you have not created any settings file, ModelScan will scan using default settings." ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -310,6 +356,7 @@ } ], "source": [ + "#!modelscan -p TensorFlowModels/unsafe_model --settings-file my-modelscan-settings.toml\n", "!modelscan -p TensorFlowModels/unsafe_model" ] }, @@ -332,7 +379,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -344,7 +391,7 @@ "Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/TensorFlowModels/unsafe_model/fingerprint.pb using modelscan.scanners.SavedModelTensorflowOpScan model scan\n", "Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/TensorFlowModels/unsafe_model/keras_metadata.pb using modelscan.scanners.SavedModelLambdaDetectScan model scan\n", "Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/TensorFlowModels/unsafe_model/saved_model.pb using modelscan.scanners.SavedModelTensorflowOpScan model scan\n", - "\u001b[1m{\u001b[0m\u001b[32m\"modelscan_version\"\u001b[0m: \u001b[32m\"0.5.0\"\u001b[0m, \u001b[32m\"timestamp\"\u001b[0m: \u001b[32m\"2024-01-25T17:56:46.559473\"\u001b[0m, \n", + "\u001b[1m{\u001b[0m\u001b[32m\"modelscan_version\"\u001b[0m: \u001b[32m\"0.5.1\"\u001b[0m, \u001b[32m\"timestamp\"\u001b[0m: \u001b[32m\"2024-02-06T10:59:34.374646\"\u001b[0m, \n", "\u001b[32m\"input_path\"\u001b[0m: \n", "\u001b[32m\"/Users/mehrinkiani/Documents/modelscan/notebooks/TensorFlowModels/unsafe_model\"\u001b[0m\n", ", \u001b[32m\"total_issues\"\u001b[0m: \u001b[1;36m2\u001b[0m, \u001b[32m\"summary\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"total_issues_by_severity\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"LOW\"\u001b[0m: \u001b[1;36m0\u001b[0m, \n", @@ -382,7 +429,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10.13 ('py310')", + "display_name": "Python 3.10.13 ('modelscan')", "language": "python", "name": "python3" }, @@ -401,7 +448,7 @@ "orig_nbformat": 4, "vscode": { "interpreter": { - "hash": "bd638e2064d9001d4ca93bc8e56e039dad230900dd235e8a6196f1614960903a" + "hash": "4dc76e75b8580a33bdc5a5a988b23a4143f70625528e602e356063408d11834c" } } }, diff --git a/notebooks/xgboost_diabetes_classification.ipynb b/notebooks/xgboost_diabetes_classification.ipynb index 4638d19..ccf0bf9 100644 --- a/notebooks/xgboost_diabetes_classification.ipynb +++ b/notebooks/xgboost_diabetes_classification.ipynb @@ -18,7 +18,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "modelscan, version 0.5.0\n" + "modelscan, version 0.5.1\n" ] } ], @@ -53,19 +53,52 @@ "from utils.xgboost_diabetes_model import train_model, get_predictions" ] }, + { + "cell_type": "markdown", + "id": "ea9cdc7f", + "metadata": {}, + "source": [ + "## Optional Settings File for ModelScan\n", + "\n", + "ModelScan scan settings can be configured using a settings file. \n", + "\n", + "- To create a configurable settings file use: `modelscan create-settings-file` (creates a `modelscan-settings.toml` file in current directory). \n", + "\n", + "- The location and name of a settings file can also be specified with `modelscan create-settings-file -l ../path-to/settings-file.toml` \n", + "\n", + "Configurations:\n", + "- A settings file would allow enabling/disabling particular scans such as H5LambdaDetectScan, and PickleUnsafeOpScan. \n", + "\n", + "- A settings file can also be configured to set particular severity level (CRITICAL, HIGH, MEDIUM, or LOW) for unsafe operators. \n", + "\n", + "- Reporting format for ModelScan results can also be specified in a settings file.\n", + "\n", + "If you would like to create a configurable settings file, uncomment the code in the next cell and run it. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f6e95e5", + "metadata": {}, + "outputs": [], + "source": [ + "#!modelscan create-settings-file -l my-modelscan-settings.toml" + ] + }, { "cell_type": "markdown", "id": "063dd649", "metadata": {}, "source": [ - "# Save a XGBoost Model\n", + "# Saving Model\n", "\n", "The model is trained on a diabetes dataset, and predicts whether a person has diabetes or not. The dataset can be found here: [Link to PIMA Indian diabetes dataset](https://www.kaggle.com/datasets/uciml/pima-indians-diabetes-database). The model is saved at ```./XGBoostModels/safe_model.pkl```" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "015f415a", "metadata": {}, "outputs": [], @@ -85,12 +118,12 @@ "id": "51812303", "metadata": {}, "source": [ - "# Predict using Safe Model" + "# Safe Model Prediction" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "8b8d0327", "metadata": {}, "outputs": [ @@ -113,14 +146,21 @@ "id": "fff6510d", "metadata": {}, "source": [ - "# Scan the safe model\n", + "# Scan Safe Model\n", + "\n", + "The scan results include information on the files scanned, and any issues if found. For the safe model scanned, modelscan finds no code injections in it, as expected.\n", + "\n", + "### ModelScan Settings File\n", + "- If you have created a settings file with default name and location (`modelscan-settings.toml`) it would automatically be used when scanning a model. \n", "\n", - "The scan results include information on the files scanned, and any issues if found. For the safe model scanned, modelscan finds no code injections in it, as expected." + "- If you want to use a specific name and/or location of settings file, please specify it with `--settings-file` as outlined in the next cell. \n", + "\n", + "- If you have not created any settings file, ModelScan will scan using default settings. " ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "ccfeee08", "metadata": {}, "outputs": [ @@ -139,6 +179,7 @@ } ], "source": [ + "#!modelscan -p XGBoostModels/safe_model.pkl --settings-file my-modelscan-settings.toml\n", "!modelscan -p XGBoostModels/safe_model.pkl" ] }, @@ -154,7 +195,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "d0e70069", "metadata": {}, "outputs": [], @@ -167,7 +208,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "7bde73cd", "metadata": {}, "outputs": [], @@ -184,7 +225,7 @@ "id": "1798152e", "metadata": {}, "source": [ - "# Predict using Unsafe Model\n", + "# Unsafe Model Prediction\n", "\n", "The malicious code gets executed when the model is loaded. The aws secret keys are displayed. \n", "\n", @@ -193,7 +234,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "49d6c62f", "metadata": {}, "outputs": [ @@ -220,16 +261,23 @@ "id": "72584048", "metadata": {}, "source": [ - "# Scan the Unsafe Model\n", + "# Scan Unsafe Model\n", "\n", "The scan results include information on the files scanned, and any issues if found. In this case, a critical severity level issue is found in the unsafe model scanned. \n", "\n", - "modelscan also outlines the found operator(s) and module(s) deemed unsafe. " + "ModelScan also outlines the found operator(s) and module(s) deemed unsafe. \n", + "\n", + "### ModelScan Settings File\n", + "- If you have created a settings file with default name and location (`modelscan-settings.toml`) it would automatically be used when scanning a model. \n", + "\n", + "- If you want to use a specific name and/or location of settings file, please specify it with `--settings-file` as outlined in the next cell. \n", + "\n", + "- If you have not created any settings file, ModelScan will scan using default settings." ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "9ee3393e", "metadata": {}, "outputs": [ @@ -264,6 +312,7 @@ } ], "source": [ + "#!modelscan -p XGBoostModels/unsafe_model.pkl --settings-file my-modelscan-settings.toml\n", "!modelscan -p XGBoostModels/unsafe_model.pkl" ] }, @@ -288,7 +337,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "id": "6df55b3e", "metadata": {}, "outputs": [ @@ -299,7 +348,7 @@ "No settings file detected at /Users/mehrinkiani/Documents/modelscan/notebooks/modelscan-settings.toml. Using defaults. \n", "\n", "Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/XGBoostModels/unsafe_model.pkl using modelscan.scanners.PickleUnsafeOpScan model scan\n", - "\u001b[1m{\u001b[0m\u001b[32m\"modelscan_version\"\u001b[0m: \u001b[32m\"0.5.0\"\u001b[0m, \u001b[32m\"timestamp\"\u001b[0m: \u001b[32m\"2024-01-25T17:56:00.855056\"\u001b[0m, \n", + "\u001b[1m{\u001b[0m\u001b[32m\"modelscan_version\"\u001b[0m: \u001b[32m\"0.5.1\"\u001b[0m, \u001b[32m\"timestamp\"\u001b[0m: \u001b[32m\"2024-02-06T10:56:13.862502\"\u001b[0m, \n", "\u001b[32m\"input_path\"\u001b[0m: \n", "\u001b[32m\"/Users/mehrinkiani/Documents/modelscan/notebooks/XGBoostModels/unsafe_model.pkl\u001b[0m\n", "\u001b[32m\"\u001b[0m, \u001b[32m\"total_issues\"\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m\"summary\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"total_issues_by_severity\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"LOW\"\u001b[0m: \u001b[1;36m0\u001b[0m, \n", @@ -322,7 +371,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.10.13 ('py310')", + "display_name": "Python 3.10.13 ('modelscan')", "language": "python", "name": "python3" }, @@ -340,7 +389,7 @@ }, "vscode": { "interpreter": { - "hash": "bd638e2064d9001d4ca93bc8e56e039dad230900dd235e8a6196f1614960903a" + "hash": "4dc76e75b8580a33bdc5a5a988b23a4143f70625528e602e356063408d11834c" } } },