Skip to content

Commit

Permalink
Merge branch 'main' into h5-no-config
Browse files Browse the repository at this point in the history
  • Loading branch information
@swashko
swashko committed Feb 1, 2024
2 parents 8cfb1b1 + 01d8cbd commit cc63104
Show file tree
Hide file tree
Showing 5 changed files with 322 additions and 102 deletions.
3 changes: 3 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@ ModelScan supports the following arguments via the CLI:
| ```modelscan -p /path/to/model_file``` | -p or --path | Scan a locally stored model |
| ```modelscan -p /path/to/model_file --settings-file ./modelscan-settings.toml``` | --settings-file | Scan a locally stored model using custom configurations |
| ```modelscan create-settings-file``` | -l or --location | Create a configurable settings file |
| ```modelscan -r``` | -r or --reporting-format | Format of the output. Options are console, json, or custom (to be defined in settings-file). Default is console |
| ```modelscan -r reporting-format -o file-name``` | -o or --output-file | Optional file name for output report |
| ```modelscan --show-skipped``` | --show-skipped | Print a list of files that were skipped during the scan |


Remember models are just like any other form of digital media, you should scan content from any untrusted source before use.
Expand Down
110 changes: 86 additions & 24 deletions notebooks/keras_fashion_mnist.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,18 @@
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"modelscan, version 0.5.0\n"
]
}
],
"source": [
"!pip install -q modelscan"
"!pip install -q modelscan\n",
"!modelscan -v"
]
},
{
Expand Down Expand Up @@ -58,35 +67,35 @@
"output_type": "stream",
"text": [
"Epoch 1/10\n",
"1875/1875 [==============================] - 2s 773us/step - loss: 0.5034 - accuracy: 0.8228\n",
"1875/1875 [==============================] - 2s 1ms/step - loss: 0.4986 - accuracy: 0.8258\n",
"Epoch 2/10\n",
"1875/1875 [==============================] - 1s 738us/step - loss: 0.3762 - accuracy: 0.8640\n",
"1875/1875 [==============================] - 2s 800us/step - loss: 0.3741 - accuracy: 0.8649\n",
"Epoch 3/10\n",
"1875/1875 [==============================] - 1s 760us/step - loss: 0.3358 - accuracy: 0.8769\n",
"1875/1875 [==============================] - 1s 772us/step - loss: 0.3391 - accuracy: 0.8766\n",
"Epoch 4/10\n",
"1875/1875 [==============================] - 1s 733us/step - loss: 0.3114 - accuracy: 0.8854\n",
"1875/1875 [==============================] - 1s 755us/step - loss: 0.3125 - accuracy: 0.8852\n",
"Epoch 5/10\n",
"1875/1875 [==============================] - 1s 743us/step - loss: 0.2949 - accuracy: 0.8909\n",
"1875/1875 [==============================] - 2s 965us/step - loss: 0.2955 - accuracy: 0.8908\n",
"Epoch 6/10\n",
"1875/1875 [==============================] - 1s 731us/step - loss: 0.2816 - accuracy: 0.8962\n",
"1875/1875 [==============================] - 2s 905us/step - loss: 0.2808 - accuracy: 0.8963\n",
"Epoch 7/10\n",
"1875/1875 [==============================] - 1s 743us/step - loss: 0.2687 - accuracy: 0.9000\n",
"1875/1875 [==============================] - 2s 1ms/step - loss: 0.2696 - accuracy: 0.9004\n",
"Epoch 8/10\n",
"1875/1875 [==============================] - 1s 762us/step - loss: 0.2567 - accuracy: 0.9044\n",
"1875/1875 [==============================] - 2s 895us/step - loss: 0.2587 - accuracy: 0.9039\n",
"Epoch 9/10\n",
"1875/1875 [==============================] - 1s 766us/step - loss: 0.2464 - accuracy: 0.9075\n",
"1875/1875 [==============================] - 2s 925us/step - loss: 0.2479 - accuracy: 0.9068\n",
"Epoch 10/10\n",
"1875/1875 [==============================] - 1s 737us/step - loss: 0.2372 - accuracy: 0.9113\n",
"313/313 [==============================] - 0s 490us/step - loss: 0.3440 - accuracy: 0.8827\n",
"1875/1875 [==============================] - 2s 893us/step - loss: 0.2406 - accuracy: 0.9112\n",
"313/313 [==============================] - 0s 555us/step - loss: 0.3496 - accuracy: 0.8790\n",
"\n",
"Model trained with test accuracy: 0.8827000260353088\n"
"Model trained with test accuracy: 0.8790000081062317\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/mehrinkiani/mambaforge/envs/notebooks-xgb-2/lib/python3.9/site-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
"/Users/mehrinkiani/mambaforge/envs/py310/lib/python3.10/site-packages/keras/src/engine/training.py:3000: UserWarning: You are saving your model as an HDF5 file via `model.save()`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')`.\n",
" saving_api.save_model(\n"
]
}
Expand Down Expand Up @@ -117,9 +126,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
"1/1 [==============================] - 0s 35ms/step\n",
"1/1 [==============================] - 0s 42ms/step\n",
"\n",
"The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 98.059 99.993996 100. ]\n",
"The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 98.948 99.926 100. ]\n",
"\n",
"The true labels are ['Ankle boot', 'Pullover', 'Trouser']\n"
]
Expand Down Expand Up @@ -159,7 +168,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/safe_model.h5 using hdf5 model scan\n",
"No settings file detected at /Users/mehrinkiani/Documents/modelscan/notebooks/modelscan-settings.toml. Using defaults. \n",
"\n",
"Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/safe_model.h5 using modelscan.scanners.H5LambdaDetectScan model scan\n",
"\n",
"\u001b[34m--- Summary ---\u001b[0m\n",
"\n",
Expand Down Expand Up @@ -244,9 +255,9 @@
"aws_secret_access_key=<aws_secret_key>\n",
"aws_access_key_id=<access_key_id>\n",
"aws_secret_access_key=<aws_secret_key>\n",
"1/1 [==============================] - 0s 29ms/step\n",
"1/1 [==============================] - 0s 35ms/step\n",
"\n",
"The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 98.059 99.993996 100. ]\n",
"The model predicts: ['Ankle boot', 'Pullover', 'Trouser'] with probabilities: [ 98.948 99.926 100. ]\n",
"\n",
"The true labels are ['Ankle boot', 'Pullover', 'Trouser']\n"
]
Expand Down Expand Up @@ -289,7 +300,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/unsafe_model.h5 using hdf5 model scan\n",
"No settings file detected at /Users/mehrinkiani/Documents/modelscan/notebooks/modelscan-settings.toml. Using defaults. \n",
"\n",
"Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/unsafe_model.h5 using modelscan.scanners.H5LambdaDetectScan model scan\n",
"\n",
"\u001b[34m--- Summary ---\u001b[0m\n",
"\n",
Expand Down Expand Up @@ -317,6 +330,55 @@
"!modelscan -p KerasModels/unsafe_model.h5"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Reporting Format\n",
"ModelScan can report scan results in console (default), JSON, or custom report (to be defined by user in settings-file). For mode details, please see: ` modelscan -h` "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## JSON Report\n",
"\n",
"For JSON reporting: `modelscan -p ./path-to/file -r json -o output-file-name.json` \n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"No settings file detected at /Users/mehrinkiani/Documents/modelscan/notebooks/modelscan-settings.toml. Using defaults. \n",
"\n",
"Scanning /Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/unsafe_model.h5 using modelscan.scanners.H5LambdaDetectScan model scan\n",
"\u001b[1m{\u001b[0m\u001b[32m\"modelscan_version\"\u001b[0m: \u001b[32m\"0.5.0\"\u001b[0m, \u001b[32m\"timestamp\"\u001b[0m: \u001b[32m\"2024-01-25T17:56:36.776966\"\u001b[0m, \n",
"\u001b[32m\"input_path\"\u001b[0m: \n",
"\u001b[32m\"/Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/unsafe_model.h5\"\u001b[0m, \n",
"\u001b[32m\"total_issues\"\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m\"summary\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"total_issues_by_severity\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"LOW\"\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m\"MEDIUM\"\u001b[0m: \n",
"\u001b[1;36m1\u001b[0m, \u001b[32m\"HIGH\"\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m\"CRITICAL\"\u001b[0m: \u001b[1;36m0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m, \u001b[32m\"issues_by_severity\"\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"MEDIUM\"\u001b[0m: \u001b[1m[\u001b[0m\u001b[1m{\u001b[0m\u001b[32m\"description\"\u001b[0m:\n",
"\u001b[32m\"Use of unsafe operator 'Lambda' from module 'Keras'\"\u001b[0m, \u001b[32m\"operator\"\u001b[0m: \u001b[32m\"Lambda\"\u001b[0m, \n",
"\u001b[32m\"module\"\u001b[0m: \u001b[32m\"Keras\"\u001b[0m, \u001b[32m\"source\"\u001b[0m: \n",
"\u001b[32m\"/Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/unsafe_model.h5\"\u001b[0m, \n",
"\u001b[32m\"scanner\"\u001b[0m: \u001b[32m\"modelscan.scanners.H5LambdaDetectScan\"\u001b[0m\u001b[1m}\u001b[0m\u001b[1m]\u001b[0m\u001b[1m}\u001b[0m, \u001b[32m\"errors\"\u001b[0m: \u001b[1m[\u001b[0m\u001b[1m]\u001b[0m, \u001b[32m\"scanned\"\u001b[0m: \n",
"\u001b[1m{\u001b[0m\u001b[32m\"total_scanned\"\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m\"scanned_files\"\u001b[0m: \n",
"\u001b[1m[\u001b[0m\u001b[32m\"/Users/mehrinkiani/Documents/modelscan/notebooks/KerasModels/unsafe_model.h5\"\u001b[0m\u001b[1m]\u001b[0m\n",
"\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\n"
]
}
],
"source": [
"# This will save the scan results in file: keras-model-scan-results.json\n",
"!modelscan --path KerasModels/unsafe_model.h5 -r json -o keras-model-scan-results.json"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand All @@ -327,7 +389,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.9.16 ('notebooks-xgb-2')",
"display_name": "Python 3.10.13 ('py310')",
"language": "python",
"name": "python3"
},
Expand All @@ -341,12 +403,12 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
"version": "3.10.13"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "8792b9cba3843e3778a70fb92fd4a6e11f0e29f4d97b60d08b903ca0b490cce6"
"hash": "bd638e2064d9001d4ca93bc8e56e039dad230900dd235e8a6196f1614960903a"
}
}
},
Expand Down
Loading

0 comments on commit cc63104

Please sign in to comment.