Skip to content

Commit

Permalink
Merge pull request #15 from mlrun/development
Browse files Browse the repository at this point in the history 
merge development, match mlrun v0.4.1
  • Loading branch information
@yaronha
yaronha authored Dec 30, 2019
2 parents 00451b6 + b788bd9 commit ffe99a3
Show file tree
Hide file tree
Showing 10 changed files with 3,177 additions and 1,303 deletions.
614 changes: 298 additions & 316 deletions image_classification/mlrun_mpijob_classify.ipynb
View file

Large diffs are not rendered by default.

216 changes: 168 additions & 48 deletions image_classification/mlrun_mpijob_pipe.ipynb
View file

Large diffs are not rendered by default.

524 changes: 265 additions & 259 deletions image_classification/nuclio-serving-tf-images.ipynb
View file

Large diffs are not rendered by default.

994 changes: 931 additions & 63 deletions netops/mlrun.ipynb
View file

Large diffs are not rendered by default.

192 changes: 149 additions & 43 deletions netops/nuclio-data-preperations.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
},
{
"cell_type": "code",
"execution_count": 18,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -33,7 +33,7 @@
},
{
"cell_type": "code",
"execution_count": 19,
"execution_count": 2,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -66,11 +66,79 @@
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"execution_count": 6,
"metadata": {
"collapsed": true,
"jupyter": {
"outputs_hidden": true
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: pyarrow in /User/.pythonlibs/lib/python3.6/site-packages (0.15.1)\n",
"Requirement already satisfied: six>=1.0.0 in /conda/lib/python3.6/site-packages (from pyarrow) (1.12.0)\n",
"Requirement already satisfied: numpy>=1.14 in /User/.pythonlibs/lib/python3.6/site-packages (from pyarrow) (1.17.4)\n",
"Requirement already satisfied: pandas in /conda/lib/python3.6/site-packages (0.24.2)\n",
"Requirement already satisfied: numpy>=1.12.0 in /User/.pythonlibs/lib/python3.6/site-packages (from pandas) (1.17.4)\n",
"Requirement already satisfied: pytz>=2011k in /conda/lib/python3.6/site-packages (from pandas) (2019.3)\n",
"Requirement already satisfied: python-dateutil>=2.5.0 in /conda/lib/python3.6/site-packages (from pandas) (2.8.0)\n",
"Requirement already satisfied: six>=1.5 in /conda/lib/python3.6/site-packages (from python-dateutil>=2.5.0->pandas) (1.12.0)\n",
"Requirement already up-to-date: v3io_frames in /User/.pythonlibs/lib/python3.6/site-packages (0.6.6)\n",
"Requirement already satisfied, skipping upgrade: requests>=2.19.1 in /conda/lib/python3.6/site-packages (from v3io_frames) (2.22.0)\n",
"Requirement already satisfied, skipping upgrade: pandas>=0.23.4 in /conda/lib/python3.6/site-packages (from v3io_frames) (0.24.2)\n",
"Requirement already satisfied, skipping upgrade: grpcio-tools>=1.16.0 in /conda/lib/python3.6/site-packages (from v3io_frames) (1.24.3)\n",
"Requirement already satisfied, skipping upgrade: googleapis-common-protos>=1.5.3 in /conda/lib/python3.6/site-packages (from v3io_frames) (1.6.0)\n",
"Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /conda/lib/python3.6/site-packages (from requests>=2.19.1->v3io_frames) (2019.9.11)\n",
"Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /conda/lib/python3.6/site-packages (from requests>=2.19.1->v3io_frames) (2.8)\n",
"Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /conda/lib/python3.6/site-packages (from requests>=2.19.1->v3io_frames) (3.0.4)\n",
"Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /conda/lib/python3.6/site-packages (from requests>=2.19.1->v3io_frames) (1.24.2)\n",
"Requirement already satisfied, skipping upgrade: python-dateutil>=2.5.0 in /conda/lib/python3.6/site-packages (from pandas>=0.23.4->v3io_frames) (2.8.0)\n",
"Requirement already satisfied, skipping upgrade: numpy>=1.12.0 in /User/.pythonlibs/lib/python3.6/site-packages (from pandas>=0.23.4->v3io_frames) (1.17.4)\n",
"Requirement already satisfied, skipping upgrade: pytz>=2011k in /conda/lib/python3.6/site-packages (from pandas>=0.23.4->v3io_frames) (2019.3)\n",
"Requirement already satisfied, skipping upgrade: grpcio>=1.24.3 in /conda/lib/python3.6/site-packages (from grpcio-tools>=1.16.0->v3io_frames) (1.24.3)\n",
"Requirement already satisfied, skipping upgrade: protobuf>=3.5.0.post1 in /conda/lib/python3.6/site-packages (from grpcio-tools>=1.16.0->v3io_frames) (3.10.0)\n",
"Requirement already satisfied, skipping upgrade: six>=1.5 in /conda/lib/python3.6/site-packages (from python-dateutil>=2.5.0->pandas>=0.23.4->v3io_frames) (1.12.0)\n",
"Requirement already satisfied, skipping upgrade: setuptools in /conda/lib/python3.6/site-packages (from protobuf>=3.5.0.post1->grpcio-tools>=1.16.0->v3io_frames) (41.4.0)\n",
"Requirement already satisfied: dask[complete] in /User/.pythonlibs/lib/python3.6/site-packages (2.9.0)\n",
"Requirement already satisfied: numpy>=1.13.0; extra == \"complete\" in /User/.pythonlibs/lib/python3.6/site-packages (from dask[complete]) (1.17.4)\n",
"Requirement already satisfied: cloudpickle>=0.2.1; extra == \"complete\" in /conda/lib/python3.6/site-packages (from dask[complete]) (1.2.2)\n",
"Requirement already satisfied: pandas>=0.21.0; extra == \"complete\" in /conda/lib/python3.6/site-packages (from dask[complete]) (0.24.2)\n",
"Requirement already satisfied: distributed>=2.0; extra == \"complete\" in /User/.pythonlibs/lib/python3.6/site-packages (from dask[complete]) (2.9.0)\n",
"Requirement already satisfied: toolz>=0.7.3; extra == \"complete\" in /conda/lib/python3.6/site-packages (from dask[complete]) (0.10.0)\n",
"Requirement already satisfied: partd>=0.3.10; extra == \"complete\" in /conda/lib/python3.6/site-packages (from dask[complete]) (1.0.0)\n",
"Requirement already satisfied: bokeh>=1.0.0; extra == \"complete\" in /conda/lib/python3.6/site-packages (from dask[complete]) (1.0.3)\n",
"Requirement already satisfied: PyYaml; extra == \"complete\" in /conda/lib/python3.6/site-packages (from dask[complete]) (5.1.2)\n",
"Collecting fsspec>=0.6.0; extra == \"complete\" (from dask[complete])\n",
"\u001b[?25l Downloading https://files.pythonhosted.org/packages/dd/1f/7028dacd3c28f34ce48130aae73a88fa5cc27b6b0e494fcf2739f7954d9d/fsspec-0.6.2-py3-none-any.whl (62kB)\n",
"\u001b[K 100% |████████████████████████████████| 71kB 3.0MB/s ta 0:00:011\n",
"\u001b[?25hRequirement already satisfied: python-dateutil>=2.5.0 in /conda/lib/python3.6/site-packages (from pandas>=0.21.0; extra == \"complete\"->dask[complete]) (2.8.0)\n",
"Requirement already satisfied: pytz>=2011k in /conda/lib/python3.6/site-packages (from pandas>=0.21.0; extra == \"complete\"->dask[complete]) (2019.3)\n",
"Requirement already satisfied: zict>=0.1.3 in /conda/lib/python3.6/site-packages (from distributed>=2.0; extra == \"complete\"->dask[complete]) (1.0.0)\n",
"Requirement already satisfied: msgpack in /conda/lib/python3.6/site-packages (from distributed>=2.0; extra == \"complete\"->dask[complete]) (0.6.2)\n",
"Requirement already satisfied: psutil>=5.0 in /conda/lib/python3.6/site-packages (from distributed>=2.0; extra == \"complete\"->dask[complete]) (5.6.3)\n",
"Requirement already satisfied: click>=6.6 in /conda/lib/python3.6/site-packages (from distributed>=2.0; extra == \"complete\"->dask[complete]) (7.0)\n",
"Requirement already satisfied: tblib in /conda/lib/python3.6/site-packages (from distributed>=2.0; extra == \"complete\"->dask[complete]) (1.5.0)\n",
"Requirement already satisfied: sortedcontainers!=2.0.0,!=2.0.1 in /conda/lib/python3.6/site-packages (from distributed>=2.0; extra == \"complete\"->dask[complete]) (2.1.0)\n",
"Requirement already satisfied: tornado>=5 in /User/.pythonlibs/lib/python3.6/site-packages (from distributed>=2.0; extra == \"complete\"->dask[complete]) (5.1.1)\n",
"Requirement already satisfied: locket in /conda/lib/python3.6/site-packages (from partd>=0.3.10; extra == \"complete\"->dask[complete]) (0.2.0)\n",
"Requirement already satisfied: packaging>=16.8 in /conda/lib/python3.6/site-packages (from bokeh>=1.0.0; extra == \"complete\"->dask[complete]) (19.2)\n",
"Requirement already satisfied: pillow>=4.0 in /conda/lib/python3.6/site-packages (from bokeh>=1.0.0; extra == \"complete\"->dask[complete]) (6.2.0)\n",
"Requirement already satisfied: Jinja2>=2.7 in /conda/lib/python3.6/site-packages (from bokeh>=1.0.0; extra == \"complete\"->dask[complete]) (2.10.3)\n",
"Requirement already satisfied: six>=1.5.2 in /conda/lib/python3.6/site-packages (from bokeh>=1.0.0; extra == \"complete\"->dask[complete]) (1.12.0)\n",
"Requirement already satisfied: heapdict in /conda/lib/python3.6/site-packages (from zict>=0.1.3->distributed>=2.0; extra == \"complete\"->dask[complete]) (1.0.1)\n",
"Requirement already satisfied: pyparsing>=2.0.2 in /conda/lib/python3.6/site-packages (from packaging>=16.8->bokeh>=1.0.0; extra == \"complete\"->dask[complete]) (2.4.2)\n",
"Requirement already satisfied: MarkupSafe>=0.23 in /conda/lib/python3.6/site-packages (from Jinja2>=2.7->bokeh>=1.0.0; extra == \"complete\"->dask[complete]) (1.1.1)\n",
"Installing collected packages: fsspec\n",
"Successfully installed fsspec-0.6.2\n",
"Requirement already satisfied: fsspec>=0.3.3 in /User/.pythonlibs/lib/python3.6/site-packages (0.6.2)\n"
]
}
],
"source": [
"%%nuclio cmd -c\n",
"%%nuclio cmd\n",
"\n",
"############\n",
"# installs #\n",
Expand All @@ -84,7 +152,8 @@
"pip install v3io_frames --upgrade\n",
"\n",
"# Function\n",
"pip install dask[\"complete\"]"
"pip install dask[\"complete\"]\n",
"pip install 'fsspec>=0.3.3'"
]
},
{
Expand All @@ -96,21 +165,28 @@
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"# DB Config\n",
"%nuclio env %v3io"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"%nuclio: setting 'V3IO_FRAMESD' environment variable\n",
"%nuclio: setting 'V3IO_USERNAME' environment variable\n",
"%nuclio: setting 'V3IO_ACCESS_KEY' environment variable\n",
"%nuclio: setting 'SAVE_TO_TSDB' environment variable\n",
"%nuclio: setting 'METRICS_TABLE' environment variable\n",
"%nuclio: setting '# METRICS_TABLE' environment variable\n",
"%nuclio: setting 'FEATURES_TABLE' environment variable\n",
"%nuclio: setting '# FEATURES_TABLE' environment variable\n",
"%nuclio: setting '# metrics_table' environment variable\n",
"%nuclio: setting 'metrics_table' environment variable\n",
"%nuclio: setting '# features_table' environment variable\n",
"%nuclio: setting 'features_table' environment variable\n",
"%nuclio: setting 'NUMBER_OF_SHARDS' environment variable\n"
]
},
Expand All @@ -121,29 +197,23 @@
"%nuclio: cannot find \"=\" in line\n",
"%nuclio: cannot find \"=\" in line\n",
"%nuclio: cannot find \"=\" in line\n",
"%nuclio: cannot find \"=\" in line\n",
"%nuclio: cannot find \"=\" in line\n"
]
}
],
"source": [
"%%nuclio env\n",
"\n",
"# DB Config\n",
"V3IO_FRAMESD=${V3IO_FRAMESD}\n",
"V3IO_USERNAME=${V3IO_USERNAME}\n",
"V3IO_ACCESS_KEY=${V3IO_ACCESS_KEY}\n",
"\n",
"# Save as\n",
"SAVE_TO_TSDB=1\n",
"SAVE_TO_TSDB=0\n",
"\n",
"# Metrics\n",
"METRICS_TABLE=netops_metrics\n",
"# METRICS_TABLE=/v3io/bigdata/netops_metrics_parquet\n",
"# metrics_table=netops_metrics\n",
"metrics_table=/v3io/bigdata/netops_metrics_parquet\n",
"\n",
"# Features\n",
"FEATURES_TABLE=netops_features\n",
"# FEATURES_TABLE=/v3io/bigdata/netops_features_parquet\n",
"# features_table=netops_features\n",
"features_table=/v3io/bigdata/netops_features_parquet\n",
"\n",
"\n",
"# Parallelizem\n",
Expand All @@ -166,7 +236,7 @@
},
{
"cell_type": "code",
"execution_count": 22,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -193,7 +263,7 @@
},
{
"cell_type": "code",
"execution_count": 23,
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -206,7 +276,7 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -219,7 +289,7 @@
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -240,7 +310,7 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -262,7 +332,7 @@
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -273,7 +343,7 @@
},
{
"cell_type": "code",
"execution_count": 28,
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -283,7 +353,7 @@
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -315,7 +385,7 @@
},
{
"cell_type": "code",
"execution_count": 35,
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -332,10 +402,10 @@
" is_save_to_tsdb = (int(os.getenv('SAVE_TO_TSDB', 1)) == 1)\n",
" \n",
" # Netops metrics table\n",
" setattr(context, 'metrics_table', os.getenv('METRICS_TABLE', 'netops_metrics'))\n",
" setattr(context, 'metrics_table', os.getenv('metrics_table', 'netops_metrics'))\n",
" \n",
" # Netops feautres table\n",
" setattr(context, 'features_table', os.getenv('FEATURES_TABLE', 'netops_features'))\n",
" setattr(context, 'features_table', os.getenv('features_table', 'netops_features'))\n",
" \n",
" \n",
" # Save to TSDB\n",
Expand Down Expand Up @@ -376,7 +446,7 @@
},
{
"cell_type": "code",
"execution_count": 42,
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -423,6 +493,15 @@
" context.write(context, features)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"# nuclio: end-code"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -432,19 +511,39 @@
},
{
"cell_type": "code",
"execution_count": 48,
"execution_count": 22,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/User/.pythonlibs/lib/python3.6/site-packages/distributed/dashboard/core.py:72: UserWarning: \n",
"Port 8787 is already in use. \n",
"Perhaps you already have a cluster running?\n",
"Hosting the diagnostics dashboard on a random port instead.\n",
" warnings.warn(\"\\n\" + msg)\n"
]
}
],
"source": [
"# nuclio: ignore\n",
"init_context(context)"
]
},
{
"cell_type": "code",
"execution_count": 49,
"execution_count": 23,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Saving features to Parquet\n"
]
}
],
"source": [
"# nuclio: ignore\n",
"# init_context(context)\n",
Expand All @@ -453,6 +552,13 @@
"output"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down Expand Up @@ -510,5 +616,5 @@
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}
Loading

0 comments on commit ffe99a3

Please sign in to comment.