From 47f6496e3fd995e7f483dd8f500b0af18aae2062 Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Wed, 1 Mar 2023 17:47:01 +0200 Subject: [PATCH 01/10] moving context to from ./src -> ./ --- stocks-prediction/01_ingest_news.ipynb | 2 +- stocks-prediction/02_ingest_stocks.ipynb | 2 +- stocks-prediction/03_train_model.ipynb | 4 +- stocks-prediction/05_stocks_pipeline.ipynb | 53 ++++++++++++++++++++-- stocks-prediction/06_grafana_view.ipynb | 4 +- 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/stocks-prediction/01_ingest_news.ipynb b/stocks-prediction/01_ingest_news.ipynb index a023e218..ef3fc7c4 100644 --- a/stocks-prediction/01_ingest_news.ipynb +++ b/stocks-prediction/01_ingest_news.ipynb @@ -76,7 +76,7 @@ ], "source": [ "import mlrun\n", - "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"src/\")" + "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"./\")" ] }, { diff --git a/stocks-prediction/02_ingest_stocks.ipynb b/stocks-prediction/02_ingest_stocks.ipynb index a62b6415..25cea197 100644 --- a/stocks-prediction/02_ingest_stocks.ipynb +++ b/stocks-prediction/02_ingest_stocks.ipynb @@ -24,7 +24,7 @@ ], "source": [ "import mlrun\n", - "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"src/\")" + "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"./\")" ] }, { diff --git a/stocks-prediction/03_train_model.ipynb b/stocks-prediction/03_train_model.ipynb index e3c9798f..3808ef0d 100644 --- a/stocks-prediction/03_train_model.ipynb +++ b/stocks-prediction/03_train_model.ipynb @@ -71,7 +71,7 @@ ], "source": [ "import mlrun\n", - "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"src/\")" + "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"./\")" ] }, { @@ -773,4 +773,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/stocks-prediction/05_stocks_pipeline.ipynb b/stocks-prediction/05_stocks_pipeline.ipynb index 0f0b6c6a..89277c5b 100644 --- a/stocks-prediction/05_stocks_pipeline.ipynb +++ b/stocks-prediction/05_stocks_pipeline.ipynb @@ -38,7 +38,7 @@ ], "source": [ "import mlrun\n", - "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"src/\")" + "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"./\")" ] }, { @@ -294,11 +294,11 @@ "import os\n", "\n", "# getting our model training function\n", - "project.set_function(os.path.join(os.getcwd(),'src','train_stocks.py'),\n", + "project.set_function('./src/train_stocks.py',\n", " name='train_stocks', kind='job', image='mlrun/ml-models')\n", "\n", "\n", - "project.set_function(os.path.join(os.getcwd(),'src','serving_stocks.py'),\n", + "project.set_function('./src/serving_stocks.py',\n", " name='serving_stocks', kind='serving', image='mlrun/ml-models')" ] }, @@ -546,9 +546,52 @@ " \"epochs\": 3,\n", " \"start_time\":59,\n", " \"end_time\":0,\n", - " \"model_filepath\":os.path.join(os.getcwd(),'src')},\n", + " \"model_filepath\":'./src'},\n", " watch=True)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "import shutil\n", + "shutil.make_archive('project_tar','gztar','./')\n", + "source = 'v3io:///users/avia/demos/stocks-prediction/project_tar.tar.gz'\n", + "project.set_source(source)\n", + "project.save()" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "project.run(name=workflow_name,\n", + " arguments={\n", + " \"vector_name\":\"stocks\",\n", + " \"seq_size\": 5,\n", + " \"batch_size\": 1,\n", + " \"hidden_dim\": 2,\n", + " \"n_layers\": 1,\n", + " \"epochs\": 3,\n", + " \"start_time\":59,\n", + " \"end_time\":0,\n", + " \"model_filepath\":'./src'},\n", + " watch=True, schedule='0 0 * * *',override=True)" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } } ], "metadata": { @@ -572,4 +615,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file diff --git a/stocks-prediction/06_grafana_view.ipynb b/stocks-prediction/06_grafana_view.ipynb index 917a9fc7..83de39c2 100644 --- a/stocks-prediction/06_grafana_view.ipynb +++ b/stocks-prediction/06_grafana_view.ipynb @@ -38,7 +38,7 @@ ], "source": [ "import mlrun\n", - "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"src/\")" + "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"./\")" ] }, { @@ -101,4 +101,4 @@ }, "nbformat": 4, "nbformat_minor": 5 -} +} \ No newline at end of file From 2b03971d45224e0f70ae4a0f22c51e6ee0c28099 Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Tue, 14 Mar 2023 11:17:04 +0200 Subject: [PATCH 02/10] [Stocks] add workflow --- stocks-prediction/05_stocks_pipeline.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stocks-prediction/05_stocks_pipeline.ipynb b/stocks-prediction/05_stocks_pipeline.ipynb index 89277c5b..0219b78d 100644 --- a/stocks-prediction/05_stocks_pipeline.ipynb +++ b/stocks-prediction/05_stocks_pipeline.ipynb @@ -546,7 +546,7 @@ " \"epochs\": 3,\n", " \"start_time\":59,\n", " \"end_time\":0,\n", - " \"model_filepath\":'./src'},\n", + " \"model_filepath\":'./'},\n", " watch=True)" ] }, From 7b38839ae17576329a000f620c2bc0dacc65a82e Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Thu, 16 Nov 2023 12:58:33 +0200 Subject: [PATCH 03/10] [Stocks] save number of stocks as project parameter --- stocks-prediction/01_ingest_news.ipynb | 8 +++++--- stocks-prediction/02_ingest_stocks.ipynb | 25 ++++++++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/stocks-prediction/01_ingest_news.ipynb b/stocks-prediction/01_ingest_news.ipynb index db0f5e50..6d048362 100644 --- a/stocks-prediction/01_ingest_news.ipynb +++ b/stocks-prediction/01_ingest_news.ipynb @@ -86,7 +86,9 @@ "metadata": {}, "outputs": [], "source": [ - "NUMBER_OF_STOCKS = 3" + "# saving number of stocks as project parameter\n", + "project.params['NUMBER_OF_STOCKS'] = 10\n", + "number_of_stocks = project.params['NUMBER_OF_STOCKS']" ] }, { @@ -986,7 +988,7 @@ "fstore.ingest(news_set,\n", " pd.DataFrame.from_dict({'ticker':['AMZN'],\n", " 'Datetime': now,\n", - " 'n_stocks':NUMBER_OF_STOCKS}),\n", + " 'n_stocks':number_of_stocks}),\n", " overwrite=True)" ] }, @@ -1074,7 +1076,7 @@ "\n", "t = requests.post(news_set_endpoint,json={'ticker':['news'],\n", " 'Datetime': now,\n", - " 'n_stocks':NUMBER_OF_STOCKS})\n", + " 'n_stocks':number_of_stocks})\n", "t.text" ] }, diff --git a/stocks-prediction/02_ingest_stocks.ipynb b/stocks-prediction/02_ingest_stocks.ipynb index 41614292..4ea5fd02 100644 --- a/stocks-prediction/02_ingest_stocks.ipynb +++ b/stocks-prediction/02_ingest_stocks.ipynb @@ -27,6 +27,19 @@ "project = mlrun.get_or_create_project(name='stocks',user_project=True, context=\"./\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "#loading the number of stocks as project parameter\n", + "\n", + "number_of_stocks = project.params['NUMBER_OF_STOCKS']" + ], + "metadata": { + "collapsed": false + } + }, { "cell_type": "markdown", "id": "4b54f299", @@ -170,15 +183,11 @@ " timestamp_key='Datetime', \n", " description=\"stocks feature set\")\n", "\n", - "# how many tickers data we ingest (make sure same number used for ingesting news)\n", - "n_tickers = 10\n", - "\n", - "\n", "info_set.graph\\\n", " .to(name='get_stocks',handler='get_stocks')\\\n", " .to(\"storey.steps.Flatten\", name=\"flatten_stocks\")\\\n", " .to(name='gen_event_key',handler='gen_event_key',full_event=True)\\\n", - " .to(OneHotEncoder(mapping={'ticker2onehot':{ticker:str(idx) for idx,ticker in enumerate(si.tickers_sp500()[:n_tickers])}}))\\\n", + " .to(OneHotEncoder(mapping={'ticker2onehot':{ticker:str(idx) for idx,ticker in enumerate(si.tickers_sp500()[:number_of_stocks])}}))\\\n", " \n", "# Setting default targets (nosql & parquet)\n", "info_set.set_targets([ParquetTarget(flush_after_seconds=5)], with_defaults=False) \n", @@ -567,7 +576,7 @@ " 'start_delta':59,\n", " 'end_delta':31,\n", " 'interval':'5m',\n", - " 'n_stocks':n_tickers}),\n", + " 'n_stocks':number_of_stocks}),\n", " overwrite=True)" ] }, @@ -654,7 +663,7 @@ " 'start_delta':29,\n", " 'end_delta':0,\n", " 'interval':'5m',\n", - " 'n_stocks':n_tickers})\n", + " 'n_stocks':number_of_stocks})\n", "t.text" ] }, @@ -691,7 +700,7 @@ " 'start_delta':1,\n", " 'end_delta':0,\n", " 'interval':'5m',\n", - " 'n_stocks':n_tickers}\n", + " 'n_stocks':NUMBER_OF_STOCKS}\n", "\n", "# specifying '0 8 * * *' as schedule will trigger the function every day at 08:00 AM\n", "fn = mlrun.code_to_function(name='ingestion_service_stocks',kind='job',image='mlrun/mlrun',handler='ingestion_service_invoker', filename='src/invoker.py')\n", From 62daf0cef4bf0bef63632d648551757a1fa6b50b Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Thu, 16 Nov 2023 13:18:45 +0200 Subject: [PATCH 04/10] [Stocks] save number of stocks as project parameter --- stocks-prediction/01_ingest_news.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/stocks-prediction/01_ingest_news.ipynb b/stocks-prediction/01_ingest_news.ipynb index 6d048362..e9dd49fb 100644 --- a/stocks-prediction/01_ingest_news.ipynb +++ b/stocks-prediction/01_ingest_news.ipynb @@ -88,6 +88,7 @@ "source": [ "# saving number of stocks as project parameter\n", "project.params['NUMBER_OF_STOCKS'] = 10\n", + "project.save()\n", "number_of_stocks = project.params['NUMBER_OF_STOCKS']" ] }, From df3da3cd55be8931923aacbbc8536c2d1b42c459 Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Thu, 16 Nov 2023 13:24:52 +0200 Subject: [PATCH 05/10] [Stocks] save number of stocks as project parameter --- stocks-prediction/02_ingest_stocks.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stocks-prediction/02_ingest_stocks.ipynb b/stocks-prediction/02_ingest_stocks.ipynb index 4ea5fd02..ec2499db 100644 --- a/stocks-prediction/02_ingest_stocks.ipynb +++ b/stocks-prediction/02_ingest_stocks.ipynb @@ -700,7 +700,7 @@ " 'start_delta':1,\n", " 'end_delta':0,\n", " 'interval':'5m',\n", - " 'n_stocks':NUMBER_OF_STOCKS}\n", + " 'n_stocks':number_of_stocks}\n", "\n", "# specifying '0 8 * * *' as schedule will trigger the function every day at 08:00 AM\n", "fn = mlrun.code_to_function(name='ingestion_service_stocks',kind='job',image='mlrun/mlrun',handler='ingestion_service_invoker', filename='src/invoker.py')\n", From ce4a35269caafd70d94718f00cea0b2f725e2c4e Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Thu, 16 Nov 2023 13:27:36 +0200 Subject: [PATCH 06/10] [Stocks] save number of stocks as project parameter --- stocks-prediction/02_ingest_stocks.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stocks-prediction/02_ingest_stocks.ipynb b/stocks-prediction/02_ingest_stocks.ipynb index ec2499db..18ac261e 100644 --- a/stocks-prediction/02_ingest_stocks.ipynb +++ b/stocks-prediction/02_ingest_stocks.ipynb @@ -34,7 +34,7 @@ "source": [ "#loading the number of stocks as project parameter\n", "\n", - "number_of_stocks = project.params['NUMBER_OF_STOCKS']" + "number_of_stocks = project.params.get('NUMBER_OF_STOCKS',10)" ], "metadata": { "collapsed": false From 67b9e6749acbd8fd02c1c18a12d5f1ab64e34e22 Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Mon, 20 Nov 2023 15:38:43 +0200 Subject: [PATCH 07/10] [Stocks] increase serving readinessTimeoutSeconds time out --- stocks-prediction/04_model_serving.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/stocks-prediction/04_model_serving.ipynb b/stocks-prediction/04_model_serving.ipynb index cce6ca41..dfe4549e 100644 --- a/stocks-prediction/04_model_serving.ipynb +++ b/stocks-prediction/04_model_serving.ipynb @@ -298,6 +298,7 @@ } ], "source": [ + "fn.spec.readinessTimeoutSeconds = 60 * 20 # 20 minutes\n", "address = fn.deploy()" ] }, From 4207a8a4df3e0988efa8e96d7dc7d8158779459e Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Tue, 21 Nov 2023 17:59:20 +0200 Subject: [PATCH 08/10] [Stocks] increase serving readinessTimeoutSeconds time out --- stocks-prediction/04_model_serving.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stocks-prediction/04_model_serving.ipynb b/stocks-prediction/04_model_serving.ipynb index dfe4549e..0ac142f1 100644 --- a/stocks-prediction/04_model_serving.ipynb +++ b/stocks-prediction/04_model_serving.ipynb @@ -298,7 +298,7 @@ } ], "source": [ - "fn.spec.readinessTimeoutSeconds = 60 * 20 # 20 minutes\n", + "fn.spec.readiness_timeout = 60 * 20 # 20 minutes\n", "address = fn.deploy()" ] }, From 140c6d862a475dda48a3cd0a8dadcd79b6220b01 Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Wed, 22 Nov 2023 15:10:20 +0200 Subject: [PATCH 09/10] [Stocks] skip log results in case of CE --- stocks-prediction/src/serving_stocks.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/stocks-prediction/src/serving_stocks.py b/stocks-prediction/src/serving_stocks.py index f4f72feb..1c99de6b 100644 --- a/stocks-prediction/src/serving_stocks.py +++ b/stocks-prediction/src/serving_stocks.py @@ -29,6 +29,16 @@ warnings.filterwarnings("ignore") +def log_results(df: pd.DataFrame): + # writing to tsdb + if "V3IO_USERNAME" in os.environ: + framesd = os.getenv("V3IO_FRAMESD",'framesd:8081') + client = v3f.Client(framesd, container=os.getenv('V3IO_CONTAINER', 'projects')) + kv_table_path = '/stocks-'+ os.environ['V3IO_USERNAME'] + '/artifacts/stocks_prediction' + client.write('kv', kv_table_path, dfs=df, index_cols=['datetime','tickers']) + print(f'writing prediction to kv at projects{kv_table_path} ...') + + def preprocess(event): vector_name = event['vector_name'] start_time = datetime.datetime.now()-datetime.timedelta(event['start_time']) @@ -85,14 +95,8 @@ def postprocess(event): df['true'] = event['outputs']['labels'] df['prediction'] = (df['prediction']*event['outputs']['price_std']) + event['outputs']['price_mean'] df['true'] = (df['true']*event['outputs']['price_std']) + event['outputs']['price_mean'] - df2 = df.copy() df['datetime'] = df['datetime'].apply(lambda x: x.strftime('%Y-%m-%d %H:%M:%S')) - # writing to tsdb - framesd = os.getenv("V3IO_FRAMESD",'framesd:8081') - client = v3f.Client(framesd, container=os.getenv('V3IO_CONTAINER', 'projects')) - kv_table_path = '/stocks-'+ os.environ['V3IO_USERNAME'] + '/artifacts/stocks_prediction' - client.write('kv', kv_table_path, dfs=df, index_cols=['datetime','tickers']) - print(f'writing prediction to kv at projects{kv_table_path} ...') + log_results(df) return [df.values.tolist(),list(df.columns)] class StocksModel(PyTorchModelServer): From 0a3c1de320e965bd42a2224308f06e85e0efb482 Mon Sep 17 00:00:00 2001 From: Avi Asulin Date: Wed, 22 Nov 2023 15:25:05 +0200 Subject: [PATCH 10/10] [Stocks] skip log results in case of CE --- stocks-prediction/src/serving_stocks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stocks-prediction/src/serving_stocks.py b/stocks-prediction/src/serving_stocks.py index 1c99de6b..ef7d09ed 100644 --- a/stocks-prediction/src/serving_stocks.py +++ b/stocks-prediction/src/serving_stocks.py @@ -27,8 +27,6 @@ import os import time -warnings.filterwarnings("ignore") - def log_results(df: pd.DataFrame): # writing to tsdb if "V3IO_USERNAME" in os.environ: