diff --git a/README.md b/README.md
index 41155acd..8fbbbb7b 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,9 @@ MAST-ML is an open-source Python package designed to broaden and accelerate the
* Tutorial 7: Model predictions with calibrated error bars, model hosting on Foundry/DLHub:
[](https://colab.research.google.com/github/uw-cmg/MAST-ML/blob/master/examples/MASTML_Tutorial_7_ModelPredictions_with_CalibratedErrorBars_HostModelonFoundry.ipynb)
+* Tutorial 8: Model predictions with guide rails:
+[](https://colab.research.google.com/github/uw-cmg/MAST-ML/blob/master/examples/MASTML_Tutorial_8_ModelPredictions_with_Guide_Rails.ipynb)
+
## MAST-ML version 3.1.x Major Updates from July 2022
* Refinement of tutorials, addition of Tutorial 7, Colab links as badges added for easier use.
diff --git a/VERSION b/VERSION
index c848fb9c..252fb772 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1,2 @@
-3.1.8
+3.2.0
+
diff --git a/examples/MASTML_Tutorial_1_GettingStarted.ipynb b/examples/MASTML_Tutorial_1_GettingStarted.ipynb
index e53ebc72..89dc27aa 100644
--- a/examples/MASTML_Tutorial_1_GettingStarted.ipynb
+++ b/examples/MASTML_Tutorial_1_GettingStarted.ipynb
@@ -22,7 +22,7 @@
"## In this notebook, we will perform a first, basic run where we: \n",
"\n",
"1. [Set up MAST-ML on Colab and begin session](#task1)\n",
- "2. [Import example data of Boston housing prices](#task2)\n",
+ "2. [Import example data of California housing prices](#task2)\n",
"3. [Define a data preprocessor to normalize the data](#task3)\n",
"4. [Define a linear regression model and kernel ridge model to fit the data](#task4)\n",
"5. [Evaluate each of our models with 5-fold cross validation](#task5)\n",
@@ -76,15 +76,6 @@
"!pip install mastml "
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!pip install pyyaml==5.4.1"
- ]
- },
{
"cell_type": "markdown",
"metadata": {
@@ -134,7 +125,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@@ -153,7 +144,34 @@
"id": "x8GIei1qQoWL",
"outputId": "5b15e778-2f39-4505-bae0-13f94baffb6a"
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Figshare is an optional dependency. To import data from figshare, manually install figshare via git clone of git clone https://github.com/cognoma/figshare.git\n",
+ "XGBoost is an optional dependency. If you want to use XGBoost models, please manually install xgboost package with pip install xgboost. If have error with finding libxgboost.dylib library, dobrew install libomp. If do not have brew on your system, first do ruby -e \"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)\" from the Terminal\n",
+ "scikit-lego is an optional dependency, enabling use of the LowessRegression model. If you want to use this model, do \"pip install scikit-lego\"\n",
+ "linear-tree is an optional dependency, enabling use of Linear tree, forest, and boosting models. If you want to use this model, do \"pip install linear-tree\"\n",
+ "gplearn is an optional dependency, enabling the use of genetic programming SymbolicRegressor model. If you want to use this model, do \"pip install gplearn\"\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/ryanjacobs/anaconda3/envs/tensorflow_env_v2/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+ " from .autonotebook import tqdm as notebook_tqdm\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "forestci is an optional dependency. To install latest forestci compatabilty with scikit-learn>=0.24, run pip install git+git://github.com/scikit-learn-contrib/forest-confidence-interval.git\n"
+ ]
+ }
+ ],
"source": [
"from mastml.mastml import Mastml\n",
"from mastml.datasets import SklearnDatasets\n",
@@ -179,7 +197,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {
"executionInfo": {
"elapsed": 454,
@@ -208,7 +226,7 @@
"id": "Ffv07xeKqlzU"
},
"source": [
- "## Task 2: Import example data of Boston housing prices "
+ "## Task 2: Import example data of California housing prices "
]
},
{
@@ -218,12 +236,12 @@
},
"source": [
"Here, we use the SklearnDatasets module to load in the data used in this tutorial.\n",
- "In this tutorial, we examine the model dataset of Boston housing prices."
+ "In this tutorial, we examine the model dataset of California housing prices."
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"metadata": {
"executionInfo": {
"elapsed": 451,
@@ -240,7 +258,7 @@
},
"outputs": [],
"source": [
- "X, y = SklearnDatasets(as_frame=True).load_boston()"
+ "X, y = SklearnDatasets(as_frame=True).load_housing()"
]
},
{
@@ -254,7 +272,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 4,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
@@ -274,7 +292,200 @@
"id": "3K0H7z7wqlzV",
"outputId": "1350c5c8-042f-4330-efdc-4068c1df9bed"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
MedInc
\n",
+ "
HouseAge
\n",
+ "
AveRooms
\n",
+ "
AveBedrms
\n",
+ "
Population
\n",
+ "
AveOccup
\n",
+ "
Latitude
\n",
+ "
Longitude
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
0
\n",
+ "
8.3252
\n",
+ "
41.0
\n",
+ "
6.984127
\n",
+ "
1.023810
\n",
+ "
322.0
\n",
+ "
2.555556
\n",
+ "
37.88
\n",
+ "
-122.23
\n",
+ "
\n",
+ "
\n",
+ "
1
\n",
+ "
8.3014
\n",
+ "
21.0
\n",
+ "
6.238137
\n",
+ "
0.971880
\n",
+ "
2401.0
\n",
+ "
2.109842
\n",
+ "
37.86
\n",
+ "
-122.22
\n",
+ "
\n",
+ "
\n",
+ "
2
\n",
+ "
7.2574
\n",
+ "
52.0
\n",
+ "
8.288136
\n",
+ "
1.073446
\n",
+ "
496.0
\n",
+ "
2.802260
\n",
+ "
37.85
\n",
+ "
-122.24
\n",
+ "
\n",
+ "
\n",
+ "
3
\n",
+ "
5.6431
\n",
+ "
52.0
\n",
+ "
5.817352
\n",
+ "
1.073059
\n",
+ "
558.0
\n",
+ "
2.547945
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
4
\n",
+ "
3.8462
\n",
+ "
52.0
\n",
+ "
6.281853
\n",
+ "
1.081081
\n",
+ "
565.0
\n",
+ "
2.181467
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
\n",
+ "
\n",
+ "
20635
\n",
+ "
1.5603
\n",
+ "
25.0
\n",
+ "
5.045455
\n",
+ "
1.133333
\n",
+ "
845.0
\n",
+ "
2.560606
\n",
+ "
39.48
\n",
+ "
-121.09
\n",
+ "
\n",
+ "
\n",
+ "
20636
\n",
+ "
2.5568
\n",
+ "
18.0
\n",
+ "
6.114035
\n",
+ "
1.315789
\n",
+ "
356.0
\n",
+ "
3.122807
\n",
+ "
39.49
\n",
+ "
-121.21
\n",
+ "
\n",
+ "
\n",
+ "
20637
\n",
+ "
1.7000
\n",
+ "
17.0
\n",
+ "
5.205543
\n",
+ "
1.120092
\n",
+ "
1007.0
\n",
+ "
2.325635
\n",
+ "
39.43
\n",
+ "
-121.22
\n",
+ "
\n",
+ "
\n",
+ "
20638
\n",
+ "
1.8672
\n",
+ "
18.0
\n",
+ "
5.329513
\n",
+ "
1.171920
\n",
+ "
741.0
\n",
+ "
2.123209
\n",
+ "
39.43
\n",
+ "
-121.32
\n",
+ "
\n",
+ "
\n",
+ "
20639
\n",
+ "
2.3886
\n",
+ "
16.0
\n",
+ "
5.254717
\n",
+ "
1.162264
\n",
+ "
1387.0
\n",
+ "
2.616981
\n",
+ "
39.37
\n",
+ "
-121.24
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
20640 rows × 8 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " MedInc HouseAge AveRooms AveBedrms Population AveOccup Latitude \\\n",
+ "0 8.3252 41.0 6.984127 1.023810 322.0 2.555556 37.88 \n",
+ "1 8.3014 21.0 6.238137 0.971880 2401.0 2.109842 37.86 \n",
+ "2 7.2574 52.0 8.288136 1.073446 496.0 2.802260 37.85 \n",
+ "3 5.6431 52.0 5.817352 1.073059 558.0 2.547945 37.85 \n",
+ "4 3.8462 52.0 6.281853 1.081081 565.0 2.181467 37.85 \n",
+ "... ... ... ... ... ... ... ... \n",
+ "20635 1.5603 25.0 5.045455 1.133333 845.0 2.560606 39.48 \n",
+ "20636 2.5568 18.0 6.114035 1.315789 356.0 3.122807 39.49 \n",
+ "20637 1.7000 17.0 5.205543 1.120092 1007.0 2.325635 39.43 \n",
+ "20638 1.8672 18.0 5.329513 1.171920 741.0 2.123209 39.43 \n",
+ "20639 2.3886 16.0 5.254717 1.162264 1387.0 2.616981 39.37 \n",
+ "\n",
+ " Longitude \n",
+ "0 -122.23 \n",
+ "1 -122.22 \n",
+ "2 -122.24 \n",
+ "3 -122.25 \n",
+ "4 -122.25 \n",
+ "... ... \n",
+ "20635 -121.09 \n",
+ "20636 -121.21 \n",
+ "20637 -121.22 \n",
+ "20638 -121.32 \n",
+ "20639 -121.24 \n",
+ "\n",
+ "[20640 rows x 8 columns]"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"X"
]
@@ -285,17 +496,15 @@
"id": "AMj2V94JqlzW"
},
"source": [
- "We have 506 data points (houses) and 13 columns (features). The features aren't the\n",
- "most straightforward to discern at first glance, but they include data such as square footage, age, zoning\n",
- "area, etc.\n",
+ "We have 20,640 data points (houses) and 8 columns (features). The features include data such as\n",
+ "location, population, house age, median income, etc.\n",
"\n",
- "From looking at the y-data, we see that the target data are home prices, in thousands. This\n",
- "data is from a while ago, which is why the prices are so low by today's standards"
+ "From looking at the y-data, we see that the target data are home prices, in units of $100k."
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 5,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
@@ -315,7 +524,103 @@
"id": "8RJ8llMgqlzW",
"outputId": "183b90e4-4f97-4ade-82fe-625b49fa65ec"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
MedHouseVal
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
0
\n",
+ "
4.526
\n",
+ "
\n",
+ "
\n",
+ "
1
\n",
+ "
3.585
\n",
+ "
\n",
+ "
\n",
+ "
2
\n",
+ "
3.521
\n",
+ "
\n",
+ "
\n",
+ "
3
\n",
+ "
3.413
\n",
+ "
\n",
+ "
\n",
+ "
4
\n",
+ "
3.422
\n",
+ "
\n",
+ "
\n",
+ "
...
\n",
+ "
...
\n",
+ "
\n",
+ "
\n",
+ "
20635
\n",
+ "
0.781
\n",
+ "
\n",
+ "
\n",
+ "
20636
\n",
+ "
0.771
\n",
+ "
\n",
+ "
\n",
+ "
20637
\n",
+ "
0.923
\n",
+ "
\n",
+ "
\n",
+ "
20638
\n",
+ "
0.847
\n",
+ "
\n",
+ "
\n",
+ "
20639
\n",
+ "
0.894
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
20640 rows × 1 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " MedHouseVal\n",
+ "0 4.526\n",
+ "1 3.585\n",
+ "2 3.521\n",
+ "3 3.413\n",
+ "4 3.422\n",
+ "... ...\n",
+ "20635 0.781\n",
+ "20636 0.771\n",
+ "20637 0.923\n",
+ "20638 0.847\n",
+ "20639 0.894\n",
+ "\n",
+ "[20640 rows x 1 columns]"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"y"
]
@@ -335,7 +640,7 @@
"id": "t-cISwwPqlzX"
},
"source": [
- "This data set already has a set of 13 features, so we don't need to generate\n",
+ "This data set already has a set of 8 features, so we don't need to generate\n",
"any more for the purposes of this example. However, we do want to normalize \n",
"our data for improved model fitting. Here, we define our preprocessing function. \n",
"We are just going to use the basic StandardScaler in scikit-learn to normalize \n",
@@ -344,7 +649,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 6,
"metadata": {
"executionInfo": {
"elapsed": 473,
@@ -379,7 +684,7 @@
"id": "zWsaU_vqqlzX"
},
"source": [
- "Now we need to decide what types of models to fit to the Boston housing data.\n",
+ "Now we need to decide what types of models to fit to the housing data.\n",
"Here, we define two models. The first is a basic linear regression model, and\n",
"the second is a kernel ridge model with Gaussian kernel. The scikit-learn model name can \n",
"just be given as a string matching the model name in the \"model\" field. \n",
@@ -389,7 +694,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 7,
"metadata": {
"executionInfo": {
"elapsed": 907,
@@ -421,7 +726,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 8,
"metadata": {
"executionInfo": {
"elapsed": 371,
@@ -467,7 +772,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 9,
"metadata": {
"executionInfo": {
"elapsed": 28652,
@@ -482,13 +787,28 @@
},
"id": "3-urmYsIzU7H"
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Running splits: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [00:05<00:00, 1.07s/it]\n",
+ "Running splits: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [01:22<00:00, 16.59s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Warning: unable to make Scatter.plot_best_worst_per_point plot. Skipping...\n"
+ ]
+ }
+ ],
"source": [
"splitter = SklearnDataSplitter(splitter='RepeatedKFold', n_repeats=1, n_splits=5)\n",
"splitter.evaluate(X=X,\n",
" y=y, \n",
" models=models,\n",
- " mastml=mastml_instance,\n",
" preprocessor=preprocessor,\n",
" savepath=savepath,\n",
" verbosity=3)"
@@ -512,7 +832,7 @@
"folder. This plot displays the true vs predicted values for all test data. \n",
"\n",
"\n",
- ""
+ ""
]
},
{
@@ -527,7 +847,7 @@
"test data, parity plots showing graphically the model performance on train and test data, and \n",
"the saved model and preprocessor files (in .pkl format).\n",
"\n",
- ""
+ ""
]
},
{
@@ -554,7 +874,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 11,
"metadata": {
"executionInfo": {
"elapsed": 34563,
@@ -569,14 +889,29 @@
},
"id": "Pz4LhAHRqlzb"
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Running splits: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████| 5/5 [05:16<00:00, 63.22s/it]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Warning: unable to make Scatter.plot_best_worst_per_point plot. Skipping...\n",
+ "Warning: unable to make Scatter.plot_best_worst_per_point plot. Skipping...\n"
+ ]
+ }
+ ],
"source": [
"model3 = SklearnModel(model='RandomForestRegressor')\n",
"splitter = SklearnDataSplitter(splitter='RepeatedKFold', n_repeats=1, n_splits=5)\n",
"splitter.evaluate(X=X,\n",
" y=y, \n",
" models=[model3],\n",
- " mastml=mastml_instance,\n",
" preprocessor=preprocessor,\n",
" savepath=savepath,\n",
" verbosity=3)"
@@ -588,7 +923,7 @@
"id": "oyjkI848qlzb"
},
"source": [
- "We can now see that the random forest analysis folder has been added to our main MAST-ML save path. Which of our models performed the best? Parity plots of true vs. predicted values on the test data from cross validation are shown below. It looks like the random forest model has the lowest RMSE!"
+ "We can now see that the random forest analysis folder has been added to our main MAST-ML save path. Which of our models performed the best? Parity plots of true vs. predicted values on the test data from cross validation are shown below, averaged over all of the CV splits. It looks like the random forest model has the lowest RMSE! However, there is still significant spread in the predicted model values, which may indicate that additional features or model types may be needed to get an improved fit for this data. Exploring feature engineering and model types is the subject of Tutorial 3"
]
},
{
@@ -598,13 +933,23 @@
},
"source": [
"## Linear model test data parity plot:\n",
- "\n",
- "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
"## Kernel ridge model test data parity plot:\n",
- "\n",
- "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
"## Random forest model test data parity plot:\n",
- ""
+ ""
]
},
{
@@ -617,7 +962,7 @@
"from a particular test. Below is the best and worst splits on test data for our random forest\n",
"model:\n",
"\n",
- ""
+ ""
]
},
{
@@ -654,7 +999,7 @@
"toc_visible": true
},
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -668,9 +1013,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.4"
+ "version": "3.11.7"
}
},
"nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
}
diff --git a/examples/MASTML_Tutorial_2_DataImport.ipynb b/examples/MASTML_Tutorial_2_DataImport.ipynb
index 2a113db7..98ff815c 100644
--- a/examples/MASTML_Tutorial_2_DataImport.ipynb
+++ b/examples/MASTML_Tutorial_2_DataImport.ipynb
@@ -75,15 +75,6 @@
"!pip install mastml"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!pip install pyyaml==5.4.1"
- ]
- },
{
"cell_type": "markdown",
"metadata": {
@@ -133,7 +124,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"metadata": {
"executionInfo": {
"elapsed": 569,
@@ -148,10 +139,19 @@
},
"id": "x8GIei1qQoWL"
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Figshare is an optional dependency. To import data from figshare, manually install figshare via git clone of git clone https://github.com/cognoma/figshare.git\n",
+ "forestci is an optional dependency. To install latest forestci compatabilty with scikit-learn>=0.24, run pip install git+git://github.com/scikit-learn-contrib/forest-confidence-interval.git\n"
+ ]
+ }
+ ],
"source": [
"from mastml.mastml import Mastml\n",
- "from mastml.datasets import SklearnDatasets, LocalDatasets, MatminerDatasets, FigshareDatasets, FoundryDatasets\n",
+ "from mastml.datasets import SklearnDatasets, LocalDatasets, MatminerDatasets,\n",
"from mastml.data_cleaning import DataCleaning\n",
"import numpy as np\n",
"from copy import copy\n",
@@ -180,7 +180,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {
"executionInfo": {
"elapsed": 694,
@@ -221,13 +221,13 @@
"Let's begin by first showing how to load some common model datasets that\n",
"come with the scikit-learn package. As in the first tutorial, we use the SklearnDatasets module \n",
"to load in the data. There are a few regression datasets that come with scikit-learn, such as\n",
- "the Boston housing data, the diabetes dataset, and the Friedman data set. For now,\n",
- "let's once again load in the Boston housing dataset."
+ "the California housing data, the diabetes dataset, and the Friedman data set. For now,\n",
+ "let's once again load in the California housing dataset."
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"metadata": {
"executionInfo": {
"elapsed": 604,
@@ -244,7 +244,7 @@
},
"outputs": [],
"source": [
- "X, y = SklearnDatasets(as_frame=True).load_boston()"
+ "X, y = SklearnDatasets(as_frame=True).load_housing()"
]
},
{
@@ -258,7 +258,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 4,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
@@ -278,7 +278,200 @@
"id": "X2REuNJ8-MCh",
"outputId": "cd3ac5bd-c7f8-41bb-a224-5ec549b44bc2"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
MedInc
\n",
+ "
HouseAge
\n",
+ "
AveRooms
\n",
+ "
AveBedrms
\n",
+ "
Population
\n",
+ "
AveOccup
\n",
+ "
Latitude
\n",
+ "
Longitude
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
0
\n",
+ "
8.3252
\n",
+ "
41.0
\n",
+ "
6.984127
\n",
+ "
1.023810
\n",
+ "
322.0
\n",
+ "
2.555556
\n",
+ "
37.88
\n",
+ "
-122.23
\n",
+ "
\n",
+ "
\n",
+ "
1
\n",
+ "
8.3014
\n",
+ "
21.0
\n",
+ "
6.238137
\n",
+ "
0.971880
\n",
+ "
2401.0
\n",
+ "
2.109842
\n",
+ "
37.86
\n",
+ "
-122.22
\n",
+ "
\n",
+ "
\n",
+ "
2
\n",
+ "
7.2574
\n",
+ "
52.0
\n",
+ "
8.288136
\n",
+ "
1.073446
\n",
+ "
496.0
\n",
+ "
2.802260
\n",
+ "
37.85
\n",
+ "
-122.24
\n",
+ "
\n",
+ "
\n",
+ "
3
\n",
+ "
5.6431
\n",
+ "
52.0
\n",
+ "
5.817352
\n",
+ "
1.073059
\n",
+ "
558.0
\n",
+ "
2.547945
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
4
\n",
+ "
3.8462
\n",
+ "
52.0
\n",
+ "
6.281853
\n",
+ "
1.081081
\n",
+ "
565.0
\n",
+ "
2.181467
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
\n",
+ "
\n",
+ "
20635
\n",
+ "
1.5603
\n",
+ "
25.0
\n",
+ "
5.045455
\n",
+ "
1.133333
\n",
+ "
845.0
\n",
+ "
2.560606
\n",
+ "
39.48
\n",
+ "
-121.09
\n",
+ "
\n",
+ "
\n",
+ "
20636
\n",
+ "
2.5568
\n",
+ "
18.0
\n",
+ "
6.114035
\n",
+ "
1.315789
\n",
+ "
356.0
\n",
+ "
3.122807
\n",
+ "
39.49
\n",
+ "
-121.21
\n",
+ "
\n",
+ "
\n",
+ "
20637
\n",
+ "
1.7000
\n",
+ "
17.0
\n",
+ "
5.205543
\n",
+ "
1.120092
\n",
+ "
1007.0
\n",
+ "
2.325635
\n",
+ "
39.43
\n",
+ "
-121.22
\n",
+ "
\n",
+ "
\n",
+ "
20638
\n",
+ "
1.8672
\n",
+ "
18.0
\n",
+ "
5.329513
\n",
+ "
1.171920
\n",
+ "
741.0
\n",
+ "
2.123209
\n",
+ "
39.43
\n",
+ "
-121.32
\n",
+ "
\n",
+ "
\n",
+ "
20639
\n",
+ "
2.3886
\n",
+ "
16.0
\n",
+ "
5.254717
\n",
+ "
1.162264
\n",
+ "
1387.0
\n",
+ "
2.616981
\n",
+ "
39.37
\n",
+ "
-121.24
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
20640 rows × 8 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " MedInc HouseAge AveRooms AveBedrms Population AveOccup Latitude \\\n",
+ "0 8.3252 41.0 6.984127 1.023810 322.0 2.555556 37.88 \n",
+ "1 8.3014 21.0 6.238137 0.971880 2401.0 2.109842 37.86 \n",
+ "2 7.2574 52.0 8.288136 1.073446 496.0 2.802260 37.85 \n",
+ "3 5.6431 52.0 5.817352 1.073059 558.0 2.547945 37.85 \n",
+ "4 3.8462 52.0 6.281853 1.081081 565.0 2.181467 37.85 \n",
+ "... ... ... ... ... ... ... ... \n",
+ "20635 1.5603 25.0 5.045455 1.133333 845.0 2.560606 39.48 \n",
+ "20636 2.5568 18.0 6.114035 1.315789 356.0 3.122807 39.49 \n",
+ "20637 1.7000 17.0 5.205543 1.120092 1007.0 2.325635 39.43 \n",
+ "20638 1.8672 18.0 5.329513 1.171920 741.0 2.123209 39.43 \n",
+ "20639 2.3886 16.0 5.254717 1.162264 1387.0 2.616981 39.37 \n",
+ "\n",
+ " Longitude \n",
+ "0 -122.23 \n",
+ "1 -122.22 \n",
+ "2 -122.24 \n",
+ "3 -122.25 \n",
+ "4 -122.25 \n",
+ "... ... \n",
+ "20635 -121.09 \n",
+ "20636 -121.21 \n",
+ "20637 -121.22 \n",
+ "20638 -121.32 \n",
+ "20639 -121.24 \n",
+ "\n",
+ "[20640 rows x 8 columns]"
+ ]
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"X"
]
@@ -298,16 +491,16 @@
"id": "TkykUBFX-MCi"
},
"source": [
- "We have 506 data points (houses) and 13 columns (features). From inspecting the dataset,\n",
+ "We have 20640 data points (houses) and 8 columns (features). From inspecting the dataset,\n",
"we can see there are no missing values in the dataset. However, many datasets in the real\n",
"world aren't this neat and tidy, and may contain values that are missing. To illustrate\n",
- "how we can go about cleaning our data, let's make the first few rows of values in the AGE\n",
+ "how we can go about cleaning our data, let's make the first few rows of values in the HouseAge\n",
"column not-a-number (NaN), which is what they will be if you import a dataset with missing values:"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 5,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
@@ -327,9 +520,202 @@
"id": "naiMVLaj-MCi",
"outputId": "ae239309-9b9f-4404-80dd-281c30bef08a"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
MedInc
\n",
+ "
HouseAge
\n",
+ "
AveRooms
\n",
+ "
AveBedrms
\n",
+ "
Population
\n",
+ "
AveOccup
\n",
+ "
Latitude
\n",
+ "
Longitude
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
0
\n",
+ "
8.3252
\n",
+ "
NaN
\n",
+ "
6.984127
\n",
+ "
1.023810
\n",
+ "
322.0
\n",
+ "
2.555556
\n",
+ "
37.88
\n",
+ "
-122.23
\n",
+ "
\n",
+ "
\n",
+ "
1
\n",
+ "
8.3014
\n",
+ "
NaN
\n",
+ "
6.238137
\n",
+ "
0.971880
\n",
+ "
2401.0
\n",
+ "
2.109842
\n",
+ "
37.86
\n",
+ "
-122.22
\n",
+ "
\n",
+ "
\n",
+ "
2
\n",
+ "
7.2574
\n",
+ "
NaN
\n",
+ "
8.288136
\n",
+ "
1.073446
\n",
+ "
496.0
\n",
+ "
2.802260
\n",
+ "
37.85
\n",
+ "
-122.24
\n",
+ "
\n",
+ "
\n",
+ "
3
\n",
+ "
5.6431
\n",
+ "
NaN
\n",
+ "
5.817352
\n",
+ "
1.073059
\n",
+ "
558.0
\n",
+ "
2.547945
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
4
\n",
+ "
3.8462
\n",
+ "
52.0
\n",
+ "
6.281853
\n",
+ "
1.081081
\n",
+ "
565.0
\n",
+ "
2.181467
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
\n",
+ "
\n",
+ "
20635
\n",
+ "
1.5603
\n",
+ "
25.0
\n",
+ "
5.045455
\n",
+ "
1.133333
\n",
+ "
845.0
\n",
+ "
2.560606
\n",
+ "
39.48
\n",
+ "
-121.09
\n",
+ "
\n",
+ "
\n",
+ "
20636
\n",
+ "
2.5568
\n",
+ "
18.0
\n",
+ "
6.114035
\n",
+ "
1.315789
\n",
+ "
356.0
\n",
+ "
3.122807
\n",
+ "
39.49
\n",
+ "
-121.21
\n",
+ "
\n",
+ "
\n",
+ "
20637
\n",
+ "
1.7000
\n",
+ "
17.0
\n",
+ "
5.205543
\n",
+ "
1.120092
\n",
+ "
1007.0
\n",
+ "
2.325635
\n",
+ "
39.43
\n",
+ "
-121.22
\n",
+ "
\n",
+ "
\n",
+ "
20638
\n",
+ "
1.8672
\n",
+ "
18.0
\n",
+ "
5.329513
\n",
+ "
1.171920
\n",
+ "
741.0
\n",
+ "
2.123209
\n",
+ "
39.43
\n",
+ "
-121.32
\n",
+ "
\n",
+ "
\n",
+ "
20639
\n",
+ "
2.3886
\n",
+ "
16.0
\n",
+ "
5.254717
\n",
+ "
1.162264
\n",
+ "
1387.0
\n",
+ "
2.616981
\n",
+ "
39.37
\n",
+ "
-121.24
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
20640 rows × 8 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " MedInc HouseAge AveRooms AveBedrms Population AveOccup Latitude \\\n",
+ "0 8.3252 NaN 6.984127 1.023810 322.0 2.555556 37.88 \n",
+ "1 8.3014 NaN 6.238137 0.971880 2401.0 2.109842 37.86 \n",
+ "2 7.2574 NaN 8.288136 1.073446 496.0 2.802260 37.85 \n",
+ "3 5.6431 NaN 5.817352 1.073059 558.0 2.547945 37.85 \n",
+ "4 3.8462 52.0 6.281853 1.081081 565.0 2.181467 37.85 \n",
+ "... ... ... ... ... ... ... ... \n",
+ "20635 1.5603 25.0 5.045455 1.133333 845.0 2.560606 39.48 \n",
+ "20636 2.5568 18.0 6.114035 1.315789 356.0 3.122807 39.49 \n",
+ "20637 1.7000 17.0 5.205543 1.120092 1007.0 2.325635 39.43 \n",
+ "20638 1.8672 18.0 5.329513 1.171920 741.0 2.123209 39.43 \n",
+ "20639 2.3886 16.0 5.254717 1.162264 1387.0 2.616981 39.37 \n",
+ "\n",
+ " Longitude \n",
+ "0 -122.23 \n",
+ "1 -122.22 \n",
+ "2 -122.24 \n",
+ "3 -122.25 \n",
+ "4 -122.25 \n",
+ "... ... \n",
+ "20635 -121.09 \n",
+ "20636 -121.21 \n",
+ "20637 -121.22 \n",
+ "20638 -121.32 \n",
+ "20639 -121.24 \n",
+ "\n",
+ "[20640 rows x 8 columns]"
+ ]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "X['AGE'][0:4] = np.nan\n",
+ "X['HouseAge'][0:4] = np.nan\n",
"X"
]
},
@@ -339,7 +725,7 @@
"id": "w7H8KtTI-MCj"
},
"source": [
- "We can now see that the first four rows of the 'AGE' column are NaN instead\n",
+ "We can now see that the first four rows of the 'HouseAge' column are NaN instead\n",
"of their original values. Let's clean the data using a few different methods\n",
"in the DataCleaning class. First, let's copy the X data so we can more easily\n",
"see the changes we make to each dataset"
@@ -347,7 +733,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 6,
"metadata": {
"executionInfo": {
"elapsed": 537,
@@ -382,7 +768,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
@@ -402,7 +788,200 @@
"id": "8_zjf-cR-MCk",
"outputId": "694dc306-86ce-4f8e-aa15-65d39dc9b5ab"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
MedInc
\n",
+ "
HouseAge
\n",
+ "
AveRooms
\n",
+ "
AveBedrms
\n",
+ "
Population
\n",
+ "
AveOccup
\n",
+ "
Latitude
\n",
+ "
Longitude
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
4
\n",
+ "
3.8462
\n",
+ "
52.0
\n",
+ "
6.281853
\n",
+ "
1.081081
\n",
+ "
565.0
\n",
+ "
2.181467
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
5
\n",
+ "
4.0368
\n",
+ "
52.0
\n",
+ "
4.761658
\n",
+ "
1.103627
\n",
+ "
413.0
\n",
+ "
2.139896
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
6
\n",
+ "
3.6591
\n",
+ "
52.0
\n",
+ "
4.931907
\n",
+ "
0.951362
\n",
+ "
1094.0
\n",
+ "
2.128405
\n",
+ "
37.84
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
7
\n",
+ "
3.1200
\n",
+ "
52.0
\n",
+ "
4.797527
\n",
+ "
1.061824
\n",
+ "
1157.0
\n",
+ "
1.788253
\n",
+ "
37.84
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
8
\n",
+ "
2.0804
\n",
+ "
42.0
\n",
+ "
4.294118
\n",
+ "
1.117647
\n",
+ "
1206.0
\n",
+ "
2.026891
\n",
+ "
37.84
\n",
+ "
-122.26
\n",
+ "
\n",
+ "
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
\n",
+ "
\n",
+ "
20635
\n",
+ "
1.5603
\n",
+ "
25.0
\n",
+ "
5.045455
\n",
+ "
1.133333
\n",
+ "
845.0
\n",
+ "
2.560606
\n",
+ "
39.48
\n",
+ "
-121.09
\n",
+ "
\n",
+ "
\n",
+ "
20636
\n",
+ "
2.5568
\n",
+ "
18.0
\n",
+ "
6.114035
\n",
+ "
1.315789
\n",
+ "
356.0
\n",
+ "
3.122807
\n",
+ "
39.49
\n",
+ "
-121.21
\n",
+ "
\n",
+ "
\n",
+ "
20637
\n",
+ "
1.7000
\n",
+ "
17.0
\n",
+ "
5.205543
\n",
+ "
1.120092
\n",
+ "
1007.0
\n",
+ "
2.325635
\n",
+ "
39.43
\n",
+ "
-121.22
\n",
+ "
\n",
+ "
\n",
+ "
20638
\n",
+ "
1.8672
\n",
+ "
18.0
\n",
+ "
5.329513
\n",
+ "
1.171920
\n",
+ "
741.0
\n",
+ "
2.123209
\n",
+ "
39.43
\n",
+ "
-121.32
\n",
+ "
\n",
+ "
\n",
+ "
20639
\n",
+ "
2.3886
\n",
+ "
16.0
\n",
+ "
5.254717
\n",
+ "
1.162264
\n",
+ "
1387.0
\n",
+ "
2.616981
\n",
+ "
39.37
\n",
+ "
-121.24
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
20636 rows × 8 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " MedInc HouseAge AveRooms AveBedrms Population AveOccup Latitude \\\n",
+ "4 3.8462 52.0 6.281853 1.081081 565.0 2.181467 37.85 \n",
+ "5 4.0368 52.0 4.761658 1.103627 413.0 2.139896 37.85 \n",
+ "6 3.6591 52.0 4.931907 0.951362 1094.0 2.128405 37.84 \n",
+ "7 3.1200 52.0 4.797527 1.061824 1157.0 1.788253 37.84 \n",
+ "8 2.0804 42.0 4.294118 1.117647 1206.0 2.026891 37.84 \n",
+ "... ... ... ... ... ... ... ... \n",
+ "20635 1.5603 25.0 5.045455 1.133333 845.0 2.560606 39.48 \n",
+ "20636 2.5568 18.0 6.114035 1.315789 356.0 3.122807 39.49 \n",
+ "20637 1.7000 17.0 5.205543 1.120092 1007.0 2.325635 39.43 \n",
+ "20638 1.8672 18.0 5.329513 1.171920 741.0 2.123209 39.43 \n",
+ "20639 2.3886 16.0 5.254717 1.162264 1387.0 2.616981 39.37 \n",
+ "\n",
+ " Longitude \n",
+ "4 -122.25 \n",
+ "5 -122.25 \n",
+ "6 -122.25 \n",
+ "7 -122.25 \n",
+ "8 -122.26 \n",
+ "... ... \n",
+ "20635 -121.09 \n",
+ "20636 -121.21 \n",
+ "20637 -121.22 \n",
+ "20638 -121.32 \n",
+ "20639 -121.24 \n",
+ "\n",
+ "[20636 rows x 8 columns]"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"X1row, y1 = DataCleaning().remove(X=X1row, \n",
" y=y, \n",
@@ -416,8 +995,8 @@
"id": "jqI9p2JJ-MCk"
},
"source": [
- "What we can see is that we now have 502 rows instead of the original 506. \n",
- "The four data points that had missing values in the AGE column have been removed.\n",
+ "What we can see is that we now have 20636 rows instead of the original 20640. \n",
+ "The four data points that had missing values in the HouseAge column have been removed.\n",
"\n",
"Note that we could have instead removed the column (feature) containing\n",
"the missing values, as in our case all of the missing values were part of\n",
@@ -426,7 +1005,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 8,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
@@ -446,7 +1025,175 @@
"id": "yWXwUdff-MCk",
"outputId": "138f9abb-6fdc-4ed9-842e-46c7ba6e77dc"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
MedInc
\n",
+ "
AveRooms
\n",
+ "
AveBedrms
\n",
+ "
Population
\n",
+ "
AveOccup
\n",
+ "
Latitude
\n",
+ "
Longitude
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
0
\n",
+ "
8.3252
\n",
+ "
6.984127
\n",
+ "
1.023810
\n",
+ "
322.0
\n",
+ "
2.555556
\n",
+ "
37.88
\n",
+ "
-122.23
\n",
+ "
\n",
+ "
\n",
+ "
1
\n",
+ "
8.3014
\n",
+ "
6.238137
\n",
+ "
0.971880
\n",
+ "
2401.0
\n",
+ "
2.109842
\n",
+ "
37.86
\n",
+ "
-122.22
\n",
+ "
\n",
+ "
\n",
+ "
2
\n",
+ "
7.2574
\n",
+ "
8.288136
\n",
+ "
1.073446
\n",
+ "
496.0
\n",
+ "
2.802260
\n",
+ "
37.85
\n",
+ "
-122.24
\n",
+ "
\n",
+ "
\n",
+ "
3
\n",
+ "
5.6431
\n",
+ "
5.817352
\n",
+ "
1.073059
\n",
+ "
558.0
\n",
+ "
2.547945
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
4
\n",
+ "
3.8462
\n",
+ "
6.281853
\n",
+ "
1.081081
\n",
+ "
565.0
\n",
+ "
2.181467
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
\n",
+ "
\n",
+ "
20635
\n",
+ "
1.5603
\n",
+ "
5.045455
\n",
+ "
1.133333
\n",
+ "
845.0
\n",
+ "
2.560606
\n",
+ "
39.48
\n",
+ "
-121.09
\n",
+ "
\n",
+ "
\n",
+ "
20636
\n",
+ "
2.5568
\n",
+ "
6.114035
\n",
+ "
1.315789
\n",
+ "
356.0
\n",
+ "
3.122807
\n",
+ "
39.49
\n",
+ "
-121.21
\n",
+ "
\n",
+ "
\n",
+ "
20637
\n",
+ "
1.7000
\n",
+ "
5.205543
\n",
+ "
1.120092
\n",
+ "
1007.0
\n",
+ "
2.325635
\n",
+ "
39.43
\n",
+ "
-121.22
\n",
+ "
\n",
+ "
\n",
+ "
20638
\n",
+ "
1.8672
\n",
+ "
5.329513
\n",
+ "
1.171920
\n",
+ "
741.0
\n",
+ "
2.123209
\n",
+ "
39.43
\n",
+ "
-121.32
\n",
+ "
\n",
+ "
\n",
+ "
20639
\n",
+ "
2.3886
\n",
+ "
5.254717
\n",
+ "
1.162264
\n",
+ "
1387.0
\n",
+ "
2.616981
\n",
+ "
39.37
\n",
+ "
-121.24
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
20640 rows × 7 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " MedInc AveRooms AveBedrms Population AveOccup Latitude Longitude\n",
+ "0 8.3252 6.984127 1.023810 322.0 2.555556 37.88 -122.23\n",
+ "1 8.3014 6.238137 0.971880 2401.0 2.109842 37.86 -122.22\n",
+ "2 7.2574 8.288136 1.073446 496.0 2.802260 37.85 -122.24\n",
+ "3 5.6431 5.817352 1.073059 558.0 2.547945 37.85 -122.25\n",
+ "4 3.8462 6.281853 1.081081 565.0 2.181467 37.85 -122.25\n",
+ "... ... ... ... ... ... ... ...\n",
+ "20635 1.5603 5.045455 1.133333 845.0 2.560606 39.48 -121.09\n",
+ "20636 2.5568 6.114035 1.315789 356.0 3.122807 39.49 -121.21\n",
+ "20637 1.7000 5.205543 1.120092 1007.0 2.325635 39.43 -121.22\n",
+ "20638 1.8672 5.329513 1.171920 741.0 2.123209 39.43 -121.32\n",
+ "20639 2.3886 5.254717 1.162264 1387.0 2.616981 39.37 -121.24\n",
+ "\n",
+ "[20640 rows x 7 columns]"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"X1col, y1 = DataCleaning().remove(X=X1col, \n",
" y=y, \n",
@@ -460,8 +1207,8 @@
"id": "LQF2J6Ra-MCl"
},
"source": [
- "What we can see is that we now have all 506 of our original data points, \n",
- "but we now just have 12 of the 13 features as the AGE feature has been removed."
+ "What we can see is that we now have all 20640 of our original data points, \n",
+ "but we now just have 7 of the 8 features as the HouseAge feature has been removed."
]
},
{
@@ -477,7 +1224,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 9,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
@@ -497,7 +1244,200 @@
"id": "3D4Iy6Y0-MCl",
"outputId": "c056d3dd-7274-4342-9248-c7c18ad9e122"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
MedInc
\n",
+ "
HouseAge
\n",
+ "
AveRooms
\n",
+ "
AveBedrms
\n",
+ "
Population
\n",
+ "
AveOccup
\n",
+ "
Latitude
\n",
+ "
Longitude
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
0
\n",
+ "
8.3252
\n",
+ "
28.636994
\n",
+ "
6.984127
\n",
+ "
1.023810
\n",
+ "
322.0
\n",
+ "
2.555556
\n",
+ "
37.88
\n",
+ "
-122.23
\n",
+ "
\n",
+ "
\n",
+ "
1
\n",
+ "
8.3014
\n",
+ "
28.636994
\n",
+ "
6.238137
\n",
+ "
0.971880
\n",
+ "
2401.0
\n",
+ "
2.109842
\n",
+ "
37.86
\n",
+ "
-122.22
\n",
+ "
\n",
+ "
\n",
+ "
2
\n",
+ "
7.2574
\n",
+ "
28.636994
\n",
+ "
8.288136
\n",
+ "
1.073446
\n",
+ "
496.0
\n",
+ "
2.802260
\n",
+ "
37.85
\n",
+ "
-122.24
\n",
+ "
\n",
+ "
\n",
+ "
3
\n",
+ "
5.6431
\n",
+ "
28.636994
\n",
+ "
5.817352
\n",
+ "
1.073059
\n",
+ "
558.0
\n",
+ "
2.547945
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
4
\n",
+ "
3.8462
\n",
+ "
52.000000
\n",
+ "
6.281853
\n",
+ "
1.081081
\n",
+ "
565.0
\n",
+ "
2.181467
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
\n",
+ "
\n",
+ "
20635
\n",
+ "
1.5603
\n",
+ "
25.000000
\n",
+ "
5.045455
\n",
+ "
1.133333
\n",
+ "
845.0
\n",
+ "
2.560606
\n",
+ "
39.48
\n",
+ "
-121.09
\n",
+ "
\n",
+ "
\n",
+ "
20636
\n",
+ "
2.5568
\n",
+ "
18.000000
\n",
+ "
6.114035
\n",
+ "
1.315789
\n",
+ "
356.0
\n",
+ "
3.122807
\n",
+ "
39.49
\n",
+ "
-121.21
\n",
+ "
\n",
+ "
\n",
+ "
20637
\n",
+ "
1.7000
\n",
+ "
17.000000
\n",
+ "
5.205543
\n",
+ "
1.120092
\n",
+ "
1007.0
\n",
+ "
2.325635
\n",
+ "
39.43
\n",
+ "
-121.22
\n",
+ "
\n",
+ "
\n",
+ "
20638
\n",
+ "
1.8672
\n",
+ "
18.000000
\n",
+ "
5.329513
\n",
+ "
1.171920
\n",
+ "
741.0
\n",
+ "
2.123209
\n",
+ "
39.43
\n",
+ "
-121.32
\n",
+ "
\n",
+ "
\n",
+ "
20639
\n",
+ "
2.3886
\n",
+ "
16.000000
\n",
+ "
5.254717
\n",
+ "
1.162264
\n",
+ "
1387.0
\n",
+ "
2.616981
\n",
+ "
39.37
\n",
+ "
-121.24
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
20640 rows × 8 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " MedInc HouseAge AveRooms AveBedrms Population AveOccup Latitude \\\n",
+ "0 8.3252 28.636994 6.984127 1.023810 322.0 2.555556 37.88 \n",
+ "1 8.3014 28.636994 6.238137 0.971880 2401.0 2.109842 37.86 \n",
+ "2 7.2574 28.636994 8.288136 1.073446 496.0 2.802260 37.85 \n",
+ "3 5.6431 28.636994 5.817352 1.073059 558.0 2.547945 37.85 \n",
+ "4 3.8462 52.000000 6.281853 1.081081 565.0 2.181467 37.85 \n",
+ "... ... ... ... ... ... ... ... \n",
+ "20635 1.5603 25.000000 5.045455 1.133333 845.0 2.560606 39.48 \n",
+ "20636 2.5568 18.000000 6.114035 1.315789 356.0 3.122807 39.49 \n",
+ "20637 1.7000 17.000000 5.205543 1.120092 1007.0 2.325635 39.43 \n",
+ "20638 1.8672 18.000000 5.329513 1.171920 741.0 2.123209 39.43 \n",
+ "20639 2.3886 16.000000 5.254717 1.162264 1387.0 2.616981 39.37 \n",
+ "\n",
+ " Longitude \n",
+ "0 -122.23 \n",
+ "1 -122.22 \n",
+ "2 -122.24 \n",
+ "3 -122.25 \n",
+ "4 -122.25 \n",
+ "... ... \n",
+ "20635 -121.09 \n",
+ "20636 -121.21 \n",
+ "20637 -121.22 \n",
+ "20638 -121.32 \n",
+ "20639 -121.24 \n",
+ "\n",
+ "[20640 rows x 8 columns]"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"X2, y2 = DataCleaning().imputation(X=X2, \n",
" y=y, \n",
@@ -511,8 +1451,8 @@
"id": "B6qDIrMD-MCm"
},
"source": [
- "What we can see is that we now have all 506 of our original data points and \n",
- "all 13 of our feature columns, and the first four rows of the AGE feature\n",
+ "What we can see is that we now have all 20640 of our original data points and \n",
+ "all 8 of our feature columns, and the first four rows of the HousAge feature\n",
"now contain the average value for that feature.\n",
"\n",
"Finally, let's do imputation again but use the median instead of \n",
@@ -521,7 +1461,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 10,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
@@ -541,7 +1481,200 @@
"id": "PRRE8_1h-MCm",
"outputId": "99241928-c592-4863-c73a-d1f733bf9852"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
MedInc
\n",
+ "
HouseAge
\n",
+ "
AveRooms
\n",
+ "
AveBedrms
\n",
+ "
Population
\n",
+ "
AveOccup
\n",
+ "
Latitude
\n",
+ "
Longitude
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
0
\n",
+ "
8.3252
\n",
+ "
29.0
\n",
+ "
6.984127
\n",
+ "
1.023810
\n",
+ "
322.0
\n",
+ "
2.555556
\n",
+ "
37.88
\n",
+ "
-122.23
\n",
+ "
\n",
+ "
\n",
+ "
1
\n",
+ "
8.3014
\n",
+ "
29.0
\n",
+ "
6.238137
\n",
+ "
0.971880
\n",
+ "
2401.0
\n",
+ "
2.109842
\n",
+ "
37.86
\n",
+ "
-122.22
\n",
+ "
\n",
+ "
\n",
+ "
2
\n",
+ "
7.2574
\n",
+ "
29.0
\n",
+ "
8.288136
\n",
+ "
1.073446
\n",
+ "
496.0
\n",
+ "
2.802260
\n",
+ "
37.85
\n",
+ "
-122.24
\n",
+ "
\n",
+ "
\n",
+ "
3
\n",
+ "
5.6431
\n",
+ "
29.0
\n",
+ "
5.817352
\n",
+ "
1.073059
\n",
+ "
558.0
\n",
+ "
2.547945
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
4
\n",
+ "
3.8462
\n",
+ "
52.0
\n",
+ "
6.281853
\n",
+ "
1.081081
\n",
+ "
565.0
\n",
+ "
2.181467
\n",
+ "
37.85
\n",
+ "
-122.25
\n",
+ "
\n",
+ "
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
\n",
+ "
\n",
+ "
20635
\n",
+ "
1.5603
\n",
+ "
25.0
\n",
+ "
5.045455
\n",
+ "
1.133333
\n",
+ "
845.0
\n",
+ "
2.560606
\n",
+ "
39.48
\n",
+ "
-121.09
\n",
+ "
\n",
+ "
\n",
+ "
20636
\n",
+ "
2.5568
\n",
+ "
18.0
\n",
+ "
6.114035
\n",
+ "
1.315789
\n",
+ "
356.0
\n",
+ "
3.122807
\n",
+ "
39.49
\n",
+ "
-121.21
\n",
+ "
\n",
+ "
\n",
+ "
20637
\n",
+ "
1.7000
\n",
+ "
17.0
\n",
+ "
5.205543
\n",
+ "
1.120092
\n",
+ "
1007.0
\n",
+ "
2.325635
\n",
+ "
39.43
\n",
+ "
-121.22
\n",
+ "
\n",
+ "
\n",
+ "
20638
\n",
+ "
1.8672
\n",
+ "
18.0
\n",
+ "
5.329513
\n",
+ "
1.171920
\n",
+ "
741.0
\n",
+ "
2.123209
\n",
+ "
39.43
\n",
+ "
-121.32
\n",
+ "
\n",
+ "
\n",
+ "
20639
\n",
+ "
2.3886
\n",
+ "
16.0
\n",
+ "
5.254717
\n",
+ "
1.162264
\n",
+ "
1387.0
\n",
+ "
2.616981
\n",
+ "
39.37
\n",
+ "
-121.24
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
20640 rows × 8 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " MedInc HouseAge AveRooms AveBedrms Population AveOccup Latitude \\\n",
+ "0 8.3252 29.0 6.984127 1.023810 322.0 2.555556 37.88 \n",
+ "1 8.3014 29.0 6.238137 0.971880 2401.0 2.109842 37.86 \n",
+ "2 7.2574 29.0 8.288136 1.073446 496.0 2.802260 37.85 \n",
+ "3 5.6431 29.0 5.817352 1.073059 558.0 2.547945 37.85 \n",
+ "4 3.8462 52.0 6.281853 1.081081 565.0 2.181467 37.85 \n",
+ "... ... ... ... ... ... ... ... \n",
+ "20635 1.5603 25.0 5.045455 1.133333 845.0 2.560606 39.48 \n",
+ "20636 2.5568 18.0 6.114035 1.315789 356.0 3.122807 39.49 \n",
+ "20637 1.7000 17.0 5.205543 1.120092 1007.0 2.325635 39.43 \n",
+ "20638 1.8672 18.0 5.329513 1.171920 741.0 2.123209 39.43 \n",
+ "20639 2.3886 16.0 5.254717 1.162264 1387.0 2.616981 39.37 \n",
+ "\n",
+ " Longitude \n",
+ "0 -122.23 \n",
+ "1 -122.22 \n",
+ "2 -122.24 \n",
+ "3 -122.25 \n",
+ "4 -122.25 \n",
+ "... ... \n",
+ "20635 -121.09 \n",
+ "20636 -121.21 \n",
+ "20637 -121.22 \n",
+ "20638 -121.32 \n",
+ "20639 -121.24 \n",
+ "\n",
+ "[20640 rows x 8 columns]"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"X3, y3 = DataCleaning().imputation(X=X3, \n",
" y=y, \n",
@@ -555,8 +1688,8 @@
"id": "WunBiCC9-MCm"
},
"source": [
- "What we can see is that we now have all 506 of our original data points and \n",
- "all 13 of our feature columns, and the first four rows of the AGE feature\n",
+ "What we can see is that we now have all 20640 of our original data points and \n",
+ "all 8 of our feature columns, and the first four rows of the HouseAge feature\n",
"now contain the median value for that feature."
]
},
@@ -574,7 +1707,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 11,
"metadata": {
"executionInfo": {
"elapsed": 1873,
@@ -634,10 +1767,7 @@
"source": [
"### DataCleaning directory with files:\n",
"\n",
- "\n",
- "\n",
- "### Histogram of target values:\n",
- ""
+ ""
]
},
{
@@ -677,7 +1807,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 12,
"metadata": {
"executionInfo": {
"elapsed": 1314,
@@ -726,7 +1856,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 13,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@@ -745,7 +1875,15 @@
"id": "1CKvDJug-MCq",
"outputId": "fe856970-5678-449d-9379-e69a1add6e10"
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "WARNING: feature_names not specified but target was specified. Assuming all columns except target and extra columns are features\n"
+ ]
+ }
+ ],
"source": [
"d = LocalDatasets(file_path=data_path+'/diffusion_data_allfeatures.xlsx',\n",
" target=target, \n",
@@ -780,7 +1918,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 14,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@@ -799,7 +1937,18 @@
"id": "h-wb7YNk-MCr",
"outputId": "12f757c2-1f87-4dd9-c708-fbc9352bbfe1"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "dict_keys(['X', 'y', 'groups', 'X_extra', 'X_testdata'])"
+ ]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"data_dict.keys()"
]
@@ -815,7 +1964,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 15,
"metadata": {
"executionInfo": {
"elapsed": 1868,
@@ -851,7 +2000,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 16,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
@@ -871,7 +2020,460 @@
"id": "MMq8D6RK-MCs",
"outputId": "e5ec7bd1-98b6-4923-b522-2b1fb3145920"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
"
+ ],
+ "text/plain": [
+ " c mn si cr ni mo v n nb co w \\\n",
+ "0 0.02 0.05 0.05 0.01 19.70 2.95 0.01 0.00 0.01 15.00 0.00 \n",
+ "1 0.18 0.01 0.01 13.44 0.01 3.01 0.46 0.04 0.01 19.46 2.35 \n",
+ "2 0.00 0.01 0.01 8.67 13.45 0.82 0.01 0.00 0.01 13.90 0.00 \n",
+ "3 0.01 0.05 0.05 0.01 17.70 3.95 0.01 0.00 0.01 15.00 0.00 \n",
+ "4 0.01 0.05 0.05 0.01 19.40 1.45 0.01 0.00 0.01 14.90 0.00 \n",
+ ".. ... ... ... ... ... ... ... ... ... ... ... \n",
+ "307 0.38 0.18 0.01 7.27 0.01 3.77 0.96 0.00 0.01 4.90 0.00 \n",
+ "308 0.00 0.06 0.05 5.15 10.20 3.20 0.01 0.00 0.01 0.01 0.00 \n",
+ "309 0.37 0.17 0.01 5.20 0.01 5.84 1.05 0.00 0.91 4.88 0.00 \n",
+ "310 0.41 0.19 0.01 6.99 0.01 5.84 0.92 0.00 0.01 0.02 0.00 \n",
+ "311 0.27 0.27 0.01 0.41 8.28 0.49 0.07 0.00 0.01 3.90 0.00 \n",
+ "\n",
+ " al ti \n",
+ "0 0.15 1.55 \n",
+ "1 0.04 0.00 \n",
+ "2 0.39 0.57 \n",
+ "3 0.13 1.47 \n",
+ "4 0.13 1.55 \n",
+ ".. ... ... \n",
+ "307 0.03 0.00 \n",
+ "308 0.03 0.09 \n",
+ "309 0.03 0.00 \n",
+ "310 0.03 0.00 \n",
+ "311 0.03 0.00 \n",
+ "\n",
+ "[312 rows x 13 columns]"
+ ]
+ },
+ "execution_count": 22,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"X"
]
@@ -1134,7 +3154,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 23,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@@ -1153,7 +3173,29 @@
"id": "0t5ClFcm-MCv",
"outputId": "188ace7c-44e0-438f-de67-bfa81dbebde5"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "0 2411.5\n",
+ "1 1123.1\n",
+ "2 1736.3\n",
+ "3 2487.3\n",
+ "4 2249.6\n",
+ " ... \n",
+ "307 1722.5\n",
+ "308 1019.0\n",
+ "309 1860.3\n",
+ "310 1812.1\n",
+ "311 1139.7\n",
+ "Name: yield strength, Length: 312, dtype: float64"
+ ]
+ },
+ "execution_count": 23,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
"y"
]
@@ -1171,7 +3213,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 24,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@@ -1190,7 +3232,105 @@
"id": "rD-MiXlR-MCv",
"outputId": "e0ae241c-8ea0-4db6-ab89-69fe88a4562a"
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "boltztrap_mp: Effective mass and thermoelectric properties of 8924 compounds in The Materials Project database that are calculated by the BoltzTraP software package run on the GGA-PBE or GGA+U density functional theory calculation results. The properties are reported at the temperature of 300 Kelvin and the carrier concentration of 1e18 1/cm3.\n",
+ "\n",
+ "brgoch_superhard_training: 2574 materials used for training regressors that predict shear and bulk modulus.\n",
+ "\n",
+ "castelli_perovskites: 18,928 perovskites generated with ABX combinatorics, calculating gllbsc band gap and pbe structure, and also reporting absolute band edge positions and heat of formation.\n",
+ "\n",
+ "citrine_thermal_conductivity: Thermal conductivity of 872 compounds measured experimentally and retrieved from Citrine database from various references. The reported values are measured at various temperatures of which 295 are at room temperature.\n",
+ "\n",
+ "dielectric_constant: 1,056 structures with dielectric properties, calculated with DFPT-PBE.\n",
+ "\n",
+ "double_perovskites_gap: Band gap of 1306 double perovskites (a_1-b_1-a_2-b_2-O6) calculated using Gritsenko, van Leeuwen, van Lenthe and Baerends potential (gllbsc) in GPAW.\n",
+ "\n",
+ "double_perovskites_gap_lumo: Supplementary lumo data of 55 atoms for the double_perovskites_gap dataset.\n",
+ "\n",
+ "elastic_tensor_2015: 1,181 structures with elastic properties calculated with DFT-PBE.\n",
+ "\n",
+ "expt_formation_enthalpy: Experimental formation enthalpies for inorganic compounds, collected from years of calorimetric experiments. There are 1,276 entries in this dataset, mostly binary compounds. Matching mpids or oqmdids as well as the DFT-computed formation energies are also added (if any).\n",
+ "\n",
+ "expt_formation_enthalpy_kingsbury: Dataset containing experimental standard formation enthalpies for solids. Formation enthalpies were compiled primarily from Kim et al., Kubaschewski, and the NIST JANAF tables (see references). Elements, liquids, and gases were excluded. Data were deduplicated such that each material is associated with a single formation enthalpy value. Refer to Wang et al. (see references) for a complete desciption of the methods used. Materials Project database IDs (mp-ids) were assigned to materials from among computed materials in the Materials Project database (version 2021.03.22) that were 1) not marked 'theoretical', 2) had structures matching at least one ICSD material, and 3) were within 200 meV of the DFT-computed stable energy hull (e_above_hull < 0.2 eV). Among these candidates, we chose the mp-id with the lowest e_above_hull that matched the reported spacegroup (where available).\n",
+ "\n",
+ "expt_gap: Experimental band gap of 6354 inorganic semiconductors.\n",
+ "\n",
+ "expt_gap_kingsbury: Identical to the matbench_expt_gap dataset, except that Materials Project database IDs (mp-ids) have been associated with each material using the same method as described for the expt_formation_enthalpy_kingsbury dataset. Columns have also been renamed for consistency with the formation enthalpy data.\n",
+ "\n",
+ "flla: 3938 structures and computed formation energies from \"Crystal Structure Representations for Machine Learning Models of Formation Energies.\"\n",
+ "\n",
+ "glass_binary: Metallic glass formation data for binary alloys, collected from various experimental techniques such as melt-spinning or mechanical alloying. This dataset covers all compositions with an interval of 5 at. % in 59 binary systems, containing a total of 5959 alloys in the dataset. The target property of this dataset is the glass forming ability (GFA), i.e. whether the composition can form monolithic glass or not, which is either 1 for glass forming or 0 for non-full glass forming.\n",
+ "\n",
+ "glass_binary_v2: Identical to glass_binary dataset, but with duplicate entries merged. If there was a disagreement in gfa when merging the class was defaulted to 1.\n",
+ "\n",
+ "glass_ternary_hipt: Metallic glass formation dataset for ternary alloys, collected from the high-throughput sputtering experiments measuring whether it is possible to form a glass using sputtering. The hipt experimental data are of the Co-Fe-Zr, Co-Ti-Zr, Co-V-Zr and Fe-Ti-Nb ternary systems.\n",
+ "\n",
+ "glass_ternary_landolt: Metallic glass formation dataset for ternary alloys, collected from the \"Nonequilibrium Phase Diagrams of Ternary Amorphous Alloys,’ a volume of the Landolt– Börnstein collection. This dataset contains experimental measurements of whether it is possible to form a glass using a variety of processing techniques at thousands of compositions from hundreds of ternary systems. The processing techniques are designated in the \"processing\" column. There are originally 7191 experiments in this dataset, will be reduced to 6203 after deduplicated, and will be further reduced to 6118 if combining multiple data for one composition. There are originally 6780 melt-spinning experiments in this dataset, will be reduced to 5800 if deduplicated, and will be further reduced to 5736 if combining multiple experimental data for one composition.\n",
+ "\n",
+ "heusler_magnetic: 1153 Heusler alloys with DFT-calculated magnetic and electronic properties. The 1153 alloys include 576 full, 449 half and 128 inverse Heusler alloys. The data are extracted and cleaned (including de-duplicating) from Citrine.\n",
+ "\n",
+ "jarvis_dft_2d: Various properties of 636 2D materials computed with the OptB88vdW and TBmBJ functionals taken from the JARVIS DFT database.\n",
+ "\n",
+ "jarvis_dft_3d: Various properties of 25,923 bulk materials computed with the OptB88vdW and TBmBJ functionals taken from the JARVIS DFT database.\n",
+ "\n",
+ "jarvis_ml_dft_training: Various properties of 24,759 bulk and 2D materials computed with the OptB88vdW and TBmBJ functionals taken from the JARVIS DFT database.\n",
+ "\n",
+ "m2ax: Elastic properties of 223 stable M2AX compounds from \"A comprehensive survey of M2AX phase elastic properties\" by Cover et al. Calculations are PAW PW91.\n",
+ "\n",
+ "matbench_dielectric: Matbench v0.1 test dataset for predicting refractive index from structure. Adapted from Materials Project database. Removed entries having a formation energy (or energy above the convex hull) more than 150meV and those having refractive indices less than 1 and those containing noble gases. Retrieved April 2, 2019. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_expt_gap: Matbench v0.1 test dataset for predicting experimental band gap from composition alone. Retrieved from Zhuo et al. supplementary information. Deduplicated according to composition, removing compositions with reported band gaps spanning more than a 0.1eV range; remaining compositions were assigned values based on the closest experimental value to the mean experimental value for that composition among all reports. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_expt_is_metal: Matbench v0.1 test dataset for classifying metallicity from composition alone. Retrieved from Zhuo et al. supplementary information. Deduplicated according to composition, ensuring no conflicting reports were entered for any compositions (i.e., no reported compositions were both metal and nonmetal). For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_glass: Matbench v0.1 test dataset for predicting full bulk metallic glass formation ability from chemical formula. Retrieved from \"Nonequilibrium Phase Diagrams of Ternary Amorphous Alloys,’ a volume of the Landolt– Börnstein collection. Deduplicated according to composition, ensuring no compositions were reported as both GFA and not GFA (i.e., all reports agreed on the classification designation). For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_jdft2d: Matbench v0.1 test dataset for predicting exfoliation energies from crystal structure (computed with the OptB88vdW and TBmBJ functionals). Adapted from the JARVIS DFT database. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_log_gvrh: Matbench v0.1 test dataset for predicting DFT log10 VRH-average shear modulus from structure. Adapted from Materials Project database. Removed entries having a formation energy (or energy above the convex hull) more than 150meV and those having negative G_Voigt, G_Reuss, G_VRH, K_Voigt, K_Reuss, or K_VRH and those failing G_Reuss <= G_VRH <= G_Voigt or K_Reuss <= K_VRH <= K_Voigt and those containing noble gases. Retrieved April 2, 2019. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_log_kvrh: Matbench v0.1 test dataset for predicting DFT log10 VRH-average bulk modulus from structure. Adapted from Materials Project database. Removed entries having a formation energy (or energy above the convex hull) more than 150meV and those having negative G_Voigt, G_Reuss, G_VRH, K_Voigt, K_Reuss, or K_VRH and those failing G_Reuss <= G_VRH <= G_Voigt or K_Reuss <= K_VRH <= K_Voigt and those containing noble gases. Retrieved April 2, 2019. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_mp_e_form: Matbench v0.1 test dataset for predicting DFT formation energy from structure. Adapted from Materials Project database. Removed entries having formation energy more than 2.5eV and those containing noble gases. Retrieved April 2, 2019. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_mp_gap: Matbench v0.1 test dataset for predicting DFT PBE band gap from structure. Adapted from Materials Project database. Removed entries having a formation energy (or energy above the convex hull) more than 150meV and those containing noble gases. Retrieved April 2, 2019. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_mp_is_metal: Matbench v0.1 test dataset for predicting DFT metallicity from structure. Adapted from Materials Project database. Removed entries having a formation energy (or energy above the convex hull) more than 150meV and those containing noble gases. Retrieved April 2, 2019. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_perovskites: Matbench v0.1 test dataset for predicting formation energy from crystal structure. Adapted from an original dataset generated by Castelli et al. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_phonons: Matbench v0.1 test dataset for predicting vibration properties from crystal structure. Original data retrieved from Petretto et al. Original calculations done via ABINIT in the harmonic approximation based on density functional perturbation theory. Removed entries having a formation energy (or energy above the convex hull) more than 150meV. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "matbench_steels: Matbench v0.1 test dataset for predicting steel yield strengths from chemical composition alone. Retrieved from Citrine informatics. Deduplicated. For benchmarking w/ nested cross validation, the order of the dataset must be identical to the retrieved data; refer to the Automatminer/Matbench publication for more details.\n",
+ "\n",
+ "mp_all_20181018: A complete copy of the Materials Project database as of 10/18/2018. mp_all files contain structure data for each material while mp_nostruct does not.\n",
+ "\n",
+ "mp_nostruct_20181018: A complete copy of the Materials Project database as of 10/18/2018. mp_all files contain structure data for each material while mp_nostruct does not.\n",
+ "\n",
+ "phonon_dielectric_mp: Phonon (lattice/atoms vibrations) and dielectric properties of 1296 compounds computed via ABINIT software package in the harmonic approximation based on density functional perturbation theory.\n",
+ "\n",
+ "piezoelectric_tensor: 941 structures with piezoelectric properties, calculated with DFT-PBE.\n",
+ "\n",
+ "ricci_boltztrap_mp_tabular: Ab-initio electronic transport database for inorganic materials. Complex multivariable BoltzTraP simulation data is condensed down into tabular form of two main motifs: average eigenvalues at set moderate carrier concentrations and temperatures, and optimal values among all carrier concentrations and temperatures within certain ranges. Here are reported the average of the eigenvalues of conductivity effective mass (mₑᶜᵒⁿᵈ), the Seebeck coefficient (S), the conductivity (σ), the electronic thermal conductivity (κₑ), and the Power Factor (PF) at a doping level of 10¹⁸ cm⁻³ and at a temperature of 300 K for n- and p-type. Also, the maximum values for S, σ, PF, and the minimum value for κₑ chosen among the temperatures [100, 1300] K, the doping levels [10¹⁶, 10²¹] cm⁻³, and doping types are reported. The properties that depend on the relaxation time are reported divided by the constant value 10⁻¹⁴. The average of the eigenvalues for all the properties at all the temperatures, doping levels, and doping types are reported in the tables for each entry. Data is indexed by materials project id (mpid)\n",
+ "\n",
+ "steel_strength: 312 steels with experimental yield strength and ultimate tensile strength, extracted and cleaned (including de-duplicating) from Citrine.\n",
+ "\n",
+ "superconductivity2018: Dataset of ~16,000 experimental superconductivity records (critical temperatures) from Stanev et al., originally from the Japanese National Institute for Materials Science. Does not include structural data. Includes ~300 measurements from materials found without superconductivity (Tc=0). No modifications were made to the core dataset, aside from basic file type change to json for (un)packaging with matminer. Reproduced under the Creative Commons 4.0 license, which can be found here: http://creativecommons.org/licenses/by/4.0/.\n",
+ "\n",
+ "tholander_nitrides: A challenging data set for quantum machine learning containing a diverse set of 12.8k polymorphs in the Zn-Ti-N, Zn-Zr-N and Zn-Hf-N chemical systems. The phase diagrams of the Ti-Zn-N, Zr-Zn-N, and Hf-Zn-N systems are determined using large-scale high-throughput density functional calculations (DFT-GGA) (PBE). In total 12,815 relaxed structures are shared alongside their energy calculated using the VASP DFT code. The High-Throughput Toolkit was used to manage the calculations. Data adapted and deduplicated from the original data on Zenodo at https://zenodo.org/record/5530535#.YjJ3ZhDMJLQ, published under MIT licence. Collated from separate files of chemical systems and deduplicated according to identical structures matching ht_ids. Prepared in collaboration with Rhys Goodall.\n",
+ "\n",
+ "ucsb_thermoelectrics: Database of ~1,100 experimental thermoelectric materials from UCSB aggregated from 108 source publications and personal communications. Downloaded from Citrine. Source UCSB webpage is http://www.mrl.ucsb.edu:8080/datamine/thermoelectric.jsp. See reference for more information on original data aggregation. No duplicate entries are present, but each src may result in multiple measurements of the same materials' properties at different temperatures or conditions.\n",
+ "\n",
+ "wolverton_oxides: 4,914 perovskite oxides containing composition data, lattice constants, and formation + vacancy formation energies. All perovskites are of the form ABO3. Adapted from a dataset presented by Emery and Wolverton.\n",
+ "\n",
+ "\n"
+ ]
+ }
+ ],
"source": [
"MatminerDatasets().get_available_datasets()"
]
@@ -1202,98 +3342,541 @@
},
"source": [
"As one last example, let's look at how to download datasets that are \n",
- "hosted on Figshare. To use the Figshare data import, we need to \n",
- "first install the python package to query Figshare datasets:"
+ "hosted on Foundry-ML. More information on Foundry-ML can be found online here (https://foundry-ml.org/#/) or in the publication here (https://joss.theoj.org/papers/10.21105/joss.05467)"
]
},
{
"cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "executionInfo": {
- "elapsed": 1993,
- "status": "ok",
- "timestamp": 1618264012601,
- "user": {
- "displayName": "Ryan Jacobs",
- "photoUrl": "",
- "userId": "08020825119070462259"
- },
- "user_tz": 300
- },
- "id": "gBtlWD6k-MCv",
- "outputId": "106f574c-2bdc-4dbe-c22f-c2dd2c8b661c"
- },
- "outputs": [],
- "source": [
- "!git clone https://github.com/cognoma/figshare.git\n",
- "import sys\n",
- "sys.path.append('figshare')"
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Starting Download of: https://data.materialsdatafacility.org/foundry/diffusion_v1.4/Dataset_diffusion_v2.csv\n",
+ "Starting Download of: https://data.materialsdatafacility.org/foundry/diffusion_v1.4/.DS_Store\n",
+ "Starting Download of: https://data.materialsdatafacility.org/foundry/diffusion_v1.4/.ipynb_checkpoints/Foundry_dataset_upload_diffusion-checkpoint.ipynb\n",
+ "Downloading... 0.03 MB"
+ ]
+ }
+ ],
+ "source": [
+ "from foundry import Foundry\n",
+ "\n",
+ "f = Foundry()\n",
+ "\n",
+ "# Load the data here!\n",
+ "f.load('10.18126/uppe-p8p1', globus=False)\n",
+ "res = f.load_data()"
]
},
{
"cell_type": "markdown",
- "metadata": {
- "id": "XGBcXLSWDtYM"
- },
+ "metadata": {},
"source": [
- "Note that you may need to restart your runtime (if on Colab) so that Colab can find the Figshare module"
+ "A new directory /data/diffusion_v1.4/Dataset_diffusion_v2.csv has appeared. This has downloaded the same diffusion\n",
+ "data we used in the above example. We can view the X data (just compositions, not featurized yet), and target y data:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " Material compositions 1 Material compositions 2\n",
+ "0 Ag Ag\n",
+ "1 Ag Co\n",
+ "2 Ag Cr\n",
+ "3 Ag Cu\n",
+ "4 Ag Fe\n",
+ ".. ... ...\n",
+ "403 Zr Nb\n",
+ "404 Zr Ta\n",
+ "405 Zr Ti\n",
+ "406 Zr Hf\n",
+ "407 Zr Sn\n",
+ "\n",
+ "[408 rows x 2 columns] E_regression_shift\n",
+ "0 0.000000\n",
+ "1 -0.090142\n",
+ "2 0.259139\n",
+ "3 -0.022200\n",
+ "4 0.317672\n",
+ ".. ...\n",
+ "403 -0.067020\n",
+ "404 0.153850\n",
+ "405 0.248110\n",
+ "406 0.204140\n",
+ "407 0.248040\n",
+ "\n",
+ "[408 rows x 1 columns]\n"
+ ]
+ }
+ ],
+ "source": [
+ "X = res['train'][0]\n",
+ "y = res['train'][1]\n",
+ "\n",
+ "print(X, y)"
]
},
{
"cell_type": "markdown",
"metadata": {
- "id": "vMtDoxLe-MCv"
+ "id": "2cvV8zI6-MCw"
},
"source": [
- "Now we can download a dataset from Figshare. The diffusion dataset\n",
- "we examined earlier is hosted on Figshare, with article ID number 7418492. \n",
- "Let's download it to our working environment:"
+ "As with this file or other datasets downloaded\n",
+ "from Foundry-ML or other remote datasets, after downloading the data files\n",
+ "you can import them into MAST-ML using the LocalDatasets class as we \n",
+ "learned in this tutorial."
]
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "executionInfo": {
- "elapsed": 33567,
- "status": "ok",
- "timestamp": 1618264106311,
- "user": {
- "displayName": "Ryan Jacobs",
- "photoUrl": "",
- "userId": "08020825119070462259"
- },
- "user_tz": 300
- },
- "id": "UPSRQ3PG-MCw",
- "outputId": "a45709cb-d7b2-4fa7-f75f-c4afed92f661"
- },
- "outputs": [],
+ "cell_type": "markdown",
+ "metadata": {},
"source": [
- "from mastml.datasets import FigshareDatasets\n",
- "FigshareDatasets().download_data(article_id='7418492')"
+ "There are many datasets available on Foundry-ML. You can view them with the query below, and import the desired data following\n",
+ "the example above"
]
},
{
- "cell_type": "markdown",
- "metadata": {
- "id": "2cvV8zI6-MCw"
- },
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
source_id
\n",
+ "
name
\n",
+ "
year
\n",
+ "
DOI
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
0
\n",
+ "
semiconductor_defectlevels_v1.1
\n",
+ "
Machine learning for impurity charge-state tra...
\n",
+ "
2022
\n",
+ "
10.18126/ite0-3iah
\n",
+ "
\n",
+ "
\n",
+ "
1
\n",
+ "
voyles_mdf_dmref_glasses_v2.2
\n",
+ "
Metallic Glasses and their Properties
\n",
+ "
2021
\n",
+ "
10.18126/7yg1-osf2
\n",
+ "
\n",
+ "
\n",
+ "
2
\n",
+ "
steel_strength_v1.1
\n",
+ "
Mechanical properties of some steels
\n",
+ "
2022
\n",
+ "
10.18126/524z-vd6m
\n",
+ "
\n",
+ "
\n",
+ "
3
\n",
+ "
elwood_md_v1.2
\n",
+ "
Project Elwood: MD Simulated Monomer Properties
\n",
+ "
2022
\n",
+ "
10.18126/8p6m-e135
\n",
+ "
\n",
+ "
\n",
+ "
4
\n",
+ "
foundry_moses_v1.1
\n",
+ "
Molecular Sets (MOSES): A Benchmarking Platfor...
\n",
+ "
2022
\n",
+ "
10.18126/rp13-3k3h
\n",
+ "
\n",
+ "
\n",
+ "
5
\n",
+ "
citrine_thermalconductivity_v1.2
\n",
+ "
Experimental thermal conductivity database fro...
\n",
+ "
2022
\n",
+ "
10.18126/dhmc-darr
\n",
+ "
\n",
+ "
\n",
+ "
6
\n",
+ "
training_locating_atoms_stem_images_v1.2
\n",
+ "
Training Dataset for Locating Atoms in STEM im...
\n",
+ "
2022
\n",
+ "
10.18126/qsdl-aj6x
\n",
+ "
\n",
+ "
\n",
+ "
7
\n",
+ "
mask_rcnn_defect_detection_v1.1
\n",
+ "
Mask RCNN defect detection dataset
\n",
+ "
2022
\n",
+ "
10.18126/hok0-qsa4
\n",
+ "
\n",
+ "
\n",
+ "
8
\n",
+ "
foundry_g4mp2_solvation_v1.2
\n",
+ "
DFT Estimates of Solvation Energy in Multiple ...
\n",
+ "
2022
\n",
+ "
10.18126/jos5-wj65
\n",
+ "
\n",
+ "
\n",
+ "
9
\n",
+ "
metallic_glass_database_v1.2
\n",
+ "
Quantifying the Origin of Metallic Glass Forma...
\n",
+ "
2023
\n",
+ "
10.18126/5svl-7ruk
\n",
+ "
\n",
+ "
\n",
+ "
10
\n",
+ "
casting_thickness_database_v1.4
\n",
+ "
Metallic Glass Critical Thicknesses for Atomis...
\n",
+ "
2023
\n",
+ "
10.18126/uxu4-kvnt
\n",
+ "
\n",
+ "
\n",
+ "
11
\n",
+ "
pacbedcnn_thickness_mistilt_training_v1.1
\n",
+ "
Training Dataset of PACBED-CNN for Infering Sp...
\n",
+ "
2023
\n",
+ "
10.18126/2q0n-ce0d
\n",
+ "
\n",
+ "
\n",
+ "
12
\n",
+ "
hrtem_nanoparticles_generalization_v1.1
\n",
+ "
HRTEM images for generalization across experim...
\n",
+ "
2023
\n",
+ "
10.18126/z4mr-xwk5
\n",
+ "
\n",
+ "
\n",
+ "
13
\n",
+ "
foundry_assorted_computational_band_gaps_v1.1
\n",
+ "
Graph Network Based Deep Learning of Band Gaps...
\n",
+ "
2021
\n",
+ "
10.18126/7io9-1z9k
\n",
+ "
\n",
+ "
\n",
+ "
14
\n",
+ "
foundry_experimental_band_gaps_v1.1
\n",
+ "
Graph Network Based Deep Learning of Band Gaps...
\n",
+ "
2021
\n",
+ "
10.18126/wg3u-g8vu
\n",
+ "
\n",
+ "
\n",
+ "
15
\n",
+ "
foundry_aflow_band_gaps_v1.1
\n",
+ "
Graph Network Based Deep Learning of Band Gaps...
\n",
+ "
2021
\n",
+ "
10.18126/6fdy-bsam
\n",
+ "
\n",
+ "
\n",
+ "
16
\n",
+ "
foundry_oqmd_band_gaps_v1.1
\n",
+ "
Graph Network Based Deep Learning of Band Gaps...
\n",
+ "
2021
\n",
+ "
10.18126/w1ey-9y8b
\n",
+ "
\n",
+ "
\n",
+ "
17
\n",
+ "
foundry_mp_band_gaps_v1.1
\n",
+ "
Graph Network Based Deep Learning of Band Gaps...
\n",
+ "
2021
\n",
+ "
10.18126/vjwr-5bs9
\n",
+ "
\n",
+ "
\n",
+ "
18
\n",
+ "
foundry_wei_atom_locating_benchmark_v1.1
\n",
+ "
Benchmark Dataset for Locating Atoms in STEM i...
\n",
+ "
2021
\n",
+ "
10.18126/e73h-3w6n
\n",
+ "
\n",
+ "
\n",
+ "
19
\n",
+ "
elastic_tensor_v1.1
\n",
+ "
Charting the complete elastic properties of in...
\n",
+ "
2022
\n",
+ "
10.18126/9fg1-528u
\n",
+ "
\n",
+ "
\n",
+ "
20
\n",
+ "
dielectric_constant_v1.1
\n",
+ "
High-throughput screening of inorganic compoun...
\n",
+ "
2022
\n",
+ "
10.18126/racd-go9m
\n",
+ "
\n",
+ "
\n",
+ "
21
\n",
+ "
foundry_osdb_v1.1
\n",
+ "
Data for: Ab initio control of zeolite synthes...
\n",
+ "
2021
\n",
+ "
10.18126/c5z9-zej7
\n",
+ "
\n",
+ "
\n",
+ "
22
\n",
+ "
wolverton_oxides_v1.1
\n",
+ "
High-throughput DFT calculations of formation ...
\n",
+ "
2022
\n",
+ "
10.18126/h15n-7wu8
\n",
+ "
\n",
+ "
\n",
+ "
23
\n",
+ "
piezoelectric_tensor_v1.1
\n",
+ "
A database to enable discovery and design of p...
\n",
+ "
2022
\n",
+ "
10.18126/p280-xrvg
\n",
+ "
\n",
+ "
\n",
+ "
24
\n",
+ "
double_perovskite_bandgap_v1.1
\n",
+ "
Machine learning bandgaps of double perovskites
\n",
+ "
2022
\n",
+ "
10.18126/lss6-o5x4
\n",
+ "
\n",
+ "
\n",
+ "
25
\n",
+ "
perovskite_workfunctions_v1.2
\n",
+ "
Data-driven analysis of the electronic-structu...
\n",
+ "
2022
\n",
+ "
10.18126/kbyt-og5w
\n",
+ "
\n",
+ "
\n",
+ "
26
\n",
+ "
foundry_stan_segmentation_v1.1
\n",
+ "
Dataset for Semantic Segmentation of Dendrites...
\n",
+ "
2021
\n",
+ "
10.18126/d4kn-cz3p
\n",
+ "
\n",
+ "
\n",
+ "
27
\n",
+ "
foundry_qmc_ml_v1.1
\n",
+ "
Foundry - Approaching QMC quality energetics t...
\n",
+ "
2022
\n",
+ "
10.18126/wg30-95z0
\n",
+ "
\n",
+ "
\n",
+ "
28
\n",
+ "
perovskite_opbandcenter_v1.1
\n",
+ "
Discovery and engineering of low work function...
\n",
+ "
2022
\n",
+ "
10.18126/f3p0-2c5f
\n",
+ "
\n",
+ "
\n",
+ "
29
\n",
+ "
superconductivity_v1.1
\n",
+ "
Machine learning modeling of superconducting c...
\n",
+ "
2022
\n",
+ "
10.18126/xlfr-hjrn
\n",
+ "
\n",
+ "
\n",
+ "
30
\n",
+ "
perovskite_stability_v1.1
\n",
+ "
Predicting the thermodynamic stability of pero...
\n",
+ "
2022
\n",
+ "
10.18126/qe5y-2dnz
\n",
+ "
\n",
+ "
\n",
+ "
31
\n",
+ "
heusler_magnetization_v1.1
\n",
+ "
University of Alabama Heusler database
\n",
+ "
2022
\n",
+ "
10.18126/un63-ohqv
\n",
+ "
\n",
+ "
\n",
+ "
32
\n",
+ "
diffusion_v1.4
\n",
+ "
Error assessment and optimal cross-validation ...
\n",
+ "
2022
\n",
+ "
10.18126/uppe-p8p1
\n",
+ "
\n",
+ "
\n",
+ "
33
\n",
+ "
electromigration_v1.1
\n",
+ "
Exploring effective charge in electromigration...
\n",
+ "
2022
\n",
+ "
10.18126/abxi-r7eb
\n",
+ "
\n",
+ "
\n",
+ "
34
\n",
+ "
atomvison_atomistic_stm_images_2d_materials_un...
\n",
+ "
AtomVison: Atomistic STM images of 2D material...
\n",
+ "
2023
\n",
+ "
10.18126/r6dh-lser
\n",
+ "
\n",
+ "
\n",
+ "
35
\n",
+ "
atomvison_simulated_atomistic_stem_images_2d_m...
\n",
+ "
AtomVison: Simulated Atomistic STEM images of ...
\n",
+ "
2023
\n",
+ "
10.18126/raou-p0vg
\n",
+ "
\n",
+ "
\n",
+ "
36
\n",
+ "
direct_electron_detectorceleritas_xs_simulated...
\n",
+ "
Direct electron detector(Celeritas XS) simulat...
\n",
+ "
2023
\n",
+ "
10.18126/rgew-xbw4
\n",
+ "
\n",
+ "
\n",
+ "
37
\n",
+ "
defecttrack_training_v1.2
\n",
+ "
DefectTrack training dataset
\n",
+ "
2023
\n",
+ "
10.18126/71ek-dyij
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " source_id \\\n",
+ "0 semiconductor_defectlevels_v1.1 \n",
+ "1 voyles_mdf_dmref_glasses_v2.2 \n",
+ "2 steel_strength_v1.1 \n",
+ "3 elwood_md_v1.2 \n",
+ "4 foundry_moses_v1.1 \n",
+ "5 citrine_thermalconductivity_v1.2 \n",
+ "6 training_locating_atoms_stem_images_v1.2 \n",
+ "7 mask_rcnn_defect_detection_v1.1 \n",
+ "8 foundry_g4mp2_solvation_v1.2 \n",
+ "9 metallic_glass_database_v1.2 \n",
+ "10 casting_thickness_database_v1.4 \n",
+ "11 pacbedcnn_thickness_mistilt_training_v1.1 \n",
+ "12 hrtem_nanoparticles_generalization_v1.1 \n",
+ "13 foundry_assorted_computational_band_gaps_v1.1 \n",
+ "14 foundry_experimental_band_gaps_v1.1 \n",
+ "15 foundry_aflow_band_gaps_v1.1 \n",
+ "16 foundry_oqmd_band_gaps_v1.1 \n",
+ "17 foundry_mp_band_gaps_v1.1 \n",
+ "18 foundry_wei_atom_locating_benchmark_v1.1 \n",
+ "19 elastic_tensor_v1.1 \n",
+ "20 dielectric_constant_v1.1 \n",
+ "21 foundry_osdb_v1.1 \n",
+ "22 wolverton_oxides_v1.1 \n",
+ "23 piezoelectric_tensor_v1.1 \n",
+ "24 double_perovskite_bandgap_v1.1 \n",
+ "25 perovskite_workfunctions_v1.2 \n",
+ "26 foundry_stan_segmentation_v1.1 \n",
+ "27 foundry_qmc_ml_v1.1 \n",
+ "28 perovskite_opbandcenter_v1.1 \n",
+ "29 superconductivity_v1.1 \n",
+ "30 perovskite_stability_v1.1 \n",
+ "31 heusler_magnetization_v1.1 \n",
+ "32 diffusion_v1.4 \n",
+ "33 electromigration_v1.1 \n",
+ "34 atomvison_atomistic_stm_images_2d_materials_un... \n",
+ "35 atomvison_simulated_atomistic_stem_images_2d_m... \n",
+ "36 direct_electron_detectorceleritas_xs_simulated... \n",
+ "37 defecttrack_training_v1.2 \n",
+ "\n",
+ " name year \\\n",
+ "0 Machine learning for impurity charge-state tra... 2022 \n",
+ "1 Metallic Glasses and their Properties 2021 \n",
+ "2 Mechanical properties of some steels 2022 \n",
+ "3 Project Elwood: MD Simulated Monomer Properties 2022 \n",
+ "4 Molecular Sets (MOSES): A Benchmarking Platfor... 2022 \n",
+ "5 Experimental thermal conductivity database fro... 2022 \n",
+ "6 Training Dataset for Locating Atoms in STEM im... 2022 \n",
+ "7 Mask RCNN defect detection dataset 2022 \n",
+ "8 DFT Estimates of Solvation Energy in Multiple ... 2022 \n",
+ "9 Quantifying the Origin of Metallic Glass Forma... 2023 \n",
+ "10 Metallic Glass Critical Thicknesses for Atomis... 2023 \n",
+ "11 Training Dataset of PACBED-CNN for Infering Sp... 2023 \n",
+ "12 HRTEM images for generalization across experim... 2023 \n",
+ "13 Graph Network Based Deep Learning of Band Gaps... 2021 \n",
+ "14 Graph Network Based Deep Learning of Band Gaps... 2021 \n",
+ "15 Graph Network Based Deep Learning of Band Gaps... 2021 \n",
+ "16 Graph Network Based Deep Learning of Band Gaps... 2021 \n",
+ "17 Graph Network Based Deep Learning of Band Gaps... 2021 \n",
+ "18 Benchmark Dataset for Locating Atoms in STEM i... 2021 \n",
+ "19 Charting the complete elastic properties of in... 2022 \n",
+ "20 High-throughput screening of inorganic compoun... 2022 \n",
+ "21 Data for: Ab initio control of zeolite synthes... 2021 \n",
+ "22 High-throughput DFT calculations of formation ... 2022 \n",
+ "23 A database to enable discovery and design of p... 2022 \n",
+ "24 Machine learning bandgaps of double perovskites 2022 \n",
+ "25 Data-driven analysis of the electronic-structu... 2022 \n",
+ "26 Dataset for Semantic Segmentation of Dendrites... 2021 \n",
+ "27 Foundry - Approaching QMC quality energetics t... 2022 \n",
+ "28 Discovery and engineering of low work function... 2022 \n",
+ "29 Machine learning modeling of superconducting c... 2022 \n",
+ "30 Predicting the thermodynamic stability of pero... 2022 \n",
+ "31 University of Alabama Heusler database 2022 \n",
+ "32 Error assessment and optimal cross-validation ... 2022 \n",
+ "33 Exploring effective charge in electromigration... 2022 \n",
+ "34 AtomVison: Atomistic STM images of 2D material... 2023 \n",
+ "35 AtomVison: Simulated Atomistic STEM images of ... 2023 \n",
+ "36 Direct electron detector(Celeritas XS) simulat... 2023 \n",
+ "37 DefectTrack training dataset 2023 \n",
+ "\n",
+ " DOI \n",
+ "0 10.18126/ite0-3iah \n",
+ "1 10.18126/7yg1-osf2 \n",
+ "2 10.18126/524z-vd6m \n",
+ "3 10.18126/8p6m-e135 \n",
+ "4 10.18126/rp13-3k3h \n",
+ "5 10.18126/dhmc-darr \n",
+ "6 10.18126/qsdl-aj6x \n",
+ "7 10.18126/hok0-qsa4 \n",
+ "8 10.18126/jos5-wj65 \n",
+ "9 10.18126/5svl-7ruk \n",
+ "10 10.18126/uxu4-kvnt \n",
+ "11 10.18126/2q0n-ce0d \n",
+ "12 10.18126/z4mr-xwk5 \n",
+ "13 10.18126/7io9-1z9k \n",
+ "14 10.18126/wg3u-g8vu \n",
+ "15 10.18126/6fdy-bsam \n",
+ "16 10.18126/w1ey-9y8b \n",
+ "17 10.18126/vjwr-5bs9 \n",
+ "18 10.18126/e73h-3w6n \n",
+ "19 10.18126/9fg1-528u \n",
+ "20 10.18126/racd-go9m \n",
+ "21 10.18126/c5z9-zej7 \n",
+ "22 10.18126/h15n-7wu8 \n",
+ "23 10.18126/p280-xrvg \n",
+ "24 10.18126/lss6-o5x4 \n",
+ "25 10.18126/kbyt-og5w \n",
+ "26 10.18126/d4kn-cz3p \n",
+ "27 10.18126/wg30-95z0 \n",
+ "28 10.18126/f3p0-2c5f \n",
+ "29 10.18126/xlfr-hjrn \n",
+ "30 10.18126/qe5y-2dnz \n",
+ "31 10.18126/un63-ohqv \n",
+ "32 10.18126/uppe-p8p1 \n",
+ "33 10.18126/abxi-r7eb \n",
+ "34 10.18126/r6dh-lser \n",
+ "35 10.18126/raou-p0vg \n",
+ "36 10.18126/rgew-xbw4 \n",
+ "37 10.18126/71ek-dyij "
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "We can now see the new folder \"figshare_7418492\" has been downloaded,\n",
- "and within it is the data file \"All_Model_Data.xlsx\". This is the same\n",
- "file we imported earlier from mastml/data when demonstrating the use\n",
- "of the LocalDatasets class. As with this file or other datasets downloaded\n",
- "from Figshare or other remote datasets, after downloading the data files\n",
- "you can import them into MAST-ML using the LocalDatasets class as we \n",
- "learned in this tutorial."
+ "f.list()"
]
},
{
@@ -1318,7 +3901,7 @@
"toc_visible": true
},
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -1332,9 +3915,9 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.4"
+ "version": "3.11.7"
}
},
"nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
}
diff --git a/examples/MASTML_Tutorial_3_FeatureEngineering.ipynb b/examples/MASTML_Tutorial_3_FeatureEngineering.ipynb
index e8b966df..0e94fdeb 100644
--- a/examples/MASTML_Tutorial_3_FeatureEngineering.ipynb
+++ b/examples/MASTML_Tutorial_3_FeatureEngineering.ipynb
@@ -80,15 +80,6 @@
"!pip install mastml"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!pip install pyyaml==5.4.1"
- ]
- },
{
"cell_type": "markdown",
"metadata": {
@@ -157,7 +148,35 @@
"id": "x8GIei1qQoWL",
"outputId": "5e0e8536-ff6a-4cd3-fed7-7ffa104a1d5c"
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Figshare is an optional dependency. To import data from figshare, manually install figshare via git clone of git clone https://github.com/cognoma/figshare.git\n",
+ "XGBoost is an optional dependency. If you want to use XGBoost models, please manually install xgboost package with pip install xgboost. If have error with finding libxgboost.dylib library, dobrew install libomp. If do not have brew on your system, first do ruby -e \"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)\" from the Terminal\n",
+ "scikit-lego is an optional dependency, enabling use of the LowessRegression model. If you want to use this model, do \"pip install scikit-lego\"\n",
+ "linear-tree is an optional dependency, enabling use of Linear tree, forest, and boosting models. If you want to use this model, do \"pip install linear-tree\"\n",
+ "gplearn is an optional dependency, enabling the use of genetic programming SymbolicRegressor model. If you want to use this model, do \"pip install gplearn\"\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CBFV is an optional dependency. To install CBFV, do pip install cbfv\n",
+ "DeepChem is an optional dependency used to generate molecular descriptors from RDKit. To install Deepchem, do pip install deepchem\n",
+ "forestci is an optional dependency. To install latest forestci compatabilty with scikit-learn>=0.24, run pip install git+git://github.com/scikit-learn-contrib/forest-confidence-interval.git\n"
+ ]
+ }
+ ],
"source": [
"from mastml.mastml import Mastml\n",
"from mastml.datasets import LocalDatasets\n",
@@ -212,15 +231,7 @@
"id": "Xf8MFHjlM7Oe",
"outputId": "faa03e7a-15af-4726-c12b-c8f73deb26fe"
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "drive/MyDrive/MASTML_tutorial_3_FeatureEngineering not empty. Renaming...\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"SAVEPATH = 'drive/MyDrive/MASTML_tutorial_3_FeatureEngineering'\n",
"\n",
@@ -433,7 +444,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 6,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
@@ -453,6 +464,13 @@
"outputId": "0b65350f-60c7-4693-842f-2dcd1396d490"
},
"outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Featurizing compositions:: 408it [00:02, 152.39it/s]\n"
+ ]
+ },
{
"name": "stdout",
"output_type": "stream",
@@ -462,7 +480,7 @@
}
],
"source": [
- "X, y = ElementalFeatureGenerator(composition_df=X_extra['Material compositions joined'], \n",
+ "X, y = ElementalFeatureGenerator(featurize_df=X_extra['Material compositions joined'], \n",
" feature_types=['composition_avg', 'arithmetic_avg', 'max', 'min', 'difference'], \n",
" remove_constant_columns=True).evaluate(X=X, y=y, savepath=savepath, make_new_dir=True)"
]
@@ -489,7 +507,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
@@ -509,7 +527,434 @@
"id": "j65kSNyBD_8b",
"outputId": "31970fce-34f6-42f0-c232-de7aa967b55e"
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "