Skip to content

Commit

Permalink
delete env python
Browse files Browse the repository at this point in the history
  • Loading branch information
@Le-Zheng
Le-Zheng committed May 28, 2019
1 parent 35221cf commit 21f7ce6
Show file tree
Hide file tree
Showing 8 changed files with 81 additions and 67 deletions.
19 changes: 10 additions & 9 deletions keras/2.1-a-first-look-at-a-neural-network.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
"env: PYSPARK_PYTHON=/usr/bin/python3.5\n",
"env: PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n"
]
}
],
"source": [
"%env PYSPARK_PYTHON=/usr/bin/python3.5\n",
"%env PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n",
"\n",
"from zoo.common.nncontext import *\n",
"sc = init_nncontext(init_spark_conf().setMaster(\"local[4]\"))"
]
Expand Down Expand Up @@ -233,7 +228,9 @@
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from zoo.pipeline.api.keras import models\n",
Expand Down Expand Up @@ -329,7 +326,9 @@
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"train_images = train_images.reshape((60000, 28 * 28))\n",
Expand All @@ -350,7 +349,9 @@
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"network.fit(train_images, train_labels, nb_epoch=5, batch_size=128)"
Expand Down Expand Up @@ -417,7 +418,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
"version": "3.5.4"
}
},
"nbformat": 4,
Expand Down
36 changes: 23 additions & 13 deletions keras/3.5-classifying-movie-reviews.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
"env: SPARK_DRIVER_MEMORY=32g\n",
"env: PYSPARK_PYTHON=/usr/bin/python3.5\n",
"env: PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n"
"env: SPARK_DRIVER_MEMORY=32g\n"
]
}
],
"source": [
"%env SPARK_DRIVER_MEMORY=32g\n",
"%env PYSPARK_PYTHON=/usr/bin/python3.5\n",
"%env PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n",
"\n",
"from zoo.common.nncontext import *\n",
"sc = init_nncontext(init_spark_conf().setMaster(\"local[4]\"))"
Expand Down Expand Up @@ -74,7 +70,9 @@
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from zoo.pipeline.api.keras.datasets import imdb\n",
Expand Down Expand Up @@ -172,7 +170,9 @@
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import numpy as np\n",
Expand Down Expand Up @@ -226,7 +226,9 @@
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"y_train = np.asarray(train_labels).astype('float32')\n",
Expand Down Expand Up @@ -379,7 +381,9 @@
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"x_val = x_train[:10000]\n",
Expand Down Expand Up @@ -423,7 +427,9 @@
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import time\n",
Expand Down Expand Up @@ -1674,7 +1680,9 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"model.fit(partial_x_train,\n",
Expand All @@ -1694,7 +1702,9 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from bigdl.util.common import to_sample_rdd\n",
Expand Down Expand Up @@ -1732,7 +1742,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
"version": "3.5.4"
}
},
"nbformat": 4,
Expand Down
32 changes: 20 additions & 12 deletions keras/3.6-classifying-newswires.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
"env: SPARK_DRIVER_MEMORY=8g\n",
"env: PYSPARK_PYTHON=/usr/bin/python3.5\n",
"env: PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n"
"env: SPARK_DRIVER_MEMORY=8g\n"
]
}
],
"source": [
"%env SPARK_DRIVER_MEMORY=8g\n",
"%env PYSPARK_PYTHON=/usr/bin/python3.5\n",
"%env PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n",
"\n",
"from zoo.common.nncontext import *\n",
"sc = init_nncontext(init_spark_conf().setMaster(\"local[4]\"))"
Expand Down Expand Up @@ -65,7 +61,9 @@
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from zoo.pipeline.api.keras.datasets import reuters\n",
Expand All @@ -85,7 +83,9 @@
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"word_index = reuters.get_word_index()\n",
Expand All @@ -104,7 +104,9 @@
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import numpy as np\n",
Expand Down Expand Up @@ -225,7 +227,9 @@
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"x_val = x_train[:1000]\n",
Expand All @@ -246,7 +250,9 @@
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"import time\n",
Expand Down Expand Up @@ -410,7 +416,9 @@
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"predictions = model.predict(x_test).collect()"
Expand Down Expand Up @@ -546,7 +554,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
"version": "3.5.4"
}
},
"nbformat": 4,
Expand Down
20 changes: 11 additions & 9 deletions keras/3.7-predicting-house-prices.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
"env: SPARK_DRIVER_MEMORY=8g\n",
"env: PYSPARK_PYTHON=/usr/bin/python3.5\n",
"env: PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n"
"env: SPARK_DRIVER_MEMORY=8g\n"
]
}
],
"source": [
"%env SPARK_DRIVER_MEMORY=8g\n",
"%env PYSPARK_PYTHON=/usr/bin/python3.5\n",
"%env PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n",
"\n",
"from zoo.common.nncontext import *\n",
"sc = init_nncontext(init_spark_conf().setMaster(\"local[4]\"))"
Expand Down Expand Up @@ -71,7 +67,9 @@
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from zoo.pipeline.api.keras.datasets import boston_housing\n",
Expand Down Expand Up @@ -223,7 +221,9 @@
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"mean = train_data.mean(axis=0)\n",
Expand Down Expand Up @@ -258,7 +258,9 @@
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"from zoo.pipeline.api.keras import models\n",
Expand Down Expand Up @@ -789,7 +791,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.2"
"version": "3.5.4"
}
},
"nbformat": 4,
Expand Down
10 changes: 3 additions & 7 deletions keras/4.4-overfitting-and-underfitting.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
"env: SPARK_DRIVER_MEMORY=12g\n",
"env: PYSPARK_PYTHON=/usr/bin/python3.5\n",
"env: PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n"
"env: SPARK_DRIVER_MEMORY=14g\n"
]
}
],
"source": [
"%env SPARK_DRIVER_MEMORY=12g\n",
"%env PYSPARK_PYTHON=/usr/bin/python3.5\n",
"%env PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n",
"%env SPARK_DRIVER_MEMORY=14g\n",
"\n",
"from zoo.common.nncontext import *\n",
"sc = init_nncontext(init_spark_conf().setMaster(\"local[4]\"))"
Expand Down Expand Up @@ -117,7 +113,7 @@
},
"outputs": [],
"source": [
"#Refer to memory saving approach at Chapter 3.5,zip the training data and label into RDD, which could save your SPARK_DRIVER_MEMORY to 12g.\n",
"#Refer to memory saving approach at Chapter 3.5,zip the training data and label into RDD, which could save your SPARK_DRIVER_MEMORY to 14g.\n",
"from bigdl.util.common import to_sample_rdd\n",
"\n",
"train = to_sample_rdd(x_train, y_train)\n",
Expand Down
6 changes: 1 addition & 5 deletions keras/5.1-introduction-to-convnets.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
"env: SPARK_DRIVER_MEMORY=8g\n",
"env: PYSPARK_PYTHON=/usr/bin/python3.5\n",
"env: PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n"
"env: SPARK_DRIVER_MEMORY=8g\n"
]
}
],
"source": [
"%env SPARK_DRIVER_MEMORY=8g\n",
"%env PYSPARK_PYTHON=/usr/bin/python3.5\n",
"%env PYSPARK_DRIVER_PYTHON=/usr/bin/python3.5\n",
"\n",
"from zoo.common.nncontext import *\n",
"sc = init_nncontext(init_spark_conf().setMaster(\"local[4]\"))"
Expand Down
Loading

0 comments on commit 21f7ce6

Please sign in to comment.