From b7c356c5ce7cc7b3efcd68a68d4f07002c76d55c Mon Sep 17 00:00:00 2001 From: Ian Goodfellow Date: Mon, 19 Sep 2016 03:51:51 -0700 Subject: [PATCH 1/8] vim automatically changed the whitespace here --- cleverhans/utils_tf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cleverhans/utils_tf.py b/cleverhans/utils_tf.py index c977180bd..615d08286 100644 --- a/cleverhans/utils_tf.py +++ b/cleverhans/utils_tf.py @@ -40,8 +40,8 @@ def tf_model_train(sess, x, y, predictions, X_train, Y_train, save=False, :param X_train: numpy array with training inputs :param Y_train: numpy array with training outputs :param save: Boolean controling the save operation - :param predictions_adv: if set with the adversarial example tensor, - will run adversarial training + :param predictions_adv: if set with the adversarial example tensor, + will run adversarial training :return: True if model trained """ print "Starting model training using TensorFlow." From eb23bf622935592532e8b921cdac59b231c3672d Mon Sep 17 00:00:00 2001 From: Ian Goodfellow Date: Mon, 19 Sep 2016 03:52:32 -0700 Subject: [PATCH 2/8] check that all examples are used --- cleverhans/utils_tf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cleverhans/utils_tf.py b/cleverhans/utils_tf.py index 615d08286..1471d0d54 100644 --- a/cleverhans/utils_tf.py +++ b/cleverhans/utils_tf.py @@ -124,6 +124,7 @@ def tf_model_eval(sess, x, y, model, X_test, Y_test): accuracy += acc_value.eval(feed_dict={x: X_test[start:end], y: Y_test[start:end], keras.backend.learning_phase(): 0}) + assert end >= len(X_test) # Divide by number of batches to get final value accuracy /= nb_batches From 54a0f94c0866850f362dd92c1f48a2bb8643d75c Mon Sep 17 00:00:00 2001 From: Ian Goodfellow Date: Mon, 19 Sep 2016 03:53:01 -0700 Subject: [PATCH 3/8] vim automatically changed the whitespace here --- tests/test_mnist_accuracy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_mnist_accuracy.py b/tests/test_mnist_accuracy.py index b8024a4d3..eca428022 100644 --- a/tests/test_mnist_accuracy.py +++ b/tests/test_mnist_accuracy.py @@ -51,11 +51,11 @@ def main(argv=None): # Train an MNIST model tf_model_train(sess, x, y, predictions, X_train, Y_train) - + # Evaluate the accuracy of the MNIST model on legitimate test examples accuracy = tf_model_eval(sess, x, y, predictions, X_test, Y_test) assert float(accuracy) >= 0.97 - - + + if __name__ == '__main__': app.run() From 269121c149d207110351cda3bd994a08e5abc280 Mon Sep 17 00:00:00 2001 From: Ian Goodfellow Date: Mon, 19 Sep 2016 03:58:53 -0700 Subject: [PATCH 4/8] vim automatically changed the whitespace here --- cleverhans/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cleverhans/utils.py b/cleverhans/utils.py index 934472a05..9185615d1 100644 --- a/cleverhans/utils.py +++ b/cleverhans/utils.py @@ -73,4 +73,4 @@ def batch_indices(batch_nb, data_length, batch_size): start -= shift end -= shift - return start, end \ No newline at end of file + return start, end From e76f16b40a144e520c440c0a8014c8cd7703dcc3 Mon Sep 17 00:00:00 2001 From: Ian Goodfellow Date: Mon, 19 Sep 2016 03:59:13 -0700 Subject: [PATCH 5/8] bug fix: integer division already discarded remainder before ceil --- cleverhans/utils_tf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cleverhans/utils_tf.py b/cleverhans/utils_tf.py index 1471d0d54..48db9d1df 100644 --- a/cleverhans/utils_tf.py +++ b/cleverhans/utils_tf.py @@ -63,7 +63,8 @@ def tf_model_train(sess, x, y, predictions, X_train, Y_train, save=False, print("Epoch " + str(epoch)) # Compute number of batches - nb_batches = int(math.ceil(len(X_train) / FLAGS.batch_size)) + nb_batches = int(math.ceil(float(len(X_train)) / FLAGS.batch_size)) + assert nb_batches * FLAGS.batch_size >= len(X_train) prev = time.time() for batch in range(nb_batches): From 723b25e28a6bedea2358bb94bb5855a4bfea80ab Mon Sep 17 00:00:00 2001 From: Ian Goodfellow Date: Mon, 19 Sep 2016 03:59:45 -0700 Subject: [PATCH 6/8] check that all examples were used --- cleverhans/utils_tf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cleverhans/utils_tf.py b/cleverhans/utils_tf.py index 48db9d1df..05eda584d 100644 --- a/cleverhans/utils_tf.py +++ b/cleverhans/utils_tf.py @@ -81,6 +81,7 @@ def tf_model_train(sess, x, y, predictions, X_train, Y_train, save=False, train_step.run(feed_dict={x: X_train[start:end], y: Y_train[start:end], keras.backend.learning_phase(): 1}) + assert end >= len(X_train) # Check that all examples were used if save: From bda50c6abfea7555f5dca90504ba29a426f18cf0 Mon Sep 17 00:00:00 2001 From: Ian Goodfellow Date: Mon, 19 Sep 2016 04:06:49 -0700 Subject: [PATCH 7/8] bug fix: make sure every example is counted exactly once for eval --- cleverhans/utils_tf.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/cleverhans/utils_tf.py b/cleverhans/utils_tf.py index 05eda584d..8f9a49ad2 100644 --- a/cleverhans/utils_tf.py +++ b/cleverhans/utils_tf.py @@ -114,22 +114,29 @@ def tf_model_eval(sess, x, y, model, X_test, Y_test): with sess.as_default(): # Compute number of batches - nb_batches = int(math.ceil(len(X_test) / FLAGS.batch_size)) + nb_batches = int(math.ceil(float(len(X_test)) / FLAGS.batch_size)) + assert nb_batches * FLAGS.batch_size >= len(X_test) for batch in range(nb_batches): if batch % 100 == 0 and batch > 0: print("Batch " + str(batch)) - # Compute batch start and end indices - start, end = batch_indices(batch, len(X_test), FLAGS.batch_size) + # Must not use the `batch_indices` function here, because it + # repeats some examples. + # It's acceptable to repeat during training, but not eval. + start = batch * FLAGS.batch_size + end = min(len(X_test), start + FLAGS.batch_size) + cur_batch_size = end - start + 1 - accuracy += acc_value.eval(feed_dict={x: X_test[start:end], + # The last batch may be smaller than all others, so we need to + # account for variable batch size here + accuracy += cur_batch_size * acc_value.eval(feed_dict={x: X_test[start:end], y: Y_test[start:end], keras.backend.learning_phase(): 0}) assert end >= len(X_test) - # Divide by number of batches to get final value - accuracy /= nb_batches + # Divide by number of examples to get final value + accuracy /= len(X_test) return accuracy From 4a60b39f4214fd8724400f203248a06716ccad67 Mon Sep 17 00:00:00 2001 From: Ian Goodfellow Date: Mon, 19 Sep 2016 04:18:35 -0700 Subject: [PATCH 8/8] print out accuracy on failure --- tests/test_mnist_accuracy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_mnist_accuracy.py b/tests/test_mnist_accuracy.py index eca428022..52d4f1f6c 100644 --- a/tests/test_mnist_accuracy.py +++ b/tests/test_mnist_accuracy.py @@ -54,7 +54,7 @@ def main(argv=None): # Evaluate the accuracy of the MNIST model on legitimate test examples accuracy = tf_model_eval(sess, x, y, predictions, X_test, Y_test) - assert float(accuracy) >= 0.97 + assert float(accuracy) >= 0.97, accuracy if __name__ == '__main__':