From 10d246d8addb211c1a268bed75a3bb0181970193 Mon Sep 17 00:00:00 2001
From: Akifumi Imanishi <imanishi@preferred.jp>
Date: Tue, 28 Aug 2018 18:35:12 +0900
Subject: [PATCH 1/4] Add benchmark for Adam

---
 benchmarks/optimizers/__init__.py | 54 +++++++++++++++++++++++++++++++
 benchmarks/optimizers/adam.py     | 23 +++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 benchmarks/optimizers/__init__.py
 create mode 100644 benchmarks/optimizers/adam.py

diff --git a/benchmarks/optimizers/__init__.py b/benchmarks/optimizers/__init__.py
new file mode 100644
index 0000000..753623e
--- /dev/null
+++ b/benchmarks/optimizers/__init__.py
@@ -0,0 +1,54 @@
+import six
+
+import chainer
+
+from benchmarks import BenchmarkBase
+
+
+class Link(chainer.Link):
+    def __init__(self, param):
+        super(Link, self).__init__()
+        with self.init_scope():
+            self.p = chainer.Parameter(param)
+
+    def __call__(self, x):
+        return x * self.p
+
+
+class OptimizerBenchmark(BenchmarkBase):
+
+    """The base class for benchmark of optimizers."""
+
+    # Call `test_*` methods only once as `backward()` has a side-effect.
+    number = 1
+
+    # Repeat the test for 10 times instead of 3 (`timeit.default_repeat`).
+    repeat = 10
+
+    def setup_benchmark(self, optimizer, batch_size, unit_num, dtype):
+        """Performs setup of benchmark for optimizers.
+
+        Call this in `setup` method of your benchmark class.
+        Note that this function performs forward computation.
+        """
+
+        xp = self.xp
+        self.optimizer = optimizer
+
+        x = xp.random.uniform(-1, 1, (batch_size, unit_num)).astype(dtype)
+        param = xp.random.uniform(-1, 1, unit_num).astype(dtype)
+        model = Link(param)
+
+        x = chainer.Variable(x)
+        y = model(x)
+        y.zerograd()
+        y.backward()
+        optimizer.setup(model)
+
+    def update(self, n_times):
+        """Runs optimizer.update()."""
+
+        optimizer = self.optimizer
+
+        for i in six.moves.range(n_times):
+            optimizer.update()
diff --git a/benchmarks/optimizers/adam.py b/benchmarks/optimizers/adam.py
new file mode 100644
index 0000000..2df9c1d
--- /dev/null
+++ b/benchmarks/optimizers/adam.py
@@ -0,0 +1,23 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize(
+    [('dtype', [numpy.float32, numpy.float64]),
+     ('amsgrad', [True, False])])
+class Adam(OptimizerBenchmark):
+    def setup(self, dtype, amsgrad):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.Adam(0.05, amsgrad=amsgrad)
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype, amsgrad):
+        self.update(1000)

From 787be391bb64ff66b876111f416078009d328b4c Mon Sep 17 00:00:00 2001
From: Akifumi Imanishi <imanishi@preferred.jp>
Date: Wed, 29 Aug 2018 14:36:07 +0900
Subject: [PATCH 2/4] Add other benchmarks for optimizers

---
 benchmarks/optimizers/ada_delta.py            | 21 +++++++++++++++++++
 benchmarks/optimizers/ada_grad.py             | 21 +++++++++++++++++++
 .../optimizers/corrected_momentum_sgd.py      | 21 +++++++++++++++++++
 benchmarks/optimizers/momentum_sgd.py         | 21 +++++++++++++++++++
 benchmarks/optimizers/msvag.py                | 21 +++++++++++++++++++
 benchmarks/optimizers/nesterov_ag.py          | 21 +++++++++++++++++++
 benchmarks/optimizers/rmsprop.py              | 21 +++++++++++++++++++
 benchmarks/optimizers/rmsprop_graves.py       | 21 +++++++++++++++++++
 benchmarks/optimizers/sgd.py                  | 21 +++++++++++++++++++
 benchmarks/optimizers/smorms3.py              | 21 +++++++++++++++++++
 10 files changed, 210 insertions(+)
 create mode 100644 benchmarks/optimizers/ada_delta.py
 create mode 100644 benchmarks/optimizers/ada_grad.py
 create mode 100644 benchmarks/optimizers/corrected_momentum_sgd.py
 create mode 100644 benchmarks/optimizers/momentum_sgd.py
 create mode 100644 benchmarks/optimizers/msvag.py
 create mode 100644 benchmarks/optimizers/nesterov_ag.py
 create mode 100644 benchmarks/optimizers/rmsprop.py
 create mode 100644 benchmarks/optimizers/rmsprop_graves.py
 create mode 100644 benchmarks/optimizers/sgd.py
 create mode 100644 benchmarks/optimizers/smorms3.py

diff --git a/benchmarks/optimizers/ada_delta.py b/benchmarks/optimizers/ada_delta.py
new file mode 100644
index 0000000..32267f8
--- /dev/null
+++ b/benchmarks/optimizers/ada_delta.py
@@ -0,0 +1,21 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize([('dtype', [numpy.float32, numpy.float64])])
+class AdaDelta(OptimizerBenchmark):
+    def setup(self, dtype):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.AdaDelta()
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype):
+        self.update(1000)
diff --git a/benchmarks/optimizers/ada_grad.py b/benchmarks/optimizers/ada_grad.py
new file mode 100644
index 0000000..8d81e4b
--- /dev/null
+++ b/benchmarks/optimizers/ada_grad.py
@@ -0,0 +1,21 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize([('dtype', [numpy.float32, numpy.float64])])
+class AdaGrad(OptimizerBenchmark):
+    def setup(self, dtype):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.AdaGrad()
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype):
+        self.update(1000)
diff --git a/benchmarks/optimizers/corrected_momentum_sgd.py b/benchmarks/optimizers/corrected_momentum_sgd.py
new file mode 100644
index 0000000..411121d
--- /dev/null
+++ b/benchmarks/optimizers/corrected_momentum_sgd.py
@@ -0,0 +1,21 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize([('dtype', [numpy.float32, numpy.float64])])
+class CorrectedMomentumSGD(OptimizerBenchmark):
+    def setup(self, dtype):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.CorrectedMomentumSGD()
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype):
+        self.update(1000)
diff --git a/benchmarks/optimizers/momentum_sgd.py b/benchmarks/optimizers/momentum_sgd.py
new file mode 100644
index 0000000..f3f0cd6
--- /dev/null
+++ b/benchmarks/optimizers/momentum_sgd.py
@@ -0,0 +1,21 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize([('dtype', [numpy.float32, numpy.float64])])
+class MomentumSGD(OptimizerBenchmark):
+    def setup(self, dtype):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.MomentumSGD()
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype):
+        self.update(1000)
diff --git a/benchmarks/optimizers/msvag.py b/benchmarks/optimizers/msvag.py
new file mode 100644
index 0000000..f8494df
--- /dev/null
+++ b/benchmarks/optimizers/msvag.py
@@ -0,0 +1,21 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize([('dtype', [numpy.float32, numpy.float64])])
+class MSVAG(OptimizerBenchmark):
+    def setup(self, dtype):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.MSVAG()
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype):
+        self.update(1000)
diff --git a/benchmarks/optimizers/nesterov_ag.py b/benchmarks/optimizers/nesterov_ag.py
new file mode 100644
index 0000000..4a2aa7f
--- /dev/null
+++ b/benchmarks/optimizers/nesterov_ag.py
@@ -0,0 +1,21 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize([('dtype', [numpy.float32, numpy.float64])])
+class NesterovAG(OptimizerBenchmark):
+    def setup(self, dtype):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.NesterovAG()
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype):
+        self.update(1000)
diff --git a/benchmarks/optimizers/rmsprop.py b/benchmarks/optimizers/rmsprop.py
new file mode 100644
index 0000000..0cf5840
--- /dev/null
+++ b/benchmarks/optimizers/rmsprop.py
@@ -0,0 +1,21 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize([('dtype', [numpy.float32, numpy.float64])])
+class RMSprop(OptimizerBenchmark):
+    def setup(self, dtype):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.RMSprop()
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype):
+        self.update(1000)
diff --git a/benchmarks/optimizers/rmsprop_graves.py b/benchmarks/optimizers/rmsprop_graves.py
new file mode 100644
index 0000000..a483f67
--- /dev/null
+++ b/benchmarks/optimizers/rmsprop_graves.py
@@ -0,0 +1,21 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize([('dtype', [numpy.float32, numpy.float64])])
+class RMSpropGraves(OptimizerBenchmark):
+    def setup(self, dtype):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.RMSpropGraves()
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype):
+        self.update(1000)
diff --git a/benchmarks/optimizers/sgd.py b/benchmarks/optimizers/sgd.py
new file mode 100644
index 0000000..1f9a518
--- /dev/null
+++ b/benchmarks/optimizers/sgd.py
@@ -0,0 +1,21 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize([('dtype', [numpy.float32, numpy.float64])])
+class SGD(OptimizerBenchmark):
+    def setup(self, dtype):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.SGD()
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype):
+        self.update(1000)
diff --git a/benchmarks/optimizers/smorms3.py b/benchmarks/optimizers/smorms3.py
new file mode 100644
index 0000000..71a2990
--- /dev/null
+++ b/benchmarks/optimizers/smorms3.py
@@ -0,0 +1,21 @@
+import numpy
+
+from chainer import optimizers
+
+from benchmarks.optimizers import OptimizerBenchmark
+from benchmarks.utils import backends
+from benchmarks.utils import parameterize
+
+
+@backends('gpu', 'cpu')
+@parameterize([('dtype', [numpy.float32, numpy.float64])])
+class SMORMS3(OptimizerBenchmark):
+    def setup(self, dtype):
+        unit_num = 100000
+        batch_size = 32
+        optimizer = optimizers.SMORMS3()
+
+        self.setup_benchmark(optimizer, batch_size, unit_num, dtype)
+
+    def time_update(self, dtype):
+        self.update(1000)

From 83d107692913f060787e65c506c22dcc43adaba0 Mon Sep 17 00:00:00 2001
From: Akifumi Imanishi <imanishi@preferred.jp>
Date: Tue, 18 Dec 2018 18:40:01 +0900
Subject: [PATCH 3/4] Call to_gpu before initialization

---
 benchmarks/optimizers/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/benchmarks/optimizers/__init__.py b/benchmarks/optimizers/__init__.py
index 753623e..42be292 100644
--- a/benchmarks/optimizers/__init__.py
+++ b/benchmarks/optimizers/__init__.py
@@ -1,6 +1,7 @@
 import six
 
 import chainer
+import cupy
 
 from benchmarks import BenchmarkBase
 
@@ -38,6 +39,8 @@ def setup_benchmark(self, optimizer, batch_size, unit_num, dtype):
         x = xp.random.uniform(-1, 1, (batch_size, unit_num)).astype(dtype)
         param = xp.random.uniform(-1, 1, unit_num).astype(dtype)
         model = Link(param)
+        if xp is cupy:
+            model.to_gpu()
 
         x = chainer.Variable(x)
         y = model(x)

From 622c41ed93aa1e18ffc81a3120e0f3cdf40f9d47 Mon Sep 17 00:00:00 2001
From: Akifumi Imanishi <imanishi@preferred.jp>
Date: Fri, 18 Jan 2019 15:51:16 +0900
Subject: [PATCH 4/4] Update once beforehand

---
 benchmarks/optimizers/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/benchmarks/optimizers/__init__.py b/benchmarks/optimizers/__init__.py
index 42be292..b73470a 100644
--- a/benchmarks/optimizers/__init__.py
+++ b/benchmarks/optimizers/__init__.py
@@ -47,6 +47,7 @@ def setup_benchmark(self, optimizer, batch_size, unit_num, dtype):
         y.zerograd()
         y.backward()
         optimizer.setup(model)
+        optimizer.update()
 
     def update(self, n_times):
         """Runs optimizer.update()."""