From 08f122762c7cee50708b009385970bea554420d7 Mon Sep 17 00:00:00 2001 From: Tammo van der Heide Date: Fri, 15 Oct 2021 17:34:32 +0200 Subject: [PATCH 1/3] Add documentation covering additional activation functions --- doc/recipes/docs/_figures/transfer/atan.svg | 690 ++++++++++ doc/recipes/docs/_figures/transfer/bent.svg | 540 ++++++++ .../docs/_figures/transfer/gaussian.svg | 1105 ++++++++-------- .../docs/_figures/transfer/heaviside.svg | 981 +++++++------- doc/recipes/docs/_figures/transfer/linear.svg | 668 +++++----- doc/recipes/docs/_figures/transfer/lrelu.svg | 546 ++++++++ .../docs/_figures/transfer/plot_transfer.py | 100 +- doc/recipes/docs/_figures/transfer/relu.svg | 687 +++++----- .../docs/_figures/transfer/sigmoid.svg | 1122 ++++++++-------- .../docs/_figures/transfer/softplus.svg | 684 ++++++++++ doc/recipes/docs/_figures/transfer/tanh.svg | 1140 +++++++++-------- doc/recipes/docs/transfer/index.rst | 61 +- 12 files changed, 5514 insertions(+), 2810 deletions(-) create mode 100644 doc/recipes/docs/_figures/transfer/atan.svg create mode 100644 doc/recipes/docs/_figures/transfer/bent.svg create mode 100644 doc/recipes/docs/_figures/transfer/lrelu.svg create mode 100644 doc/recipes/docs/_figures/transfer/softplus.svg diff --git a/doc/recipes/docs/_figures/transfer/atan.svg b/doc/recipes/docs/_figures/transfer/atan.svg new file mode 100644 index 0000000..c294108 --- /dev/null +++ b/doc/recipes/docs/_figures/transfer/atan.svg @@ -0,0 +1,690 @@ + + + + + + + + 2021-10-15T17:26:13.330253 + image/svg+xml + + + Matplotlib v3.4.3, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/recipes/docs/_figures/transfer/bent.svg b/doc/recipes/docs/_figures/transfer/bent.svg new file mode 100644 index 0000000..1865b19 --- /dev/null +++ b/doc/recipes/docs/_figures/transfer/bent.svg @@ -0,0 +1,540 @@ + + + + + + + + 2021-10-15T17:26:16.455719 + image/svg+xml + + + Matplotlib v3.4.3, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/recipes/docs/_figures/transfer/gaussian.svg b/doc/recipes/docs/_figures/transfer/gaussian.svg index 5ed5be6..9b45949 100644 --- a/doc/recipes/docs/_figures/transfer/gaussian.svg +++ b/doc/recipes/docs/_figures/transfer/gaussian.svg @@ -1,12 +1,23 @@ - + + + + + 2021-10-15T17:26:14.659938 + image/svg+xml + + + Matplotlib v3.4.3, https://matplotlib.org/ + + + + + - + @@ -21,8 +32,8 @@ z @@ -32,224 +43,227 @@ z +" id="m5eb690e32c" style="stroke:#000000;stroke-width:0.8;"/> - + - - + + - + - - - - +" id="DejaVuSans-33" transform="scale(0.015625)"/> + + + - + - - - - - + + + + + - + - - - - - + + + + + - + - - + + - - - +" id="DejaVuSans-30" transform="scale(0.015625)"/> + + - + - + - + - + - + - + - - - - + + + + @@ -259,220 +273,219 @@ z +" id="m696e15c043" style="stroke:#000000;stroke-width:0.8;"/> - + - - + + - + - - + + - - - + + - + - + - - - - - - +" id="DejaVuSans-29" transform="scale(0.015625)"/> + + + + + - - @@ -481,253 +494,259 @@ L 493.2 318.04 " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/> - - - + + - + - + - + - + - + - - + + - + - + - + - - - - - - - - - - - - - - - - - - - +" id="DejaVuSans-6f" transform="scale(0.015625)"/> + + + + + + + + + + + + + + + + + + - - + + diff --git a/doc/recipes/docs/_figures/transfer/heaviside.svg b/doc/recipes/docs/_figures/transfer/heaviside.svg index 4535bca..8118ba5 100644 --- a/doc/recipes/docs/_figures/transfer/heaviside.svg +++ b/doc/recipes/docs/_figures/transfer/heaviside.svg @@ -1,12 +1,23 @@ - + + + + + 2021-10-15T17:26:17.038417 + image/svg+xml + + + Matplotlib v3.4.3, https://matplotlib.org/ + + + + + - + @@ -21,8 +32,8 @@ z @@ -32,110 +43,113 @@ z +" id="m11018b4887" style="stroke:#000000;stroke-width:0.8;"/> - + - - - - - - + + + + + + - + - - - - + + + + - + - + - - - - + + + + @@ -145,164 +159,168 @@ z +" id="m915a2a331e" style="stroke:#000000;stroke-width:0.8;"/> - + - - - - - - - + + + + + + + - + - - - - - - - + + + + + + + - + - - - - + + + + - - - - - - - - - - + + + + + + + + + + - @@ -311,302 +329,311 @@ L 493.2 318.04 " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + diff --git a/doc/recipes/docs/_figures/transfer/linear.svg b/doc/recipes/docs/_figures/transfer/linear.svg index b347824..558bce8 100644 --- a/doc/recipes/docs/_figures/transfer/linear.svg +++ b/doc/recipes/docs/_figures/transfer/linear.svg @@ -1,12 +1,23 @@ - + + + + + 2021-10-15T17:26:15.074841 + image/svg+xml + + + Matplotlib v3.4.3, https://matplotlib.org/ + + + + + - + @@ -21,8 +32,8 @@ z @@ -32,112 +43,115 @@ z +" id="mbdf6066967" style="stroke:#000000;stroke-width:0.8;"/> - + - - + + - + - - - - +" id="DejaVuSans-31" transform="scale(0.015625)"/> + + + - + - - + + - - - +" id="DejaVuSans-30" transform="scale(0.015625)"/> + + - + - + - - - - + + + + @@ -147,88 +161,89 @@ z +" id="m717f2283eb" style="stroke:#000000;stroke-width:0.8;"/> - + - - - + + + - + - - + + - + - - + + - - + + - - - +" id="DejaVuSans-Oblique-78" transform="scale(0.015625)"/> + + - - @@ -237,243 +252,252 @@ L 493.2 317.32 " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/> - - - + + - + - + - + - + - + - - + + - + - + - + - + - - - - - - - - - - - - - - - - - +" id="DejaVuSans-6f" transform="scale(0.015625)"/> + + + + + + + + + + + + + + + + - - + + diff --git a/doc/recipes/docs/_figures/transfer/lrelu.svg b/doc/recipes/docs/_figures/transfer/lrelu.svg new file mode 100644 index 0000000..dc10551 --- /dev/null +++ b/doc/recipes/docs/_figures/transfer/lrelu.svg @@ -0,0 +1,546 @@ + + + + + + + + 2021-10-15T17:26:15.941242 + image/svg+xml + + + Matplotlib v3.4.3, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/recipes/docs/_figures/transfer/plot_transfer.py b/doc/recipes/docs/_figures/transfer/plot_transfer.py index 83298af..a7f18ca 100644 --- a/doc/recipes/docs/_figures/transfer/plot_transfer.py +++ b/doc/recipes/docs/_figures/transfer/plot_transfer.py @@ -17,10 +17,14 @@ def main(): '''Main driver routine.''' plot_tanh() + plot_atan() plot_sigmoid() + plot_softplus() plot_gaussian() plot_linear() plot_relu() + plot_lrelu() + plot_bent() plot_heaviside() @@ -46,6 +50,30 @@ def plot_tanh(): plt.show() +def plot_atan(): + '''Plots the arcus tangent in given interval.''' + + xx = np.linspace(-10.0, 10.0, 1000) + atan = np.arctan(xx) + + plt.figure(1, figsize=[7, 5]) + plt.title('Arcus Tangent') + plt.xlabel(r'$x$') + plt.ylabel(r'$\arctan(x)$') + + plt.xticks((-10.0, 0.0, 10.0)) + plt.ylim((-np.pi / 2.0, np.pi / 2.0)) + plt.yticks((-np.pi / 2.0, 0.0, np.pi / 2.0), + (r'$-\frac{\pi}{2}$', '0', r'$\frac{\pi}{2}$')) + + plt.hlines(0.0, np.min(xx), np.max(xx), color='gray', linestyle='dashed') + plt.plot(xx, atan) + plt.tight_layout() + + plt.savefig('atan.svg', dpi=900, format='svg') + plt.show() + + def plot_sigmoid(): '''Plots the sigmoid function in given interval.''' @@ -67,6 +95,26 @@ def plot_sigmoid(): plt.show() +def plot_softplus(): + '''Plots the SoftPlus function in given interval.''' + + xx = np.linspace(-3.0, 3.0, 1000) + softplus = np.log(1.0 + np.exp(xx)) + + plt.figure(1, figsize=[7, 5]) + plt.title('SoftPlus Function') + plt.xlabel(r'$x$') + plt.ylabel(r'SoftPlus') + + plt.yticks((-1.0, 0.0, 1.0, 2.0, 3.0)) + + plt.plot(xx, softplus) + plt.tight_layout() + + plt.savefig('softplus.svg', dpi=900, format='svg') + plt.show() + + def plot_gaussian(): '''Plots gaussian function in given interval.''' @@ -76,7 +124,7 @@ def plot_gaussian(): plt.figure(1, figsize=[7, 5]) plt.title('Gaussian Function') plt.xlabel(r'$x$') - plt.ylabel(r'$S(x)$') + plt.ylabel(r'$G(x)$') plt.yticks((0.0, 1.0)) @@ -111,7 +159,7 @@ def plot_linear(): def plot_relu(): - '''Plots relu function in given interval.''' + '''Plots ReLU function in given interval.''' xx = np.linspace(-1.0, 1.0, 1000) relu = xx * (xx > 0.0) @@ -119,7 +167,7 @@ def plot_relu(): plt.figure(1, figsize=[7, 5]) plt.title('ReLU Function') plt.xlabel(r'$x$') - plt.ylabel(r'$R(x)$') + plt.ylabel('ReLU') plt.xticks((-1.0, 0.0, 1.0)) plt.yticks((-1.0, 0.0, 1.0)) @@ -131,6 +179,52 @@ def plot_relu(): plt.show() +def plot_lrelu(): + '''Plots leaky ReLU function in given interval.''' + + xx = np.linspace(-1.0, 1.0, 1000) + lrelu = np.array([max(0.01 * val, val) for val in xx], dtype=float) + + plt.figure(1, figsize=[7, 5]) + plt.title('Leaky ReLU Function') + plt.xlabel(r'$x$') + plt.ylabel('Leaky ReLU') + + plt.xticks((-1.0, 0.0, 1.0)) + plt.yticks((-1.0, 0.0, 1.0)) + + plt.hlines(0.0, np.min(xx), np.max(xx), color='gray', linestyle='dashed') + + plt.plot(xx, lrelu) + plt.tight_layout() + + plt.savefig('lrelu.svg', dpi=900, format='svg') + plt.show() + + +def plot_bent(): + '''Plots Bent identity function in given interval.''' + + xx = np.linspace(-1.0, 1.0, 1000) + bent = (np.sqrt(xx**2 + 1.0) - 1.0) / 2.0 + xx + + plt.figure(1, figsize=[7, 5]) + plt.title('Bent Identity Function') + plt.xlabel(r'$x$') + plt.ylabel('Bent identity') + + plt.xticks((-2.0, -1.0, 0.0, 1.0)) + plt.yticks((-1.0, 0.0, 1.0)) + + plt.hlines(0.0, np.min(xx), np.max(xx), color='gray', linestyle='dashed') + + plt.plot(xx, bent) + plt.tight_layout() + + plt.savefig('bent.svg', dpi=900, format='svg') + plt.show() + + def plot_heaviside(): '''Plots heaviside function in given interval.''' diff --git a/doc/recipes/docs/_figures/transfer/relu.svg b/doc/recipes/docs/_figures/transfer/relu.svg index 47da324..a5d84c7 100644 --- a/doc/recipes/docs/_figures/transfer/relu.svg +++ b/doc/recipes/docs/_figures/transfer/relu.svg @@ -1,12 +1,23 @@ - + + + + + 2021-10-15T17:26:15.497497 + image/svg+xml + + + Matplotlib v3.4.3, https://matplotlib.org/ + + + + + - + @@ -19,10 +30,10 @@ z - @@ -32,110 +43,113 @@ z +" id="mfc14b0e98a" style="stroke:#000000;stroke-width:0.8;"/> - + - - + + - + - - - - +" id="DejaVuSans-31" transform="scale(0.015625)"/> + + + - + - - + + - - - +" id="DejaVuSans-30" transform="scale(0.015625)"/> + + - + - - + + - - + + - - - +" id="DejaVuSans-Oblique-78" transform="scale(0.015625)"/> + + @@ -145,350 +159,307 @@ z +" id="m956957b8ee" style="stroke:#000000;stroke-width:0.8;"/> - + - - + + - + - - + + - - - + + - + - - - - - - - +" id="DejaVuSans-65" transform="scale(0.015625)"/> + + + + + + + - - - - - - - - - - - + + + - + - + - + - + - + - + - - - - - - - - - - - - - - - +" id="DejaVuSans-6f" transform="scale(0.015625)"/> + + + + + + + + + + + + + + - - + + diff --git a/doc/recipes/docs/_figures/transfer/sigmoid.svg b/doc/recipes/docs/_figures/transfer/sigmoid.svg index 75290bd..c9aacf0 100644 --- a/doc/recipes/docs/_figures/transfer/sigmoid.svg +++ b/doc/recipes/docs/_figures/transfer/sigmoid.svg @@ -1,12 +1,23 @@ - + + + + + 2021-10-15T17:26:13.789400 + image/svg+xml + + + Matplotlib v3.4.3, https://matplotlib.org/ + + + + + - + @@ -21,8 +32,8 @@ z @@ -32,226 +43,230 @@ z +" id="m7c8eceab2a" style="stroke:#000000;stroke-width:0.8;"/> - + - - + + - + - - - - +" id="DejaVuSans-36" transform="scale(0.015625)"/> + + + - + - - + + - - - - +" id="DejaVuSans-34" transform="scale(0.015625)"/> + + + - + - - - - - + + + + + - + - - + + - - - +" id="DejaVuSans-30" transform="scale(0.015625)"/> + + - + - + - + - + - + - + - - - - + + + + @@ -261,200 +276,201 @@ z +" id="m40b6c5f60f" style="stroke:#000000;stroke-width:0.8;"/> - + - - + + - + - - + + - - - +" id="DejaVuSans-31" transform="scale(0.015625)"/> + + - - + + - + - + - - - - - - +" id="DejaVuSans-29" transform="scale(0.015625)"/> + + + + + - - @@ -463,284 +479,290 @@ L 493.2 318.04 " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/> - - - + + - + - + - + - + - + - - + + - + - + - + - + - - - - - - - - - - - - - - - - - - +" id="DejaVuSans-74" transform="scale(0.015625)"/> + + + + + + + + + + + + + + + + + - - + + diff --git a/doc/recipes/docs/_figures/transfer/softplus.svg b/doc/recipes/docs/_figures/transfer/softplus.svg new file mode 100644 index 0000000..47a5a0a --- /dev/null +++ b/doc/recipes/docs/_figures/transfer/softplus.svg @@ -0,0 +1,684 @@ + + + + + + + + 2021-10-15T17:26:14.252116 + image/svg+xml + + + Matplotlib v3.4.3, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/recipes/docs/_figures/transfer/tanh.svg b/doc/recipes/docs/_figures/transfer/tanh.svg index c086e88..389da91 100644 --- a/doc/recipes/docs/_figures/transfer/tanh.svg +++ b/doc/recipes/docs/_figures/transfer/tanh.svg @@ -1,12 +1,23 @@ - + + + + + 2021-10-15T17:26:12.073454 + image/svg+xml + + + Matplotlib v3.4.3, https://matplotlib.org/ + + + + + - + @@ -21,8 +32,8 @@ z @@ -32,119 +43,122 @@ z +" id="me6d5d33929" style="stroke:#000000;stroke-width:0.8;"/> - + - - - - - - + + + + + + - + - - - - + + + + - + - + - - - - + + + + @@ -154,275 +168,279 @@ z +" id="mf6aa033a18" style="stroke:#000000;stroke-width:0.8;"/> - + - - - - - - + + + + + + - + - - + + - + - - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - @@ -431,264 +449,274 @@ L 493.2 318.04 " style="fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;"/> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + diff --git a/doc/recipes/docs/transfer/index.rst b/doc/recipes/docs/transfer/index.rst index cfef15b..225c974 100644 --- a/doc/recipes/docs/transfer/index.rst +++ b/doc/recipes/docs/transfer/index.rst @@ -10,9 +10,12 @@ significant difference, depending on the situation or dataset. In order to give you a certain freedom in this sense, Fortnet implements the following functions: - hyperbolic tangent + - arcus tangent - sigmoid function + - softplus function - gaussian function - - relu function + - (leaky) ReLU function + - Bent identity function - heaviside function - linear function @@ -38,6 +41,20 @@ Hyperbolic Tangent :align: center :alt: Plot of the hyperbolic tangent. +Arcus Tangent +================== +:: + + Network = BPNN { + Hidden = 2 2 + Activation = atan + } + +.. figure:: ../_figures/transfer/atan.svg + :width: 100% + :align: center + :alt: Plot of the arcus tangent. + Sigmoid ======= :: @@ -52,6 +69,20 @@ Sigmoid :align: center :alt: Plot of sigmoid activation function. +SoftPlus +======== +:: + + Network = BPNN { + Hidden = 2 2 + Activation = softplus + } + +.. figure:: ../_figures/transfer/softplus.svg + :width: 100% + :align: center + :alt: Plot of softplus activation function. + Gaussian ======== :: @@ -80,6 +111,34 @@ ReLU :align: center :alt: Plot of relu activation function. +Leaky ReLU +========== +:: + + Network = BPNN { + Hidden = 2 2 + Activation = lrelu + } + +.. figure:: ../_figures/transfer/lrelu.svg + :width: 100% + :align: center + :alt: Plot of leaky ReLU activation function. + +Bent Identity +============= +:: + + Network = BPNN { + Hidden = 2 2 + Activation = bent + } + +.. figure:: ../_figures/transfer/bent.svg + :width: 100% + :align: center + :alt: Plot of Bent identity activation function. + Heaviside ========= :: From 94fe759a57e5c222b1f8b321ce36dd3edb0436da Mon Sep 17 00:00:00 2001 From: Tammo van der Heide Date: Fri, 15 Oct 2021 17:35:27 +0200 Subject: [PATCH 2/3] Implement some additional activation functions --- prog/fortnet/lib_fortnet/initprogram.F90 | 15 ++- prog/fortnet/lib_nn/layer.F90 | 25 ++++- prog/fortnet/lib_nn/transfer.F90 | 122 ++++++++++++++++++++++- 3 files changed, 157 insertions(+), 5 deletions(-) diff --git a/prog/fortnet/lib_fortnet/initprogram.F90 b/prog/fortnet/lib_fortnet/initprogram.F90 index c597273..1c9cf0b 100644 --- a/prog/fortnet/lib_fortnet/initprogram.F90 +++ b/prog/fortnet/lib_fortnet/initprogram.F90 @@ -697,6 +697,15 @@ subroutine readNetworkBlock(network, node, case, nFeatures, nTargets) call getChildValue(node, 'Activation', strBuffer) network%activation = tolower(trim(unquote(char(strBuffer)))) + associate (activation => network%activation) + if (.not. (activation == 'gaussian' .or. activation == 'relu' .or. activation == 'lrelu'& + & .or. activation == 'softplus' .or. activation == 'bent' .or. activation == 'atan'& + & .or. activation == 'sigmoid' .or. activation == 'heaviside' .or. activation == 'tanh'& + & .or. activation == 'linear')) then + call detailedError(node, 'Invalid activation function descriptor obtained.') + end if + end associate + case default call detailedError(node, 'Invalid network type.') @@ -1305,8 +1314,10 @@ subroutine TInput_checkInputConsistency(this) @:ASSERT(tolower(this%network%type) == 'bpnn') associate (descriptor => this%network%activation) - @:ASSERT(descriptor == 'gaussian' .or. descriptor == 'relu' .or. descriptor == 'sigmoid'& - & .or. descriptor == 'heaviside' .or. descriptor == 'tanh' .or. descriptor == 'linear') + @:ASSERT(descriptor == 'gaussian' .or. descriptor == 'relu' .or. descriptor == 'lrelu'& + & .or. descriptor == 'softplus' .or. descriptor == 'bent' .or. descriptor == 'atan'& + & .or. descriptor == 'sigmoid' .or. descriptor == 'heaviside' .or. descriptor == 'tanh'& + & .or. descriptor == 'linear') end associate if (tolower(this%option%mode) == 'train') then diff --git a/prog/fortnet/lib_nn/layer.F90 b/prog/fortnet/lib_nn/layer.F90 index cb54e87..0740d01 100644 --- a/prog/fortnet/lib_nn/layer.F90 +++ b/prog/fortnet/lib_nn/layer.F90 @@ -13,8 +13,9 @@ module fnet_layer use dftbp_accuracy, only: dp use dftbp_ranlux, only : TRanlux - use fnet_transfer, only : transferFunc, gaussian, gaussianDeriv, relu, reluDeriv, sigmoid,& - & sigmoidDeriv, heaviside, heavisideDeriv, tanhf, tanhDeriv, linear, linearDeriv + use fnet_transfer, only : transferFunc, gaussian, gaussianDeriv, relu, reluDeriv, lrelu,& + & lreluDeriv, softPlus, softPlusDeriv, bentIdentity, bentIdentityDeriv, arctan, arctanDeriv,& + & sigmoid, sigmoidDeriv, heaviside, heavisideDeriv, tanhf, tanhDeriv, linear, linearDeriv use fnet_random, only : normalXavier implicit none @@ -108,6 +109,26 @@ subroutine TLayer_setTransferFunc(this, descriptor) this%transferDeriv => reluDeriv this%descriptor = 'relu' + case('lrelu') + this%transfer => lrelu + this%transferDeriv => lreluDeriv + this%descriptor = 'lrelu' + + case('softplus') + this%transfer => softPlus + this%transferDeriv => softPlusDeriv + this%descriptor = 'softplus' + + case('bent') + this%transfer => bentIdentity + this%transferDeriv => bentIdentityDeriv + this%descriptor = 'bent' + + case('atan') + this%transfer => arctan + this%transferDeriv => arctanDeriv + this%descriptor = 'atan' + case('sigmoid') this%transfer => sigmoid this%transferDeriv => sigmoidDeriv diff --git a/prog/fortnet/lib_nn/transfer.F90 b/prog/fortnet/lib_nn/transfer.F90 index b49ada6..b011ee9 100644 --- a/prog/fortnet/lib_nn/transfer.F90 +++ b/prog/fortnet/lib_nn/transfer.F90 @@ -19,6 +19,10 @@ module fnet_transfer public :: transferFunc public :: gaussian, gaussianDeriv public :: relu, reluDeriv + public :: lrelu, lreluDeriv + public :: softPlus, softPlusDeriv + public :: bentIdentity, bentIdentityDeriv + public :: arctan, arctanDeriv public :: sigmoid, sigmoidDeriv public :: heaviside, heavisideDeriv public :: tanhf, tanhDeriv @@ -74,6 +78,62 @@ pure function gaussianDeriv(xx) result(res) end function gaussianDeriv + !> Calculates a SoftPlus transfer for given arguments. + pure function softplus(xx) result(res) + + !> array to calculate the transfer function for + real(dp), intent(in) :: xx(:) + + !> corresponding transfer function values + real(dp) :: res(size(xx)) + + res(:) = log(1.0_dp + exp(xx)) + + end function softplus + + + !> Calculates the derivatives of a SoftPlus transfer. + pure function softPlusDeriv(xx) result(res) + + !> array to calculate the transfer function for + real(dp), intent(in) :: xx(:) + + !> corresponding transfer function values + real(dp) :: res(size(xx)) + + res(:) = 1.0_dp / (1.0_dp + exp(- xx)) + + end function softPlusDeriv + + + !> Calculates a Bent identity transfer for given arguments. + pure function bentIdentity(xx) result(res) + + !> array to calculate the transfer function for + real(dp), intent(in) :: xx(:) + + !> corresponding transfer function values + real(dp) :: res(size(xx)) + + res(:) = (sqrt(xx**2 + 1.0_dp) - 1.0_dp) / 2.0_dp + xx + + end function bentIdentity + + + !> Calculates the derivatives of a Bent identity transfer. + pure function bentIdentityDeriv(xx) result(res) + + !> array to calculate the transfer function for + real(dp), intent(in) :: xx(:) + + !> corresponding transfer function values + real(dp) :: res(size(xx)) + + res(:) = xx / (2.0_dp * sqrt(xx**2 + 1.0_dp)) + 1.0_dp + + end function bentIdentityDeriv + + !> Calculates a ReLU transfer for given arguments. pure function relu(xx) result(res) @@ -97,7 +157,7 @@ pure function reluDeriv(xx) result(res) !> corresponding transfer function values real(dp) :: res(size(xx)) - where (xx > 0.0_dp) + where (xx >= 0.0_dp) res(:) = 1.0_dp elsewhere res(:) = 0.0_dp @@ -106,6 +166,38 @@ pure function reluDeriv(xx) result(res) end function reluDeriv + !> Calculates a leaky ReLU transfer for given arguments. + pure function lrelu(xx) result(res) + + !> array to calculate the transfer function for + real(dp), intent(in) :: xx(:) + + !> corresponding transfer function values + real(dp) :: res(size(xx)) + + res(:) = max(0.01_dp * xx, xx) + + end function lrelu + + + !> Calculates the derivatives of a leaky ReLU transfer. + pure function lreluDeriv(xx) result(res) + + !> array to calculate the transfer function for + real(dp), intent(in) :: xx(:) + + !> corresponding transfer function values + real(dp) :: res(size(xx)) + + where (xx >= 0.0_dp) + res(:) = 1.0_dp + elsewhere + res(:) = 0.01_dp + end where + + end function lreluDeriv + + !> Calculates a logistic, sigmoidal transfer for given arguments. pure function sigmoid(xx) result(res) @@ -194,6 +286,34 @@ pure function tanhDeriv(xx) result(res) end function tanhDeriv + !> Calculates an arcus tangent transfer for given arguments. + pure function arctan(xx) result(res) + + !> array to calculate the transfer function for + real(dp), intent(in) :: xx(:) + + !> corresponding transfer function values + real(dp) :: res(size(xx)) + + res(:) = atan(xx) + + end function arctan + + + !> Calculates the derivatives of an arcus tangent transfer. + pure function arctanDeriv(xx) result(res) + + !> array to calculate the transfer function for + real(dp), intent(in) :: xx(:) + + !> corresponding transfer function values + real(dp) :: res(size(xx)) + + res(:) = 1.0_dp / (xx**2 + 1.0_dp) + + end function arctanDeriv + + !> Calculates a linear transfer for given arguments. pure function linear(xx) result(res) From 6bcdcc2831b5a0b555ebb2227c394119a3128182 Mon Sep 17 00:00:00 2001 From: Tammo van der Heide Date: Fri, 15 Oct 2021 17:36:36 +0200 Subject: [PATCH 3/3] Add regression tests covering activation functions --- .../input/activation/arctan/_fortnet.hdf5 | Bin 0 -> 57176 bytes .../input/activation/arctan/fortnet_in.hsd | 54 ++++++++++++++++++ .../input/activation/bent/_fortnet.hdf5 | Bin 0 -> 57176 bytes .../input/activation/bent/fortnet_in.hsd | 54 ++++++++++++++++++ .../input/activation/lrelu/_fortnet.hdf5 | Bin 0 -> 57176 bytes .../input/activation/lrelu/fortnet_in.hsd | 54 ++++++++++++++++++ .../input/activation/relu/_fortnet.hdf5 | Bin 57176 -> 57176 bytes .../input/activation/softplus/_fortnet.hdf5 | Bin 0 -> 57192 bytes .../input/activation/softplus/fortnet_in.hsd | 54 ++++++++++++++++++ test/prog/fortnet/tests | 4 ++ 10 files changed, 220 insertions(+) create mode 100644 test/prog/fortnet/input/activation/arctan/_fortnet.hdf5 create mode 100644 test/prog/fortnet/input/activation/arctan/fortnet_in.hsd create mode 100644 test/prog/fortnet/input/activation/bent/_fortnet.hdf5 create mode 100644 test/prog/fortnet/input/activation/bent/fortnet_in.hsd create mode 100644 test/prog/fortnet/input/activation/lrelu/_fortnet.hdf5 create mode 100644 test/prog/fortnet/input/activation/lrelu/fortnet_in.hsd create mode 100644 test/prog/fortnet/input/activation/softplus/_fortnet.hdf5 create mode 100644 test/prog/fortnet/input/activation/softplus/fortnet_in.hsd diff --git a/test/prog/fortnet/input/activation/arctan/_fortnet.hdf5 b/test/prog/fortnet/input/activation/arctan/_fortnet.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..9189faa1b6dff0d675cd77f869e3274cc9e9fbbb GIT binary patch literal 57176 zcmeHQ3tUvy);|NN_(m;4y?UqY8rch#y!e%_@p-Waz7^|UV?cx{80NyDn0dwWcJW$D zmNuCd>8D}ljb2LG7nm>1G#_QCX{2_AM8yYcO8U(?Yb|ija0bBv!P&!)vu2;O&ffd{ z_uBup_CEWZnH4&4aLZ;Lno07<*H`k9{FJY}xk8`tpKuSS`FI3<(UD9endyaJbdvlh zNddIK4dt)a8#Q9kAjz)D`Lm?cq=H8H&6jrw=}`!T4(c~dzTj}WYPQlmvNHz59Boaq z8m)@W$@@b2KnJv-~~Y zXt9{gG39n_V1;VRkzqpz$~MfrsVBQ-zE<0DHM92^b)Dp`v<0`DAN#SE2!TRQ_ODg( zfT0EI2YL*0;zF-RK+OXobTFO#1M3>C3Gt>Vb8>uSbYhZyXY74R$&q&FHaQ{j8CfSm zmFE7emk(C*K_2ib=YhZ`sKnSaVBw zxWQYrEAR5!1~qqGA?D9~;!Wk^?WY9ZM(4Fc$@~~}C#G31oYE;>N&hAhJT6lA!xmDI za)mmnxgBF)$!biDiMCp&SfcU5+u4QObs3+SxP(Y!oSa5GWxUy6r4YWEc)FzKBP>ml z^rS~>k~P7S5SI`$h0}4yDbb1DB}tKXPEcgeih6MSvA$YzeKI=Y5%!-gB!jX;j#ygQ z`$TkHbbPeg>ePR@zwvdeWko(slzhxe1)ck$RzBEQD{3w0Z|}mAzx`kC+w#A>Po1{! z5+8wpKtLcM5D*9m1a1-psv6gacfbpkV+8T>JOnu2=0IFMF64NUW%b8}K{PJ(a{#w~ zj|+Xu@B5TcoX>H3z#s53LOP?=(^X|rn)UK2eSs8sTw%E#PhdS8rL()>e(r~%?J461 zDtKI|9v`d6g~lkWX_C=uN>FBxB}pC^%2UgXkI`yWj%eM_P`5`_e3!o-0cD&UD7>QD zHAFx?Z@}{m;nZI_+sV#dF2wCkI$rntFqtBKvWd` z%Pr+uQfIG6d^a0#YG$iFxn)559v@hN7=JvJS#ThqD$>T}&eN=akC)Mpyl|KmDgLpE| z4HI2a>zX2<_6Izk%%uLz`MN)zkw=uy>yy&Jd2Y4$7B7F$n6fcsTA0l@LCfr*p$ zb3ftvl)y3D?d`3{wTqHIZ(y^ITN2HDRKi z0o&+-+S(umHr&X4F%S;woCqm(A| zutE98>yNF8#tA0-f@-aN*v{0lqFzmvdf9Kk$0WxY$(pVQHy;JJXsc1FXLWHKu>NGo zvagGvqJG{=e#%28_8hC5pHd0?M$6y$0%{0r90j%U>tPulunIoVX6;7P=T zd6ID#JXt-S#JK)4Xz^tD5X4#Vq+>r9smc-1;z^0@h|l8w>h5?lcQBSv5Q~Tgw zi>EJyJL5BWj;(j~6dqJ->46g&=_Gih~QGr|iy5Bzd-hkg9 zUgfXJlPTlz3ItDbOA6@<0WF@?lO6H7+&|nE&oWOIJV_-`d4eYq4<7f6coOlbZai5P zPZrwOuE`(4lZb=Zx<){YCqttg&vQ3V;yT#I<-cQ)OY@{$j(Xmd!`nh4YSvSa!(*x1 z=@h53o;q$#A-!zkY1UKob{gpwMBo?J^EF~RkZ|JXE5bi<)Jvsw#%kQ}?k)Qpb$x`t zL8RH=aJyB^)8Kn}#X4BBS9;GkG%Xywz|2~$GK?PhhPttQ575s5?&hyH!$g#Orf6w?3@yljn#bo*OV(RBoD&`yN z*F23Fe_+E^cl=S6KhRqo?L+tj>L5Z?ih!0s$fM`g_#}7x1CE>3&&LUWfQU1_%ec$` z6@IT~aWl7Lg+I8y=f~I{D)PhUy;3NSs4lNz{YtKSehmFpoH0Ifg0ZI0k6oSL_06|` z`(W42otAuJUq4*=dJh{tx5<84@Ge`Nkd+~z#k&U8^S|8B+!cp2@2a0S7QBnCbT#kF zF{aB}I6zh(6{ib4xf_D)Iv2~4r7Vl<1>v&#m@ooxDeA}oz!Mlh%^RCRff_LkG zyt`utwwvHxN8S*r$`R1w-Jn!QeRuONzwalHiWx@y%jKx~IG^ly(q|5ZXf&u6p0aQA$T+OG)SMZ~Y>T`_K`@wlPRcsG~g9l^WI z7(%*2K#O-{RdI7|@vb^f7rg7pyE5kr-mU+<8!`{uP4KQOwh_k|0WIDwc!f1k))wz( z%*Fh3sXW2Eh{x6OuIJ1%1Q{yk1sc}6kP9|!H^IAzgV?%8K#O<7>32r>YYgi-J3z(1T5UOt=9msmqQ7@m;7Z&6G02ERvy(gLTQ@=Nv)atF^ zd2%%l!hd0tUG2X-XWYq}Y&YS*>MiBetBPs)uK=)yu$Afr?ZRH4P@viR*M}0CU$zAa-=gvRp*`0Q9ntxW5 z(>+zkgH>sIyk25``WeLX^q*DwA*wX}{Aam-_aIc@{JK*G%hOcB@-$VjJWUlWPg4cU z(^SFoG*z%XO%*IpQw2HQLsh>Xs`~Yy`dJdAqY}&$O!7~rn#?f@XUJC;$_+|52vI!* zc%Ce8Ii6RTiBkB-`mt_O_`T}cv!`im`8_Gi5ecSAclkZMPtiyUp^O9H!Sc$>p)`*} zn5Q#{XIM|oJK=P|K<`0kz1&rpf58VR1L^WC{T$0x|85!kN84aWJ@xPQP8`nNfE>cTD9SRpS?~?!!!3EfAAOGkRIu;N=;mJ)hY5A86h) z^}aSQzXjI*wkG=NrO-U^vG-;_asZ02tS(+0c?A5{jVL|wXffP3ciPy4WjkQ_yg!cJ z^p72Ir@vvw@_zQ;kuPlBZse_Nz$^O2nLX}5443};%d9tAy$g3_Jod|mY0Kf=4?1jG zxph7C*Cmhod%H|nJTP&~YbR}x{lBN5U%cQblzkkLKDqOcu)F_zQ{PQL07p*^zTB-~ z8~g;n9S!fd71BPoygk0t2AKLU7QfS?o39Q@8!Bs?PCh z(yYaQ@+k$M7Y{u(PIm$x*nhlq>EAcQ{1L-DbZEB)+MACZU-rv4u<~fg=$#AlVcEQ* zRw4aQ!`jNEs3xGCAi97`}1GpyKJ9;hvDp zRt2L69fiLP7<#JD%mNs1Gvs81IRcr6s3feZTkG`Mj_Jdd=w-{Kn^dVd-ZqM%jF=V_nddK2?f~Ka@m?3{U2cs{p+LyTMG<1C;r^&YQ=b)SE>EeKXJK?=05g*-eJ`ACTPreTO zc>^@HrkTp-mcq4LbvOc0RhQ;}-?+)}cp>%(`Rn?KAP21Kv0d zJ0b?%b+OAi_;b>wj=Q%Xf$ihk-qt9w7}_m5cy!&9C&9=6g>FT2PD1-R&4Ra&ISI{1 z^lfc>%HHms!VZ=_RRWjBlx05K`V4F;9`R}FibHTPZtSw;$4gFL%yBZXX#n!V|ZxhbN@XHU4J|94TeF@(jc*uD6g>CRduZ*{QeR>KS?@7;Z1uy4}%jkH-cKFqt zge|MHS3I!~<_`>Mll8@Z2%K_iv9$9L{Pe?L<9=TFGpu}lRr?K>-^uBeb9~O6GfQ&r zztunF!m-`(umAgU>cdI-;CuHjOTeO?IaT#Xaa-4(B+NWzFNv6pAysB@k)n@m#|g@4A0pDV6Z4-}6c-Ke+B8+`9VjUF&(;(W?3x zkLy_0yq^_pLu|#olFV7o>n2JFAH9PKI`dH7j(64mtYDu*^_&vi&(iXp4teyBy6b0t z+Y#u7-Ot#sGw-O=HRmDyHR5w)`&sBg*WHEtS;>CadY*Q)s(!|H#Iojg6hwB!ErCco K^Yx958~-2EZ(WQ4 literal 0 HcmV?d00001 diff --git a/test/prog/fortnet/input/activation/arctan/fortnet_in.hsd b/test/prog/fortnet/input/activation/arctan/fortnet_in.hsd new file mode 100644 index 0000000..f0c4cd2 --- /dev/null +++ b/test/prog/fortnet/input/activation/arctan/fortnet_in.hsd @@ -0,0 +1,54 @@ +Network = BPNN { + + Hidden = 2 2 + Activation = 'atan' + +} + +Features { + + Mapping = ACSF { + Reduce = No + Standardization = Yes + Function = Auto { + AtomID = 0 + RCut = 4.0 + NRadial = 5 + NAngular = 4 + } + } + +} + +Training = SD { + + Threshold = 1e-20 + NIterations = 1 + NPrintout = 1 + NSaveNet = 1 + MinDisplacement = 1e-10 + MaxDisplacement = 5e-01 + LearningRate = 0.001 + Loss = 'mse' + + Shuffle = No + +} + +Data { + + Dataset = '../../../datasets/fnetdata_oh_c_global.hdf5' + Validset = '../../../datasets/fnetdata_oh_c_global.hdf5' + NetstatFile = 'fortnet.hdf5' + +} + +Options { + + ReadNetStats = No + Mode = 'train' + + RandomSeed = 123456 + WriteIterationTrajectory = No + +} \ No newline at end of file diff --git a/test/prog/fortnet/input/activation/bent/_fortnet.hdf5 b/test/prog/fortnet/input/activation/bent/_fortnet.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..5a1d8aed7f988452f5348423c84e9c554a2e2a27 GIT binary patch literal 57176 zcmeHQ4O~=J+P{1VqmYf}8mq0%Mk#Aci;BDWR-Vz+#Vkeq!mM5gM23=Kf@B9^WcZlb$6NQoqw+Q*9DDD{dJN*MlpMRVug=XnO^4tGXyKyc3R%PvF{fY;$#i@=e(}OI6Vp6z z_~Ipqe}WK%`@3QOa=nSsBS#8WP0F8AJWZ(R1m9G&gQp&aK;+0_QQ`&1;8l~AmZF^z z7>0OLs!3;(Y);%4Ne5b(D%o$O4`irEyL4s0Rh>bf>fOTX@Y6ad0&@E$KFmb>q05Rj z8Vut1B%RTiXh^Wxv6d8?nNAoxZiHyV#G4wlTjFcE9XB(3k5Ts$e5AIZc2kody8|GQ z$jSQE6%S|wK|jP}kmDEH7y&sCMBu?}^be$~GbJY_#u?I*V&hX%tvmIhscErR=Qbxf z<#|yjSte6|)`$n2_y7-h+jt1zSsj)E#9}S)Y}t z6NC&naErHKS3b7dh6OmU5bA_6#`C&DO8gjhKggsPgK~Ob)W1y<8W+j? zVW1EuU7?o{V8vJ^m~<%#@g~z;V?12=UDibGx`a={tmIhTEKvqKCA`UBr6az1aJnGp zBPdM}G^mGaf+^XUJS#b2F3GcWbK_G62!fPaIYCMXHPnOJkM!j!)hDMn93lT1C}^b} zV#E???Gy2{;*;VHCWrn*{f(|$o;Kv;FXdy9D(Kh`mGXhU+E8mTe`^;OdRqU*zAgTX z`{c597yobsI0762jsQo1BXFA_;A&hS(+e(Cj1l<9^AMnTn*wq9xRByWN}E3}48w7u z+78_IJudXM-S?@%IG^J5pnKplBKkn8!K=zgne@t0zKjYqt}y-@P9Qxq%7wwOpZa0s zFEQf|n$WmVK0cO@3w3d(#MwGiVzM-QEC}MbP@GyOd}75xlq5HOKTh5ruK2F}V=*@v z6TvsW^&|x3^9D4}5QF`du$U8{n4mYM3bBbgaT?QRk6Kb_W~y{QPR{q)N3MQUt zW;M8a2S-53euL1D(B-w2{mT89EBh_|0&2+Zms-lRQYHH>$M}cPaKHUFJ+EAY^YM9D zuq*q`3jsxLztmEmmCEgx?mL)e_jOw9zC+D#0B?8QcOW}<*L{c9A5UsK^mtO+AKH(` zH+qyae+&CHkO)Gky#$M;Uj_l*W_t`>5_7Ps-cF zl|Kmk58z2ScT9A7ty_wK+#k?*G7tMRDA-7*@DbGrk>{pL| zgwSX!`<44ISN1EM;Py*xJ zK(P?@Gc(}JWt8(*!IvPEWAfn(o+jkPfvl$h5cPP{F%Tg8Ey9JD>DZ3bH$nZ$u3{#F z9D#a)0*nB>tanSK^>w(iPP~8OWBn&{NRagd$VCt+o|dnRb(U}A`!c$?)R|&lf%v3~ zxXw>p_hy)xX0YDUNj6Bc<~P1>OtQ}RqJ19sCeY-_C$9w3BY7b{`?D6P+ZCOd8X_(%F^8}jQa@uvK_3HiEv$`9#J zTE{DuBb6rduvYp;>yJ$-x*3Vq1=ULVke$iXhI)0D>SfXy64GYr(3)-rH(v?2c#}@5 zXLE6T?Bfd&8}^-J4fXSp@{=AivF6y^{G>`)H%k6S#Xy7m8$ybw&Ja-YH~B;D&!Yb3 zE`LKjX?z6A$;R?HPXZpqm*oFBPd3kAk@YI^WY`GM;5;o~#-JC8)4G&Xa%# z@g(8S<@ej+-3uD2VQ3uywLhg~9Syj( zulwzzej9Y{ku5zHc{2T3xB||T)RH`PhJX@JYS518TjxO+BNaRc@l8oTjvNU@nmG2{dw-@NmvK#XZsz4BAh3sa^&--6y8=L zQInp09InS|XJed7dh)n69rX&4r%6xF+nK0WF&(~;o=OLD2;xM~SHx_y*ULaT=R?@< z?k)Knd40IQ0i?;_P`fqE)4+Rp`8rs%SA5Sm;Y2B|M^>6=&cpeyY&2~C+=aV)`0bUS zlD{d$=NRZ}$^P6Gm(x76W;2wKg9Y3$PvUbL4g7I?&hv_}h_Sg+f6sXz@GB%@1=04r znEbhvhWUo}HBZCGAJA~k9e=p;2O6WjeQx%u`3gZu9N)jQj)>xY|O?_tL0Hpwq@-X)9UX;TO&@vc_({4e!0cg5kvyYlCa zIqyO%HRfG0M&-QQymfJaV%zbqdLER>c^7ctTjvNU@owRZ_UE-0@22C#w;9XhybHJ! z?~0tudAI$?y9XCQyK&yN=M643jervGhGp36yPJ3EeLtmGOce4jl_Tfla?~q*3BHhC z4a({a*k6TunXlUG<)eIXA?&}5dctCRy%3ZmmykmHPHEgA$ASAVXgBg-qP>X!eB98Y zaf78{KB;x*Ya(BRcH{mFaNt|#2q^il8svF8oBV~l=WFDCh4U^T-k5j!xS_@4hBo8f zB8+!9?-FD1)ENRwysMYR&8@||@;IIIu08LHoXdH){qt_bGH5r>yUy5#A0q^mc(>vW z(txzJcsC~t@-M>jIPU@;jpJRK+STf5$Of3067Bd8-fS&-?G;$NBQz<*dGLm5Q*}jXUZ2Wc72dUD>0!2?=+4?5`;ZGrqNwn7?$muDPiU#|Rj%g8^PN7?Jif42v-Yx9Ko zSAKnb+y+fS<2F8Dc2nKe|ahX{=Bge6xFED6VF0G?!Pp6 z`9-q#L?*@S45`**vvnznIzwE%_+9*ve;}TRy{@;!yYlx^I^$jcM&-Sj8kd|BFP85I zb-;>v;V;n8`^S4?RVekb9v4-W-pkU-x{6ag?(OX@<*35^e7p*GKvwpSy}PvP6L8RH zCF%qr9ZS453B(27SSvx8j(nh08|wFkyOlINbAR>VZ@oqjT*vfDi-JF>&tWNd^vTKo z*E!a8Qew*P+)C#C&ABIkE~;T+CyyWM9R4MHxx7j{E%qdfyZ*`eD}Vlq{bfPjpr;P~ zg9QxA2<`TlHOw^J?5~+$#{xp0c=z?kPp}`ZfB3`d*l(D6cXZwPF+Z@-te2*oyn2w0 zUv}@5l0P40_xIE;$Q@?=9r>y|@0#$7?aVuV;j055s$}&~f4k_-t{d5XIZu51$xFFx zgKHmshVm%dQqo&z{rgIQw$=yYn_?pI~Qy z9QAYmiv8>q`~R~s!^&7@fpP7#zxssDTi)L|xn~Yj?|J^G5$0tq(Nb2{OZ5{Q@V71f zKU{KW9QW z{0>{aV&?py@MFxlJ!kC1e);UuTO%f2Gi+r;k7fl6_51Nx>pj!9_*HWRI0762jzIfG zpzY#H{Z2rkeaDrKc)$#h7lg?I%@a_ZEtgv!cbD!1e5OnBI9ZHi+^wfWZIV))P4Tm4 zkMm;1`4jO|S_~=Q0s@Y96(k*?@f+C@r7gFkkk6n*YB7);?daBy-yHF1x}Omq^0ejn zL?AvI00y^?_}tolmRV|7JqLmMSy+j~A&=hSXjiS(&th(ien!rQ`j=d8c|X(NAU_S0g;(**zR;zmF!7o18%U1TUymIFwW$a@36B9rD z$6+=^|C^6a1=q5X)#~1H@9$)lJ@mh8&wCp3c<8{A(&)-$Vc0IQ3%Cp6F zY>sZp^CN#(&L%HTsvoiBC`${ObUJc>9ow-b%lCnl&zRm2eDJ%Wm)Je$F3h=d`4sEs z6MAm&p*j|H0blN6bCdhOQTNjK?A?dTe%3`_WY@pD z^PWEh{=l9}m_7G{8UM$M=0EfJv6_=?iF$v@=hYY3`hEw)|7gC*uD;gOzjMxRHgK}< zoV`IiS@fBm57+lP#9qHR;oh!WE7>>iU(&pmRLe5%omLHO^a-s>%IBx!>uPbg;E`Zrh3)g@Y< ztjqee-;slC|B~awbBo`$eEpHv-I|^oEKy5`Rvvg|nWd)W=rgI^c3BG6@9(a@vfZ*| z)VzTymDOxm_|l~TeJ`>>8#=EDsyNNgMEHHR{P8lDb9Me(-5=h=hK4OPYgQg+tIia! zOZF>beNWBWT&$~My&rfl^w^vq*?p$Vew!vxW_Q{4XbIc(>V}0+@2+Eyem(iHY4HJ;{IXxd!o?R^y?5!u*WymI z;FZ1mt`NRw!^f(od{9)&^lwhJWRJ1te_rL2ySI|1PE-}VnOe=p)vhSs^5l8eUDbO^ z)|eU=5W6ZgW$OhtX6xq6f^q+0{_p!_pM122sfC2^bsKA0p(Y|=7I?xE*@V!xk~@Xxv)Ke5BZ{+QeMfjahXTJ(&-J$qQ? zz2`>%YwAZ#_289N`q|a2{O^M%_3iXMo0|3P;06A>SnaU2b(0bgupQ~^qSs&ikA8)X%U7j)d>R*qu`j* z=qE>RupAhfTYG)xCd<{V;d8?`A7c@|PYu6s`)SsvYV_uPrC(TF^+$f3Bft^h2(%Rf z^6{?vE9WIs9B%&9xt^yTbrpwGoJncRdHHeSK&IBEKsBOo)a(`BF)UJ9C z0`;@Za)(16y`$~=nfh}8x?}e<^6SJqa=GO^q`5(SZf!q{Jn6i?+>XM~j;JL7X-B@kwQ=Kr0ioSfKL7v# literal 0 HcmV?d00001 diff --git a/test/prog/fortnet/input/activation/bent/fortnet_in.hsd b/test/prog/fortnet/input/activation/bent/fortnet_in.hsd new file mode 100644 index 0000000..cda2964 --- /dev/null +++ b/test/prog/fortnet/input/activation/bent/fortnet_in.hsd @@ -0,0 +1,54 @@ +Network = BPNN { + + Hidden = 2 2 + Activation = 'bent' + +} + +Features { + + Mapping = ACSF { + Reduce = No + Standardization = Yes + Function = Auto { + AtomID = 0 + RCut = 4.0 + NRadial = 5 + NAngular = 4 + } + } + +} + +Training = SD { + + Threshold = 1e-20 + NIterations = 1 + NPrintout = 1 + NSaveNet = 1 + MinDisplacement = 1e-10 + MaxDisplacement = 5e-01 + LearningRate = 0.001 + Loss = 'mse' + + Shuffle = No + +} + +Data { + + Dataset = '../../../datasets/fnetdata_oh_c_global.hdf5' + Validset = '../../../datasets/fnetdata_oh_c_global.hdf5' + NetstatFile = 'fortnet.hdf5' + +} + +Options { + + ReadNetStats = No + Mode = 'train' + + RandomSeed = 123456 + WriteIterationTrajectory = No + +} \ No newline at end of file diff --git a/test/prog/fortnet/input/activation/lrelu/_fortnet.hdf5 b/test/prog/fortnet/input/activation/lrelu/_fortnet.hdf5 new file mode 100644 index 0000000000000000000000000000000000000000..0eb689bcc66d3ab4a9c00f3d9fe8a758dacb64ba GIT binary patch literal 57176 zcmeHQ4O~=J7Jmb%sHCWAsH=@>hHkbh zt1BMR_Mv+QiuoaQVA`AlMy%-Mz$&y z?4WRjQf~`Sh)4V_1neW>qn5TUEG;4K_ww?RPV~imyj6kiB?W6sfz%OYQTes`@j5}s zfCIOA3wGsgt!+?S=M^IUEJfawF5YrV;5~R=J4lHigZh9>dRmmz`=b6GlF+zF-VgnR zAn6KKLR$;QzJgJg92;XaPB+BBg?F+fV%H^nViS_0bP1vic1n1Yze-1ZbK!JB&PPz1 zAgECf)dXXbAt@m#b~?!ky6G{=0|h}!Eu0{w58LWN?MM3Zlpj|(ZDq_pni!XO+M`rCoq zw8w=$*84tH80S-*9`GBujL@Ets`0AwP$s=nlrNzIjVlbj-~`e$p`70v_ESF$>xvmS z(}c!_^6{~JT&RmS#=odD#wSU$$ATb^3tL+L37>@In1obGuKj+Tyggj;UFi-0lyC-# zZ#@M8`Md$mGib5D5)NrG@v(8n6d@{JCr)Ep?NLJtbxf7+$H{rZmHleIf`YmI*0BNZ z5OpQ{4M0Cam)ThMEB9Zn?6>#;w_j=_PfV5Uw-nadTl>6n70$=!V!^KLH#ZOz zIsZ~ic~&a7U%KyLlHJ#7sQV68{Qz%w-FF~6cGrD}h96IAn)P^6(;wQ8#y4>&XFdq~ z)sP6n0DHYal*4`p`;mr(LX;`L3Y2~EemAwR=>f{Pd8Ra;l;1~n_jpp?9p^V!vwUH;LO7ifEiegL@$0>#tvb+OL!6_@-|zPjj?sbXG!_@s)s&QDzT zrk|Rsx7^Z6(o3`EH@|L8T#qJxR6?6t`;~bx9FN26hkhj}_AAilTTexR`jr*|p!ilp z#EJXKo@qYiR`G&$#Xi!SOEIrf^aoUq@OvmhjrA~Lf#I?6h4j)arQ)Zg6YZ-^%i zL!g{2ERXXf;6Z#z{-5(?-TW01J0Z$?fW^GYDYkcCk3=4I*an ztWP}IP9xq|6EO@xL#F$I75E+-bkF$yCVoE*Vd9Al0VSRcA8)Tn{ncIZB-yJ$di#?g zRAB+Pizn@`)={GQRR6JXEC}(j@fW<8j0?zDJAQ$;nuY7w~u@< z;OdZdT@-mTJp!(P^CYz-Pn{v4#FJ{YBRZG*hr8lg;z{#lECI{oJPCNvxSz+9fJa^9 zNmo2sVOhH-emGA84t(nz0VSRci?%<{-8>2FU|U*$$Dk1BNvRz9yeWmZ6-d;iCm)B$ zVYRa`P9;5g+?tMh`N-3xC+F=<)H6rI7t-_9fgFf9(eoAB&+PRwP|n#1``x`IeCEe0+|9 zu9oc2U2!?hGpqjrCFEcMH_emyoQ91*?#y{!5f(8vSL*LMp8$UOM64iMpBIxqmtvc5 zXj=0$eEb0oSJ&}}D}SIi*xLv92iQS)QY!*V{-7A2SEG~M?GGq!mOmfI{Q)3O^e*Bq z{@eUs{o-b7N1H#mwdcpk9&Gu+^Iqu~N7R+qpng?1JU<5hDnXYRHAPq7=f`f$@7Ctq zzaG^0s~#&pv#cMkeZ7YXpW7tA%z2kAj;FODpv1cx+4H~D&)gM<6Yt8OH|D$xt#l*r ziZLqZ-MX!d0~8yNcl}?25;^Yz4t(nz0VUqepJ9JqL-B4pPJElNJkGm-JMpf_xtw>K ze!N>U3)+qIu03yXsdfaEcsD4+UfpAlbK^oh)ek#(5WT;9KViDDkcqe`kcwb~o?RJh?As zoR0iU<;eLs01s3xgc^}vz#@=CvA%(LAn$#9y;77fErtC7kO*P;o@B~T{@!F%YqAN= zlgn}7{tKGyM*rnG}Avc&K1o+p?470$bWIPtD$ z&DS(v-mO>(?Z$bR7=x$I5K!XXEVLs!x1o49d^wb0!16fn0v^P>BJSe9XRHfxJMT8Y z{YgKmJ!srX&nNpo5A7C&9fd!~19kO-z_8mXc}p+C;#0Z%&yTB;$Qjo@o^h8 z1&!PIeAz9%f1v@!ZQiwx+X6O0yMP^ zqtmBYj=iW$j@RjQXI?*^dg@wX6$>i=et+wbgY4DP3eDuGau$7k=h(}C{f50gtNP(T@Bb%j`*6m9 z4sS1G#z#%9)HADD+rTG3esfeAJ9B;GnWa%bGXL%4s?R=gh7Fki%EaV#>Xa@o@1$-5R-nppma=U-jA_$2#vLu6K3&+pm6M?ZdPbygWWd2Zyj{^ot` z2lmTJ?XW#8bA#c7h#ouHOAGrOp6Qaq{J(tR;&4+oi!a!-rA&D z2sz3On{q}^cp#6}yg&TutNQh9=#lxog)95;SL;2~w)j=|zRkPQ#yKK$e`#7Y6=m8(TT-g{upu6|$*|(w<7)Tg}FAiMo5u z#8TEn*jBbS?g+bT=((Z$p=x&Y)a=e%Htc1p%`;Pb4ys}Mx{Qm=Zf9l#RF`^BI$F(i z<@3n z9`-Z+#7aI|8GA0|2Nv=CFGt$6u3{lGH-GlK_b#xt5tF_4^xe)jojp3}q0cw7HNE;A zDWAHZ{hI&f*8683XDjAR-2L&~^Q`QJn$@#+ZD)H=91zB49%0wMO*`20xvgwZaHnPW zcd2AE?x}E^=ERVTIR)y7!%y5>mQ(Os{e7vbH+L}gJzIQMESkekJw500npOwc z*PVL}DOWEm*xRP=(F@ZtC{;J^SED+vCVIESl{(z5gYLL=nvQI+rfT5n0I*ooNrlGhpb(zuNShJ z3+4^&da#Ne9Fl3iciTC3d1lH}F>PvCyZySoLrH!-!-xJuT(WBY@5#T{a6*Ik=DD!htm%*lrbu!9XB zIwNEKtbeoO%Wu#7V)q&LX)FIlVV!?wUyd6Wy?Em`Hg;T1c-oqMtYyFU|LVB)99z2b zkv`?;rWN#9oVR*O_r(QEl66B@PAFyb%J+@>IpGLf*mlL=+1LMN^JiC{TYlZl3ZHp@ z-t?Cj7R;Vx45;qFqpsp`iZdy#f1H_)aVE6{AniDwYaZiW_s=V3AU;idUMc-M z=RJg4SAEF2o~Ipk)z4^LM``{0nRy>z%jcCu&T?EgQHA)Z?M%>-hZ=XhEB9ySBX-qu z5U8JJmO32r=pBvM&;0)bKsWDxMt+@mM=sZ&htxNT&+YAJVdc)d3-z