From 6954663fa94932a7d6472a1dd3cabd32650f1047 Mon Sep 17 00:00:00 2001 From: Patrick Lavin Date: Tue, 1 Aug 2017 20:05:23 -0400 Subject: [PATCH 1/7] Add auxiliary functions needed for conv2d --- include/af/autograd/Functions.hpp | 4 ++++ src/autograd/Functions.cpp | 36 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/include/af/autograd/Functions.hpp b/include/af/autograd/Functions.hpp index e4f471d..971903b 100644 --- a/include/af/autograd/Functions.hpp +++ b/include/af/autograd/Functions.hpp @@ -79,5 +79,9 @@ namespace af { Variable flat(const Variable &input); Variable moddims(const Variable &input, const dim4 &dims); + Variable reorder(const Variable &input, int d0, int d1, int d2, int d3); + + Variable unwrap(const Variable &input, int wx, int wy, int sx, int sy, int px, int py); + Variable wrap(const Variable &input, int ox, int oy, int wx, int wy, int sx, int sy, int px, int py); } } diff --git a/src/autograd/Functions.cpp b/src/autograd/Functions.cpp index 938dba6..8e83cd4 100644 --- a/src/autograd/Functions.cpp +++ b/src/autograd/Functions.cpp @@ -414,5 +414,41 @@ namespace af { }; return Variable(result, {input}, grad_func); } + + Variable reorder(const Variable &input, int d0, int d1, int d2, int d3) + { + array res = reorder(input.array(), d0, d1, d2, d3); + + int tmp[] = {d0, d1, d2, d3}; + int tmp2[4]; + for(int i = 0; i < 4; i++){ + tmp2[tmp[i]] = i; + } + auto reverse = Variable(array(4, tmp2), false); + + auto grad_func = [tmp2](std::vector &inputs, const Variable &grad_output){ + inputs[0].addGrad(reorder(grad_output, tmp2[0], tmp2[1], tmp2[2], tmp2[3])); + }; + return Variable(res, {input, reverse}, grad_func); + } + + Variable unwrap(const Variable &input, int wx, int wy, int sx, int sy, int px, int py) + { + array res = unwrap(input.array(), wx, wy, sx, sy, px, py); + auto grad_func = [wx, wy, sx, sy, px, py](std::vector &inputs, const Variable &grad_output) { + dim4 d = inputs[0].dims(); + inputs[0].addGrad(wrap(grad_output, d[0], d[1], wx, wy, sx, sy, px, py)); + }; + return Variable(res, {input}, grad_func); + } + + Variable wrap(const Variable &input, int ox, int oy, int wx, int wy, int sx, int sy, int px, int py) + { + array res = wrap(input.array(), ox, oy, wx, wy, sx, sy, px, py); + auto grad_func = [wx, wy, sx, sy, px, py](std::vector &inputs, const Variable &grad_output) { + inputs[0].addGrad(unwrap(grad_output, wx, wy, sx, sy, px, py)); + }; + return Variable(res, {input}, grad_func); + } } } From 4beea0fdb3181827b1e09fa48035f6a6ac61ef08 Mon Sep 17 00:00:00 2001 From: Patrick Lavin Date: Tue, 1 Aug 2017 20:48:45 -0400 Subject: [PATCH 2/7] add conv2d function --- include/af/autograd/Functions.hpp | 2 + src/autograd/Functions.cpp | 110 ++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/include/af/autograd/Functions.hpp b/include/af/autograd/Functions.hpp index 971903b..b2c93f3 100644 --- a/include/af/autograd/Functions.hpp +++ b/include/af/autograd/Functions.hpp @@ -83,5 +83,7 @@ namespace af { Variable unwrap(const Variable &input, int wx, int wy, int sx, int sy, int px, int py); Variable wrap(const Variable &input, int ox, int oy, int wx, int wy, int sx, int sy, int px, int py); + + Variable conv2d(const Variable &input, const Variable &weights, int wx, int wy, int sx, int sy, int px, int py); } } diff --git a/src/autograd/Functions.cpp b/src/autograd/Functions.cpp index 8e83cd4..f6d37ec 100644 --- a/src/autograd/Functions.cpp +++ b/src/autograd/Functions.cpp @@ -450,5 +450,115 @@ namespace af { }; return Variable(res, {input}, grad_func); } + + Variable conv2d(const Variable &input, const Variable &weights, int wx, int wy, int sx, int sy, int px, int py) + { + dim4 idims = input.array().dims(); // (x_i, y_i, c_i, n ) + dim4 wdims = weights.array().dims(); // (wx, wy, c_i, c_o) + + int x_i = idims[0]; //size of x dim of input + int y_i = idims[1]; //size of y dim of input + int c_i = idims[2]; //number of input channels + int n = idims[3]; //batch size (1 for now) + + int x_o = (x_i + 2 * px - wx) / sx + 1; //size of x dim of output + int y_o = (y_i + 2 * py - wy) / sy + 1; //size of x dim of output + int c_o = wdims[3]; //number of output channels + + array windows = unwrap(input.array(), wx, wy, sx, sy, px, py); + + array lhs = moddims( + reorder(windows, 1, 0, 2, 3), + dim4(x_o * y_o, wx * wy * c_i, n, 1)); + array rhs = moddims(weights.array(), dim4(wx * wy * c_i, c_o, 1, 1)); + + //TODO: This loop can be replaced with a batched matmult as soon as + //that is added to arrayfire + std::vector out; + for(int i = 0; i < n; i++){ + array res = matmul(lhs(span, span, i), rhs); + out.push_back(moddims(res , dim4(x_o, y_o, c_o, 1))); + } + + //LOL @ C++ API - need this loop to have arbitrary batch size + array result = out[0]; + for(int i = 1; i < n; i+=3){ + int rem = n - i; + if(rem >= 3){ + result = join(3, result, out[i], out[i+1], out[i+2]); + }else if(rem == 2){ + result = join(3, result, out[i], out[i+1]); + break; + }else if(rem == 1){ + result = join(3, result, out[i]); + break; + }else{ + break; + } + } + + auto grad_func = [wx, wy, sx, sy, px, py, c_i, n](std::vector &inputs, const Variable &grad_output) { + dim4 odims = grad_output.array().dims(); + dim4 wdims = inputs[1].array().dims(); + dim4 idims = inputs[0].array().dims(); + + auto grad_out_reshape = moddims(grad_output, dim4(odims[0]*odims[1], odims[2], odims[3], 1)); + + auto weights_reshape = moddims(inputs[1], dim4(wdims[0]*wdims[1]*wdims[2], wdims[3], 1, 1)); + + //TODO: This really needs batched matmul... + //TODO: This doesn't work for n > 1 + //TODO: Can these lines be shortened? - This seems like a large grad function - perhaps this + // could all be implemented in Conv2D::forward(). I had to implement the helper functions anyways + /* + std::vector out; + for(int i = 0; i < n; i++){ + auto a = matmulNT(grad_out_reshape(span, span, i), weights_reshape); //Problem is here - can't call () on Variable + auto adims = a.array().dims(); + auto b = moddims(a, dim4(adims[0], wx*wy, c_i, adims[3])); + auto c = reorder(b, 1, 0, 2, 3); + out.push_pack(wrap(c, idims[0], idims[1], wx, wy, sx, sy, px, py)); + } + + array result = out[0]; + for(int i = 1; i < n; i+=3){ + int rem = n - i; + if(rem >= 3){ + result = join(3, result, out[i], out[i+1], out[i+2]); + }else if(rem == 2){ + result = join(3, result, out[i], out[i+1]); + break; + }else if(rem == 1){ + result = join(3, result, out[i]); + break; + }else{ + break; + } + } + */ + auto a = matmulNT(grad_out_reshape, weights_reshape); + auto adims = a.array().dims(); + auto b = moddims(a, dim4(adims[0], wx*wy, c_i, adims[3])); + auto c = reorder(b, 1, 0, 2, 3); + inputs[0].addGrad(wrap(c, idims[0], idims[1], wx, wy, sx, sy, px, py)); + + auto d = matmulTN(inputs[2],grad_out_reshape); + inputs[1].addGrad(moddims(d, dim4(wx, wy, c_i, d.dims()[1]))); + + /* + for(int i = 0; i < odims[3]; i++){ + inputs[0].addGrad(wrap(), p[0], p[1], p[2], p[3], p[4], p[5]); + inputs[0].addGrad(wrap(matmulNT(Variable(lhs(span, span, span, i), false), p_tmp), p[0], p[1], p[2], p[3], p[4], p[5])); + inputs[1].addGrad(matmulTN(inputs[0], Variable(lhs(span, span, span, i), true))); + } + */ + + }; + return Variable(result, {input, weights, Variable(lhs, false)}, grad_func); + + } + + + } } From 42da3a80ff945688ce066aa99354853f367d977a Mon Sep 17 00:00:00 2001 From: Patrick Lavin Date: Tue, 1 Aug 2017 21:19:59 -0400 Subject: [PATCH 3/7] added Conv2D layer --- CMakeLists.txt | 1 + include/af/nn/Modules.hpp | 1 + include/af/nn/Modules/Conv2D.hpp | 37 +++++++++++++++ src/nn/Modules/Conv2D.cpp | 78 ++++++++++++++++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 include/af/nn/Modules/Conv2D.hpp create mode 100644 src/nn/Modules/Conv2D.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f31cd8a..c2c0b4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,7 @@ target_sources(afml src/nn/Modules/Container.cpp src/nn/Modules/Linear.cpp src/nn/Modules/Loss.cpp + src/nn/Modules/Conv2D.cpp src/nn/Modules/Module.cpp src/nn/Init.cpp ) diff --git a/include/af/nn/Modules.hpp b/include/af/nn/Modules.hpp index eeb22a7..4fa3506 100644 --- a/include/af/nn/Modules.hpp +++ b/include/af/nn/Modules.hpp @@ -13,3 +13,4 @@ #include #include #include +#include diff --git a/include/af/nn/Modules/Conv2D.hpp b/include/af/nn/Modules/Conv2D.hpp new file mode 100644 index 0000000..57c9255 --- /dev/null +++ b/include/af/nn/Modules/Conv2D.hpp @@ -0,0 +1,37 @@ +/******************************************************* + * Copyright (c) 2017, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ +#pragma once + +#include + +namespace af +{ + namespace nn + { + class Conv2D : public Module + { + private: + bool m_bias; + int m_wx; + int m_wy; + int m_sx; + int m_sy; + int m_px; + int m_py; + public: + Conv2D(int wx, int wy, int sx, int sy, int px, int py, int n_in, int n_out, bool bias = true); + + Conv2D(const autograd::Variable &w, int sx = 1, int sy = 1, int px = 0, int py = 0); + + Conv2D(const autograd::Variable &w, const autograd::Variable &b, int sx = 1, int sy = 1, int px = 0, int py = 0); + + autograd::Variable forward(const autograd::Variable &input); + }; + } +} diff --git a/src/nn/Modules/Conv2D.cpp b/src/nn/Modules/Conv2D.cpp new file mode 100644 index 0000000..3de0507 --- /dev/null +++ b/src/nn/Modules/Conv2D.cpp @@ -0,0 +1,78 @@ +/******************************************************* + * Copyright (c) 2017, ArrayFire + * All rights reserved. + * + * This file is distributed under 3-clause BSD license. + * The complete license agreement can be obtained at: + * http://arrayfire.com/licenses/BSD-3-Clause + ********************************************************/ +#include +#include +#include +//output will be ho x wo x no x n +namespace af +{ + namespace nn + { + using namespace autograd; + + Conv2D::Conv2D(int wx, int wy, int sx, int sy, int px, int py, int n_in, int n_out, bool bias) : + m_wx(wx), + m_wy(wy), + m_sx(sx), + m_sy(sy), + m_px(px), + m_py(py), + m_bias(bias) + { + auto w = nn::lecunNormal(dim4(wx, wy, n_in, n_out)); + if (bias) { + auto b = nn::lecunNormal(dim4(1, 1, n_out, 1)); + setParams({w, b}); + } else { + setParams({w}); + } + } + + Conv2D::Conv2D(const Variable &w, int sx, int sy, int px, int py) : + m_sx(sx), + m_sy(sy), + m_px(px), + m_py(py), + m_bias(false), + Module({w}) + { + dim4 pdims = w.array().dims(); + m_wx = pdims[0]; + m_wy = pdims[1]; + } + + Conv2D::Conv2D(const Variable &w, const Variable &b, int sx, int sy, int px, int py) : + m_sx(sx), + m_sy(sy), + m_px(px), + m_py(py), + m_bias(true), + Module({w, b}) + { + /*if (b.array().dims(0) != w.array().dims(0)) { + throw af::exception("nn:Linear: Dimension mismatch between weight and bias."); + }*/ + if (b.array().dims(1) != 1) { + throw af::exception("nn::Linear: Bias must be a vector."); + } + dim4 pdims = w.array().dims(); + m_wx = pdims[0]; + m_wy = pdims[1]; + } + + Variable Conv2D::forward(const Variable &input) + { + auto res = conv2d(input, m_parameters[0], m_wx, m_wy, m_sx, m_sy, m_px, m_py); + if (m_bias) { + res = res + tileAs(m_parameters[1], res); + } + return res; + } + } +} From 34ac18eae3e7fe1be9528345895cdbbe9fad2575 Mon Sep 17 00:00:00 2001 From: Patrick Lavin Date: Tue, 1 Aug 2017 21:26:32 -0400 Subject: [PATCH 4/7] cleanup --- src/autograd/Functions.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/autograd/Functions.cpp b/src/autograd/Functions.cpp index f6d37ec..e35d975 100644 --- a/src/autograd/Functions.cpp +++ b/src/autograd/Functions.cpp @@ -545,14 +545,6 @@ namespace af { auto d = matmulTN(inputs[2],grad_out_reshape); inputs[1].addGrad(moddims(d, dim4(wx, wy, c_i, d.dims()[1]))); - /* - for(int i = 0; i < odims[3]; i++){ - inputs[0].addGrad(wrap(), p[0], p[1], p[2], p[3], p[4], p[5]); - inputs[0].addGrad(wrap(matmulNT(Variable(lhs(span, span, span, i), false), p_tmp), p[0], p[1], p[2], p[3], p[4], p[5])); - inputs[1].addGrad(matmulTN(inputs[0], Variable(lhs(span, span, span, i), true))); - } - */ - }; return Variable(result, {input, weights, Variable(lhs, false)}, grad_func); From 93b3554072a9502e457d294bee15119229796127 Mon Sep 17 00:00:00 2001 From: Patrick Lavin Date: Wed, 2 Aug 2017 09:46:04 -0400 Subject: [PATCH 5/7] cleanup --- src/autograd/Functions.cpp | 3 +-- src/nn/Modules/Conv2D.cpp | 9 +++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/autograd/Functions.cpp b/src/autograd/Functions.cpp index e35d975..446c305 100644 --- a/src/autograd/Functions.cpp +++ b/src/autograd/Functions.cpp @@ -424,12 +424,11 @@ namespace af { for(int i = 0; i < 4; i++){ tmp2[tmp[i]] = i; } - auto reverse = Variable(array(4, tmp2), false); auto grad_func = [tmp2](std::vector &inputs, const Variable &grad_output){ inputs[0].addGrad(reorder(grad_output, tmp2[0], tmp2[1], tmp2[2], tmp2[3])); }; - return Variable(res, {input, reverse}, grad_func); + return Variable(res, {input}, grad_func); } Variable unwrap(const Variable &input, int wx, int wy, int sx, int sy, int px, int py) diff --git a/src/nn/Modules/Conv2D.cpp b/src/nn/Modules/Conv2D.cpp index 3de0507..72d4a2e 100644 --- a/src/nn/Modules/Conv2D.cpp +++ b/src/nn/Modules/Conv2D.cpp @@ -42,7 +42,7 @@ namespace af m_bias(false), Module({w}) { - dim4 pdims = w.array().dims(); + dim4 pdims = w.dims(); m_wx = pdims[0]; m_wy = pdims[1]; } @@ -55,13 +55,10 @@ namespace af m_bias(true), Module({w, b}) { - /*if (b.array().dims(0) != w.array().dims(0)) { - throw af::exception("nn:Linear: Dimension mismatch between weight and bias."); - }*/ - if (b.array().dims(1) != 1) { + if (b.dims()[1] != 1) { throw af::exception("nn::Linear: Bias must be a vector."); } - dim4 pdims = w.array().dims(); + dim4 pdims = w.dims(); m_wx = pdims[0]; m_wy = pdims[1]; } From a037c0a85ff8a353d11a8418c9a4425b6399220d Mon Sep 17 00:00:00 2001 From: Patrick Lavin Date: Wed, 2 Aug 2017 09:50:37 -0400 Subject: [PATCH 6/7] changed name of Conv2D to Colvolve2 --- CMakeLists.txt | 2 +- include/af/nn/Modules.hpp | 2 +- include/af/nn/Modules/{Conv2D.hpp => Convolve2.hpp} | 8 ++++---- src/nn/Modules/{Conv2D.cpp => Convolve2.cpp} | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) rename include/af/nn/Modules/{Conv2D.hpp => Convolve2.hpp} (65%) rename src/nn/Modules/{Conv2D.cpp => Convolve2.cpp} (82%) diff --git a/CMakeLists.txt b/CMakeLists.txt index c2c0b4e..dcbc69e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ target_sources(afml src/nn/Modules/Container.cpp src/nn/Modules/Linear.cpp src/nn/Modules/Loss.cpp - src/nn/Modules/Conv2D.cpp + src/nn/Modules/Convolve2.cpp src/nn/Modules/Module.cpp src/nn/Init.cpp ) diff --git a/include/af/nn/Modules.hpp b/include/af/nn/Modules.hpp index 4fa3506..bd70b84 100644 --- a/include/af/nn/Modules.hpp +++ b/include/af/nn/Modules.hpp @@ -13,4 +13,4 @@ #include #include #include -#include +#include diff --git a/include/af/nn/Modules/Conv2D.hpp b/include/af/nn/Modules/Convolve2.hpp similarity index 65% rename from include/af/nn/Modules/Conv2D.hpp rename to include/af/nn/Modules/Convolve2.hpp index 57c9255..2ac336d 100644 --- a/include/af/nn/Modules/Conv2D.hpp +++ b/include/af/nn/Modules/Convolve2.hpp @@ -14,7 +14,7 @@ namespace af { namespace nn { - class Conv2D : public Module + class Convolve2 : public Module { private: bool m_bias; @@ -25,11 +25,11 @@ namespace af int m_px; int m_py; public: - Conv2D(int wx, int wy, int sx, int sy, int px, int py, int n_in, int n_out, bool bias = true); + Convolve2(int wx, int wy, int sx, int sy, int px, int py, int n_in, int n_out, bool bias = true); - Conv2D(const autograd::Variable &w, int sx = 1, int sy = 1, int px = 0, int py = 0); + Convolve2(const autograd::Variable &w, int sx = 1, int sy = 1, int px = 0, int py = 0); - Conv2D(const autograd::Variable &w, const autograd::Variable &b, int sx = 1, int sy = 1, int px = 0, int py = 0); + Convolve2(const autograd::Variable &w, const autograd::Variable &b, int sx = 1, int sy = 1, int px = 0, int py = 0); autograd::Variable forward(const autograd::Variable &input); }; diff --git a/src/nn/Modules/Conv2D.cpp b/src/nn/Modules/Convolve2.cpp similarity index 82% rename from src/nn/Modules/Conv2D.cpp rename to src/nn/Modules/Convolve2.cpp index 72d4a2e..2f964ec 100644 --- a/src/nn/Modules/Conv2D.cpp +++ b/src/nn/Modules/Convolve2.cpp @@ -8,7 +8,7 @@ ********************************************************/ #include #include -#include +#include //output will be ho x wo x no x n namespace af { @@ -16,7 +16,7 @@ namespace af { using namespace autograd; - Conv2D::Conv2D(int wx, int wy, int sx, int sy, int px, int py, int n_in, int n_out, bool bias) : + Convolve2::Convolve2(int wx, int wy, int sx, int sy, int px, int py, int n_in, int n_out, bool bias) : m_wx(wx), m_wy(wy), m_sx(sx), @@ -34,7 +34,7 @@ namespace af } } - Conv2D::Conv2D(const Variable &w, int sx, int sy, int px, int py) : + Convolve2::Convolve2(const Variable &w, int sx, int sy, int px, int py) : m_sx(sx), m_sy(sy), m_px(px), @@ -47,7 +47,7 @@ namespace af m_wy = pdims[1]; } - Conv2D::Conv2D(const Variable &w, const Variable &b, int sx, int sy, int px, int py) : + Convolve2::Convolve2(const Variable &w, const Variable &b, int sx, int sy, int px, int py) : m_sx(sx), m_sy(sy), m_px(px), @@ -63,7 +63,7 @@ namespace af m_wy = pdims[1]; } - Variable Conv2D::forward(const Variable &input) + Variable Convolve2::forward(const Variable &input) { auto res = conv2d(input, m_parameters[0], m_wx, m_wy, m_sx, m_sy, m_px, m_py); if (m_bias) { From 49e077e9337d1158f5e1c92745a49d64615b22ac Mon Sep 17 00:00:00 2001 From: Patrick Lavin Date: Wed, 2 Aug 2017 14:22:31 -0400 Subject: [PATCH 7/7] rename conv2d to convolve2 --- include/af/autograd/Functions.hpp | 2 +- src/autograd/Functions.cpp | 2 +- src/nn/Modules/Convolve2.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/af/autograd/Functions.hpp b/include/af/autograd/Functions.hpp index b2c93f3..1e78a44 100644 --- a/include/af/autograd/Functions.hpp +++ b/include/af/autograd/Functions.hpp @@ -84,6 +84,6 @@ namespace af { Variable unwrap(const Variable &input, int wx, int wy, int sx, int sy, int px, int py); Variable wrap(const Variable &input, int ox, int oy, int wx, int wy, int sx, int sy, int px, int py); - Variable conv2d(const Variable &input, const Variable &weights, int wx, int wy, int sx, int sy, int px, int py); + Variable convolve2(const Variable &input, const Variable &weights, int wx, int wy, int sx, int sy, int px, int py); } } diff --git a/src/autograd/Functions.cpp b/src/autograd/Functions.cpp index 446c305..6bced08 100644 --- a/src/autograd/Functions.cpp +++ b/src/autograd/Functions.cpp @@ -450,7 +450,7 @@ namespace af { return Variable(res, {input}, grad_func); } - Variable conv2d(const Variable &input, const Variable &weights, int wx, int wy, int sx, int sy, int px, int py) + Variable convolve2(const Variable &input, const Variable &weights, int wx, int wy, int sx, int sy, int px, int py) { dim4 idims = input.array().dims(); // (x_i, y_i, c_i, n ) dim4 wdims = weights.array().dims(); // (wx, wy, c_i, c_o) diff --git a/src/nn/Modules/Convolve2.cpp b/src/nn/Modules/Convolve2.cpp index 2f964ec..b360468 100644 --- a/src/nn/Modules/Convolve2.cpp +++ b/src/nn/Modules/Convolve2.cpp @@ -65,7 +65,7 @@ namespace af Variable Convolve2::forward(const Variable &input) { - auto res = conv2d(input, m_parameters[0], m_wx, m_wy, m_sx, m_sy, m_px, m_py); + auto res = convolve2(input, m_parameters[0], m_wx, m_wy, m_sx, m_sy, m_px, m_py); if (m_bias) { res = res + tileAs(m_parameters[1], res); }