Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

komi + tensorbatch + squeeze-excitation + ladder/legality/liberty #91

Open
wants to merge 9 commits into
base: komi+tensorbatch-se
Choose a base branch
from
Open

komi + tensorbatch + squeeze-excitation + ladder/legality/liberty #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bc2e02f
SE
Ttl Apr 22, 2018
9cf5e19
Xgemv half storage support
Ttl Aug 5, 2018
5d80b05
Simplify kernel workgroup sizes
Ttl Aug 5, 2018
a6a4218
Move ladder and liberty display to commands
Ttl Aug 5, 2018
193ebf4
Fix merge
Ttl Sep 10, 2018
c9919e5
Speed up current "add_features" implementation
AncalagonX Sep 10, 2018
63e0e00
Add Ladder.h and Ladder.cpp to VS2017 project.
alreadydone Jan 12, 2019
1207030
Merge pull request #3 from alreadydone/ladder-liberty-se
Ttl Jan 12, 2019
ec81fdf
copy branch
alreadydone Jan 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion FAQ.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Leela Zero常见问题解答 #
# Leela Zero常见问题解答 #
# Frequently Asked Questions about Leela Zero #

## 为什么网络不是每次都变强的 ##
Expand Down
6 changes: 4 additions & 2 deletions msvc/VS2017/leela-zero.vcxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
<ClInclude Include="..\..\src\GTP.h" />
<ClInclude Include="..\..\src\Im2Col.h" />
<ClInclude Include="..\..\src\KoState.h" />
<ClInclude Include="..\..\src\Ladder.h" />
<ClInclude Include="..\..\src\Network.h" />
<ClInclude Include="..\..\src\NNCache.h" />
<ClInclude Include="..\..\src\ForwardPipe.h" />
Expand Down Expand Up @@ -57,6 +58,7 @@
<ClCompile Include="..\..\src\GameState.cpp" />
<ClCompile Include="..\..\src\GTP.cpp" />
<ClCompile Include="..\..\src\KoState.cpp" />
<ClCompile Include="..\..\src\Ladder.cpp" />
<ClCompile Include="..\..\src\Leela.cpp" />
<ClCompile Include="..\..\src\Network.cpp" />
<ClCompile Include="..\..\src\NNCache.cpp" />
Expand All @@ -83,7 +85,7 @@
<ProjectGuid>{7B887BFE-8D2C-46CD-B139-5213434BF218}</ProjectGuid>
<Keyword>Win32Proj</Keyword>
<RootNamespace>leelazero</RootNamespace>
<WindowsTargetPlatformVersion>10.0.16299.0</WindowsTargetPlatformVersion>
<WindowsTargetPlatformVersion>10.0.17134.0</WindowsTargetPlatformVersion>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
Expand Down Expand Up @@ -224,4 +226,4 @@
<Error Condition="!Exists('..\packages\opencl-nug.0.777.12\build\native\opencl-nug.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\opencl-nug.0.777.12\build\native\opencl-nug.targets'))" />
<Error Condition="!Exists('..\packages\zlib-msvc14-x64.1.2.11.7795\build\native\zlib-msvc14-x64.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\zlib-msvc14-x64.1.2.11.7795\build\native\zlib-msvc14-x64.targets'))" />
</Target>
</Project>
</Project>
8 changes: 7 additions & 1 deletion msvc/VS2017/leela-zero.vcxproj.filters
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,9 @@
<ClInclude Include="..\..\src\UCTNodePointer.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\..\src\Ladder.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\..\src\FastBoard.cpp">
Expand Down Expand Up @@ -182,8 +185,11 @@
<ClCompile Include="..\..\src\UCTNodePointer.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\src\Ladder.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
</Project>
</Project>
131 changes: 120 additions & 11 deletions src/CPUPipe.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,29 +266,104 @@ void batchnorm(const size_t channels,
std::vector<float>& data,
const float* const means,
const float* const stddivs,
const float* const eltwise = nullptr) {
const auto lambda_ReLU = [](const auto val) { return (val > 0.0f) ?
val : 0.0f; };
const float* const prelu_alphas,
const bool relu = true,
const float* const eltwise = nullptr)
{
const auto lambda_PReLU = [](const auto val, const auto alpha) { return (val > 0.0f) ?
val : alpha * val; };
for (auto c = size_t{0}; c < channels; ++c) {
const auto mean = means[c];
const auto scale_stddiv = stddivs[c];
const auto arr = &data[c * spatial_size];
const auto prelu_alpha = prelu_alphas[c];

if (eltwise == nullptr) {
// Classical BN
for (auto b = size_t{0}; b < spatial_size; b++) {
arr[b] = lambda_ReLU(scale_stddiv * (arr[b] - mean));
auto val = scale_stddiv * (arr[b] - mean);
if (relu) {
val = lambda_PReLU(val, prelu_alpha);
}
arr[b] = val;
}
} else {
// BN + residual add
const auto res = &eltwise[c * spatial_size];
for (auto b = size_t{0}; b < spatial_size; b++) {
arr[b] = lambda_ReLU((scale_stddiv * (arr[b] - mean)) + res[b]);
auto val = scale_stddiv * (arr[b] - mean) + res[b];
if (relu) {
val = lambda_PReLU(val, prelu_alpha);
}
arr[b] = val;
}
}
}
}

std::vector<float> innerproduct(const size_t inputs,
const size_t outputs,
const bool ReLU,
const std::vector<float>& input,
const std::vector<float>& weights,
const std::vector<float>& biases) {
std::vector<float> output(outputs);

cblas_sgemv(CblasRowMajor, CblasNoTrans,
// M K
outputs, inputs,
1.0f, &weights[0], inputs,
&input[0], 1,
0.0f, &output[0], 1);

const auto lambda_ReLU = [](const auto val) { return (val > 0.0f) ?
val : 0.0f; };
for (unsigned int o = 0; o < outputs; o++) {
auto val = biases[o] + output[o];
if (ReLU) {
val = lambda_ReLU(val);
}
output[o] = val;
}

return output;
}

void global_avg_pooling(const size_t channels,
const std::vector<float>& input,
std::vector<float>& output) {

for ( auto c = size_t{0}; c < channels; c++) {
auto acc = 0.0f;
for ( auto i = size_t{0}; i < BOARD_SQUARES; i++) {
acc += input[c * BOARD_SQUARES + i];
}
output[c] = acc/BOARD_SQUARES;
}
}

void apply_se(const size_t channels,
const std::vector<float>& input,
const std::vector<float>& res,
const std::vector<float>& scale,
std::vector<float>& output,
const std::vector<float>& prelu_alphas) {

const auto lambda_ReLU = [](const auto val, const auto alpha) { return (val > 0.0f) ?
val : alpha * val; };

const auto lambda_sigmoid = [](const auto val) { return 1.0f/(1.0f + exp(-val)); };

for ( auto c = size_t{0}; c < channels; c++) {
auto sig_scale = lambda_sigmoid(scale[c]);
auto alpha = prelu_alphas[c];
for ( auto i = size_t{0}; i < BOARD_SQUARES; i++) {
output[c * BOARD_SQUARES + i] = lambda_ReLU(sig_scale * input[c * BOARD_SQUARES + i]
+ res[c * BOARD_SQUARES + i], alpha);
}
}
}

void CPUPipe::forward(const std::vector<float>& input,
std::vector<float>& output_pol,
std::vector<float>& output_val) {
Expand All @@ -309,19 +384,24 @@ void CPUPipe::forward(const std::vector<float>& input,
winograd_convolve3(output_channels, input, m_conv_weights[0], V, M, conv_out);
batchnorm<BOARD_SQUARES>(output_channels, conv_out,
m_batchnorm_means[0].data(),
m_batchnorm_stddivs[0].data());
m_batchnorm_stddivs[0].data(),
m_prelu_alphas[0].data());


// Residual tower
auto pooling = std::vector<float>(output_channels);
auto conv_in = std::vector<float>(output_channels * BOARD_SQUARES);
auto res = std::vector<float>(output_channels * BOARD_SQUARES);
auto block = 0;
for (auto i = size_t{1}; i < m_conv_weights.size(); i += 2) {
auto output_channels = m_input_channels;
auto output_channels = m_batchnorm_stddivs[i].size();
std::swap(conv_out, conv_in);
winograd_convolve3(output_channels, conv_in,
m_conv_weights[i], V, M, conv_out);
batchnorm<BOARD_SQUARES>(output_channels, conv_out,
m_batchnorm_means[i].data(),
m_batchnorm_stddivs[i].data());
m_batchnorm_stddivs[i].data(),
m_prelu_alphas[i].data());

std::swap(conv_in, res);
std::swap(conv_out, conv_in);
Expand All @@ -330,7 +410,20 @@ void CPUPipe::forward(const std::vector<float>& input,
batchnorm<BOARD_SQUARES>(output_channels, conv_out,
m_batchnorm_means[i + 1].data(),
m_batchnorm_stddivs[i + 1].data(),
res.data());
m_prelu_alphas[i + 1].data(),
false);
std::swap(conv_out, conv_in);

global_avg_pooling(output_channels, conv_in, pooling);

auto fc_outputs = m_se_fc1_w[block].size() / output_channels;
auto se1 = innerproduct(output_channels, fc_outputs, true, pooling, m_se_fc1_w[block], m_se_fc1_b[block]);
auto se2 = innerproduct(fc_outputs, output_channels, false, se1, m_se_fc2_w[block], m_se_fc2_b[block]);

apply_se(output_channels, conv_in, res, se2, conv_out, m_prelu_alphas[i + 1]);

block++;

}
convolve<1>(Network::OUTPUTS_POLICY, conv_out, m_conv_pol_w, m_conv_pol_b, output_pol);
convolve<1>(Network::OUTPUTS_VALUE, conv_out, m_conv_val_w, m_conv_val_b, output_val);
Expand All @@ -342,28 +435,44 @@ void CPUPipe::push_input_convolution(unsigned int /*filter_size*/,
unsigned int /*outputs*/,
const std::vector<float>& weights,
const std::vector<float>& means,
const std::vector<float>& variances) {
const std::vector<float>& variances,
const std::vector<float>& prelu_alphas) {
m_conv_weights.push_back(weights);
m_batchnorm_means.push_back(means);
m_batchnorm_stddivs.push_back(variances);
m_prelu_alphas.push_back(prelu_alphas);
}

void CPUPipe::push_residual(unsigned int /*filter_size*/,
unsigned int /*channels*/,
unsigned int /*outputs*/,
unsigned int /*se_fc_outputs*/,
const std::vector<float>& weights_1,
const std::vector<float>& means_1,
const std::vector<float>& variances_1,
const std::vector<float>& prelu_alphas_1,
const std::vector<float>& weights_2,
const std::vector<float>& means_2,
const std::vector<float>& variances_2) {
const std::vector<float>& variances_2,
const std::vector<float>& prelu_alphas_2,
const std::vector<float>& se_fc1_w,
const std::vector<float>& se_fc1_b,
const std::vector<float>& se_fc2_w,
const std::vector<float>& se_fc2_b) {
m_conv_weights.push_back(weights_1);
m_batchnorm_means.push_back(means_1);
m_batchnorm_stddivs.push_back(variances_1);
m_prelu_alphas.push_back(prelu_alphas_1);

m_conv_weights.push_back(weights_2);
m_batchnorm_means.push_back(means_2);
m_batchnorm_stddivs.push_back(variances_2);
m_prelu_alphas.push_back(prelu_alphas_2);

m_se_fc1_w.push_back(se_fc1_w);
m_se_fc1_b.push_back(se_fc1_b);
m_se_fc2_w.push_back(se_fc2_w);
m_se_fc2_b.push_back(se_fc2_b);
}

void CPUPipe::push_convolve(unsigned int filter_size,
Expand Down
18 changes: 16 additions & 2 deletions src/CPUPipe.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,25 @@ class CPUPipe : public ForwardPipe {
unsigned int outputs,
const std::vector<float>& weights,
const std::vector<float>& means,
const std::vector<float>& variances);
const std::vector<float>& variances,
const std::vector<float>& prelu_alphas);

virtual void push_residual(unsigned int filter_size,
unsigned int channels,
unsigned int outputs,
unsigned int se_fc_outputs,
const std::vector<float>& weights_1,
const std::vector<float>& means_1,
const std::vector<float>& variances_1,
const std::vector<float>& prelu_alphas_1,
const std::vector<float>& weights_2,
const std::vector<float>& means_2,
const std::vector<float>& variances_2);
const std::vector<float>& variances_2,
const std::vector<float>& prelu_alphas_2,
const std::vector<float>& se_fc1_w,
const std::vector<float>& se_fc1_b,
const std::vector<float>& se_fc2_w,
const std::vector<float>& se_fc2_b);

virtual void push_convolve(unsigned int filter_size,
unsigned int channels,
Expand Down Expand Up @@ -83,6 +91,12 @@ class CPUPipe : public ForwardPipe {
std::vector<std::vector<float>> m_conv_weights;
std::vector<std::vector<float>> m_batchnorm_means;
std::vector<std::vector<float>> m_batchnorm_stddivs;
std::vector<std::vector<float>> m_prelu_alphas;

std::vector<std::vector<float>> m_se_fc1_w;
std::vector<std::vector<float>> m_se_fc1_b;
std::vector<std::vector<float>> m_se_fc2_w;
std::vector<std::vector<float>> m_se_fc2_b;

std::vector<float> m_conv_pol_w;
std::vector<float> m_conv_val_w;
Expand Down
Loading