From 246dedebd3c34260d2469f4a880310ba45c6b2af Mon Sep 17 00:00:00 2001 From: 1aerostorm Date: Mon, 28 Feb 2022 00:27:58 +0300 Subject: [PATCH 01/10] Improve Elastic Search --- .../elastic_search/elastic_search_state.hpp | 121 +++++++++--------- 1 file changed, 62 insertions(+), 59 deletions(-) diff --git a/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp b/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp index 53ffb01b68..9f0b7e8b2e 100644 --- a/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp +++ b/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp @@ -19,7 +19,7 @@ namespace golos { namespace plugins { namespace elastic_search { using boost::locale::conv::utf_to_utf; #ifdef STEEMIT_BUILD_TESTNET - #define ELASTIC_WRITE_EACH_N_OP 1 + #define ELASTIC_WRITE_EACH_N_OP 3 #else #define ELASTIC_WRITE_EACH_N_OP 100 #endif @@ -72,7 +72,7 @@ class elastic_search_state_writer { return headers; } - bool find_in_es(const std::string& id, optional& res) { + bool find_in_es(const std::string& id, variant_object& res) { auto headers = get_es_headers(); auto reply0 = conn.request("GET", url + "/blog/post/" + id + "?pretty", "", headers); auto reply_body = std::string(reply0.body.data(), reply0.body.size()); @@ -83,7 +83,22 @@ class elastic_search_state_writer { return true; } } - res = optional(); + res = variant_object(); + return false; + } + + bool find_post(const std::string& id, fc::mutable_variant_object& found, bool& from_buffer) { + auto found_buf = buffer.find(id); + variant_object found_es; + from_buffer = false; + if (found_buf != buffer.end()) { + from_buffer = true; + found = found_buf->second; + return true; + } else if (find_in_es(id, found_es)) { + found = fc::mutable_variant_object(found_es); + return true; + } return false; } @@ -94,34 +109,28 @@ class elastic_search_state_writer { return; } + fc::mutable_variant_object doc; + bool from_buffer = false; + bool exists = find_post(id, doc, from_buffer); + std::string base_body = doc["body"].as_string(); + std::string body = op.body; try { diff_match_patch dmp; auto patch = dmp.patch_fromText(utf8_to_wstring(body)); - if (patch.size()) { - std::string base_body; - auto found = buffer.find(id); - optional found2; - if (found != buffer.end()) { - base_body = found->second["body"].as_string(); - } else if (find_in_es(id, found2)) { - base_body = (*found2)["body"].as_string(); - } - if (base_body.size()) { - auto result = dmp.patch_apply(patch, utf8_to_wstring(base_body)); - auto patched_body = wstring_to_utf8(result.first); - if(!fc::is_utf8(patched_body)) { - body = fc::prune_invalid_utf8(patched_body); - } else { - body = patched_body; - } + if (patch.size() && base_body.size()) { + auto result = dmp.patch_apply(patch, utf8_to_wstring(base_body)); + auto patched_body = wstring_to_utf8(result.first); + if(!fc::is_utf8(patched_body)) { + body = fc::prune_invalid_utf8(patched_body); + } else { + body = patched_body; } } } catch ( ... ) { } const auto& cmt = _db.get_comment(op.author, op.permlink); - fc::mutable_variant_object doc; doc["id"] = cmt.id; doc["created"] = _db.head_block_time(); @@ -150,14 +159,19 @@ class elastic_search_state_writer { doc["tags"] = golos::plugins::tags::get_metadata(op.json_metadata, TAGS_NUMBER, TAG_MAX_LENGTH).tags; doc["json_metadata"] = op.json_metadata; - doc["total_votes"] = uint32_t(0); - doc["net_rshares"] = share_type(0); - doc["donates"] = asset(0, STEEM_SYMBOL); - doc["donates_uia"] = share_type(); + if (!exists) { + doc["net_votes"] = int32_t(0); + doc["total_votes"] = uint32_t(0); + doc["net_rshares"] = share_type(0); + doc["donates"] = asset(0, STEEM_SYMBOL); + doc["donates_uia"] = share_type(); + } + + doc["author_reputation"] = std::string(_db.get_account_reputation(op.author)); buffer[id] = std::move(doc); - ++op_num; + ++op_num; if (op_num % ELASTIC_WRITE_EACH_N_OP != 0) { return; } @@ -166,34 +180,30 @@ class elastic_search_state_writer { } result_type operator()(const vote_operation& op) { +#ifndef STEEMIT_BUILD_TESTNET if (_db.head_block_num() < 35000000) { // Speed up replay return; } +#endif if (_db.is_account_vote(op)) { return; } auto id = std::string(op.author) + "." + op.permlink; - auto found = buffer.find(id); - optional found_es; - fc::mutable_variant_object doc_es; - fc::mutable_variant_object* doc = &doc_es; - if (found != buffer.end()) { - doc = &found->second; - } else if (find_in_es(id, found_es)) { - doc_es = fc::mutable_variant_object(*found_es); - } - auto& o = *doc; + fc::mutable_variant_object doc; + bool from_buffer = false; + find_post(id, doc, from_buffer); const auto& cmt = _db.get_comment(op.author, op.permlink); - o["total_votes"] = cmt.total_votes; - o["net_rshares"] = cmt.net_rshares; + doc["total_votes"] = cmt.total_votes; + doc["net_votes"] = cmt.net_votes; + doc["net_rshares"] = cmt.net_rshares; - if (!!found_es) { - buffer[id] = std::move(o); - } + doc["author_reputation"] = std::string(_db.get_account_reputation(op.author)); + + buffer[id] = std::move(doc); ++op_num; if (op_num % ELASTIC_WRITE_EACH_N_OP != 0) { @@ -214,31 +224,24 @@ class elastic_search_state_writer { const auto* comment = _db.find_comment(author, permlink); if (comment) { auto id = author_str + "." + permlink; - auto found = buffer.find(id); - optional found_es; - - fc::mutable_variant_object doc_es; - fc::mutable_variant_object* doc = &doc_es; - if (found != buffer.end()) { - doc = &found->second; - } else if (find_in_es(id, found_es)) { - doc_es = fc::mutable_variant_object(*found_es); - } - auto& o = *doc; + + fc::mutable_variant_object doc; + bool from_buffer = false; + find_post(id, doc, from_buffer); if (op.amount.symbol == STEEM_SYMBOL) { - auto donates = o["donates"].as(); + auto donates = doc["donates"].as(); donates += op.amount; - o["donates"] = donates; + doc["donates"] = donates; } else { - auto donates_uia = o["donates_uia"].as(); + auto donates_uia = doc["donates_uia"].as(); donates_uia += (op.amount.amount / op.amount.precision()); - o["donates_uia"] = donates_uia; + doc["donates_uia"] = donates_uia; } - if (!!found_es) { - buffer[id] = std::move(o); - } + doc["author_reputation"] = std::string(_db.get_account_reputation(author_str)); + + buffer[id] = std::move(doc); } } catch (...) {} From 435cb925ec06d7d4b42dedea3ac85bdb677a780d Mon Sep 17 00:00:00 2001 From: 1aerostorm Date: Mon, 28 Feb 2022 06:06:28 +0000 Subject: [PATCH 02/10] Update fc --- thirdparty/fc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/fc b/thirdparty/fc index 07615fe58b..139327cdca 160000 --- a/thirdparty/fc +++ b/thirdparty/fc @@ -1 +1 @@ -Subproject commit 07615fe58b0d9dcec927ee8e4cbe3826b4e0e4ea +Subproject commit 139327cdca02def819e422cfdf130f741a5d8e31 From c733755c0aca71d4f90a9fccf919d9c2dd6cbf73 Mon Sep 17 00:00:00 2001 From: 1aerostorm Date: Mon, 28 Feb 2022 06:08:39 +0000 Subject: [PATCH 03/10] Remove rand_pseudo_bytes --- libraries/network/node.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/network/node.cpp b/libraries/network/node.cpp index 0ef57eae17..d151749c0a 100644 --- a/libraries/network/node.cpp +++ b/libraries/network/node.cpp @@ -901,7 +901,7 @@ namespace golos { _maximum_number_of_sync_blocks_to_prefetch(MAXIMUM_NUMBER_OF_BLOCKS_TO_PREFETCH), _maximum_blocks_per_peer_during_syncing(GRAPHENE_NET_MAX_BLOCKS_PER_PEER_DURING_SYNCING) { _rate_limiter.set_actual_rate_time_constant(fc::seconds(2)); - fc::rand_pseudo_bytes(&_node_id.data[0], (int)_node_id.size()); + fc::rand_bytes(&_node_id.data[0], (int)_node_id.size()); } node_impl::~node_impl() { From 925917ef1895515dc1e9eb6a83a83cebd40288f8 Mon Sep 17 00:00:00 2001 From: 1aerostorm Date: Mon, 28 Feb 2022 06:09:14 +0000 Subject: [PATCH 04/10] Fix Doxygen problem --- libraries/wallet/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/wallet/CMakeLists.txt b/libraries/wallet/CMakeLists.txt index dd18237500..5eae955566 100644 --- a/libraries/wallet/CMakeLists.txt +++ b/libraries/wallet/CMakeLists.txt @@ -11,7 +11,7 @@ if(PERL_FOUND AND DOXYGEN_FOUND AND NOT "${CMAKE_GENERATOR}" STREQUAL "Ninja") DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile ${CMAKE_CURRENT_SOURCE_DIR}/include/golos/wallet/wallet.hpp) add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp - COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_api_documentation.pl ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp.new + COMMAND PERLLIB=${CMAKE_CURRENT_BINARY_DIR} ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_api_documentation.pl ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp.new COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp.new ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp.new From 8aa5183282625f15d30f6d8c57617eda8f36de74 Mon Sep 17 00:00:00 2001 From: 1aerostorm Date: Mon, 28 Feb 2022 06:23:02 +0000 Subject: [PATCH 05/10] Update phusion/baseimage in Dockerfiles --- Dockerfile | 9 ++++----- share/golosd/docker/Dockerfile-livetest | 9 ++++----- share/golosd/docker/Dockerfile-mongo | 9 ++++----- share/golosd/docker/Dockerfile-small | 9 ++++----- share/golosd/docker/Dockerfile-test | 2 +- share/golosd/docker/Dockerfile-testnet | 9 ++++----- share/golosd/docker/Dockerfile-testnet-mongo | 9 ++++----- 7 files changed, 25 insertions(+), 31 deletions(-) diff --git a/Dockerfile b/Dockerfile index 04d4519417..58e3fb704f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.9.19 as builder +FROM phusion/baseimage:0.11 as builder ARG TYPE=Release @@ -63,11 +63,10 @@ RUN \ cmake \ doxygen \ dpkg-dev \ - git \ libboost-all-dev \ libc6-dev \ libexpat1-dev \ - libgcc-5-dev \ + libgcc-7-dev \ libhwloc-dev \ libibverbs-dev \ libicu-dev \ @@ -80,7 +79,7 @@ RUN \ libreadline-dev \ libreadline6-dev \ libssl-dev \ - libstdc++-5-dev \ + libstdc++-7-dev \ libtinfo-dev \ libtool \ linux-libc-dev \ @@ -102,7 +101,7 @@ RUN \ /usr/include \ /usr/local/include -FROM phusion/baseimage:0.9.19 as production +FROM phusion/baseimage:0.11 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl diff --git a/share/golosd/docker/Dockerfile-livetest b/share/golosd/docker/Dockerfile-livetest index 8bfa2a99a1..5c8dd2890d 100644 --- a/share/golosd/docker/Dockerfile-livetest +++ b/share/golosd/docker/Dockerfile-livetest @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.9.19 as builder +FROM phusion/baseimage:0.11 as builder ARG TYPE=Release @@ -70,11 +70,10 @@ RUN \ cmake \ doxygen \ dpkg-dev \ - git \ libboost-all-dev \ libc6-dev \ libexpat1-dev \ - libgcc-5-dev \ + libgcc-7-dev \ libhwloc-dev \ libibverbs-dev \ libicu-dev \ @@ -87,7 +86,7 @@ RUN \ libreadline-dev \ libreadline6-dev \ libssl-dev \ - libstdc++-5-dev \ + libstdc++-7-dev \ libtinfo-dev \ libtool \ linux-libc-dev \ @@ -109,7 +108,7 @@ RUN \ /usr/include \ /usr/local/include -FROM phusion/baseimage:0.9.19 as production +FROM phusion/baseimage:0.11 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl diff --git a/share/golosd/docker/Dockerfile-mongo b/share/golosd/docker/Dockerfile-mongo index 575b908496..790b442906 100644 --- a/share/golosd/docker/Dockerfile-mongo +++ b/share/golosd/docker/Dockerfile-mongo @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.9.19 as builder +FROM phusion/baseimage:0.11 as builder ARG TYPE=Release @@ -92,11 +92,10 @@ RUN \ cmake \ doxygen \ dpkg-dev \ - git \ libboost-all-dev \ libc6-dev \ libexpat1-dev \ - libgcc-5-dev \ + libgcc-7-dev \ libhwloc-dev \ libibverbs-dev \ libicu-dev \ @@ -109,7 +108,7 @@ RUN \ libreadline-dev \ libreadline6-dev \ libssl-dev \ - libstdc++-5-dev \ + libstdc++-7-dev \ libtinfo-dev \ libtool \ linux-libc-dev \ @@ -131,7 +130,7 @@ RUN \ /usr/include \ /usr/local/include -FROM phusion/baseimage:0.9.19 as production +FROM phusion/baseimage:0.11 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl diff --git a/share/golosd/docker/Dockerfile-small b/share/golosd/docker/Dockerfile-small index 296b85129d..2e474ad103 100644 --- a/share/golosd/docker/Dockerfile-small +++ b/share/golosd/docker/Dockerfile-small @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.9.19 as builder +FROM phusion/baseimage:0.11 as builder ARG TYPE=Release @@ -59,11 +59,10 @@ RUN \ cmake \ doxygen \ dpkg-dev \ - git \ libboost-all-dev \ libc6-dev \ libexpat1-dev \ - libgcc-5-dev \ + libgcc-7-dev \ libhwloc-dev \ libibverbs-dev \ libicu-dev \ @@ -76,7 +75,7 @@ RUN \ libreadline-dev \ libreadline6-dev \ libssl-dev \ - libstdc++-5-dev \ + libstdc++-7-dev \ libtinfo-dev \ libtool \ linux-libc-dev \ @@ -103,7 +102,7 @@ RUN \ mkdir /var/cache/golosd && \ chown golosd:golosd -R /var/cache/golosd -FROM phusion/baseimage:0.9.19 as production +FROM phusion/baseimage:0.11 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl diff --git a/share/golosd/docker/Dockerfile-test b/share/golosd/docker/Dockerfile-test index 0d031055f9..04789d9525 100644 --- a/share/golosd/docker/Dockerfile-test +++ b/share/golosd/docker/Dockerfile-test @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.9.19 +FROM phusion/baseimage:0.11 ENV LANG=en_US.UTF-8 diff --git a/share/golosd/docker/Dockerfile-testnet b/share/golosd/docker/Dockerfile-testnet index 78394d5606..e177672d7d 100644 --- a/share/golosd/docker/Dockerfile-testnet +++ b/share/golosd/docker/Dockerfile-testnet @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.9.19 as builder +FROM phusion/baseimage:0.11 as builder ARG TYPE=Release @@ -62,11 +62,10 @@ RUN \ cmake \ doxygen \ dpkg-dev \ - git \ libboost-all-dev \ libc6-dev \ libexpat1-dev \ - libgcc-5-dev \ + libgcc-7-dev \ libhwloc-dev \ libibverbs-dev \ libicu-dev \ @@ -79,7 +78,7 @@ RUN \ libreadline-dev \ libreadline6-dev \ libssl-dev \ - libstdc++-5-dev \ + libstdc++-7-dev \ libtinfo-dev \ libtool \ linux-libc-dev \ @@ -101,7 +100,7 @@ RUN \ /usr/include \ /usr/local/include -FROM phusion/baseimage:0.9.19 as production +FROM phusion/baseimage:0.11 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl diff --git a/share/golosd/docker/Dockerfile-testnet-mongo b/share/golosd/docker/Dockerfile-testnet-mongo index 35bb49acc0..ba287e5148 100644 --- a/share/golosd/docker/Dockerfile-testnet-mongo +++ b/share/golosd/docker/Dockerfile-testnet-mongo @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.9.19 as builder +FROM phusion/baseimage:0.11 as builder ARG TYPE=Release @@ -91,11 +91,10 @@ RUN \ cmake \ doxygen \ dpkg-dev \ - git \ libboost-all-dev \ libc6-dev \ libexpat1-dev \ - libgcc-5-dev \ + libgcc-7-dev \ libhwloc-dev \ libibverbs-dev \ libicu-dev \ @@ -108,7 +107,7 @@ RUN \ libreadline-dev \ libreadline6-dev \ libssl-dev \ - libstdc++-5-dev \ + libstdc++-7-dev \ libtinfo-dev \ libtool \ linux-libc-dev \ @@ -130,7 +129,7 @@ RUN \ /usr/include \ /usr/local/include -FROM phusion/baseimage:0.9.19 as production +FROM phusion/baseimage:0.11 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl From 74114b5aedc598b3077fe4b91e7cd9edece83aba Mon Sep 17 00:00:00 2001 From: 1aerostorm Date: Mon, 28 Feb 2022 15:12:08 +0000 Subject: [PATCH 06/10] Optimize ElasticSearch --- .../elastic_search/elastic_search_state.hpp | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp b/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp index 9f0b7e8b2e..4e8aba0a62 100644 --- a/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp +++ b/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp @@ -111,20 +111,23 @@ class elastic_search_state_writer { fc::mutable_variant_object doc; bool from_buffer = false; - bool exists = find_post(id, doc, from_buffer); - std::string base_body = doc["body"].as_string(); + bool exists = false; std::string body = op.body; try { diff_match_patch dmp; auto patch = dmp.patch_fromText(utf8_to_wstring(body)); - if (patch.size() && base_body.size()) { - auto result = dmp.patch_apply(patch, utf8_to_wstring(base_body)); - auto patched_body = wstring_to_utf8(result.first); - if(!fc::is_utf8(patched_body)) { - body = fc::prune_invalid_utf8(patched_body); - } else { - body = patched_body; + if (patch.size()) { + exists = find_post(id, doc, from_buffer); + std::string base_body = doc["body"].as_string(); + if (base_body.size()) { + auto result = dmp.patch_apply(patch, utf8_to_wstring(base_body)); + auto patched_body = wstring_to_utf8(result.first); + if(!fc::is_utf8(patched_body)) { + body = fc::prune_invalid_utf8(patched_body); + } else { + body = patched_body; + } } } } catch ( ... ) { @@ -161,7 +164,6 @@ class elastic_search_state_writer { if (!exists) { doc["net_votes"] = int32_t(0); - doc["total_votes"] = uint32_t(0); doc["net_rshares"] = share_type(0); doc["donates"] = asset(0, STEEM_SYMBOL); doc["donates_uia"] = share_type(); @@ -181,7 +183,7 @@ class elastic_search_state_writer { result_type operator()(const vote_operation& op) { #ifndef STEEMIT_BUILD_TESTNET - if (_db.head_block_num() < 35000000) { // Speed up replay + if (_db.head_block_num() < 45000000) { // Speed up replay return; } #endif @@ -197,7 +199,6 @@ class elastic_search_state_writer { const auto& cmt = _db.get_comment(op.author, op.permlink); - doc["total_votes"] = cmt.total_votes; doc["net_votes"] = cmt.net_votes; doc["net_rshares"] = cmt.net_rshares; From 7d492a2c7864af59939a33c55f92eb10e0b78eb2 Mon Sep 17 00:00:00 2001 From: 1aerostorm <78693755+1aerostorm@users.noreply.github.com> Date: Tue, 1 Mar 2022 00:26:35 +0300 Subject: [PATCH 07/10] Update Docker image to phusion/baseimage:bionic-1.0.0 --- share/golosd/docker/Dockerfile-livetest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/golosd/docker/Dockerfile-livetest b/share/golosd/docker/Dockerfile-livetest index 5c8dd2890d..56d792a734 100644 --- a/share/golosd/docker/Dockerfile-livetest +++ b/share/golosd/docker/Dockerfile-livetest @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.11 as builder +FROM phusion/baseimage:bionic-1.0.0 as builder ARG TYPE=Release @@ -108,7 +108,7 @@ RUN \ /usr/include \ /usr/local/include -FROM phusion/baseimage:0.11 as production +FROM phusion/baseimage:bionic-1.0.0 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl From 55c8f9a642bbd5b4c49108e38e3865172e882aee Mon Sep 17 00:00:00 2001 From: 1aerostorm Date: Tue, 1 Mar 2022 22:51:08 +0000 Subject: [PATCH 08/10] Improve ElasticSearch --- .../elastic_search/elastic_search_state.hpp | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp b/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp index 4e8aba0a62..2192ab770c 100644 --- a/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp +++ b/plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp @@ -111,14 +111,13 @@ class elastic_search_state_writer { fc::mutable_variant_object doc; bool from_buffer = false; - bool exists = false; std::string body = op.body; try { diff_match_patch dmp; auto patch = dmp.patch_fromText(utf8_to_wstring(body)); if (patch.size()) { - exists = find_post(id, doc, from_buffer); + find_post(id, doc, from_buffer); std::string base_body = doc["body"].as_string(); if (base_body.size()) { auto result = dmp.patch_apply(patch, utf8_to_wstring(base_body)); @@ -151,7 +150,8 @@ class elastic_search_state_writer { const auto& root_cmt = _db.get(cmt.root_comment); doc["category"] = root_cmt.parent_permlink; doc["root_author"] = root_cmt.author; - doc["root_permlink"] = to_string(root_cmt.permlink); + auto root_permlink = to_string(root_cmt.permlink); + doc["root_permlink"] = root_permlink; const auto* root_cnt = appbase::app().get_plugin().find_comment_content(root_cmt.id); doc["root_title"] = root_cnt ? to_string(root_cnt->title) : ""; } @@ -162,13 +162,6 @@ class elastic_search_state_writer { doc["tags"] = golos::plugins::tags::get_metadata(op.json_metadata, TAGS_NUMBER, TAG_MAX_LENGTH).tags; doc["json_metadata"] = op.json_metadata; - if (!exists) { - doc["net_votes"] = int32_t(0); - doc["net_rshares"] = share_type(0); - doc["donates"] = asset(0, STEEM_SYMBOL); - doc["donates_uia"] = share_type(); - } - doc["author_reputation"] = std::string(_db.get_account_reputation(op.author)); buffer[id] = std::move(doc); @@ -181,26 +174,28 @@ class elastic_search_state_writer { write_buffer(); } - result_type operator()(const vote_operation& op) { + result_type operator()(const comment_reward_operation& op) { #ifndef STEEMIT_BUILD_TESTNET - if (_db.head_block_num() < 45000000) { // Speed up replay + if (_db.head_block_num() < 35000000) { // Speed up replay return; } #endif - if (_db.is_account_vote(op)) { - return; - } auto id = std::string(op.author) + "." + op.permlink; fc::mutable_variant_object doc; bool from_buffer = false; - find_post(id, doc, from_buffer); + bool exists = find_post(id, doc, from_buffer); + + if (!exists) { + return; + } const auto& cmt = _db.get_comment(op.author, op.permlink); doc["net_votes"] = cmt.net_votes; doc["net_rshares"] = cmt.net_rshares; + doc["children"] = cmt.children; doc["author_reputation"] = std::string(_db.get_account_reputation(op.author)); @@ -231,12 +226,18 @@ class elastic_search_state_writer { find_post(id, doc, from_buffer); if (op.amount.symbol == STEEM_SYMBOL) { - auto donates = doc["donates"].as(); - donates += op.amount; + auto donates = op.amount; + auto itr = doc.find("donates"); + if (itr != doc.end()) { + donates += itr->value().as(); + } doc["donates"] = donates; } else { - auto donates_uia = doc["donates_uia"].as(); - donates_uia += (op.amount.amount / op.amount.precision()); + auto donates_uia = (op.amount.amount / op.amount.precision()); + auto itr = doc.find("donates_uia"); + if (itr != doc.end()) { + donates_uia += itr->value().as(); + } doc["donates_uia"] = donates_uia; } From 6aebb8229071c4bb71c93fb1c8564ca0ffeb389e Mon Sep 17 00:00:00 2001 From: 1aerostorm Date: Thu, 3 Mar 2022 18:12:49 +0000 Subject: [PATCH 09/10] Update all Docker files to phusion/baseimage:bionic-1.0.0 --- Dockerfile | 4 ++-- share/golosd/docker/Dockerfile-mongo | 4 ++-- share/golosd/docker/Dockerfile-small | 4 ++-- share/golosd/docker/Dockerfile-test | 2 +- share/golosd/docker/Dockerfile-testnet | 4 ++-- share/golosd/docker/Dockerfile-testnet-mongo | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 58e3fb704f..26ab54cb44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.11 as builder +FROM phusion/baseimage:bionic-1.0.0 as builder ARG TYPE=Release @@ -101,7 +101,7 @@ RUN \ /usr/include \ /usr/local/include -FROM phusion/baseimage:0.11 as production +FROM phusion/baseimage:bionic-1.0.0 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl diff --git a/share/golosd/docker/Dockerfile-mongo b/share/golosd/docker/Dockerfile-mongo index 790b442906..8686d788d0 100644 --- a/share/golosd/docker/Dockerfile-mongo +++ b/share/golosd/docker/Dockerfile-mongo @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.11 as builder +FROM phusion/baseimage:bionic-1.0.0 as builder ARG TYPE=Release @@ -130,7 +130,7 @@ RUN \ /usr/include \ /usr/local/include -FROM phusion/baseimage:0.11 as production +FROM phusion/baseimage:bionic-1.0.0 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl diff --git a/share/golosd/docker/Dockerfile-small b/share/golosd/docker/Dockerfile-small index 2e474ad103..d73fdb3679 100644 --- a/share/golosd/docker/Dockerfile-small +++ b/share/golosd/docker/Dockerfile-small @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.11 as builder +FROM phusion/baseimage:bionic-1.0.0 as builder ARG TYPE=Release @@ -102,7 +102,7 @@ RUN \ mkdir /var/cache/golosd && \ chown golosd:golosd -R /var/cache/golosd -FROM phusion/baseimage:0.11 as production +FROM phusion/baseimage:bionic-1.0.0 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl diff --git a/share/golosd/docker/Dockerfile-test b/share/golosd/docker/Dockerfile-test index 04789d9525..0d15b30c71 100644 --- a/share/golosd/docker/Dockerfile-test +++ b/share/golosd/docker/Dockerfile-test @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.11 +FROM phusion/baseimage:bionic-1.0.0 ENV LANG=en_US.UTF-8 diff --git a/share/golosd/docker/Dockerfile-testnet b/share/golosd/docker/Dockerfile-testnet index e177672d7d..99a921cf3a 100644 --- a/share/golosd/docker/Dockerfile-testnet +++ b/share/golosd/docker/Dockerfile-testnet @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.11 as builder +FROM phusion/baseimage:bionic-1.0.0 as builder ARG TYPE=Release @@ -100,7 +100,7 @@ RUN \ /usr/include \ /usr/local/include -FROM phusion/baseimage:0.11 as production +FROM phusion/baseimage:bionic-1.0.0 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl diff --git a/share/golosd/docker/Dockerfile-testnet-mongo b/share/golosd/docker/Dockerfile-testnet-mongo index ba287e5148..37f2f0d908 100644 --- a/share/golosd/docker/Dockerfile-testnet-mongo +++ b/share/golosd/docker/Dockerfile-testnet-mongo @@ -1,4 +1,4 @@ -FROM phusion/baseimage:0.11 as builder +FROM phusion/baseimage:bionic-1.0.0 as builder ARG TYPE=Release @@ -129,7 +129,7 @@ RUN \ /usr/include \ /usr/local/include -FROM phusion/baseimage:0.11 as production +FROM phusion/baseimage:bionic-1.0.0 as production COPY --from=builder /usr/local /usr/local ADD share/golosd/golosdctl /usr/local/bin/golosdctl From fb3687c1c695e05b410545b9ca9f694cb61d0bdf Mon Sep 17 00:00:00 2001 From: 1aerostorm Date: Sat, 5 Mar 2022 01:20:36 +0000 Subject: [PATCH 10/10] Fix time backward compatibility --- thirdparty/fc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/fc b/thirdparty/fc index 139327cdca..1a7f30393e 160000 --- a/thirdparty/fc +++ b/thirdparty/fc @@ -1 +1 @@ -Subproject commit 139327cdca02def819e422cfdf130f741a5d8e31 +Subproject commit 1a7f30393e44534abfdbdb01c98237c0713934bc