Skip to content

Commit

Permalink
Merge pull request #193 from golos-blockchain/livetest
Browse files Browse the repository at this point in the history 
Livetest
  • Loading branch information
@Lex-Ai
Lex-Ai authored Mar 9, 2022
2 parents 7bb825a + fb3687c commit a0fe1e6
Show file tree
Hide file tree
Showing 11 changed files with 89 additions and 90 deletions.
9 changes: 4 additions & 5 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM phusion/baseimage:0.9.19 as builder
FROM phusion/baseimage:bionic-1.0.0 as builder

ARG TYPE=Release

Expand Down Expand Up @@ -63,11 +63,10 @@ RUN \
cmake \
doxygen \
dpkg-dev \
git \
libboost-all-dev \
libc6-dev \
libexpat1-dev \
libgcc-5-dev \
libgcc-7-dev \
libhwloc-dev \
libibverbs-dev \
libicu-dev \
Expand All @@ -80,7 +79,7 @@ RUN \
libreadline-dev \
libreadline6-dev \
libssl-dev \
libstdc++-5-dev \
libstdc++-7-dev \
libtinfo-dev \
libtool \
linux-libc-dev \
Expand All @@ -102,7 +101,7 @@ RUN \
/usr/include \
/usr/local/include

FROM phusion/baseimage:0.9.19 as production
FROM phusion/baseimage:bionic-1.0.0 as production
COPY --from=builder /usr/local /usr/local

ADD share/golosd/golosdctl /usr/local/bin/golosdctl
Expand Down
2 changes: 1 addition & 1 deletion libraries/network/node.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -901,7 +901,7 @@ namespace golos {
_maximum_number_of_sync_blocks_to_prefetch(MAXIMUM_NUMBER_OF_BLOCKS_TO_PREFETCH),
_maximum_blocks_per_peer_during_syncing(GRAPHENE_NET_MAX_BLOCKS_PER_PEER_DURING_SYNCING) {
_rate_limiter.set_actual_rate_time_constant(fc::seconds(2));
fc::rand_pseudo_bytes(&_node_id.data[0], (int)_node_id.size());
fc::rand_bytes(&_node_id.data[0], (int)_node_id.size());
}

node_impl::~node_impl() {
Expand Down
2 changes: 1 addition & 1 deletion libraries/wallet/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ if(PERL_FOUND AND DOXYGEN_FOUND AND NOT "${CMAKE_GENERATOR}" STREQUAL "Ninja")
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile ${CMAKE_CURRENT_SOURCE_DIR}/include/golos/wallet/wallet.hpp)

add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp
COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_api_documentation.pl ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp.new
COMMAND PERLLIB=${CMAKE_CURRENT_BINARY_DIR} ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_api_documentation.pl ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp.new

COMMAND ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp.new ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp
COMMAND ${CMAKE_COMMAND} -E remove ${CMAKE_CURRENT_BINARY_DIR}/api_documentation.cpp.new
Expand Down
117 changes: 61 additions & 56 deletions plugins/elastic_search/include/golos/plugins/elastic_search/elastic_search_state.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ namespace golos { namespace plugins { namespace elastic_search {
using boost::locale::conv::utf_to_utf;

#ifdef STEEMIT_BUILD_TESTNET
#define ELASTIC_WRITE_EACH_N_OP 1
#define ELASTIC_WRITE_EACH_N_OP 3
#else
#define ELASTIC_WRITE_EACH_N_OP 100
#endif
Expand Down Expand Up @@ -72,7 +72,7 @@ class elastic_search_state_writer {
return headers;
}

bool find_in_es(const std::string& id, optional<variant_object>& res) {
bool find_in_es(const std::string& id, variant_object& res) {
auto headers = get_es_headers();
auto reply0 = conn.request("GET", url + "/blog/post/" + id + "?pretty", "", headers);
auto reply_body = std::string(reply0.body.data(), reply0.body.size());
Expand All @@ -83,7 +83,22 @@ class elastic_search_state_writer {
return true;
}
}
res = optional<variant_object>();
res = variant_object();
return false;
}

bool find_post(const std::string& id, fc::mutable_variant_object& found, bool& from_buffer) {
auto found_buf = buffer.find(id);
variant_object found_es;
from_buffer = false;
if (found_buf != buffer.end()) {
from_buffer = true;
found = found_buf->second;
return true;
} else if (find_in_es(id, found_es)) {
found = fc::mutable_variant_object(found_es);
return true;
}
return false;
}

Expand All @@ -94,19 +109,16 @@ class elastic_search_state_writer {
return;
}

fc::mutable_variant_object doc;
bool from_buffer = false;

std::string body = op.body;
try {
diff_match_patch<std::wstring> dmp;
auto patch = dmp.patch_fromText(utf8_to_wstring(body));
if (patch.size()) {
std::string base_body;
auto found = buffer.find(id);
optional<variant_object> found2;
if (found != buffer.end()) {
base_body = found->second["body"].as_string();
} else if (find_in_es(id, found2)) {
base_body = (*found2)["body"].as_string();
}
find_post(id, doc, from_buffer);
std::string base_body = doc["body"].as_string();
if (base_body.size()) {
auto result = dmp.patch_apply(patch, utf8_to_wstring(base_body));
auto patched_body = wstring_to_utf8(result.first);
Expand All @@ -121,7 +133,6 @@ class elastic_search_state_writer {
}

const auto& cmt = _db.get_comment(op.author, op.permlink);
fc::mutable_variant_object doc;

doc["id"] = cmt.id;
doc["created"] = _db.head_block_time();
Expand All @@ -139,7 +150,8 @@ class elastic_search_state_writer {
const auto& root_cmt = _db.get<comment_object, by_id>(cmt.root_comment);
doc["category"] = root_cmt.parent_permlink;
doc["root_author"] = root_cmt.author;
doc["root_permlink"] = to_string(root_cmt.permlink);
auto root_permlink = to_string(root_cmt.permlink);
doc["root_permlink"] = root_permlink;
const auto* root_cnt = appbase::app().get_plugin<golos::plugins::social_network::social_network>().find_comment_content(root_cmt.id);
doc["root_title"] = root_cnt ? to_string(root_cnt->title) : "";
}
Expand All @@ -150,50 +162,44 @@ class elastic_search_state_writer {
doc["tags"] = golos::plugins::tags::get_metadata(op.json_metadata, TAGS_NUMBER, TAG_MAX_LENGTH).tags;
doc["json_metadata"] = op.json_metadata;

doc["total_votes"] = uint32_t(0);
doc["net_rshares"] = share_type(0);
doc["donates"] = asset(0, STEEM_SYMBOL);
doc["donates_uia"] = share_type();
doc["author_reputation"] = std::string(_db.get_account_reputation(op.author));

buffer[id] = std::move(doc);
++op_num;

++op_num;
if (op_num % ELASTIC_WRITE_EACH_N_OP != 0) {
return;
}

write_buffer();
}

result_type operator()(const vote_operation& op) {
result_type operator()(const comment_reward_operation& op) {
#ifndef STEEMIT_BUILD_TESTNET
if (_db.head_block_num() < 35000000) { // Speed up replay
return;
}
if (_db.is_account_vote(op)) {
return;
}
#endif

auto id = std::string(op.author) + "." + op.permlink;
auto found = buffer.find(id);
optional<variant_object> found_es;

fc::mutable_variant_object doc_es;
fc::mutable_variant_object* doc = &doc_es;
if (found != buffer.end()) {
doc = &found->second;
} else if (find_in_es(id, found_es)) {
doc_es = fc::mutable_variant_object(*found_es);
fc::mutable_variant_object doc;
bool from_buffer = false;
bool exists = find_post(id, doc, from_buffer);

if (!exists) {
return;
}
auto& o = *doc;

const auto& cmt = _db.get_comment(op.author, op.permlink);

o["total_votes"] = cmt.total_votes;
o["net_rshares"] = cmt.net_rshares;
doc["net_votes"] = cmt.net_votes;
doc["net_rshares"] = cmt.net_rshares;
doc["children"] = cmt.children;

if (!!found_es) {
buffer[id] = std::move(o);
}
doc["author_reputation"] = std::string(_db.get_account_reputation(op.author));

buffer[id] = std::move(doc);

++op_num;
if (op_num % ELASTIC_WRITE_EACH_N_OP != 0) {
Expand All @@ -214,31 +220,30 @@ class elastic_search_state_writer {
const auto* comment = _db.find_comment(author, permlink);
if (comment) {
auto id = author_str + "." + permlink;
auto found = buffer.find(id);
optional<variant_object> found_es;

fc::mutable_variant_object doc_es;
fc::mutable_variant_object* doc = &doc_es;
if (found != buffer.end()) {
doc = &found->second;
} else if (find_in_es(id, found_es)) {
doc_es = fc::mutable_variant_object(*found_es);
}
auto& o = *doc;

fc::mutable_variant_object doc;
bool from_buffer = false;
find_post(id, doc, from_buffer);

if (op.amount.symbol == STEEM_SYMBOL) {
auto donates = o["donates"].as<asset>();
donates += op.amount;
o["donates"] = donates;
auto donates = op.amount;
auto itr = doc.find("donates");
if (itr != doc.end()) {
donates += itr->value().as<asset>();
}
doc["donates"] = donates;
} else {
auto donates_uia = o["donates_uia"].as<share_type>();
donates_uia += (op.amount.amount / op.amount.precision());
o["donates_uia"] = donates_uia;
auto donates_uia = (op.amount.amount / op.amount.precision());
auto itr = doc.find("donates_uia");
if (itr != doc.end()) {
donates_uia += itr->value().as<share_type>();
}
doc["donates_uia"] = donates_uia;
}

if (!!found_es) {
buffer[id] = std::move(o);
}
doc["author_reputation"] = std::string(_db.get_account_reputation(author_str));

buffer[id] = std::move(doc);
}
} catch (...) {}

Expand Down
9 changes: 4 additions & 5 deletions share/golosd/docker/Dockerfile-livetest
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM phusion/baseimage:0.9.19 as builder
FROM phusion/baseimage:bionic-1.0.0 as builder

ARG TYPE=Release

Expand Down Expand Up @@ -70,11 +70,10 @@ RUN \
cmake \
doxygen \
dpkg-dev \
git \
libboost-all-dev \
libc6-dev \
libexpat1-dev \
libgcc-5-dev \
libgcc-7-dev \
libhwloc-dev \
libibverbs-dev \
libicu-dev \
Expand All @@ -87,7 +86,7 @@ RUN \
libreadline-dev \
libreadline6-dev \
libssl-dev \
libstdc++-5-dev \
libstdc++-7-dev \
libtinfo-dev \
libtool \
linux-libc-dev \
Expand All @@ -109,7 +108,7 @@ RUN \
/usr/include \
/usr/local/include

FROM phusion/baseimage:0.9.19 as production
FROM phusion/baseimage:bionic-1.0.0 as production
COPY --from=builder /usr/local /usr/local

ADD share/golosd/golosdctl /usr/local/bin/golosdctl
Expand Down
9 changes: 4 additions & 5 deletions share/golosd/docker/Dockerfile-mongo
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM phusion/baseimage:0.9.19 as builder
FROM phusion/baseimage:bionic-1.0.0 as builder

ARG TYPE=Release

Expand Down Expand Up @@ -92,11 +92,10 @@ RUN \
cmake \
doxygen \
dpkg-dev \
git \
libboost-all-dev \
libc6-dev \
libexpat1-dev \
libgcc-5-dev \
libgcc-7-dev \
libhwloc-dev \
libibverbs-dev \
libicu-dev \
Expand All @@ -109,7 +108,7 @@ RUN \
libreadline-dev \
libreadline6-dev \
libssl-dev \
libstdc++-5-dev \
libstdc++-7-dev \
libtinfo-dev \
libtool \
linux-libc-dev \
Expand All @@ -131,7 +130,7 @@ RUN \
/usr/include \
/usr/local/include

FROM phusion/baseimage:0.9.19 as production
FROM phusion/baseimage:bionic-1.0.0 as production
COPY --from=builder /usr/local /usr/local

ADD share/golosd/golosdctl /usr/local/bin/golosdctl
Expand Down
9 changes: 4 additions & 5 deletions share/golosd/docker/Dockerfile-small
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM phusion/baseimage:0.9.19 as builder
FROM phusion/baseimage:bionic-1.0.0 as builder

ARG TYPE=Release

Expand Down Expand Up @@ -59,11 +59,10 @@ RUN \
cmake \
doxygen \
dpkg-dev \
git \
libboost-all-dev \
libc6-dev \
libexpat1-dev \
libgcc-5-dev \
libgcc-7-dev \
libhwloc-dev \
libibverbs-dev \
libicu-dev \
Expand All @@ -76,7 +75,7 @@ RUN \
libreadline-dev \
libreadline6-dev \
libssl-dev \
libstdc++-5-dev \
libstdc++-7-dev \
libtinfo-dev \
libtool \
linux-libc-dev \
Expand All @@ -103,7 +102,7 @@ RUN \
mkdir /var/cache/golosd && \
chown golosd:golosd -R /var/cache/golosd

FROM phusion/baseimage:0.9.19 as production
FROM phusion/baseimage:bionic-1.0.0 as production
COPY --from=builder /usr/local /usr/local

ADD share/golosd/golosdctl /usr/local/bin/golosdctl
Expand Down
2 changes: 1 addition & 1 deletion share/golosd/docker/Dockerfile-test
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM phusion/baseimage:0.9.19
FROM phusion/baseimage:bionic-1.0.0

ENV LANG=en_US.UTF-8

Expand Down
Loading

0 comments on commit a0fe1e6

Please sign in to comment.