Merge pull request #194 from golos-blockchain/livetest

Add historian util and Fix delete_reblog bug

plugins/account_history/plugin.cpp

@@ -149,7 +149,7 @@ if (options.count(name)) { \
             history_operations result;
             const auto& idx = db.get_index<account_history_index>().indices().get<by_account>();
             auto itr = idx.lower_bound(std::make_tuple(account, from));
-            auto end = idx.upper_bound(std::make_tuple(account, std::max(int64_t(0), int64_t(itr->sequence) - limit)));
+            auto end = idx.upper_bound(std::make_tuple(account, std::max(int64_t(0), int64_t(itr->sequence) - limit + 1)));
             for (; itr != end; ++itr) {
                 result[itr->sequence] = db.get(itr->op);
                 result[itr->sequence].json_metadata = to_string(itr->json_metadata);
@@ -245,7 +245,7 @@ if (options.count(name)) { \
             }
 
             history_operations result;
-            while (!itrs.empty() && result.size() <= limit) {
+            while (!itrs.empty() && result.size() < limit) {
                 auto itr = itrs.top().itr;
                 itrs.pop();
                 result[itr->sequence] = db.get(itr->op);

plugins/follow/follow_evaluators.cpp

@@ -239,12 +239,15 @@ namespace golos {
                             auto feed_itr = comment_idx.find(boost::make_tuple(c.id, itr->follower));
 
                             if (feed_itr != comment_idx.end()) {
+                                auto pos = std::find(feed_itr->reblogged_by.begin(), feed_itr->reblogged_by.end(), o.account);
+                                if (pos == feed_itr->reblogged_by.end()) {
+                                    continue;
+                                }
                                 if (feed_itr->reblogs <= 1) {
                                     db().remove(*feed_itr);
                                 } else {
                                     db().modify(*feed_itr, [&](feed_object& f) {
-                                        f.reblogged_by.erase(std::remove(f.reblogged_by.begin(), f.reblogged_by.end(),
-                                            o.account));
+                                        f.reblogged_by.erase(pos);
                                         f.reblogs--;
                                     });
                                 }

plugins/follow/include/golos/plugins/follow/follow_objects.hpp

@@ -11,13 +11,14 @@ namespace golos {
             using chainbase::object;
             using chainbase::object_id;
             using chainbase::allocator;
-            using chainbase::shared_vector;
             using golos::chain::comment_object;
             using golos::chain::by_id;
             using golos::chain::comment_vote_index;
             using golos::chain::by_comment_voter;
             using golos::chain::shared_string;
 
+            namespace bip = boost::interprocess;
+
 #ifndef FOLLOW_SPACE_ID
 #define FOLLOW_SPACE_ID 8
 #endif
@@ -53,14 +54,14 @@ namespace golos {
 
                 template<typename Constructor, typename Allocator>
                 feed_object(Constructor &&c, allocator<Allocator> a)
-                        :reblogged_by(a.get_segment_manager()) {
+                        :reblogged_by(a) {
                     c(*this);
                 }
 
                 id_type id;
 
                 account_name_type account;
-                shared_vector<account_name_type> reblogged_by;
+                bip::vector<account_name_type, allocator<account_name_type>> reblogged_by;
                 account_name_type first_reblogged_by;
                 time_point_sec first_reblogged_on;
                 comment_object::id_type comment;
@@ -233,7 +234,7 @@ FC_REFLECT((golos::plugins::follow::follow_object), (id)(follower)(following)(wh
 CHAINBASE_SET_INDEX_TYPE(golos::plugins::follow::follow_object, golos::plugins::follow::follow_index)
 
 FC_REFLECT((golos::plugins::follow::feed_object),
-           (id)(account)(first_reblogged_by)(first_reblogged_on)(reblogged_by)(comment)(reblogs)(account_feed_id))
+           (id)(account)(first_reblogged_by)(first_reblogged_on)(comment)(reblogs)(account_feed_id))
 CHAINBASE_SET_INDEX_TYPE(golos::plugins::follow::feed_object, golos::plugins::follow::feed_index)
 
 FC_REFLECT((golos::plugins::follow::blog_object), (id)(account)(comment)(reblogged_on)(blog_feed_id))

plugins/follow/plugin.cpp

@@ -197,7 +197,7 @@ namespace golos {
                                     const auto& old_feed_idx = _db.get_index<feed_index, by_old_feed>();
                                     auto old_feed = old_feed_idx.lower_bound(itr->follower);
 
-                                    while (old_feed->account == itr->follower &&
+                                    while (old_feed != old_feed_idx.end() && old_feed->account == itr->follower &&
                                            next_id - old_feed->account_feed_id > _plugin.max_feed_size()) {
                                         _db.remove(*old_feed);
                                         old_feed = old_feed_idx.lower_bound(itr->follower);

programs/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_subdirectory(build_helpers)
 add_subdirectory(cli_wallet)
 add_subdirectory(golosd)
+add_subdirectory(historian)
 #add_subdirectory( delayed_node )
 add_subdirectory(js_operation_serializer)
 add_subdirectory(meter)

programs/historian/CMakeLists.txt

@@ -0,0 +1,19 @@
+add_executable(historian main.cpp)
+if(UNIX AND NOT APPLE)
+    set(rt_library rt)
+endif()
+
+target_link_libraries(historian
+        PRIVATE golos_chain
+        golos_protocol
+        fc
+        indicators
+        ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS})
+
+install(TARGETS
+        historian
+
+        RUNTIME DESTINATION bin
+        LIBRARY DESTINATION lib
+        ARCHIVE DESTINATION lib
+        )

programs/historian/main.cpp

@@ -0,0 +1,125 @@
+#include <fc/io/json.hpp>
+#include <fc/smart_ref_impl.hpp>
+#include <fc/variant_object.hpp>
+#include <fc/stacktrace.hpp>
+#include <fc/filesystem.hpp>
+#include <boost/algorithm/string/replace.hpp>
+#include <boost/filesystem/path.hpp>
+#include <boost/program_options.hpp>
+
+#include <golos/chain/database.hpp>
+#include <golos/protocol/operation_util_impl.hpp>
+
+#include "./program_options.hpp"
+#include "./progress_bar.hpp"
+#include "./special_operation_visitor.hpp"
+
+namespace bfs = boost::filesystem;
+namespace bpo = boost::program_options;
+using bpo::options_description;
+using bpo::variables_map;
+using namespace golos::chain;
+
+int unsafe_main(int argc, char** argv);
+
+int main(int argc, char** argv) {
+    try {
+        return unsafe_main(argc, argv);
+    } catch (const fc::exception& e) {
+        std::cout << e.to_detail_string() << "\n";
+        return -1;
+    }
+}
+
+int unsafe_main(int argc, char** argv) {
+    auto po = get_program_options(argc, argv);
+
+    if (!po.proceed) {
+        return po.exit_code;
+    }
+
+    fc::install_stacktrace_crash_handler();
+
+    std::cout << "Opening block_log: " << po.blocklog_file.string() << "..." << std::endl;
+
+    golos::chain::block_log bg;
+    bg.open(po.blocklog_file);
+
+    std::cout << "Opened. Processing blocks..." << std::endl;
+
+    auto head_block = bg.head();
+    if (!head_block.valid()) {
+        std::cerr << "Blocklog is empty." << std::endl;
+        return -3;
+    }
+    auto head_block_num = head_block->block_num();
+
+    auto start_time = fc::time_point::now();
+
+    uint32_t start_block = std::stol(po.start);
+    uint32_t end_block = std::stol(po.end);
+    end_block = std::min(end_block, head_block_num);
+    uint32_t block_num = start_block;
+
+    uint64_t found = 0;
+    uint64_t old_found = 0;
+
+    progress_bar p;
+
+    uint32_t old_pct = 0;
+
+    auto save_found = [&](const std::string& str) {
+        std::ofstream output;
+        output.open(po.output_file, std::ios_base::app);
+        output << str << std::endl;
+        ++found;
+    };
+
+    special_operation_visitor sov(po, save_found);
+
+    while (block_num <= end_block) {
+        auto block = bg.read_block_by_num(block_num);
+        if (!block) {
+            break;
+        }
+
+        std::string op_name;
+        for (const auto& tx : block->transactions) {
+            for (const auto& op : tx.operations) {
+                op_name = "";
+                op.visit(fc::get_operation_name(op_name));
+                if (po.search_json.size()) {
+                    if (!po.search_op.size() || op_name.find(po.search_op) != std::string::npos) {
+                        auto json_str = fc::json::to_string(op);
+                        if (json_str.find(po.search_json) != std::string::npos) {
+                            save_found(op_name + " " + json_str);
+                        }
+                    }
+                } else if (po.search_text.size()) {
+                    if (!po.search_op.size() || op_name.find(po.search_op) != std::string::npos) {
+                        op.visit(sov);
+                    }
+                } else {
+                    if (op_name.find(po.search_op) != std::string::npos) {
+                        save_found(op_name + " " + fc::json::to_string(op));
+                    } else {
+                        op.visit(sov);
+                    }
+                }
+            }
+        }
+        ++block_num;
+
+        p.update_progress(found, &old_found, *block, end_block, &old_pct);
+    }
+
+    p.complete(*head_block);
+
+    std::cout << "Found: " << found << std::endl;
+
+    auto end_time = fc::time_point::now();
+
+    std::cout << "Elapsed: " << fc::get_approximate_relative_time_string(start_time, end_time, "") << std::endl;
+
+    return 0;
+}

programs/historian/program_options.hpp

@@ -0,0 +1,75 @@
+#pragma once
+
+#include <boost/filesystem/path.hpp>
+#include <boost/program_options.hpp>
+
+namespace bfs = boost::filesystem;
+namespace bpo = boost::program_options;
+using bpo::options_description;
+using bpo::variables_map;
+
+struct program_options {
+    bfs::path blocklog_file;
+    std::string search_op;
+    std::string search_text;
+    std::string search_json;
+    std::string output_file;
+    std::string start;
+    std::string end;
+    int exit_code = 0;
+    bool proceed = false;
+};
+
+program_options get_program_options(int argc, char** argv) {
+    program_options po;
+
+    options_description cli("historian allows you to check if some operation present in block_log.\n"
+        "\n"
+        "It requires block_log file. block_log.index is optional.\n"
+        "\n"
+        "Example of usage:\n"
+        "historian -b /home/block_log -o private_message\n"
+        "\n"
+        "Command line options");
+
+    cli.add_options()
+        ("blocklog,b", bpo::value<bfs::path>(&po.blocklog_file), "Path to block_log.")
+        ("operation,o", bpo::value<std::string>(&po.search_op), "Name of searching operation.")
+        ("text,t", bpo::value<std::string>(&po.search_text), "Search compromised text. Can be used with -o or without.")
+        ("json,j", bpo::value<std::string>(&po.search_json), "Search raw operation JSON. Examples: '\"author\":\"lex\"' or '\\\"follower\\\":\\\"lex' Can be used with -o or without.")
+        ("log,l", bpo::value<std::string>(&po.output_file)->default_value("log.txt"), "Name of output file. Default is log.txt")
+        ("start,s", bpo::value<std::string>(&po.start)->default_value("2"), "Start block number.")
+        ("end,e", bpo::value<std::string>(&po.end)->default_value("4294967295"), "End block number.")
+        ("help,h", "Print this help message and exit.")
+        ;
+
+    variables_map vmap;
+    bpo::store(bpo::parse_command_line(argc, argv, cli), vmap);
+    bpo::notify(vmap);
+    if (vmap.count("help") > 0 || vmap.count("blocklog") == 0 ||
+        (vmap.count("operation") == 0 && vmap.count("text") == 0 && vmap.count("json") == 0)) {
+        cli.print(std::cerr);
+        return po;
+    }
+
+    if (!bfs::exists(po.blocklog_file)) {
+        std::cerr << po.blocklog_file.string() << " not exists." << std::endl;
+        po.exit_code = -1;
+        return po;
+    }
+
+    if (!bfs::is_regular_file(po.blocklog_file)) {
+        std::cerr << po.blocklog_file.string() << " is not a file, it is a directory or something another." << std::endl;
+        po.exit_code = -2;
+        return po;
+    }
+
+    if (bfs::exists(po.output_file)) {
+        std::cerr << "Log file " << po.output_file << " already exists." << std::endl;
+        po.exit_code = -3;
+        return po;
+    }
+
+    po.proceed = true;
+    return po;
+}

programs/historian/progress_bar.hpp

@@ -0,0 +1,56 @@
+#include <indicators/progress_bar.hpp>
+#include <golos/chain/database.hpp>
+
+using namespace golos::chain;
+
+class progress_bar {
+private:
+    indicators::ProgressBar* p = nullptr;
+
+    std::string fmt_found(uint64_t found) {
+        if (found > 10000) {
+            return "     >10000 found ";
+        }
+        return std::string("     ") + std::to_string(found) + " found ";
+    }
+public:
+    progress_bar() {
+        p = new indicators::ProgressBar();
+        p->set_option(indicators::option::BarWidth{40});
+        p->set_option(indicators::option::Fill{"■"});
+        p->set_option(indicators::option::Lead{"■"});
+        p->set_option(indicators::option::ShowPercentage{true});
+        p->set_option(indicators::option::PrefixText{this->fmt_found(0)});
+        p->set_option(indicators::option::PostfixText{"2016-01-01T01:01:01"});
+    }
+
+    ~progress_bar() {
+        delete p;
+    }
+
+    void update_progress(uint64_t found, uint64_t* old_found,
+            const signed_block& block, uint32_t head_block_num,
+            uint32_t* old_pct) {
+        auto block_num = block.block_num();
+
+        auto pct = 100 / std::max(uint32_t(1), head_block_num / block_num);
+        pct = std::min(pct, uint32_t(99));
+        if (block_num % 50000 == 0) {
+            p->set_option(indicators::option::PostfixText{block.timestamp.to_iso_string()});
+        }
+        if (found > *old_found) {
+            p->set_option(indicators::option::PrefixText{this->fmt_found(found)});
+            p->set_progress(pct);
+            *old_found = found;
+            *old_pct = pct;
+        } else if (pct - *old_pct >= 2) {
+            p->set_progress(pct);
+            *old_pct = pct;
+        }
+    }
+
+    void complete(const signed_block& head_block) {
+        p->set_option(indicators::option::PostfixText{head_block.timestamp.to_iso_string()});
+        p->set_progress(100);
+    }
+};