Add historian util

programs/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_subdirectory(build_helpers)
 add_subdirectory(cli_wallet)
 add_subdirectory(golosd)
+add_subdirectory(historian)
 #add_subdirectory( delayed_node )
 add_subdirectory(js_operation_serializer)
 add_subdirectory(meter)

programs/historian/CMakeLists.txt

@@ -0,0 +1,19 @@
+add_executable(historian main.cpp)
+if(UNIX AND NOT APPLE)
+    set(rt_library rt)
+endif()
+
+target_link_libraries(historian
+        PRIVATE golos_chain
+        golos_protocol
+        fc
+        indicators
+        ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS})
+
+install(TARGETS
+        historian
+
+        RUNTIME DESTINATION bin
+        LIBRARY DESTINATION lib
+        ARCHIVE DESTINATION lib
+        )

programs/historian/main.cpp

@@ -0,0 +1,125 @@
+#include <fc/io/json.hpp>
+#include <fc/smart_ref_impl.hpp>
+#include <fc/variant_object.hpp>
+#include <fc/stacktrace.hpp>
+#include <fc/filesystem.hpp>
+#include <boost/algorithm/string/replace.hpp>
+#include <boost/filesystem/path.hpp>
+#include <boost/program_options.hpp>
+
+#include <golos/chain/database.hpp>
+#include <golos/protocol/operation_util_impl.hpp>
+
+#include "./program_options.hpp"
+#include "./progress_bar.hpp"
+#include "./special_operation_visitor.hpp"
+
+namespace bfs = boost::filesystem;
+namespace bpo = boost::program_options;
+using bpo::options_description;
+using bpo::variables_map;
+using namespace golos::chain;
+
+int unsafe_main(int argc, char** argv);
+
+int main(int argc, char** argv) {
+    try {
+        return unsafe_main(argc, argv);
+    } catch (const fc::exception& e) {
+        std::cout << e.to_detail_string() << "\n";
+        return -1;
+    }
+}
+
+int unsafe_main(int argc, char** argv) {
+    auto po = get_program_options(argc, argv);
+
+    if (!po.proceed) {
+        return po.exit_code;
+    }
+
+    fc::install_stacktrace_crash_handler();
+
+    std::cout << "Opening block_log: " << po.blocklog_file.string() << "..." << std::endl;
+
+    golos::chain::block_log bg;
+    bg.open(po.blocklog_file);
+
+    std::cout << "Opened. Processing blocks..." << std::endl;
+
+    auto head_block = bg.head();
+    if (!head_block.valid()) {
+        std::cerr << "Blocklog is empty." << std::endl;
+        return -3;
+    }
+    auto head_block_num = head_block->block_num();
+
+    auto start_time = fc::time_point::now();
+
+    uint32_t start_block = std::stol(po.start);
+    uint32_t end_block = std::stol(po.end);
+    end_block = std::min(end_block, head_block_num);
+    uint32_t block_num = start_block;
+
+    uint64_t found = 0;
+    uint64_t old_found = 0;
+
+    progress_bar p;
+
+    uint32_t old_pct = 0;
+
+    auto save_found = [&](const std::string& str) {
+        std::ofstream output;
+        output.open(po.output_file, std::ios_base::app);
+        output << str << std::endl;
+        ++found;
+    };
+
+    special_operation_visitor sov(po, save_found);
+
+    while (block_num <= end_block) {
+        auto block = bg.read_block_by_num(block_num);
+        if (!block) {
+            break;
+        }
+
+        std::string op_name;
+        for (const auto& tx : block->transactions) {
+            for (const auto& op : tx.operations) {
+                op_name = "";
+                op.visit(fc::get_operation_name(op_name));
+                if (po.search_json.size()) {
+                    if (!po.search_op.size() || op_name.find(po.search_op) != std::string::npos) {
+                        auto json_str = fc::json::to_string(op);
+                        if (json_str.find(po.search_json) != std::string::npos) {
+                            save_found(op_name + " " + json_str);
+                        }
+                    }
+                } else if (po.search_text.size()) {
+                    if (!po.search_op.size() || op_name.find(po.search_op) != std::string::npos) {
+                        op.visit(sov);
+                    }
+                } else {
+                    if (op_name.find(po.search_op) != std::string::npos) {
+                        save_found(op_name + " " + fc::json::to_string(op));
+                    } else {
+                        op.visit(sov);
+                    }
+                }
+            }
+        }
+        ++block_num;
+
+        p.update_progress(found, &old_found, *block, end_block, &old_pct);
+    }
+
+    p.complete(*head_block);
+
+    std::cout << "Found: " << found << std::endl;
+
+    auto end_time = fc::time_point::now();
+
+    std::cout << "Elapsed: " << fc::get_approximate_relative_time_string(start_time, end_time, "") << std::endl;
+
+    return 0;
+}

programs/historian/program_options.hpp

@@ -0,0 +1,75 @@
+#pragma once
+
+#include <boost/filesystem/path.hpp>
+#include <boost/program_options.hpp>
+
+namespace bfs = boost::filesystem;
+namespace bpo = boost::program_options;
+using bpo::options_description;
+using bpo::variables_map;
+
+struct program_options {
+    bfs::path blocklog_file;
+    std::string search_op;
+    std::string search_text;
+    std::string search_json;
+    std::string output_file;
+    std::string start;
+    std::string end;
+    int exit_code = 0;
+    bool proceed = false;
+};
+
+program_options get_program_options(int argc, char** argv) {
+    program_options po;
+
+    options_description cli("historian allows you to check if some operation present in block_log.\n"
+        "\n"
+        "It requires block_log file. block_log.index is optional.\n"
+        "\n"
+        "Example of usage:\n"
+        "historian -b /home/block_log -o private_message\n"
+        "\n"
+        "Command line options");
+
+    cli.add_options()
+        ("blocklog,b", bpo::value<bfs::path>(&po.blocklog_file), "Path to block_log.")
+        ("operation,o", bpo::value<std::string>(&po.search_op), "Name of searching operation.")
+        ("text,t", bpo::value<std::string>(&po.search_text), "Search compromised text. Can be used with -o or without.")
+        ("json,j", bpo::value<std::string>(&po.search_json), "Search raw operation JSON. Examples: '\"author\":\"lex\"' or '\\\"follower\\\":\\\"lex' Can be used with -o or without.")
+        ("log,l", bpo::value<std::string>(&po.output_file)->default_value("log.txt"), "Name of output file. Default is log.txt")
+        ("start,s", bpo::value<std::string>(&po.start)->default_value("2"), "Start block number.")
+        ("end,e", bpo::value<std::string>(&po.end)->default_value("4294967295"), "End block number.")
+        ("help,h", "Print this help message and exit.")
+        ;
+
+    variables_map vmap;
+    bpo::store(bpo::parse_command_line(argc, argv, cli), vmap);
+    bpo::notify(vmap);
+    if (vmap.count("help") > 0 || vmap.count("blocklog") == 0 ||
+        (vmap.count("operation") == 0 && vmap.count("text") == 0 && vmap.count("json") == 0)) {
+        cli.print(std::cerr);
+        return po;
+    }
+
+    if (!bfs::exists(po.blocklog_file)) {
+        std::cerr << po.blocklog_file.string() << " not exists." << std::endl;
+        po.exit_code = -1;
+        return po;
+    }
+
+    if (!bfs::is_regular_file(po.blocklog_file)) {
+        std::cerr << po.blocklog_file.string() << " is not a file, it is a directory or something another." << std::endl;
+        po.exit_code = -2;
+        return po;
+    }
+
+    if (bfs::exists(po.output_file)) {
+        std::cerr << "Log file " << po.output_file << " already exists." << std::endl;
+        po.exit_code = -3;
+        return po;
+    }
+
+    po.proceed = true;
+    return po;
+}

programs/historian/progress_bar.hpp

@@ -0,0 +1,56 @@
+#include <indicators/progress_bar.hpp>
+#include <golos/chain/database.hpp>
+
+using namespace golos::chain;
+
+class progress_bar {
+private:
+    indicators::ProgressBar* p = nullptr;
+
+    std::string fmt_found(uint64_t found) {
+        if (found > 10000) {
+            return "     >10000 found ";
+        }
+        return std::string("     ") + std::to_string(found) + " found ";
+    }
+public:
+    progress_bar() {
+        p = new indicators::ProgressBar();
+        p->set_option(indicators::option::BarWidth{40});
+        p->set_option(indicators::option::Fill{"■"});
+        p->set_option(indicators::option::Lead{"■"});
+        p->set_option(indicators::option::ShowPercentage{true});
+        p->set_option(indicators::option::PrefixText{this->fmt_found(0)});
+        p->set_option(indicators::option::PostfixText{"2016-01-01T01:01:01"});
+    }
+
+    ~progress_bar() {
+        delete p;
+    }
+
+    void update_progress(uint64_t found, uint64_t* old_found,
+            const signed_block& block, uint32_t head_block_num,
+            uint32_t* old_pct) {
+        auto block_num = block.block_num();
+
+        auto pct = 100 / std::max(uint32_t(1), head_block_num / block_num);
+        pct = std::min(pct, uint32_t(99));
+        if (block_num % 50000 == 0) {
+            p->set_option(indicators::option::PostfixText{block.timestamp.to_iso_string()});
+        }
+        if (found > *old_found) {
+            p->set_option(indicators::option::PrefixText{this->fmt_found(found)});
+            p->set_progress(pct);
+            *old_found = found;
+            *old_pct = pct;
+        } else if (pct - *old_pct >= 2) {
+            p->set_progress(pct);
+            *old_pct = pct;
+        }
+    }
+
+    void complete(const signed_block& head_block) {
+        p->set_option(indicators::option::PostfixText{head_block.timestamp.to_iso_string()});
+        p->set_progress(100);
+    }
+};

programs/historian/special_operation_visitor.hpp

@@ -0,0 +1,90 @@
+#pragma once
+
+#include <functional>
+
+#include <golos/chain/database.hpp>
+
+#include "./program_options.hpp"
+
+class special_operation_visitor {
+private:
+    const program_options& po;
+    std::function<void(const std::string&)> save_found;
+public:
+    using result_type = void;
+    special_operation_visitor(const program_options& _po,
+            std::function<void(const std::string&)> _save_found)
+            : po(_po), save_found(_save_found) {
+    }
+
+    void operator()(const golos::protocol::custom_json_operation& op) const {
+        if (po.search_text.size()) {
+            return;
+        }
+        fc::variant v = fc::json::from_string(op.json);
+        if (!v.is_array() || !v.size() || v.get_array()[0].is_array()) return;
+        auto op_type = v.get_array()[0].get_string();
+        if (op_type.find(po.search_op) != std::string::npos) {
+            auto& vo = v.get_array()[1].get_object();
+            save_found(op_type + " " + fc::json::to_string(vo));
+        }
+    }
+
+    bool check_for_compomise(const golos::protocol::operation& op, const std::string& field) const {
+        if (!po.search_text.size()) {
+            return true;
+        }
+        if (field.find(po.search_text) != std::string::npos) {
+            save_found(fc::json::to_string(op));
+            return true;
+        }
+        return false;
+    }
+
+    void operator()(const golos::protocol::comment_operation& op) const {
+        if (check_for_compomise(op, op.title)) return;
+        if (check_for_compomise(op, op.body)) return;
+        if (check_for_compomise(op, op.json_metadata)) return;
+    }
+
+    void operator()(const golos::protocol::transfer_operation& op) const {
+        if (check_for_compomise(op, op.memo)) return;
+    }
+
+    void operator()(const golos::protocol::transfer_to_savings_operation& op) const {
+        if (check_for_compomise(op, op.memo)) return;
+    }
+
+    void operator()(const golos::protocol::transfer_from_savings_operation& op) const {
+        if (check_for_compomise(op, op.memo)) return;
+    }
+
+    void operator()(const golos::protocol::donate_operation& op) const {
+        if (!op.memo.comment) return;
+        if (check_for_compomise(op, *op.memo.comment)) return;
+    }
+
+    void operator()(const golos::protocol::transfer_to_tip_operation& op) const {
+        if (check_for_compomise(op, op.memo)) return;
+    }
+
+    void operator()(const golos::protocol::transfer_from_tip_operation& op) const {
+        if (check_for_compomise(op, op.memo)) return;
+    }
+
+    void operator()(const golos::protocol::override_transfer_operation& op) const {
+        if (check_for_compomise(op, op.memo)) return;
+    }
+
+    void operator()(const golos::protocol::invite_donate_operation& op) const {
+        if (check_for_compomise(op, op.memo)) return;
+    }
+
+    void operator()(const golos::protocol::invite_transfer_operation& op) const {
+        if (check_for_compomise(op, op.memo)) return;
+    }
+
+    template<typename Op>
+    void operator()(Op&&) const {
+    } /// ignore all other ops
+};

thirdparty/CMakeLists.txt

@@ -1,3 +1,4 @@
 add_subdirectory(appbase)
 add_subdirectory(fc)
-add_subdirectory(chainbase)
+add_subdirectory(chainbase)
+add_subdirectory(indicators)

thirdparty/indicators/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_library(indicators INTERFACE)
+target_include_directories(indicators INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/include" )