bitshares · jmjatlanta · Aug 16, 2019 · Apr 20, 2019 · Apr 20, 2019 · Jun 24, 2019
diff --git a/.travis.yml b/.travis.yml
@@ -24,19 +24,9 @@ env:
     - CCACHE_MAXSIZE=1Gi
     - CCACHE_SLOPPINESS=include_file_ctime,include_file_mtime,time_macros
 
-script:
- - programs/build_helpers/buildstep -s 3500
- - ccache -s
- - programs/build_helpers/buildstep Prepare 1 "sed -i '/tests/d' libraries/fc/CMakeLists.txt"
- - programs/build_helpers/buildstep cmake 5 "cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_FLAGS=--coverage -DCMAKE_CXX_FLAGS=--coverage -DBoost_USE_STATIC_LIBS=OFF -DCMAKE_CXX_OUTPUT_EXTENSION_REPLACE=ON ."
- - programs/build_helpers/buildstep make.cli_wallet 2200 "programs/build_helpers/make_with_sonar bw-output -j 2 cli_wallet witness_node js_operation_serializer get_dev_key network_mapper"
- - programs/build_helpers/buildstep make.chain_test 1000 "make -j 2 chain_test"
- - programs/build_helpers/buildstep make.cli_test 200 "make -j 2 cli_test"
- - programs/build_helpers/buildstep make.perf_test 120 "make -j 2 performance_test"
- - set -o pipefail
- - programs/build_helpers/buildstep run.chain_test 240 "libraries/fc/tests/run-parallel-tests.sh tests/chain_test"
- - programs/build_helpers/buildstep run.cli_test 60 "libraries/fc/tests/run-parallel-tests.sh tests/cli_test"
- - 'programs/build_helpers/buildstep prepare.sonar 20 "find libraries/[acdenptuw]*/CMakeFiles/*.dir programs/[cdgjsw]*/CMakeFiles/*.dir -type d | while read d; do gcov -o \"\$d\" \"\${d/CMakeFiles*.dir//}\"/*.cpp; done >/dev/null; programs/build_helpers/set_sonar_branch sonar-project.properties" || true'
- - 'programs/build_helpers/buildstep run.sonar 1200 "which sonar-scanner && sonar-scanner" || true'
- - programs/build_helpers/buildstep end 0
- - ccache -s
+jobs:
+  include:
+    - stage: build for cache
+      script: ./programs/build_helpers/build_protocol
+    - stage: build and test
+      script: ./programs/build_helpers/build_and_test
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,6 +1,6 @@
 # Defines BitShares library target.
-project( BitShares )
-cmake_minimum_required( VERSION 3.1 )
+cmake_minimum_required( VERSION 3.2 FATAL_ERROR )
+project( BitShares LANGUAGES CXX C)
 
 set( BLOCKCHAIN_NAME "BitShares" )
 
@@ -11,7 +11,12 @@ set( INSTALLER_APP_ID "68ad7005-8eee-49c9-95ce-9eed97e5b347" )
 
 set( CMAKE_CXX_STANDARD 14 )
 set( CMAKE_CXX_STANDARD_REQUIRED ON )
-set( CMAKE_CXX_EXTENSIONS OFF )
+
+if( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" )
+  set( CMAKE_CXX_EXTENSIONS ON ) # for __int128 support
+else( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" )
+  set( CMAKE_CXX_EXTENSIONS OFF )
+endif( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" )
 
 # http://stackoverflow.com/a/18369825
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
@@ -26,6 +31,87 @@ endif()
 
 list( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules" )
 
+include(CheckCCompilerFlag)
+include(Utils)
+
+# function to help with cUrl
+macro(FIND_CURL)
+   if (NOT WIN32 AND NOT APPLE AND CURL_STATICLIB)
+      set (OLD_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+      set (CMAKE_FIND_LIBRARY_SUFFIXES .a)
+      find_package(CURL REQUIRED)
+      list(APPEND CURL_LIBRARIES ssl crypto)
+      set (CMAKE_FIND_LIBRARY_SUFFIXES ${OLD_SUFFIXES})
+   else (NOT WIN32 AND NOT APPLE AND CURL_STATICLIB)
+      find_package(CURL REQUIRED)
+   endif (NOT WIN32 AND NOT APPLE AND CURL_STATICLIB)
+endmacro()
+
+# Fortify source
+if (CMAKE_COMPILER_IS_GNUCXX)
+	if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+		message ("-- Setting optimizations for clang++")
+		set(CMAKE_CXX_FLAGS_RELEASE "-D_FORTIFY_SOURCE=2 -O3 -DNDEBUG=1")
+		set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-D_FORTIFY_SOURCE=2 -O3 -DNDEBUG=1 -g")
+
+		# check and add data execution prevention
+		message ("-- Enabling data execution prevention")
+		add_linker_flag("-fsanitize=safe-stack")
+
+		# check and add Stack-based buffer overrun detection
+		set(CMAKE_REQUIRED_FLAGS "-fstack-protector")
+		check_c_compiler_flag("" HAVE_STACKPROTECTOR)
+		if(HAVE_STACKPROTECTOR)
+			message ("-- Enabling stack-based buffer overrun detection")
+			add_flag_append(CMAKE_C_FLAGS "-fstack-protector")
+			add_flag_append(CMAKE_CXX_FLAGS "-fstack-protector")
+		endif()
+	else ()
+		message ("-- Setting optimizations for g++")
+		set(CMAKE_CXX_FLAGS_RELEASE "-D_FORTIFY_SOURCE=2 -O3 -DNDEBUG=1")
+		set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-D_FORTIFY_SOURCE=2 -O3 -DNDEBUG=1 -g")
+
+		# check and add data execution prevention
+		set(CMAKE_REQUIRED_FLAGS "-Wl,-znoexecstack")
+		check_c_compiler_flag("" HAVE_NOEXECSTACK)
+		if(HAVE_NOEXECSTACK)
+			message ("-- Enabling data execution prevention")
+			add_linker_flag("-znoexecstack")
+		endif()
+
+		# check and add Stack-based buffer overrun detection
+		set(CMAKE_REQUIRED_FLAGS "-fstack-protector-strong")
+		check_c_compiler_flag("" HAVE_STACKPROTECTOR)
+		if(HAVE_STACKPROTECTOR)
+			message ("-- Enabling stack-based buffer overrun detection")
+			add_flag_append(CMAKE_C_FLAGS "-fstack-protector-strong")
+			add_flag_append(CMAKE_CXX_FLAGS "-fstack-protector-strong")
+		endif()
+
+	endif ()
+endif ()
+
+# check for Data relocation and Protection (RELRO)
+set(CMAKE_REQUIRED_FLAGS "-Wl,-zrelro,-znow")
+check_c_compiler_flag("" HAVE_RELROFULL)
+if(HAVE_RELROFULL)
+	message ("-- Enabling full data relocation and protection")
+	add_linker_flag("-zrelro")
+	add_linker_flag("-znow")
+else()
+	#if full relro is not available, try partial relro
+	set(CMAKE_REQUIRED_FLAGS "-Wl,-zrelro")
+	check_c_compiler_flag("" HAVE_RELROPARTIAL)
+	if(HAVE_RELROPARTIAL)
+		message ("-- Enabling partial data relocation and protection")
+		add_linker_flag("-zrelro")
+	endif()
+endif()
+
+# position independent executetable (PIE)
+# position independent code (PIC)
+add_definitions (-fPIC)
+
 set(CMAKE_EXPORT_COMPILE_COMMANDS "ON")
 set(GRAPHENE_EGENESIS_JSON "${CMAKE_CURRENT_SOURCE_DIR}/libraries/egenesis/genesis.json" )
 

diff --git a/CMakeModules/Utils.cmake b/CMakeModules/Utils.cmake
@@ -0,0 +1,13 @@
+macro(add_flag_append _VAR_NAME _FLAG)
+	set(${_VAR_NAME} "${${_VAR_NAME}} ${_FLAG}")
+endmacro(add_flag_append _VAR_NAME _FLAG)
+
+macro(add_linker_flag _FLAG)
+	#executables
+	add_flag_append(CMAKE_C_LINK_FLAGS "-Wl,${_FLAG}")
+	add_flag_append(CMAKE_CXX_LINK_FLAGS "-Wl,${_FLAG}")
+	#libraries
+	add_flag_append(CMAKE_SHARED_LIBRARY_C_FLAGS "-Wl,${_FLAG}")
+	add_flag_append(CMAKE_SHARED_LIBRARY_CXX_FLAGS "-Wl,${_FLAG}")
+endmacro(add_linker_flag _FLAG)
+
diff --git a/Doxyfile b/Doxyfile
@@ -38,7 +38,7 @@ PROJECT_NAME           = "Bitshares-Core"
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "3.0.1"
+PROJECT_NUMBER         = "3.3.0"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a
@@ -758,7 +758,7 @@ WARN_LOGFILE           =
 # spaces.
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = README.md doc/main.dox libraries/chain libraries/chain/db libraries/app libraries/wallet
+INPUT                  = README.md doc/main.dox libraries/chain libraries/db libraries/app libraries/wallet libraries/protocol libraries/net libraries/plugins libraries/fc libraries/utilities libraries/egenesis
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
@@ -778,7 +778,7 @@ INPUT_ENCODING         = UTF-8
 # *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf,
 # *.qsf, *.as and *.js.
 
-FILE_PATTERNS          =
+FILE_PATTERNS          = *.cpp *.hpp *.hxx
 
 # The RECURSIVE tag can be used to specify whether or not subdirectories should
 # be searched for input files as well.
@@ -793,7 +793,7 @@ RECURSIVE              = YES
 # Note that relative paths are relative to the directory from which doxygen is
 # run.
 
-EXCLUDE                =
+EXCLUDE                = libraries/fc/vendor/editline libraries/fc/vendor/secp256k1-zkp libraries/fc/vendor/websocketpp
 
 # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
 # directories that are symbolic links (a Unix file system feature) are excluded
@@ -820,7 +820,7 @@ EXCLUDE_PATTERNS       =
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories use the pattern */test/*
 
-EXCLUDE_SYMBOLS        =
+EXCLUDE_SYMBOLS        = boost
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
 # that contain example code fragments that are included (see the \include

diff --git a/libraries/README.md b/libraries/README.md
@@ -10,11 +10,12 @@ Code in libraries is the most important part of **bitshares-core** project and i
 Folder | Name | Description | Status
 ---|---|---|---
 [app](app) | Application | Bundles component libraries (chain, network, plugins) into a useful application. Also provides API access. | Active 
-[chain](chain) | Blockchain | Defines all objects, operations and types. This include the consensus protocol, defines the whole blockchain behaviour. | Active 
+[chain](chain) | Blockchain | Blockchain implementation and business logic. Database structure in the form of objects and updates to the blockchain in the form of evaluators are implemented here. | Active 
 [db](db) | Database | Defines the internal database graphene uses. | Active 
 [egenesis](egenesis) | Genesis | Hardcodes the `genesis.json` file into the `witness_node` executable.| Active
 [fc](fc) | Fast-compiling C++ library | https://github.com/bitshares/bitshares-fc | Active 
 [net](net) | Network | The graphene p2p layer. | Active 
 [plugins](plugins) | Plugins | Collection of singleton designed modules used for extending the bitshares-core.  | Active 
+[protocol](protocol) | Protocol | Fundamental structure of the data that will be transmitted on the wire. Operations are defined and basic data integrity checks are done for each.  | Active 
 [utilities](utilities) | Utilities | Common utility calls used in applications or other libraries. | Active 
 [wallet](wallet) | Wallet | Wallet definition for the `cli_wallet` software. | Active
diff --git a/libraries/app/CMakeLists.txt b/libraries/app/CMakeLists.txt
@@ -3,6 +3,7 @@ file(GLOB EGENESIS_HEADERS "../egenesis/include/graphene/app/*.hpp")
 
 add_library( graphene_app 
              api.cpp
+             api_objects.cpp
              application.cpp
              util.cpp
              database_api.cpp
@@ -13,7 +14,10 @@ add_library( graphene_app
            )
 
 # need to link graphene_debug_witness because plugins aren't sufficiently isolated #246
-target_link_libraries( graphene_app graphene_market_history graphene_account_history graphene_grouped_orders graphene_chain fc graphene_db graphene_net graphene_utilities graphene_debug_witness )
+target_link_libraries( graphene_app
+                       graphene_market_history graphene_account_history graphene_elasticsearch graphene_grouped_orders
+                       graphene_api_helper_indexes
+                       graphene_chain fc graphene_db graphene_net graphene_utilities graphene_debug_witness )
 target_include_directories( graphene_app
                             PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include"
                             "${CMAKE_CURRENT_SOURCE_DIR}/../egenesis/include" )

diff --git a/libraries/app/api.cpp b/libraries/app/api.cpp
@@ -36,6 +36,7 @@
 #include <graphene/chain/withdraw_permission_object.hpp>
 #include <graphene/chain/worker_object.hpp>
 
+#include <fc/crypto/base64.hpp>
 #include <fc/crypto/hex.hpp>
 #include <fc/rpc/api_connection.hpp>
 #include <fc/thread/future.hpp>
@@ -180,7 +181,7 @@ namespace graphene { namespace app {
 
     fc::variant network_broadcast_api::broadcast_transaction_synchronous(const precomputable_transaction& trx)
     {
-       fc::promise<fc::variant>::ptr prom( new fc::promise<fc::variant>() );
+       fc::promise<fc::variant>::ptr prom = fc::promise<fc::variant>::create();
        broadcast_transaction_with_callback( [prom]( const fc::variant& v ){
         prom->set_value(v);
        }, trx );
@@ -340,6 +341,18 @@ namespace graphene { namespace app {
              start = node.operation_id;
        } catch(...) { return result; }
 
+       if(_app.is_plugin_enabled("elasticsearch")) {
+          auto es = _app.get_plugin<elasticsearch::elasticsearch_plugin>("elasticsearch");
+          if(es.get()->get_running_mode() != elasticsearch::mode::only_save) {
+             if(!_app.elasticsearch_thread)
+                _app.elasticsearch_thread= std::make_shared<fc::thread>("elasticsearch");
+
+             return _app.elasticsearch_thread->async([&es, &account, &stop, &limit, &start]() {
+                return es->get_account_history(account, stop, limit, start);
+             }, "thread invoke for method " BOOST_PP_STRINGIZE(method_name)).wait();
+          }
+       }
+
        const auto& hist_idx = db.get_index_type<account_transaction_history_index>();
        const auto& by_op_idx = hist_idx.indices().get<by_op>();
        auto index_start = by_op_idx.begin();

diff --git a/libraries/app/api_objects.cpp b/libraries/app/api_objects.cpp
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2017 Cryptonomex, Inc., and contributors.
+ *
+ * The MIT License
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <graphene/app/api_objects.hpp>
+#include <graphene/app/util.hpp>
+
+namespace graphene { namespace app {
+
+market_ticker::market_ticker(const market_ticker_object& mto,
+                             const fc::time_point_sec& now,
+                             const asset_object& asset_base,
+                             const asset_object& asset_quote,
+                             const order_book& orders)
+{
+   time = now;
+   base = asset_base.symbol;
+   quote = asset_quote.symbol;
+   percent_change = "0";
+   lowest_ask = "0";
+   highest_bid = "0";
+
+   fc::uint128_t bv;
+   fc::uint128_t qv;
+   price latest_price = asset( mto.latest_base, mto.base ) / asset( mto.latest_quote, mto.quote );
+   if( mto.base != asset_base.id )
+      latest_price = ~latest_price;
+   latest = price_to_string( latest_price, asset_base, asset_quote );
+   if( mto.last_day_base != 0 && mto.last_day_quote != 0 // has trade data before 24 hours
+       && ( mto.last_day_base != mto.latest_base || mto.last_day_quote != mto.latest_quote ) ) // price changed
+   {
+      price last_day_price = asset( mto.last_day_base, mto.base ) / asset( mto.last_day_quote, mto.quote );
+      if( mto.base != asset_base.id )
+         last_day_price = ~last_day_price;
+      percent_change = price_diff_percent_string( last_day_price, latest_price );
+   }
+   if( asset_base.id == mto.base )
+   {
+      bv = mto.base_volume;
+      qv = mto.quote_volume;
+   }
+   else
+   {
+      bv = mto.quote_volume;
+      qv = mto.base_volume;
+   }
+   base_volume = uint128_amount_to_string( bv, asset_base.precision );
+   quote_volume = uint128_amount_to_string( qv, asset_quote.precision );
+
+   if(!orders.asks.empty())
+      lowest_ask = orders.asks[0].price;
+   if(!orders.bids.empty())
+      highest_bid = orders.bids[0].price;
+}
+
+market_ticker::market_ticker(const fc::time_point_sec& now,
+                             const asset_object& asset_base,
+                             const asset_object& asset_quote)
+{
+   time = now;
+   base = asset_base.symbol;
+   quote = asset_quote.symbol;
+   latest = "0";
+   lowest_ask = "0";
+   highest_bid = "0";
+   percent_change = "0";
+   base_volume = "0";
+   quote_volume = "0";
+}
+
+} } // graphene::app