From d309d066872cf7b49808f6b3011dd46dbe7844d1 Mon Sep 17 00:00:00 2001 From: Jorge and Stefan Date: Tue, 6 Sep 2016 13:40:05 +0200 Subject: [PATCH] added license and readme --- LICENSE | 13 ++ README | 1 - README.org | 145 ++++++++++++++++++++++ api/src/main/scala/api/models/Block.scala | 2 +- bge_ng.org | 130 ------------------- build.sbt | 2 +- todo.org | 3 - 7 files changed, 160 insertions(+), 136 deletions(-) create mode 100644 LICENSE delete mode 100755 README create mode 100644 README.org delete mode 100644 bge_ng.org delete mode 100644 todo.org diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b662210 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright 2016 Jorge Martinez Pizarro and Stefan Richter + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/README b/README deleted file mode 100755 index bdda6ba..0000000 --- a/README +++ /dev/null @@ -1 +0,0 @@ -Probably the first project bringing together Scala and bitcoin, this exploration is growing into a tool that parses the blockchain, stores the relevant data in databases, and gives the user access to powerful analysis tools like the closure of an address with regard to probable co-ownership. diff --git a/README.org b/README.org new file mode 100644 index 0000000..799efbd --- /dev/null +++ b/README.org @@ -0,0 +1,145 @@ +* Bitcoin Graph Explorer + +Probably the first project bringing together Scala and bitcoin, Bitcoin Graph +Explorer (BGE) parses the blockchain, stores the relevant data in databases, and gives +the user access to powerful analysis tools like the closure of an address with +regard to probable co-ownership. We offer a REST API for easy access. + +Please see https://bitcoinprivacy.net/ for a reference project using BGE. There +you can also find a publicly running example of the API under +https://api.bitcoinprivacy.net. + +** REQUIREMENTS + +*** postgres + +BGE uses [[https://www.postgresql.org/][postgres]] for storing its main database. Install and run the service +according to your OS. + +*** bitcoind + +BGE uses a locally running [[https://bitcoincore.org][bitcoin daemon]] in order to read its raw files and +connect to it via bitcoin protocol. Install, run and let sync before starting +BGE. + +*** hardware + +Using LMDB for UTXOs means that this should work ok even on machines with not +too much RAM. BGE is heavily disk bound, though: + +We recommend an SSD with at least three times the space the raw blockchain +needs. At the moment that means about 270G including everything and takes about +24 hours to catch up. + + +** INSTALLATION + +*** automatic + +For easiest installation on linux 64-bit architectures use [[http://nixos.org/nix/][nix]]. Install nix +first (or use nixos), then + +#+BEGIN_SRC sh +git clone https://github.com/bitcoinprivacy/nixpkgs +nix-env -f nixpkgs -i bge +#+END_SRC + +*** manual + +If you don't want nix or need this to run on a different architecure, you can +install bge manually. Install [[http://www.scala-sbt.org/][sbt]] first, then + +#+BEGIN_SRC sh +git clone https://github.com/bitcoinprivacy/Bitcoin-Graph-Explorer bge +cd bge +#+END_SRC + +Change build.sbt in order to import the correct [[https://github.com/deephacks/lmdbjni][LMDB JNI]] architecture library. +E.g., for mac use lmdbjni-osx64 instead of lmdbjni-linux64. Then + +#+BEGIN_SRC sh +sbt assembly publish-local +cd api +sbt assembly +#+END_SRC + +Modify the bge and api/bgeapi scripts such that they point at the correct jars +in target/scala-2.11/bge-assembly-3.0.jar and +api/target/scala-2.11/bgeapi-assembly-1.0.jar. Put these scripts somewhere in your +PATH. + +LMDB JNI needs access to libstdc++.so.6, so set LD_LIBRARY_PATH accordingly +before running bge manually. Nix builds a wrapper script for this automatically. + + +** USAGE + +*** Configuration + +Per default, bge assumes user "postgres" with password "trivial" in psql. Either +configure psql like this or override configuration file reference.conf with +application.conf, changing this line +#+BEGIN_SRC +password = "trivial" +#+END_SRC +with the password you have defined in the psql installation. The configuration +is done via [[https://github.com/typesafehub/config][typesafe config]]. Read the doc for all the possibilities or simply +include -Dconfig.file= in the java -jar call. + +*** Start + +#+BEGIN_SRC +bge start +#+END_SRC +initializes everything and begins populating the database with all blocks +available in the bitcoind raw data at the moment. When it is finished, it +automatically does + +#+BEGIN_SRC +bge resume +#+END_SRC + +to keep up with the incoming new blocks. If you just want to populate the DB, do + +#+BEGIN_SRC +bge populate +#+END_SRC +. + +*** API + +#+BEGIN_SRC sh +bgeapi [port] +#+END_SRC + +starts the api on localhost, default port is 8080. + +These are all available queries: + +#+BEGIN_SRC +GET /blocks/:from/:until +GET /blocks/summary +GET /distribution/:limit +GET /inputs/:tx/:from/:until +GET /inputs/:tx/summary +GET /movements/:ad/:from/:until +GET /movements/:ad/summary +GET /outputs/:tx/:from/:until +GET /outputs/:tx/summary +GET /richlist/addresses/:block_height/:from/:until +GET /richlist/addresses/:block_height/summary +GET /richlist/wallets/:block_height/:from/:until +GET /richlist/wallets/:block_height/summary +GET /stats +GET /stats/history +GET /tx_utxos/:tx/:from/:until +GET /tx_utxos/:tx/summary +GET /txs/:block_height/:from/:until +GET /txs/:block_height/summary +GET /utxos/:ad/:from/:until +GET /utxos/:ad/summary +GET /wallet/:ad/:from/:until +GET /wallet/:ad/summary +#+END_SRC + + diff --git a/api/src/main/scala/api/models/Block.scala b/api/src/main/scala/api/models/Block.scala index 3c010a3..201d4d7 100644 --- a/api/src/main/scala/api/models/Block.scala +++ b/api/src/main/scala/api/models/Block.scala @@ -14,7 +14,7 @@ object Block extends db.BitcoinDB DB withSession { implicit session => val max = stats.map(_.block_height).max.run.getOrElse(0) - val blockslist = for (b<- blockDB.filter(_.block_height < max).sortBy(_.block_height asc).drop(from).take(until-from)) + val blockslist = for (b<- blockDB.filter(_.block_height <= max).sortBy(_.block_height asc).drop(from).take(until-from)) yield (b.hash, b.block_height, b.txs,b.btcs, b.tstamp) blockslist.run map (p => Block(Hash(p._1).toString, p._2,p._3,p._4,p._5)) diff --git a/bge_ng.org b/bge_ng.org deleted file mode 100644 index 037c783..0000000 --- a/bge_ng.org +++ /dev/null @@ -1,130 +0,0 @@ - -* Bitcoin Graph Explorer Next Generation - -Bitcoin Graph Explorer (BGE) is a set of functions that allows for parsing -the blockchain, saving important information about it in databases, -and extracting interesting data from these, e.g. for display on a -website like [[http://bitcoinprivacy.net]]. - -So far we have completed version 1.0. The aim of this document is to -rewrite BGE in literate programming style using org-mode, while -uncluttering the design. This redesign takes its inspiration from [[http://ipaper.googlecode.com/git-history/8070869c59470de474515000e3af74f8958b2161/John-Hughes/The%20Computer%20Journal-1989-Hughes-98-107.pdf][Why -functional programming matters]] by John Hughes. That is, I intend to -replace loops and global variables by a purely functional lazy style -that only has side effects in the final function which writes to a -database. This design should improve code reuse, encourage modularity, -and hopefully lead to shorter and more readable code. Because we are -writing in scala, modules should be implemented by traits which I plan -to use in a style similar to the cake pattern. That is, components -that are expected to have multiple implementations should be -configurable in an "end of the world" object. At the same time, we try -to adhere to the YAGNI principle, so not everything needs to be -generalized. - -** The Block Reader (populate) feature - -This is the part of BGE that reads the blockchain data and stores it -in a database. We have a working implementation of the batch file -reader already in [[file:src/main/scala/actions/BlocksReader.scala][BlocksReader.scala]]. Thus, we start by implementing -an update function. This needs to check for new blocks from bitcoind -and import these into the database. There should be -enough time to do this by simply updating the DB, since at the moment -there are only on the order of 1000 transactions (Tx) per block/10 -minutes. Therefore, we don't need to keep the unspent transaction -outputs (utxo) in memory, which should allow us to run the updater on -a fairly basic machine. - -*** BlockSource - -Because we want to escape loops and make it possible to swap out Block -sources, we use a stream (a lazy list) of Blocks as an abstraction: - -#+BEGIN_SRC scala :tangle src/main/scala/BlockSource.scala - import com.google.bitcoin.core.Block - - trait BlockSource { - def blockStream: Stream[Block] - } -#+END_SRC - -**** TODO BlockSource trait is unnecessary boilerplate, simply use abstract defs - -**** TODO Reading from a file - -Our first implementation reads raw bitcoind files -using some bitcoinj functions that parse these files and yield an -iterator of Blocks. - -#+BEGIN_SRC scala :tangle src/main/scala/BitcoinDRawBlockFile - import com.google.bitcoin.params.MainNetParams - import com.google.bitcoin.utils.BlockFileLoader - import scala.collection.convert.WrapAsScala._ - - trait BitcoinDRawBlockFile extends BlockSource { - private val params = MainNetParams.get - private val loader = new BlockFileLoader(params,BlockFileLoader.getReferenceClientBlockFileList) - - val blockStream = asScalaIterator(loader).toStream - } -#+END_SRC - -**** Asking for Blocks from bitcoind via json-rpc - -- uses our own json-rpc implementation -- can be delayed, as reading from the raw files of a running bitcoind - should work just the same, only slower - -*** Filtering for the longest current blockchain - -We need to provide a set of Hashes that only includes the valid blocks -in the current longest blockchain. Note that we could also include -orphaned blocks for the closure computation. However, this prevents us -from simply matching inputs/outputs that arrive with outputs/inputs in -our DB/memory. So for the moment we decide that we only save the -longest blockchain. Also, our implementation is simplified by assuming -that blocks once saved will never be invalidated. This assumption is -true with high probability when we never include, say, the latest 5 -blocks. So the set we need to get includes the longest blockchain -minus the latest 5 blocks. - -It could be implemented in a number of ways. For now, there is simply -this method in libs/package.scala, which reads a text file. - -#+BEGIN_SRC scala - def getLongestBlockChainHashSet: Set[Hash] = - { - val lines = scala.io.Source.fromFile("blockchain/blocklist.txt").getLines - val hashes = for (line <- lines) yield Hash(line) - - hashes.toSet - } -#+END_SRC - -This might be factored out in a trait and reimplemented as a DB. It -might also be implemented not as a set but a predicate, since we only -need to test for inclusion. - -*** Filtering against the blocks already in the DB - -This is now simply a query against the blocks DB. -TODO: Think about a nice design for the DB traits. - -*** the main "loop" - -We ignore the duplicate transactions, so we have to ensure that they -are already processed. 200k blocks with the FastBlockReader should be -good enough. -#+BEGIN_SRC scala :tangle src/main/scala/SlowBlockReader.scala - import libs._ // for blocks db and longestChain - - trait SlowBlockReader extends BlockSource - { - - - } - - - - - -#+END_SRC diff --git a/build.sbt b/build.sbt index a60b3f0..edba47e 100644 --- a/build.sbt +++ b/build.sbt @@ -20,9 +20,9 @@ libraryDependencies ++= Seq( "org.scalatest" %% "scalatest" % "2.1.5" % "test", "org.deephacks.lmdbjni" % "lmdbjni" % "0.4.6", "org.deephacks.lmdbjni" % "lmdbjni-linux64" % "0.4.6", + // change here for different architectures "com.typesafe.scala-logging" %% "scala-logging" % "3.4.0", "ch.qos.logback" % "logback-classic" % "1.1.7" - // change here for different architectures ) libraryDependencies ~= { _.map(_.exclude("org.slf4j", "slf4j-simple")) } diff --git a/todo.org b/todo.org deleted file mode 100644 index d0d3f9e..0000000 --- a/todo.org +++ /dev/null @@ -1,3 +0,0 @@ -* TODO match out-of-order inputs in slowblockreader -shouldn't really be necessary once we get blocks in correct order by http -