Mempool info tracking

lib/bridge.ex

@@ -9,6 +9,7 @@ defmodule BitcoinStream.Bridge do
 
   alias BitcoinStream.Protocol.Block, as: BitcoinBlock
   alias BitcoinStream.Protocol.Transaction, as: BitcoinTx
+  alias BitcoinStream.Mempool, as: Mempool
 
   def child_spec(port: port) do
     %{
@@ -54,7 +55,7 @@ defmodule BitcoinStream.Bridge do
     # IO.puts("Forwarding transaction to websocket clients")
     Registry.dispatch(Registry.BitcoinStream, "txs", fn(entries) ->
       for {pid, _} <- entries do
-        IO.puts("Forwarding to pid #{inspect pid}")
+        # IO.puts("Forwarding to pid #{inspect pid}")
         case Jason.encode(%{type: "txn", txn: txn}) do
           {:ok, payload} -> Process.send(pid, payload, []);
           {:error, reason} -> IO.puts("Error json encoding transaction: #{reason}");
@@ -63,6 +64,10 @@ defmodule BitcoinStream.Bridge do
     end)
   end
 
+  def incrementMempool() do
+    Mempool.increment(:mempool)
+  end
+
   def sendBlock(block) do
     IO.puts("Forwarding block to websocket clients")
     Registry.dispatch(Registry.BitcoinStream, "txs", fn(entries) ->
@@ -83,7 +88,8 @@ defmodule BitcoinStream.Bridge do
     case BitcoinTx.decode(payload) do
       {:ok, txn} ->
         sendTxn(txn);
-        IO.puts("new tx")
+        incrementMempool();
+        # IO.puts("new tx")
       {:error, reason} -> IO.puts("Tx decoding failed: #{reason}");
     end
 
@@ -100,6 +106,7 @@ defmodule BitcoinStream.Bridge do
     case BitcoinBlock.decode(payload) do
       {:ok, block} ->
         sendBlock(block);
+        Mempool.sync(:mempool);
         IO.puts("new block")
       {:error, reason} -> IO.puts("Block decoding failed: #{reason}");
     end

lib/mempool.ex

@@ -0,0 +1,75 @@
+Application.ensure_all_started(:hackney)
+
+defmodule BitcoinStream.Mempool do
+  @moduledoc """
+  Agent for retrieving and maintaining mempool info (primarily tx count)
+  """
+  use Agent
+
+  @doc """
+  Start a new mempool tracker,
+  connecting to a bitcoin node at RPC `port` for ground truth data
+  """
+  def start_link(opts) do
+    {port, opts} = Keyword.pop(opts, :port);
+    IO.puts("Starting mempool agent on port #{port}");
+    case Agent.start_link(fn -> %{count: 0, port: port} end, opts) do
+      {:ok, pid} ->
+        sync(pid);
+        {:ok, pid}
+
+      result -> result
+    end
+  end
+
+  def getPort(pid) do
+    Agent.get(pid, &Map.get(&1, :port))
+  end
+
+  def set(pid, n) do
+    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> n end))
+  end
+
+  def get(pid) do
+    Agent.get(pid, &Map.get(&1, :count))
+  end
+
+  def increment(pid) do
+    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x + 1 end))
+  end
+
+  def decrement(pid) do
+    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x - 1 end))
+  end
+
+  def add(pid, n) do
+    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x + n end))
+  end
+
+  def subtract(pid, n) do
+    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x - n end))
+  end
+
+  def sync(pid) do
+    port = getPort(pid);
+    user = System.get_env("BITCOIN_RPC_USER");
+    pass = System.get_env("BITCOIN_RPC_PASS");
+    IO.puts("Syncing mempool with bitcoin node on port #{port}");
+    with  {:ok, rpc_request} <- Jason.encode(%{method: "getmempoolinfo", params: [], request_id: 0}),
+          {:ok, 200, _headers, body_ref} <- :hackney.request(:post, "http://localhost:#{port}", [{"content-type", "application/json"}], rpc_request, [basic_auth: {user, pass}]),
+          {:ok, body} <- :hackney.body(body_ref),
+          {:ok, %{"result" => info}} <- Jason.decode(body),
+          %{"size" => pool_size} <- info do
+      IO.puts("Synced pool: size = #{pool_size}");
+      set(pid, pool_size)
+    else
+      {:error, reason} ->
+        IO.puts("Pool sync failed");
+        IO.inspect(reason)
+        :error
+      _ ->
+        IO.puts("Pool sync failed: (unknown reason)");
+        :error
+    end
+  end
+end

lib/server.ex

@@ -3,6 +3,7 @@ defmodule BitcoinStream.Server do
 
   def start(_type, _args) do
     children = [
+      { BitcoinStream.Mempool, [port: 9959, name: :mempool] },
       Plug.Cowboy.child_spec(
         scheme: :http,
         plug: BitcoinStream.Router,

lib/socket_handler.ex

@@ -2,6 +2,7 @@ defmodule BitcoinStream.SocketHandler do
   @behaviour :cowboy_websocket
 
   alias BitcoinStream.Protocol.Block, as: BitcoinBlock
+  alias BitcoinStream.Mempool, as: Mempool
 
   def init(request, state) do
     {:cowboy_websocket, request, state}
@@ -15,8 +16,8 @@ defmodule BitcoinStream.SocketHandler do
   end
 
   def load_block() do
-    with  {:ok, blockData} <- File.read("block.dat"),
-          {:ok, block} <- BitcoinBlock.decode(blockData),
+    with  {:ok, block_data} <- File.read("block.dat"),
+          {:ok, block} <- BitcoinBlock.decode(block_data),
           {:ok, payload} <- Jason.encode(%{type: "block", block: block})
           do
       {:ok, payload}
@@ -30,18 +31,30 @@ defmodule BitcoinStream.SocketHandler do
     end
   end
 
+  def get_mempool_count_msg() do
+    count = Mempool.get(:mempool);
+    IO.puts("Count: #{count}");
+    "{ \"type\": \"count\", \"count\": #{count}}"
+  end
+
   def websocket_handle({:text, msg}, state) do
-    IO.puts("message received: #{msg}");
+    IO.puts("message received: #{msg} | #{inspect self()}");
     case msg do
       "hb" -> {:reply, {:text, msg}, state};
+
       "block" ->
         IO.puts('block request')
         case load_block() do
-          {:ok, blockMsg} ->
-            {:reply, {:text, blockMsg}, state};
+          {:ok, block_msg} ->
+            {:reply, {:text, block_msg}, state};
 
           _ -> {:reply, {:text, "error"}, state}
         end
+
+      "count" ->
+        count = get_mempool_count_msg();
+        {:reply, {:text, count}, state};
+
       _ ->
         {:reply, {:text, "?"}, state}
     end

mix.exs

@@ -4,7 +4,7 @@ defmodule BitcoinStream.MixProject do
   def project do
     [
       app: :bitcoin_stream,
-      version: "1.0.1",
+      version: "1.1.0",
       elixir: "~> 1.11",
       start_permanent: Mix.env() == :prod,
       deps: deps(),
@@ -36,6 +36,7 @@ defmodule BitcoinStream.MixProject do
       # {:bitcoinex, "~> 0.1.0"},
       # {:bitcoinex, git: "git@github.com:mononaut/bitcoinex.git", tag: "master"},
       {:bitcoinex, path: "../bitcoinex", override: true},
+      {:hackney, "~> 1.15"},
       {:cowboy, "~> 2.4"},
       {:plug, "~> 1.7"},
       {:plug_cowboy, "~> 2.0"},

mix.lock

@@ -9,12 +9,15 @@
   "czmq": {:git, "https://github.com/gar1t/erlang-czmq.git", "530c976ee4d0892294ac0468e366ad65c70e6a50", []},
   "decimal": {:hex, :decimal, "1.9.0", "83e8daf59631d632b171faabafb4a9f4242c514b0a06ba3df493951c08f64d07", [:mix], [], "hexpm", "b1f2343568eed6928f3e751cf2dffde95bfaa19dd95d09e8a9ea92ccfd6f7d85"},
   "gettext": {:hex, :gettext, "0.18.2", "7df3ea191bb56c0309c00a783334b288d08a879f53a7014341284635850a6e55", [:mix], [], "hexpm", "f9f537b13d4fdd30f3039d33cb80144c3aa1f8d9698e47d7bcbcc8df93b1f5c5"},
+  "gun": {:hex, :gun, "2.0.0-rc.1", "b87d81dad83f41fa3f2cbf1a923eae44c5ce559a7006728d47888c3e7eb7a6ce", [:make, :rebar3], [{:cowlib, "2.10.1", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "459e7c843c894f69878df60378e7fa4a4b5504a00066c02138d084435c2c7968"},
   "hackney": {:hex, :hackney, "1.17.0", "717ea195fd2f898d9fe9f1ce0afcc2621a41ecfe137fae57e7fe6e9484b9aa99", [:rebar3], [{:certifi, "~>2.5", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "64c22225f1ea8855f584720c0e5b3cd14095703af1c9fbc845ba042811dc671c"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"},
+  "jsonrpc2": {:hex, :jsonrpc2, "2.0.0", "247277a14f8492e537c11e3e7e5580982de46afa2cd97f2d74a5f41805f021bb", [:mix], [{:cowboy, "~> 2.4 or ~> 1.1", [hex: :cowboy, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:jiffy, "~> 1.0 or ~> 0.14", [hex: :jiffy, repo: "hexpm", optional: true]}, {:plug, "~> 1.3", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:poison, "~> 4.0 or ~> 3.0 or ~> 2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ranch, "~> 1.2", [hex: :ranch, repo: "hexpm", optional: true]}, {:shackle, "~> 0.3", [hex: :shackle, repo: "hexpm", optional: true]}], "hexpm", "2b23f7b588a30b47bd0d8b51899b1f91e0358b7f072354090e5225e862d0ff45"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "1.5.0", "203ef35ef3389aae6d361918bf3f952fa17a09e8e43b5aa592b93eba05d0fb8d", [:mix], [], "hexpm", "55a94c0f552249fc1a3dd9cd2d3ab9de9d3c89b559c2bd01121f824834f24746"},
   "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
+  "mint": {:hex, :mint, "1.2.1", "369cc8fecc54afd170e11740aa7efd066709e5ef3b5a2c63f0a47d1542cbd56a", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "053fe2f48c965f31878a16272478d9299fa412bc4df86dee2678986f2e40e018"},
   "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"},
   "plug": {:hex, :plug, "1.11.0", "f17217525597628298998bc3baed9f8ea1fa3f1160aa9871aee6df47a6e4d38e", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2d9c633f0499f9dc5c2fd069161af4e2e7756890b81adcbb2ceaa074e8308876"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.4.1", "779ba386c0915027f22e14a48919a9545714f849505fa15af2631a0d298abf0f", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d72113b6dff7b37a7d9b2a5b68892808e3a9a752f2bf7e503240945385b70507"},