Merge pull request #26 from bitfeed-project/fees

API server refactor & fee features

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bitfeed-client",
-  "version": "2.1.5",
+  "version": "2.2.0",
   "scripts": {
     "build": "rollup -c",
     "dev": "rollup -c -w",

client/src/components/BlockInfo.svelte

@@ -5,7 +5,7 @@
   import { createEventDispatcher } from 'svelte'
   import Icon from '../components/Icon.svelte'
   import closeIcon from '../assets/icon/cil-x-circle.svg'
-  import { shortBtcFormat, longBtcFormat, timeFormat, integerFormat } from '../utils/format.js'
+  import { shortBtcFormat, longBtcFormat, timeFormat, numberFormat } from '../utils/format.js'
   import { exchangeRates, settings } from '../stores.js'
   import { formatCurrency } from '../utils/fx.js'
 
@@ -57,16 +57,22 @@
 
   function formatBytes (bytes) {
     if (bytes) {
-      return `${integerFormat.format(bytes)} bytes`
+      return `${numberFormat.format(bytes)} bytes`
     } else return `unknown size`
   }
 
   function formatCount (n) {
     if (n) {
-      return integerFormat.format(n)
+      return numberFormat.format(n)
     } else return '0'
   }
 
+  function formatFee (n) {
+    if (n) {
+      return numberFormat.format(n.toFixed(2))
+    } else return  '0'
+  }
+
   function hideBlock () {
     analytics.trackEvent('viz', 'block', 'hide')
     dispatch('hideBlock')
@@ -108,11 +114,36 @@
       font-size: 4.4vw;
     }
 
+    .compact {
+      display: none;
+    }
+
+    @media (max-aspect-ratio: 1/1) and (max-height: 760px) {
+      .compact {
+        display: block;
+      }
+      .full-size {
+        display: none;
+      }
+    }
+    @media (aspect-ratio: 1/1) and (max-height: 760px) {
+      .compact {
+        display: none;
+      }
+      .full-size {
+        display: block;
+      }
+    }
+
     .data-row {
       display: flex;
       flex-direction: row;
       flex-wrap: nowrap;
       justify-content: space-between;
+
+      &.spacer {
+        display: none;
+      }
     }
 
     .data-field {
@@ -157,6 +188,10 @@
         flex-direction: column;
         justify-content: flex-start;
         align-items: flex-end;
+
+        &.spacer {
+          display: flex;
+        }
       }
 
       .data-field {
@@ -196,17 +231,46 @@
   {#if block != null && visible }
     <div class="block-info" out:fly="{{ y: -50, duration: 2000, easing: linear }}" in:fly="{{ y: (restoring ? -50 : 50), duration: (restoring ? 500 : 1000), easing: linear, delay: (restoring ? 0 : newBlockDelay) }}">
         <!-- <span class="data-field">Hash: { block.id }</span> -->
-        <div class="data-row">
+        <div class="full-size">
-          <span class="data-field title-field" title="{block.miner_sig}"><b>Latest Block: </b>{ integerFormat.format(block.height) }</span>
+          <div class="data-row">
-          <button class="data-field close-button" on:click={hideBlock}><Icon icon={closeIcon} color="var(--palette-x)" /></button>
+            <span class="data-field title-field" title="{block.miner_sig}"><b>Latest Block: </b>{ numberFormat.format(block.height) }</span>
+            <button class="data-field close-button" on:click={hideBlock}><Icon icon={closeIcon} color="var(--palette-x)" /></button>
+          </div>
+          <div class="data-row">
+            <span class="data-field">Mined { formatTime(block.time) }</span>
+            <span class="data-field">{ formattedBlockValue }</span>
+          </div>
+          <div class="data-row">
+            <span class="data-field">{ formatBytes(block.bytes) }</span>
+            <span class="data-field">{ formatCount(block.txnCount) } transactions</span>
+          </div>
+          <div class="data-row spacer">&nbsp;</div>
+          <div class="data-row">
+            <span class="data-field">Avg Fee Rate</span>
+            {#if block.fees != null}
+              <span class="data-field">{ formatFee(block.avgFeerate) } sats/vbyte</span>
+            {:else}
+              <span class="data-field">unavailable</span>
+            {/if}
+          </div>
         </div>
-        <div class="data-row">
+        <div class="compact">
-          <span class="data-field">Mined { formatTime(block.time) }</span>
+          <div class="data-row">
-          <span class="data-field">{ formattedBlockValue }</span>
+            <span class="data-field title-field" title="{block.miner_sig}"><b>Latest Block: </b>{ numberFormat.format(block.height) }</span>
-        </div>
+            <button class="data-field close-button" on:click={hideBlock}><Icon icon={closeIcon} color="var(--palette-x)" /></button>
-        <div class="data-row">
+          </div>
-          <span class="data-field">{ formatBytes(block.bytes) }</span>
+          <div class="data-row">
-          <span class="data-field">{ formatCount(block.txnCount) } transactions</span>
+            <span class="data-field">Mined { formatTime(block.time) }</span>
+            <span class="data-field">{ formattedBlockValue }</span>
+          </div>
+          <div class="data-row">
+            <span class="data-field">{ formatCount(block.txnCount) } transactions</span>
+            {#if block.fees != null}
+              <span class="data-field">{ formatFee(block.avgFeerate) } sats/vb</span>
+            {:else}
+              <span class="data-field">{ formatBytes(block.bytes) }</span>
+            {/if}
+          </div>
         </div>
     </div>
     <button class="close-button standalone" on:click={hideBlock} out:fly="{{ y: -50, duration: 2000, easing: linear }}" in:fly="{{ y: (restoring ? -50 : 50), duration: (restoring ? 500 : 1000), easing: linear, delay: (restoring ? 0 : newBlockDelay) }}" >

client/src/components/MempoolLegend.svelte

@@ -1,10 +1,16 @@
 <script>
 import { onMount } from 'svelte'
-import { settings } from '../stores.js'
+import { settings, colorMode } from '../stores.js'
-import { integerFormat } from '../utils/format.js'
+import { numberFormat } from '../utils/format.js'
 import { logTxSize, byteTxSize } from '../utils/misc.js'
 import { interpolateHcl } from 'd3-interpolate'
-import { color } from 'd3-color'
+import { color, hcl } from 'd3-color'
+import { hlToHex, orange, teal, green, purple } from '../utils/color.js'
+
+const orangeHex = hlToHex(orange)
+const tealHex = hlToHex(teal)
+const greenHex = hlToHex(green)
+const purpleHex = hlToHex(purple)
 
 const sizes = [
   { value: 1000000, vbytes: 1*256, size: null },
@@ -18,8 +24,15 @@ let unitWidth
 let unitPadding
 let gridSize
 let colorScale
+let feeColorScale
 const colorScaleWidth = 200
 
+let squareColor = orangeHex
+$: {
+  if ($colorMode === 'age') squareColor = orangeHex
+  else squareColor = purpleHex
+}
+
 function resize () {
   unitWidth = Math.floor(Math.max(4, (window.innerWidth - 20) / 250))
   unitPadding = Math.floor(Math.max(1, (window.innerWidth - 20) / 1000))
@@ -29,7 +42,8 @@ resize()
 
 onMount(() => {
   resize()
-  colorScale = generateColorScale('#f7941d', '#00ffc6')
+  colorScale = generateColorScale(orangeHex, tealHex)
+  feeColorScale = generateColorScale(tealHex, purpleHex)
 })
 
 function calcSizes (gridSize, unitWidth, unitPadding) {
@@ -52,7 +66,7 @@ function calcSize ({ vbytes, value }) {
 }
 
 function formatBytes (bytes) {
-  const str = integerFormat.format(bytes) + ' vbytes'
+  const str = numberFormat.format(bytes) + ' vbytes'
   const padded = str.padStart(13, ' ')
   return padded
 }
@@ -98,6 +112,7 @@ function generateColorScale (colorA, colorB) {
     .size-legend {
       display: table;
       margin: auto;
+      margin-bottom: 5px;
 
       .size-row {
         display: table-row;
@@ -166,23 +181,33 @@ function generateColorScale (colorA, colorB) {
     {#if $settings.vbytes }
       {#each sizes as { size, vbytes } }
         <div class="size-row">
-          <span class="square-container"><div class="square" style="width: {size}px; height: {size}px" /></span>
+          <span class="square-container"><div class="square" style="width: {size}px; height: {size}px; background: {squareColor};" /></span>
           <span class="value"><span class="part left">&lt;</span><span class="part center">&nbsp;</span><span class="part right">{ formatBytes(vbytes) }</span></span>
         </div>
       {/each}
     {:else}
       {#each sizes as { size, value } }
         <div class="size-row">
-          <span class="square-container"><div class="square" style="width: {size}px; height: {size}px" /></span>
+          <span class="square-container"><div class="square" style="width: {size}px; height: {size}px; background: {squareColor}" /></span>
           <span class="value"><span class="part left">&lt;</span> <span class="part center">&#8383;</span><span class="part right">{ formatValue(value) }</span></span>
         </div>
       {/each}
     {/if}
   </div>
-  <h3 class="subheading">Age in seconds</h3>
+  {#if $colorMode === 'age'}
+    <h3 class="subheading">Age in seconds</h3>
+  {:else}
+    <h3 class="subheading">Fee rate in sats/vbyte</h3>
+  {/if}
   <div class="color-legend">
-    <span class="value left">0</span>
+    {#if $colorMode === 'age'}
-    <img src={colorScale} alt="" class="color-scale-img" width="200" height="15">
+      <span class="value left">0</span>
-    <span class="value right">60+</span>
+      <img src={colorScale} alt="" class="color-scale-img" width="200" height="15">
+      <span class="value right">60+</span>
+    {:else}
+      <span class="value left">1</span>
+      <img src={feeColorScale} alt="" class="color-scale-img" width="200" height="15">
+      <span class="value right">64+</span>
+    {/if}
   </div>
 </div>

client/src/components/SearchTab.svelte

@@ -10,16 +10,10 @@ import AddressIcon from '../assets/icon/cil-wallet.svg'
 import TxIcon from '../assets/icon/cil-arrow-circle-right.svg'
 import { matchQuery } from '../utils/search.js'
 import { highlight, newHighlightQuery, highlightingFull } from '../stores.js'
-import { hcl } from 'd3-color'
+import { hlToHex, highlightA, highlightB, highlightC, highlightD, highlightE } from '../utils/color.js'
 
-const highlightColors = [
+const highlightColors = [highlightA, highlightB, highlightC, highlightD, highlightE]
-  { h: 0.03, l: 0.35 },
+const highlightHexColors = highlightColors.map(c => hlToHex(c))
-  { h: 0.40, l: 0.35 },
-  { h: 0.65, l: 0.35 },
-  { h: 0.85, l: 0.35 },
-  { h: 0.12, l: 0.35 },
-]
-const highlightHexColors = highlightColors.map(c => hclToHex(c))
 const usedColors = [false, false, false, false, false]
 
 const queryIcons = {
@@ -36,6 +30,7 @@ export let tab
 let query
 let matchedQuery
 let queryAddress
+let queryColorIndex
 let queryColor
 let queryColorHex
 let watchlist = []
@@ -48,8 +43,9 @@ $: {
   if ($newHighlightQuery) {
     matchedQuery = matchQuery($newHighlightQuery)
     if (matchedQuery) {
-      matchedQuery.color = queryColor
+      matchedQuery.colorIndex = queryColorIndex
-      matchedQuery.colorHex = queryColorHex
+      matchedQuery.color = highlightColors[queryColorIndex]
+      matchedQuery.colorHex = highlightHexColors[queryColorIndex]
       add()
       query = null
     }
@@ -66,8 +62,9 @@ $: {
   if (query) {
     matchedQuery = matchQuery(query.trim())
     if (matchedQuery) {
-      matchedQuery.color = queryColor
+      matchedQuery.colorIndex = queryColorIndex
-      matchedQuery.colorHex = queryColorHex
+      matchedQuery.color = highlightColors[queryColorIndex]
+      matchedQuery.colorHex = highlightHexColors[queryColorIndex]
     }
   } else matchedQuery = null
 }
@@ -81,9 +78,20 @@ function init () {
     try {
       watchlist = JSON.parse(val)
       watchlist.forEach(q => {
-        const i = highlightHexColors.findIndex(c => c === q.colorHex)
+        if (q.colorIndex) {
-        if (i >= 0) usedColors[i] = q.colorHex
+          usedColors[q.colorIndex] = true
-        else console.log('unknown color')
+          q.color = highlightColors[q.colorIndex]
+          q.colorHex = highlightHexColors[q.colorIndex]
+        }
+      })
+      watchlist.forEach(q => {
+        if (!q.colorIndex){
+          const nextIndex = usedColors.findIndex(used => !used)
+          usedColors[nextIndex] = true
+          q.colorIndex = nextIndex
+          q.color = highlightColors[nextIndex]
+          q.colorHex = highlightHexColors[nextIndex]
+        }
       })
     } catch (e) {
       console.log('failed to parse cached highlight queries')
@@ -94,18 +102,14 @@ function init () {
 function setNextColor () {
   const nextIndex = usedColors.findIndex(used => !used)
   if (nextIndex >= 0) {
+    queryColorIndex = nextIndex
     queryColor = highlightColors[nextIndex]
     queryColorHex = highlightHexColors[nextIndex]
-    usedColors[nextIndex] = queryColorHex
+    usedColors[nextIndex] = true
   }
 }
-function clearUsedColor (hex) {
+function clearUsedColor (colorIndex) {
-  const clearIndex = usedColors.findIndex(used => used === hex)
+  usedColors[colorIndex] = false
-  usedColors[clearIndex] = false
-}
-
-function hclToHex (color) {
-  return hcl(color.h * 360, 78.225, color.l * 150).hex()
 }
 
 async function add () {
@@ -127,7 +131,7 @@ async function remove (index) {
   const wasFull = $highlightingFull
   const removed = watchlist.splice(index,1)
   if (removed.length) {
-    clearUsedColor(removed[0].colorHex)
+    clearUsedColor(removed[0].colorIndex)
     watchlist = watchlist
     if (tab) {
       await tick()
@@ -249,7 +253,7 @@ function searchSubmit (e) {
     </div>
   </div>
   <div class="watchlist">
-    {#each watchlist as watched, index (watched.colorHex)}
+    {#each watchlist as watched, index (watched.colorIndex)}
       <div
         class="watched"
 				transition:fade={{ duration: 200 }}

client/src/components/Settings.svelte

@@ -46,6 +46,13 @@ let settingConfig = {
     trueLabel: 'vbytes',
     valueType: 'bool'
   },
+  colorByFee: {
+    label: 'Color by',
+    type: 'pill',
+    falseLabel: 'age',
+    trueLabel: 'fee rate',
+    valueType: 'bool'
+  }
 }
 $: {
   if ($nativeAntialias) {

client/src/components/TxInfo.svelte

@@ -1,7 +1,7 @@
 <script>
 import Icon from './Icon.svelte'
 import BookmarkIcon from '../assets/icon/cil-bookmark.svg'
-import { longBtcFormat, integerFormat } from '../utils/format.js'
+import { longBtcFormat, numberFormat, feeRateFormat } from '../utils/format.js'
 import { exchangeRates, settings, sidebarToggle, newHighlightQuery, highlightingFull } from '../stores.js'
 import { formatCurrency } from '../utils/fx.js'
 
@@ -84,6 +84,24 @@ function highlight () {
       word-break: break-all;
     }
 
+    .inputs {
+      display: flex;
+      flex-direction: row;
+      justify-content: space-between;
+      align-items: baseline;
+
+      span {
+        width: 0;
+        flex-shrink: 1;
+        flex-grow: 1;
+        &.arrow {
+          min-width: 1.5em;
+          flex-shrink: 1;
+          flex-grow: 0;
+        }
+      }
+    }
+
     &:hover {
       .hash {
         white-space: pre-wrap;
@@ -119,12 +137,21 @@ function highlight () {
   <p class="field hash">
     TxID: { tx.id }
   </p>
-  {#if tx.inputs && !tx.coinbase }<p class="field inputs">{ tx.inputs.length } input{#if tx.inputs.length != 1}s{/if}</p>
+  {#if tx.inputs && tx.outputs && !tx.coinbase }
+    <p class="field inputs">
+      <span>{ tx.inputs.length } input{#if tx.inputs.length != 1}s{/if}</span>
+      <span class="arrow"> &rarr; </span>
+      <span>{ tx.outputs.length } output{#if tx.outputs.length != 1}s{/if}</span>
+    </p>
   {:else if tx.coinbase }
     <p class="field coinbase">Coinbase: { tx.coinbase.sigAscii }</p>
+    <p class="field inputs">{ tx.outputs.length } output{#if tx.outputs.length != 1}s{/if}</p>
+  {/if}
+  <p class="field vbytes">Size: { numberFormat.format(tx.vbytes) } vbytes</p>
+  {#if !tx.coinbase && tx.fee != null }
+    <p class="field feerate">Fee rate: { numberFormat.format(tx.feerate.toFixed(2)) } sats/vbyte</p>
+    <p class="field fee">Fee: { numberFormat.format(tx.fee) } sats</p>
   {/if}
-  {#if tx.outputs }<p class="field outputs">{ tx.outputs.length } output{#if tx.outputs.length != 1}s{/if}</p>{/if}
-  <p class="field vbytes">{ integerFormat.format(tx.vbytes) } vbytes</p>
   <p class="field value">
     Total value: { formatBTC(tx.value) }
     {#if formattedLocalValue != null }

client/src/components/TxViz.svelte

@@ -3,7 +3,7 @@
   import TxController from '../controllers/TxController.js'
   import TxRender from './TxRender.svelte'
   import getTxStream from '../controllers/TxStream.js'
-  import { settings, overlay, serverConnected, serverDelay, txQueueLength, txCount, mempoolCount, mempoolScreenHeight, frameRate, avgFrameRate, blockVisible, currentBlock, selectedTx, blockAreaSize, devEvents, devSettings } from '../stores.js'
+  import { settings, overlay, serverConnected, serverDelay, txCount, mempoolCount, mempoolScreenHeight, frameRate, avgFrameRate, blockVisible, currentBlock, selectedTx, blockAreaSize, devEvents, devSettings } from '../stores.js'
   import BitcoinBlock from '../models/BitcoinBlock.js'
   import BlockInfo from '../components/BlockInfo.svelte'
   import TxInfo from '../components/TxInfo.svelte'
@@ -12,7 +12,7 @@
   import DonationOverlay from '../components/DonationOverlay.svelte'
   import SupportersOverlay from '../components/SupportersOverlay.svelte'
   import Alerts from '../components/alert/Alerts.svelte'
-  import { integerFormat } from '../utils/format.js'
+  import { numberFormat } from '../utils/format.js'
   import { exchangeRates, lastBlockId, haveSupporters, sidebarToggle } from '../stores.js'
   import { formatCurrency } from '../utils/fx.js'
   import config from '../config.js'
@@ -54,14 +54,16 @@
       txStream.subscribe('tx', tx => {
         txController.addTx(tx)
       })
+      txStream.subscribe('drop_tx', txid => {
+        txController.dropTx(txid)
+      })
     }
     if (!config.noBlockFeed) {
-      txStream.subscribe('block', block => {
+      txStream.subscribe('block', ({block, realtime}) => {
         if (block) {
-          const added = txController.addBlock(block)
+          const added = txController.addBlock(block, realtime)
           if (added && added.id) $lastBlockId = added.id
         }
-        txStream.sendMempoolRequest()
       })
     }
     if (!config.noTxFeed || !config.noBlockFeed) {
@@ -398,7 +400,7 @@
 
     <div class="mempool-height" style="bottom: calc({$mempoolScreenHeight + 20}px)">
       <div class="height-bar" />
-      <span class="mempool-count">Mempool: { integerFormat.format($mempoolCount) } unconfirmed</span>
+      <span class="mempool-count">Mempool: { numberFormat.format(Math.round($mempoolCount)) } unconfirmed</span>
     </div>
 
     <div class="block-area-wrapper">

client/src/components/alert/GenericAlertv2.svelte

@@ -10,7 +10,7 @@ function chooseImgs () {
   const randomIndex = Math.floor(Math.random() * imgs.length)
   for (let i = 0; i < Math.min(3, imgs.length); i++) {
     const randomImg = imgs[(randomIndex + i) % imgs.length]
-    displayImgs.push(randomImg)
+    if (randomImg && randomImg.img) displayImgs.push(randomImg)
   }
 }
 

client/src/config.js

@@ -31,7 +31,7 @@ export default {
   noTxFeed: false,
   noBlockFeed: false,
   // Minimum delay in ms before newly recieved transactions enter the visualization
-  txDelay: 10000,
+  txDelay: 3000,
   donationsEnabled: true,
   // Enables the message bar
   messagesEnabled: true,

client/src/controllers/TxController.js

@@ -5,7 +5,7 @@ import BitcoinTx from '../models/BitcoinTx.js'
 import BitcoinBlock from '../models/BitcoinBlock.js'
 import TxSprite from '../models/TxSprite.js'
 import { FastVertexArray } from '../utils/memory.js'
-import { txQueueLength, txCount, mempoolCount, mempoolScreenHeight, blockVisible, currentBlock, selectedTx, blockAreaSize, highlight } from '../stores.js'
+import { txCount, mempoolCount, mempoolScreenHeight, blockVisible, currentBlock, selectedTx, blockAreaSize, highlight, colorMode } from '../stores.js'
 import config from "../config.js"
 
 export default class TxController {
@@ -26,17 +26,16 @@ export default class TxController {
     this.selectedTx = null
     this.selectionLocked = false
 
-    this.pendingTxs = []
+    this.lastTxTime = 0
-    this.pendingMap = {}
+    this.txDelay = 0
-    this.queueTimeout = null
-    this.queueLength = 0
 
     highlight.subscribe(criteria => {
       this.highlightCriteria = criteria
       this.applyHighlighting()
     })
-
+    colorMode.subscribe(mode => {
-    this.scheduleQueue(1000)
+      this.setColorMode(mode)
+    })
   }
 
   getVertexData () {
@@ -65,6 +64,14 @@ export default class TxController {
     this.redoLayout({ width, height })
   }
 
+  setColorMode (mode) {
+    this.colorMode = mode
+    this.poolScene.setColorMode(mode)
+    if (this.blockScene) {
+      this.blockScene.setColorMode(mode)
+    }
+  }
+
   applyHighlighting () {
     this.poolScene.applyHighlighting(this.highlightCriteria)
     if (this.blockScene) {
@@ -76,71 +83,25 @@ export default class TxController {
     const tx = new BitcoinTx(txData, this.vertexArray)
     tx.applyHighlighting(this.highlightCriteria)
     if (!this.txs[tx.id] && !this.expiredTxs[tx.id]) {
-      this.pendingTxs.push([tx, performance.now()])
+      // smooth near-simultaneous arrivals over up to three seconds
-      this.pendingTxs[tx.id] = tx
+      const dx = performance.now() - this.lastTxTime
-      txQueueLength.increment()
+      this.lastTxTime = performance.now()
+      if (dx <= 250) {
+        this.txDelay = Math.min(3000, this.txDelay + (Math.random() * 250))
+      } else {
+        this.txDelay = Math.max(0, this.txDelay - (dx-250))
+      }
+      this.txs[tx.id] = tx
+      tx.onEnterScene()
+      this.poolScene.insert(this.txs[tx.id], this.txDelay)
     }
   }
 
-  // Dual-strategy queue processing:
+  dropTx (txid) {
-  // - ensure transactions are queued for at least txDelay
+    // don't actually need to do anything, just let the tx expire
-  // - when queue length exceeds 500, process iteratively to avoid unbounded growth
-  // - while queue is small, use jittered timeouts to evenly distribute arrivals
-  //
-  // transactions tend to arrive in groups, so for smoothest
-  // animation the queue should stay short but never empty.
-  processQueue () {
-    let done
-    let delay
-    while (!done) {
-      if (this.pendingTxs && this.pendingTxs.length) {
-        if (this.txs[this.pendingTxs[0][0].id] || this.expiredTxs[this.pendingTxs[0][0].id]) {
-          // duplicate transaction, skip without delay
-          const tx = this.pendingTxs.shift()[0]
-          delete this.pendingMap[tx.id]
-          txQueueLength.decrement()
-        } else {
-          const timeSince = performance.now() - this.pendingTxs[0][1]
-          if (timeSince > this.txDelay) {
-            //process the next tx in the queue, if it arrived longer ago than txDelay
-            if (this.txDelay < this.maxTxDelay) {
-              // slowly ramp up from 0 to maxTxDelay on start, so there's no wait for the first txs on page load
-              this.txDelay += 50
-            }
-            const tx = this.pendingTxs.shift()[0]
-            delete this.pendingMap[tx.id]
-            txQueueLength.decrement()
-            this.txs[tx.id] = tx
-            this.poolScene.insert(this.txs[tx.id])
-            mempoolCount.increment()
-          } else {
-            // end the loop when the head of the queue arrived more recently than txDelay
-            done = true
-            // schedule to continue processing when head of queue matures
-            delay = this.txDelay - timeSince
-          }
-          if (this.pendingTxs.length < 500) {
-            // or the queue is under 500
-            done = true
-          }
-          // otherwise keep processing the queue
-        }
-      } else done = true
-    }
-    // randomly jitter arrival times so that txs enter more naturally
-    // with jittered delay inversely proportional to size of queue
-    let jitter = Math.random() * Math.max(1, (500 - this.pendingTxs.length))
-    this.scheduleQueue(delay || jitter)
   }
 
-  scheduleQueue (delay) {
+  addBlock (blockData, realtime=true) {
-    if (this.queueTimeout) clearTimeout(this.queueTimeout)
-    this.queueTimeout = setTimeout(() => {
-      this.processQueue()
-    }, delay)
-  }
-
-  addBlock (blockData) {
     // discard duplicate blocks
     if (!blockData || !blockData.id || this.knownBlocks[blockData.id]) {
       return
@@ -148,7 +109,7 @@ export default class TxController {
 
     this.poolScene.scrollLock = true
 
-    const block = (blockData && blockData.isBlock) ? blockData : new BitcoinBlock(blockData)
+    const block = new BitcoinBlock(blockData)
     this.knownBlocks[block.id] = true
     if (this.clearBlockTimeout) clearTimeout(this.clearBlockTimeout)
 
@@ -156,39 +117,27 @@ export default class TxController {
 
     this.clearBlock()
 
-    this.blockScene = new TxBlockScene({ width: this.blockAreaSize, height: this.blockAreaSize, blockId: block.id, controller: this })
+    this.blockScene = new TxBlockScene({ width: this.blockAreaSize, height: this.blockAreaSize, blockId: block.id, controller: this, colorMode: this.colorMode })
-    let poolCount = 0
     let knownCount = 0
     let unknownCount = 0
     for (let i = 0; i < block.txns.length; i++) {
       if (this.poolScene.remove(block.txns[i].id)) {
-        poolCount++
         knownCount++
-        this.txs[block.txns[i].id].setBlock(block.id)
+        this.txs[block.txns[i].id].setBlock(block)
-        this.blockScene.insert(this.txs[block.txns[i].id], false)
+        this.blockScene.insert(this.txs[block.txns[i].id], 0, false)
-      } else if (this.pendingMap[block.txns[i].id]) {
-        knownCount++
-        const tx = this.pendingMap[tx.id]
-        const pendingIndex = this.pendingTxs.indexOf(tx)
-        if (pendingIndex >= 0) this.pendingTxs.splice(pendingIndex, 1)
-        delete this.pendingMap[tx.id]
-        tx.setBlock(block.id)
-        this.txs[tx.id] = tx
-        this.blockScene.insert(tx, false)
       } else {
         unknownCount++
         const tx = new BitcoinTx({
           ...block.txns[i],
-          block: block.id
+          block: block
         }, this.vertexArray)
         this.txs[tx.id] = tx
         this.txs[tx.id].applyHighlighting(this.highlightCriteria)
-        this.blockScene.insert(tx, false)
+        this.blockScene.insert(tx, 0, false)
       }
       this.expiredTxs[block.txns[i].id] = true
     }
     console.log(`New block with ${knownCount} known transactions and ${unknownCount} unknown transactions`)
-    mempoolCount.subtract(poolCount)
     this.blockScene.initialLayout()
     setTimeout(() => { this.poolScene.scrollLock = false; this.poolScene.layoutAll() }, 4000)
 
@@ -198,64 +147,6 @@ export default class TxController {
     return block
   }
 
-  simulateBlock () {
-    for (var i = 0; i < 1000; i++) {
-      this.addTx(new BitcoinTx({
-        version: 'simulated',
-        time: Date.now(),
-        id: `simulated_${i}_${Math.random()}`,
-        value: Math.floor(Math.pow(2,(0.1-(Math.log(1 - Math.random()))) * 8))
-      }))
-    }
-    const simulatedTxns = this.pendingTxs.map(pending => {
-      return {
-        version: pending[0].version,
-        id: pending[0].id,
-        time: pending[0].time,
-        value: pending[0].value
-      }
-    })
-    // const simulatedTxns = []
-    Object.values(this.poolScene.txs).forEach(tx => {
-      if (Math.random() < 0.5) {
-        simulatedTxns.push({
-          version: tx.version,
-          id: tx.id,
-          time: tx.time,
-          value: tx.value
-        })
-      }
-    })
-    const block = new BitcoinBlock({
-      version: 'fake',
-      id: 'fake_block' + Math.random(),
-      value: 12345678900,
-      prev_block: 'also_fake',
-      merkle_root: 'merkle',
-      timestamp: Date.now() / 1000,
-      bits: 'none',
-      bytes: 1379334,
-      txn_count: simulatedTxns.length,
-      txns: simulatedTxns
-    })
-    setTimeout(() => {
-      this.addBlock(block)
-    }, 2500)
-    return block
-  }
-
-  simulateDumpTx (n, value) {
-    for (var i = 0; i < n; i++) {
-      this.addTx(new BitcoinTx({
-        version: 'simulated',
-        time: Date.now(),
-        id: `simulated_${i}_${Math.random()}`,
-        value: value || Math.min(10000, Math.floor(Math.pow(2,(0.1-(Math.log(1 - Math.random()))) * 8))) // horrible but plausible distribution of tx values
-        // value: value || Math.pow(10,Math.floor(Math.random() * 5))
-      }, this.vertexArray))
-    }
-  }
-
   hideBlock () {
     if (this.blockScene) {
       this.blockScene.hide()

client/src/controllers/TxStream.js

@@ -8,7 +8,9 @@ lastBlockId.subscribe(val => { lastBlockSeen = val })
 
 class TxStream {
   constructor () {
-    this.websocketUri = `${config.secureSocket ? 'wss://' : 'ws://'}${config.backend ? config.backend : window.location.host }${config.backendPort ? ':' + config.backendPort : ''}/ws/txs`
+    this.apiRoot = `${config.backend ? config.backend : window.location.host }${config.backendPort ? ':' + config.backendPort : ''}`
+    this.websocketUri = `${config.secureSocket ? 'wss://' : 'ws://'}${this.apiRoot}/ws/txs`
+    this.apiUri = `${config.secureSocket ? 'https://' : 'http://'}${this.apiRoot}`
     console.log('connecting to ', this.websocketUri)
     this.reconnectBackoff = 250
     this.websocket = null
@@ -91,13 +93,7 @@ class TxStream {
   sendBlockRequest () {
     if (config.noBlockFeed) return
     console.log('Checking for missed blocks...')
-    this.websocket.send(JSON.stringify({method: 'get_block', last: lastBlockSeen }))
+    this.websocket.send("block_id")
-  }
-
-  sendMempoolRequest () {
-    this.websocket.send('count')
-    if (mempoolTimer) clearTimeout(mempoolTimer)
-    mempoolTimer = setTimeout(() => { this.sendMempoolRequest() }, 60000)
   }
 
   disconnect () {
@@ -122,7 +118,25 @@ class TxStream {
     this.reconnectBackoff = 128
     this.sendHeartbeat()
     this.sendBlockRequest()
-    this.sendMempoolRequest()
+  }
+
+  async fetchBlock (id, calledOnLoad) {
+    if (!id) return
+    if (id !== lastBlockSeen) {
+      try {
+        console.log('downloading block', id)
+        const response = await fetch(`${this.apiUri}/api/block/${id}`, {
+          method: 'GET'
+        })
+        let blockData = await response.json()
+        console.log('downloaded block', id)
+        window.dispatchEvent(new CustomEvent('bitcoin_block', { detail: { block: blockData, realtime: !calledOnLoad} }))
+      } catch (err) {
+        console.log("failed to download block ", id)
+      }
+    } else {
+      console.log('already seen block ', lastBlockSeen)
+    }
   }
 
   onmessage (event) {
@@ -134,17 +148,38 @@ class TxStream {
     } else {
       try {
         const msg = JSON.parse(event.data)
-        if (msg && msg.type === 'count') {
+
-          window.dispatchEvent(new CustomEvent('bitcoin_mempool_count', { detail: msg.count }))
+        if (!msg) throw new Error('null websocket message')
-        } else if (msg && msg.type === 'txn') {
+
-          window.dispatchEvent(new CustomEvent('bitcoin_tx', { detail: msg.txn }))
+        switch (msg.type) {
-        } else if (msg && msg.type === 'block') {
+
-          if (msg.block && msg.block.id) window.dispatchEvent(new CustomEvent('bitcoin_block', { detail: msg.block }))
+          // reply to a last block_id request message
-        } else {
+          case 'block_id':
-          // console.log('unknown message from websocket: ', msg)
+            this.fetchBlock(msg.block_id, true)
+            break;
+
+          // notification of tx added to mempool
+          case 'txn':
+            window.dispatchEvent(new CustomEvent('bitcoin_tx', { detail: msg.txn }))
+            break;
+
+          // notification of tx dropped from mempool
+          case 'drop':
+            window.dispatchEvent(new CustomEvent('bitcoin_drop_tx', { detail: msg.txid }))
+            break;
+
+          // notification of a new block
+          case 'block':
+            if (msg.block && msg.block.id) {
+              this.fetchBlock(msg.block.id)
+            }
+            break;
         }
+
+        // all events can include a count field, with the latest mempool size
+        if (msg.count) window.dispatchEvent(new CustomEvent('bitcoin_mempool_count', { detail: msg.count }))
       } catch (err) {
-        // console.log('unknown message from websocket: ', msg)
+        console.log('error parsing msg json: ', err)
       }
     }
   }

client/src/models/BitcoinBlock.js

@@ -1,7 +1,7 @@
 import BitcoinTx from '../models/BitcoinTx.js'
 
 export default class BitcoinBlock {
-  constructor ({ version, id, value, prev_block, merkle_root, timestamp, bits, bytes, txn_count, txns }) {
+  constructor ({ version, id, value, prev_block, merkle_root, timestamp, bits, bytes, txn_count, txns, fees }) {
     this.isBlock = true
     this.version = version
     this.id = id
@@ -14,7 +14,31 @@ export default class BitcoinBlock {
     this.txnCount = txn_count
     this.txns = txns
     this.coinbase = new BitcoinTx(this.txns[0])
+    if (fees) {
+      this.fees = fees + this.coinbase.value
+    } else {
+      this.fees = null
+    }
     this.height = this.coinbase.coinbase.height
     this.miner_sig = this.coinbase.coinbase.sigAscii
+
+    this.total_vbytes = 0
+
+    if (this.fees != null) {
+      this.maxFeerate = 0
+      this.minFeerate = this.txnCount > 1 ? Infinity : 0
+      this.avgFeerate = 0
+      this.txns.forEach(txn => {
+        if (!BitcoinTx.prototype.isCoinbase(txn)) {
+          if (txn.fee <= 0) console.log(txn)
+          const txFeerate = txn.fee / txn.vbytes
+          this.maxFeerate = Math.max(this.maxFeerate, txFeerate)
+          this.minFeerate = Math.min(this.minFeerate, txFeerate)
+          this.avgFeerate += (txn.feerate / this.txnCount)
+        }
+        this.total_vbytes += txn.vbytes
+      })
+      this.avgFeerate = this.fees / this.total_vbytes
+    }
   }
 }

client/src/models/BitcoinTx.js

@@ -1,17 +1,9 @@
 import TxView from './TxView.js'
 import config from '../config.js'
-
+import { mixColor, pink, bluegreen, orange, teal, green, purple  } from '../utils/color.js'
-const highlightColor = {
-  h: 0.03,
-  l: 0.35
-}
-const hoverColor = {
-  h: 0.4,
-  l: 0.42
-}
 
 export default class BitcoinTx {
-  constructor ({ version, id, value, vbytes, inputs, outputs, time, block }, vertexArray) {
+  constructor ({ version, id, value, fee, vbytes, inputs, outputs, time, block }, vertexArray) {
     this.version = version
     this.id = id
     this.vertexArray = vertexArray
@@ -21,8 +13,11 @@ export default class BitcoinTx {
     this.inputs = inputs
     this.outputs = outputs
     this.value = value
+    this.fee = fee
     this.vbytes = vbytes
 
+    if (this.fee != null) this.feerate = fee / vbytes
+
     if (inputs && outputs && value == null) {
       this.value = this.calcValue()
     }
@@ -31,8 +26,35 @@ export default class BitcoinTx {
     this.highlight = false
 
     // is a coinbase transaction?
-    if (this.inputs && this.inputs.length === 1 && this.inputs[0].prev_txid === "0000000000000000000000000000000000000000000000000000000000000000") {
+    this.coinbase = this.isCoinbase(this)
-      const cbInfo = this.inputs[0].script_sig
+    if (this.coinbase) {
+      this.fee = null
+      this.feerate = null
+    }
+
+    this.setBlock(block)
+
+    const feeColor = (this.feerate == null
+      ? orange
+      : mixColor(teal, purple, 1, Math.log2(64), Math.log2(this.feerate))
+    )
+    this.colors = {
+      age: {
+        block: { color: orange },
+        pool: { color: orange, endColor: teal, duration: 60000 },
+      },
+      fee: {
+        block: { color: feeColor },
+        pool: { color: feeColor },
+      }
+    }
+
+    this.view = new TxView(this)
+  }
+
+  isCoinbase (txn) {
+    if (txn.inputs && txn.inputs.length === 1 && txn.inputs[0].prev_txid === "0000000000000000000000000000000000000000000000000000000000000000") {
+      const cbInfo = txn.inputs[0].script_sig
       // number of bytes encoding the block height
       const height_bytes = parseInt(cbInfo.substring(0,2), 16)
       // extract the specified number of bytes, reverse the endianness (reverse pairs of hex characters), parse as a hex string
@@ -42,15 +64,13 @@ export default class BitcoinTx {
       const sigAscii = sig.match(/../g).reduce((parsed, hexChar) => {
         return parsed + String.fromCharCode(parseInt(hexChar, 16))
       }, "")
-      this.coinbase = {
+
+      return {
         height,
         sig,
         sigAscii
       }
-    }
+    } else return false
-
-    this.setBlock(block)
-    this.view = new TxView(this)
   }
 
   destroy () {
@@ -70,7 +90,15 @@ export default class BitcoinTx {
     this.state = this.block ? 'block' : 'pool'
   }
 
-  hoverOn (color = hoverColor) {
+  onEnterScene () {
+    this.enteredTime = performance.now()
+  }
+
+  getColor (scene, mode) {
+    return this.colors[mode][scene]
+  }
+
+  hoverOn (color = bluegreen) {
     if (this.view) this.view.setHover(true, color)
   }
 
@@ -78,7 +106,7 @@ export default class BitcoinTx {
     if (this.view) this.view.setHover(false)
   }
 
-  highlightOn (color = highlightColor) {
+  highlightOn (color = pink) {
     if (this.view) this.view.setHighlight(true, color)
     this.highlight = true
   }
@@ -104,6 +132,6 @@ export default class BitcoinTx {
         })
       }
     })
-    this.view.setHighlight(this.highlight, color || highlightColor)
+    this.view.setHighlight(this.highlight, color || pink)
   }
 }

client/src/models/TxBlockScene.js

@@ -1,8 +1,8 @@
 import TxMondrianPoolScene from './TxMondrianPoolScene.js'
 
 export default class TxBlockScene extends TxMondrianPoolScene {
-  constructor ({ width, height, unit = 4, padding = 1, blockId, controller }) {
+  constructor ({ width, height, unit = 4, padding = 1, blockId, controller, heightStore, colorMode }) {
-    super({ width, height, unit, padding, controller })
+    super({ width, height, unit, padding, controller, heightStore, colorMode })
     this.heightLimit = null
     this.expired = false
     this.laidOut = false
@@ -10,6 +10,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
     this.initialised = true
     this.inverted = true
     this.hidden = false
+    this.sceneType = 'block'
   }
 
   resize ({ width = this.width, height = this.height }) {
@@ -55,7 +56,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
             y: -(Math.random() * window.innerWidth) - (this.scene.offset.y * 2) - pixelPosition.r,
             r: pixelPosition.r
           },
-          color: this.defaultColor
+          color: tx.getColor('block', this.colorMode).color
         },
         delay: 0,
         state: 'ready'
@@ -67,7 +68,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
       tx.view.update({
         display: {
           position: tx.screenPosition,
-          color: this.defaultColor
+          color: tx.getColor('block', this.colorMode).color
         },
         duration: 0,
         delay: 0,
@@ -77,7 +78,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
       tx.view.update({
         display: {
           position: tx.screenPosition,
-          color: this.defaultColor
+          color: tx.getColor('block', this.colorMode).color
         },
         duration: this.laidOut ? 1000 : 2000,
         delay: 0,
@@ -96,7 +97,10 @@ export default class TxBlockScene extends TxMondrianPoolScene {
             y: -(Math.random() * window.innerWidth) - (this.scene.offset.y * 2) - tx.pixelPosition.r,
             r: tx.pixelPosition.r
           },
-          color: this.defaultColor
+          color: {
+            ...tx.getColor('block', this.colorMode).color,
+            alpha: 1
+          }
         },
         delay: 0,
         state: 'ready'
@@ -104,7 +108,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
     }
     tx.view.update({
       display: {
-        color: this.defaultColor
+      color: tx.getColor('block', this.colorMode).color
       },
       duration: 2000,
       delay: 0
@@ -112,7 +116,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
   }
 
   prepareTx (tx, sequence) {
-    this.prepareTxOnScreen(tx, this.layoutTx(tx, sequence, false))
+    this.prepareTxOnScreen(tx, this.layoutTx(tx, sequence, 0, false))
   }
 
   hideTx (tx) {

client/src/models/TxMondrianPoolScene.js

@@ -10,8 +10,8 @@ settings.subscribe(v => {
 })
 
 export default class TxMondrianPoolScene extends TxPoolScene {
-  constructor ({ width, height, unit, padding, controller, heightStore }) {
+  constructor ({ width, height, unit, padding, controller, heightStore, colorMode }) {
-    super({ width, height, unit, padding, controller, heightStore })
+    super({ width, height, unit, padding, controller, heightStore, colorMode })
   }
 
   resize ({ width, height, unit, padding }) {

client/src/models/TxPackingScene.js

@@ -1,50 +0,0 @@
-/*
-  Interface definition for TxPackingScene type classes
-*/
-
-export default class TxPackingScene {
-  constructor ({ width, height }) {
-    this.init({ width, height })
-  }
-
-  init ({ width, height }) {
-    this.width = width
-    this.height = height
-    this.txs = {}
-    this.hiddenTxs = {}
-    this.scene = {
-      width: width,
-      height: height,
-      count: 0
-    }
-  }
-
-  // insert a transaction into the scene
-  // bool 'autoLayout' controls whether to calc & update the tx position immediately
-  insert (tx, autoLayout?) {}
-
-  // remove the transaction with the given id, return false if not present
-  remove (id) {
-    return !!success
-  }
-
-  // return a list of present transaction ids
-  getTxList () {
-    return [
-      ...this.getActiveTxList(),
-      ...this.getHiddenTxList()
-      ...Object.keys(this.hiddenTxs)
-    ]
-  }
-  getActiveTxList () {
-    return Object.keys(this.txs)
-  }
-  getHiddenTxList () {
-    return Object.keys(this.hiddenTxs)
-  }
-
-  // Return a flattened array of all vertex data of active tx sprites
-  getVertexData () {
-    return Object.values(this.txs).slice(-1000).flatMap(tx => tx.view.sprite.getVertexData())
-  }
-}

client/src/models/TxPackingSquare.js

@@ -1,5 +0,0 @@
-export default class TxPackingSquarez\ {
-  constructor () {
-
-  }
-}

client/src/models/TxPoolScene.js

@@ -1,15 +1,12 @@
 import config from '../config.js'
 
 export default class TxPoolScene {
-  constructor ({ width, height, unit, padding, controller, heightStore }) {
+  constructor ({ width, height, unit, padding, controller, heightStore, colorMode }) {
+    this.colorMode = colorMode || "age"
     this.maxHeight = 0
     this.heightStore = heightStore
+    this.sceneType = 'pool'
     this.init({ width, height, unit, padding, controller })
-    this.defaultColor = {
-      h: 0.1809005702490606,
-      l: 0.47246995247155193,
-      alpha: 1
-    }
   }
 
   init ({ width, height, controller }) {
@@ -50,9 +47,41 @@ export default class TxPoolScene {
     this.scene.offset.y = (window.innerHeight - (this.blockHeight * this.gridSize)) / 2
   }
 
-  insert (tx, autoLayout=true) {
+  setColorMode (mode) {
+    this.colorMode = mode
+    Object.values(this.txs).forEach(tx => {
+      const txColor = tx.getColor(this.sceneType, mode)
+      if (txColor.endColor) {
+        tx.view.update({
+          display: {
+            color: txColor.color
+          },
+          duration: 0,
+          delay: 0,
+        })
+        tx.view.update({
+          display: {
+            color: txColor.endColor
+          },
+          start: tx.enteredTime,
+          duration: txColor.duration,
+          delay: 0
+        })
+      } else {
+        tx.view.update({
+          display: {
+            color: txColor.color
+          },
+          duration: 500,
+          delay: 0,
+        })
+      }
+    })
+  }
+
+  insert (tx, insertDelay, autoLayout=true) {
     if (autoLayout) {
-      this.layoutTx(tx, this.scene.count++)
+      this.layoutTx(tx, this.scene.count++, insertDelay)
       this.txs[tx.id] = tx
     } else {
       this.hiddenTxs[tx.id] = tx
@@ -132,7 +161,7 @@ export default class TxPoolScene {
     return 1
   }
 
-  layoutTx (tx, sequence, setOnScreen = true) {
+  layoutTx (tx, sequence, insertDelay, setOnScreen = true) {
     const units = this.txSize(tx)
     this.place(tx, sequence, units)
     this.saveGridToPixelPosition(tx)
@@ -148,11 +177,12 @@ export default class TxPoolScene {
       if (this.heightStore) this.heightStore.set(this.maxHeight)
       this.saveGridToPixelPosition(tx)
     }
-    if (setOnScreen) this.setTxOnScreen(tx)
+    if (setOnScreen) this.setTxOnScreen(tx, insertDelay)
   }
 
-  setTxOnScreen (tx) {
+  setTxOnScreen (tx, insertDelay=0) {
     if (!tx.view.initialised) {
+      const txColor = tx.getColor(this.sceneType, this.colorMode)
       tx.view.update({
         display: {
           position: this.pixelsToScreen({
@@ -160,7 +190,10 @@ export default class TxPoolScene {
             y: window.innerHeight + 10,
             r: this.unitWidth / 2
           }),
-          color: this.defaultColor
+          color: {
+            ...txColor.color,
+            alpha: 1
+          },
         },
         delay: 0,
         state: 'ready'
@@ -168,22 +201,22 @@ export default class TxPoolScene {
       tx.view.update({
         display: {
           position: this.pixelsToScreen(tx.pixelPosition),
-          color: this.defaultColor
+          color: txColor.color
         },
         duration: 2500,
-        delay: 0,
+        delay: insertDelay,
         state: 'pool'
       })
-      tx.view.update({
+      if (txColor.endColor) {
-        display: {
+        tx.view.update({
-          color: {
+          display: {
-            h: 0.4751474394406347,
+            color: txColor.endColor
-            l: 0.5970385691107944
+          },
-          }
+          start: tx.enteredTime,
-        },
+          duration: txColor.duration,
-        duration: 60000,
+          delay: 0
-        delay: 0
+        })
-      })
+      }
     } else {
       tx.view.update({
         display: {

client/src/stores.js

@@ -1,4 +1,5 @@
 import { writable, derived } from 'svelte/store'
+import { tweened } from 'svelte/motion';
 import { makePollStore } from './utils/pollStore.js'
 import LocaleCurrency from 'locale-currency'
 import { currencies } from './utils/fx.js'
@@ -79,10 +80,9 @@ export const devEvents = writable({
 	addBlockCallback: null
 })
 
-export const txQueueLength = createCounter()
 export const txCount = createCounter()
 export const lastBlockId = writable(null)
-export const mempoolCount = createCounter()
+export const mempoolCount = tweened(0)
 export const mempoolScreenHeight = writable(0)
 export const frameRate = writable(null)
 export const avgFrameRate = writable(null)
@@ -102,10 +102,14 @@ export const settings = createCachedDict('settings', {
 	currency: localeCurrencyCode,
 	showFX: true,
 	vbytes: false,
+	colorByFee: false,
 	fancyGraphics: true,
 	showMessages: true,
 	noTrack: false
 })
+export const colorMode = derived([settings], ([$settings]) => {
+	return $settings.colorByFee ? "fee" : "age"
+})
 
 export const devSettings = (config.dev && config.debug) ? createCachedDict('dev-settings', {
 	guides: false,

client/src/utils/color.js

@@ -0,0 +1,26 @@
+import { hcl } from 'd3-color'
+
+export function hlToHex ({h, l}) {
+  return hcl(h * 360, 78.225, l * 150).hex()
+}
+
+export function mixColor (startColor, endColor, min, max, value) {
+  const dx = Math.max(0, Math.min(1, (value - min) / (max - min)))
+  return {
+    h: startColor.h + (dx *(endColor.h - startColor.h)),
+    l: startColor.l + (dx *(endColor.l - startColor.l))
+  }
+}
+
+export const pink = { h: 0.03, l: 0.35 }
+export const bluegreen = { h: 0.45, l: 0.4 }
+export const orange = { h: 0.181, l: 0.472 }
+export const teal = { h: 0.475, l: 0.55 }
+export const green = { h: 0.37, l: 0.35 }
+export const purple = { h: 0.95, l: 0.35 }
+
+export const highlightA = { h: 0.93, l: 0.5 } //pink
+export const highlightB = { h: 0.214, l: 0.62 } // green
+export const highlightC = { h: 0.30, l: 1.0 } // white
+export const highlightD = { h: 0.42, l: 0.35 } // blue
+export const highlightE = { h: 0.12, l: 0.375 } // red

client/src/utils/format.js

@@ -10,13 +10,18 @@ export const longBtcFormat = (Intl && Intl.NumberFormat) ? new Intl.NumberFormat
     return Number(number).toFixed(8)
   }
 }
+export const feeRateFormat = (Intl && Intl.NumberFormat) ? new Intl.NumberFormat(undefined, { maximumFractionDigits: 2, maximumSignificantDigits: 2 }) : {
+  format (number) {
+    return Number(number).toPrecision(2).slice(0,3)
+  }
+}
 export const timeFormat = (Intl && Intl.DateTimeFormat) ? new Intl.DateTimeFormat(undefined, { timeStyle: "short"}) : {
   format (date) {
     const d = new Date(date)
     return `${('' + d.getHours()).padStart(2, '0')}:${('' + d.getMinutes()).padStart(2, '0')}`
   }
 }
-export const integerFormat = (Intl && Intl.NumberFormat) ? new Intl.NumberFormat(undefined) : {
+export const numberFormat = (Intl && Intl.NumberFormat) ? new Intl.NumberFormat(undefined) : {
   format (number) {
     return Number(number).toLocaleString()
   }

docker-compose.yml

@@ -24,6 +24,7 @@ services:
       BITCOIN_HOST: "172.17.0.1"
       BITCOIN_ZMQ_RAWBLOCK_PORT: "29000"
       BITCOIN_ZMQ_RAWTX_PORT: "29001"
+      BITCOIN_ZMQ_SEQUENCE_PORT: "29002"
       BITCOIN_RPC_PORT: "8332"
       BITCOIN_RPC_USER: "bitcoin"
       BITCOIN_RPC_PASS: "correcthorsebatterystaple"

server/README.md

@@ -27,6 +27,7 @@ The API server expects the following environment variables to be set:
 | BITCOIN_HOST | Bitcoin node host address |
 | BITCOIN_ZMQ_RAWBLOCK_PORT | Bitcoin node ZMQ port for block events (to match `zmqpubrawblock` in bitcoin.conf) |
 | BITCOIN_ZMQ_RAWTX_PORT | Bitcoin node ZMQ port for transaction events (to match `zmqpubrawtx` in bitcoin.conf) |
+| BITCOIN_ZMQ_SEQUENCE_PORT | Bitcoin node ZMQ port for sequence events (to match `zmqpubsequence` in bitcoin.conf) |
 | BITCOIN_RPC_PORT | Bitcoin node RPC port |
 | either | |
 | BITCOIN_RPC_USER | Bitcoin node RPC user |

server/bitfeed.service

@@ -14,6 +14,12 @@ Environment=LANG=en_US.UTF-8
 Environment=PORT=<port>
 Environment=BITCOIN_RPC_USER=<rpc user>
 Environment=BITCOIN_RPC_PASS=<rpc password>
+Environment=BITCOIN_HOST=<rpc host>
+Environment=BITCOIN_RPC_PORT=<rpc port>
+Environment=BITCOIN_ZMQ_RAWBLOCK_PORT=<zmq rawblock port>
+Environment=BITCOIN_ZMQ_RAWTX_PORT=<zmq rawtx port>
+Environment=BITCOIN_ZMQ_SEQUENCE_PORT=<zmq sequence port>
+
 
 WorkingDirectory=<installation root>/server
 

server/lib/bitcoin_rpc.ex

@@ -10,14 +10,16 @@ defmodule BitcoinStream.RPC do
     {port, opts} = Keyword.pop(opts, :port);
     {host, opts} = Keyword.pop(opts, :host);
     IO.puts("Starting Bitcoin RPC server on #{host} port #{port}")
-    GenServer.start_link(__MODULE__, {host, port, nil}, opts)
+    GenServer.start_link(__MODULE__, {host, port, nil, nil}, opts)
   end
 
   @impl true
-  def init(state) do
+  def init({host, port, status, _}) do
     # start node monitoring loop
-    send(self(), :check_status)
+    creds = rpc_creds();
-    {:ok, state}
+
+    send(self(), :check_status);
+    {:ok, {host, port, status, creds}}
   end
 
   def handle_info(:check_status, state) do
@@ -28,28 +30,28 @@ defmodule BitcoinStream.RPC do
   end
 
   @impl true
-  def handle_call({:request, method, params}, _from, {host, port, status}) do
+  def handle_call({:request, method, params}, _from, {host, port, status, creds}) do
-    case make_request(host, port, method, params) do
+    case make_request(host, port, creds, method, params) do
-      {:ok, info} ->
+      {:ok, code, info} ->
-        {:reply, {:ok, info}, {host, port, status}}
+        {:reply, {:ok, code, info}, {host, port, status, creds}}
 
       {:error, reason} ->
-        {:reply, {:error, reason}, {host, port, status}}
+        {:reply, {:error, reason}, {host, port, status, creds}}
     end
   end
 
   @impl true
-  def handle_call({:get_node_status}, _from, {host, port, status}) do
+  def handle_call({:get_node_status}, _from, {host, port, status, creds}) do
-    {:reply, {:ok, status}, {host, port, status}}
+    {:reply, {:ok, status}, {host, port, status, creds}}
   end
 
-  defp make_request(host, port, method, params) do
+  defp make_request(host, port, creds, method, params) do
-    with  { user, pw } <- rpc_creds(),
+    with  { user, pw } <- creds,
           {:ok, rpc_request} <- Jason.encode(%{method: method, params: params}),
-          {:ok, 200, _headers, body_ref} <- :hackney.request(:post, "http://#{host}:#{port}", [{"content-type", "application/json"}], rpc_request, [basic_auth: { user, pw }]),
+          {:ok, code, _headers, body_ref} <- :hackney.request(:post, "http://#{host}:#{port}", [{"content-type", "application/json"}], rpc_request, [basic_auth: { user, pw }]),
           {:ok, body} <- :hackney.body(body_ref),
           {:ok, %{"result" => info}} <- Jason.decode(body) do
-      {:ok, info}
+      {:ok, code, info}
     else
       {:ok, code, _} ->
         IO.puts("RPC request #{method} failed with HTTP code #{code}")
@@ -66,7 +68,6 @@ defmodule BitcoinStream.RPC do
   end
 
   def request(pid, method, params) do
-    IO.inspect({pid, method, params});
     GenServer.call(pid, {:request, method, params}, 60000)
   catch
     :exit, reason ->
@@ -78,18 +79,18 @@ defmodule BitcoinStream.RPC do
     GenServer.call(pid, {:get_node_status})
   end
 
-  def check_status({host, port, status}) do
+  def check_status({host, port, status, creds}) do
-    with  {:ok, info} <- make_request(host, port, "getblockchaininfo", []) do
+    with  {:ok, 200, info} <- make_request(host, port, creds, "getblockchaininfo", []) do
-      {host, port, info}
+      {host, port, info, creds}
     else
       {:error, reason} ->
         IO.puts("node status check failed");
         IO.inspect(reason)
-        {host, port, status}
+        {host, port, status, creds}
       err ->
         IO.puts("node status check failed: (unknown reason)");
         IO.inspect(err);
-        {host, port, status}
+        {host, port, status, creds}
     end
   end
 

server/lib/block_data.ex

@@ -1,3 +1,5 @@
+Application.ensure_all_started(BitcoinStream.RPC)
+
 defmodule BitcoinStream.BlockData do
   @moduledoc """
   Block data module.
@@ -10,12 +12,11 @@ defmodule BitcoinStream.BlockData do
 
   def start_link(opts) do
     IO.puts("Starting block data link")
-    # load block.dat
+    # load block
 
-    with {:ok, block_data} <- File.read("data/block.dat"),
+    with {:ok, json} <- File.read("data/last_block.json"),
-         {:ok, block} <- BitcoinBlock.decode(block_data),
+         {:ok, %{"id" => id}} <- Jason.decode(json) do
-         {:ok, payload} <- Jason.encode(%{type: "block", block: block}) do
+      GenServer.start_link(__MODULE__, {id, json}, opts)
-      GenServer.start_link(__MODULE__, {block, payload}, opts)
     else
       _ -> GenServer.start_link(__MODULE__, {nil, "null"}, opts)
     end
@@ -27,41 +28,17 @@ defmodule BitcoinStream.BlockData do
   end
 
   @impl true
-  def handle_call(:block_id, _from, {block, json}) do
+  def handle_call(:block_id, _from, {id, json}) do
-    case block do
+    {:reply, id, {id, json}}
-      %{id: id} ->
-        {:reply, id, {block, json}}
-      _ -> {:reply, nil, {block, json}}
-    end
   end
 
   @impl true
-  def handle_call(:block, _from, {block, json}) do
+  def handle_call(:json_block, _from, {id, json}) do
-    {:reply, block, {block, json}}
+    {:reply, json, {id, json}}
   end
 
   @impl true
-  def handle_call(:json_block, _from, {block, json}) do
+  def handle_cast({:json, {id, json}}, _state) do
-    {:reply, json, {block, json}}
+      {:noreply, {id, json}}
   end
-
-  @impl true
-  def handle_cast({:block, block}, state) do
-    with {:ok, json } <- Jason.encode(%{type: "block", block: block}) do
-      IO.puts("Storing block data");
-      {:noreply, {block, json}}
-    else
-      {:err, reason} ->
-        IO.puts("Failed to json encode block data");
-        IO.inspect(reason);
-        {:noreply, state}
-      _ ->
-        IO.puts("Failed to json encode block data");
-        {:noreply, state}
-    end
-  end
-
-  def get_block_number() do
-  end
-
 end

server/lib/bridge.ex

@@ -12,17 +12,21 @@ defmodule BitcoinStream.Bridge do
   alias BitcoinStream.Mempool, as: Mempool
   alias BitcoinStream.RPC, as: RPC
 
-  def child_spec(host: host, tx_port: tx_port, block_port: block_port) do
+  def child_spec(host: host, tx_port: tx_port, block_port: block_port, sequence_port: sequence_port) do
     %{
       id: BitcoinStream.Bridge,
-      start: {BitcoinStream.Bridge, :start_link, [host, tx_port, block_port]}
+      start: {BitcoinStream.Bridge, :start_link, [host, tx_port, block_port, sequence_port]}
     }
   end
 
-  def start_link(host, tx_port, block_port) do
+  def start_link(host, tx_port, block_port, sequence_port) do
-    IO.puts("Starting Bitcoin bridge on #{host} ports #{tx_port}, #{block_port}")
+    IO.puts("Starting Bitcoin bridge on #{host} ports #{tx_port}, #{block_port}, #{sequence_port}")
+    IO.puts("Mempool loaded, ready to syncronize");
     Task.start(fn -> connect_tx(host, tx_port) end);
     Task.start(fn -> connect_block(host, block_port) end);
+    Task.start(fn -> connect_sequence(host, sequence_port) end);
+    :timer.sleep(2000);
+    Task.start(fn -> Mempool.sync(:mempool) end);
     GenServer.start_link(__MODULE__, %{})
   end
 
@@ -48,7 +52,7 @@ defmodule BitcoinStream.Bridge do
     end
 
     # start tx loop
-    tx_loop(socket)
+    tx_loop(socket, 0)
   end
 
   defp connect_block(host, port) do
@@ -57,9 +61,6 @@ defmodule BitcoinStream.Bridge do
     wait_for_ibd();
     IO.puts("Node is fully synced, connecting to block socket");
 
-    # sync mempool
-    Mempool.sync(:mempool);
-
     # connect to socket
     {:ok, socket} = :chumak.socket(:sub);
     IO.puts("Connected block zmq socket on #{host} port #{port}");
@@ -72,7 +73,28 @@ defmodule BitcoinStream.Bridge do
     end
 
     # start block loop
-    block_loop(socket)
+    block_loop(socket, 0)
+  end
+
+  defp connect_sequence(host, port) do
+    # check rpc online & synced
+    IO.puts("Waiting for node to come online and fully sync before connecting to sequence socket");
+    wait_for_ibd();
+    IO.puts("Node is fully synced, connecting to sequence socket");
+
+    # connect to socket
+    {:ok, socket} = :chumak.socket(:sub);
+    IO.puts("Connected sequence zmq socket on #{host} port #{port}");
+    :chumak.subscribe(socket, 'sequence')
+    IO.puts("Subscribed to sequence events")
+    case :chumak.connect(socket, :tcp, String.to_charlist(host), port) do
+      {:ok, pid} -> IO.puts("Binding ok to sequence socket pid #{inspect pid}");
+      {:error, reason} -> IO.puts("Binding sequence socket failed: #{reason}");
+      _ -> IO.puts("???");
+    end
+
+    # start tx loop
+    sequence_loop(socket)
   end
 
   defp wait_for_ibd() do
@@ -84,9 +106,9 @@ defmodule BitcoinStream.Bridge do
     end
   end
 
-  defp sendTxn(txn) do
+  defp send_txn(txn, count) do
     # IO.puts("Forwarding transaction to websocket clients")
-    case Jason.encode(%{type: "txn", txn: txn}) do
+    case Jason.encode(%{type: "txn", txn: txn, count: count}) do
       {:ok, payload} ->
         Registry.dispatch(Registry.BitcoinStream, "txs", fn(entries) ->
           for {pid, _} <- entries do
@@ -97,12 +119,8 @@ defmodule BitcoinStream.Bridge do
     end
   end
 
-  defp incrementMempool() do
+  defp send_block(block, count) do
-    Mempool.increment(:mempool)
+    case Jason.encode(%{type: "block", block: %{id: block.id}, drop: count}) do
-  end
-
-  defp sendBlock(block) do
-    case Jason.encode(%{type: "block", block: block}) do
       {:ok, payload} ->
         Registry.dispatch(Registry.BitcoinStream, "txs", fn(entries) ->
           for {pid, _} <- entries do
@@ -114,37 +132,112 @@ defmodule BitcoinStream.Bridge do
     end
   end
 
-  defp tx_loop(socket) do
+  defp send_drop_tx(txid, count) do
-    # IO.puts("client tx loop");
+    case Jason.encode(%{type: "drop", txid: txid, count: count}) do
-    with  {:ok, message} <- :chumak.recv_multipart(socket),
+      {:ok, payload} ->
-          [_topic, payload, _size] <- message,
+        Registry.dispatch(Registry.BitcoinStream, "txs", fn(entries) ->
-          {:ok, txn} <- BitcoinTx.decode(payload) do
+          for {pid, _} <- entries do
-      sendTxn(txn);
+            Process.send(pid, payload, []);
-      incrementMempool();
+          end
-    else
+        end)
-      {:error, reason} -> IO.puts("Bitcoin node transaction feed bridge error: #{reason}");
+      {:error, reason} -> IO.puts("Error json encoding drop message: #{reason}");
-      _ -> IO.puts("Bitcoin node transaction feed bridge error (unknown reason)");
     end
-
-    tx_loop(socket)
   end
 
-  defp block_loop(socket) do
+  defp tx_process(payload) do
-    IO.puts("client block loop");
+    case BitcoinTx.decode(payload) do
-    with  {:ok, message} <- :chumak.recv_multipart(socket),
+      {:ok, txn} ->
-          [_topic, payload, _size] <- message,
+        case Mempool.get_tx_status(:mempool, txn.id) do
-          :ok <- File.write("data/block.dat", payload, [:binary]),
+          # :registered and :new transactions are inflated and inserted into the mempool
-          {:ok, block} <- BitcoinBlock.decode(payload) do
+          status when (status in [:registered, :new]) ->
-      GenServer.cast(:block_data, {:block, block})
+            inflated_txn = BitcoinTx.inflate(txn);
-      sendBlock(block);
+            case Mempool.insert(:mempool, txn.id, inflated_txn) do
-      Mempool.sync(:mempool);
+              # Mempool.insert returns the size of the mempool if insertion was successful
-      IO.puts("new block")
+              # Forward tx to clients in this case
-    else
+              count when is_integer(count) -> send_txn(inflated_txn, count)
-      {:error, reason} -> IO.puts("Bitcoin node block feed bridge error: #{reason}");
-      _ -> IO.puts("Bitcoin node block feed bridge error (unknown reason)");
-    end
 
-    block_loop(socket)
+              _ -> false
+            end
+
+          # other statuses indicate duplicate or dropped transaction
+          _ -> false
+        end
+
+      {:err, reason} ->
+        IO.puts("Error decoding tx: #{reason}");
+        false
+
+      error ->
+        IO.puts("Error decoding tx");
+        IO.inspect(error);
+        false
+    end
+  end
+
+  defp tx_loop(socket, seq) do
+    with  {:ok, message} <- :chumak.recv_multipart(socket),
+          [_topic, payload, <<sequence::little-size(32)>>] <- message,
+          true <- (seq  != sequence) do
+      Task.start(fn -> tx_process(payload) end);
+      tx_loop(socket, sequence)
+    else
+      _ -> tx_loop(socket, seq)
+    end
+  end
+
+  defp block_loop(socket, seq) do
+    IO.puts("client block loop");
+    with  {:ok, message} <- :chumak.recv_multipart(socket), # wait for the next zmq message in the queue
+          [_topic, payload, <<sequence::little-size(32)>>] <- message,
+          true <- (seq != sequence), # discard contiguous duplicate messages
+          _ <- IO.puts("block received"),
+          {:ok, block} <- BitcoinBlock.decode(payload),
+          count <- Mempool.clear_block_txs(:mempool, block),
+          {:ok, json} <- Jason.encode(block),
+          :ok <- File.write("data/last_block.json", json) do
+      IO.puts("processed block #{block.id}");
+      GenServer.cast(:block_data, {:json, { block.id, json }});
+      send_block(block, count);
+      block_loop(socket, sequence)
+    else
+      _ -> block_loop(socket, seq)
+    end
+  end
+
+  defp sequence_loop(socket) do
+    with  {:ok, message} <- :chumak.recv_multipart(socket),
+          [_topic, <<hash::binary-size(32), type::binary-size(1), seq::little-size(64)>>, <<_sequence::little-size(32)>>] <- message,
+          txid <- Base.encode16(hash, case: :lower),
+          event <- to_charlist(type) do
+      # IO.puts("loop #{sequence}");
+      case event do
+        # Transaction added to mempool
+        'A' ->
+          case Mempool.register(:mempool, txid, seq, true) do
+            false -> false
+
+            { txn, count } ->
+              # IO.puts("*SEQ* #{txid}");
+              send_txn(txn, count)
+          end
+
+        # Transaction removed from mempool for non block inclusion reason
+        'R' ->
+          case Mempool.drop(:mempool, txid) do
+            count when is_integer(count) ->
+              send_drop_tx(txid, count);
+
+            _ ->
+              true
+          end
+
+        # Don't care about other events
+        _ -> true
+      end
+    else
+      _ -> false
+    end
+    sequence_loop(socket)
   end
 
 end

server/lib/exometer_report_dash.ex

@@ -6,6 +6,8 @@ defmodule BitcoinStream.ExometerReportDash do
   @impl true
   def exometer_init(opts) do
     IO.puts("Initialising dashboard exometer reporter")
+    Registry.BitcoinStream
+    |> Registry.register("metrics", {})
     {:ok, opts}
   end
 

server/lib/mempool.ex

@@ -1,9 +1,20 @@
 defmodule BitcoinStream.Mempool do
   @moduledoc """
-  Agent for retrieving and maintaining mempool info (primarily tx count)
+  Agent for retrieving and maintaining mempool info
+  Used for tracking mempool count, and maintaining an :ets cache of transaction prevouts
+
+  Transaction lifecycle:
+    Register -> a ZMQ sequence 'A' message received with this txid
+    Insert -> a ZMQ rawtx message is received
+    Drop -> EITHER a ZMQ sequence 'R' message is received
+         -> OR the transaction is included in a ZMQ rawblock message
+
+    ZMQ 'A' and 'R' messages are guaranteed to arrive in order relative to each other
+    but rawtx and rawblock messages may arrive in any order
   """
   use Agent
 
+  alias BitcoinStream.Protocol.Transaction, as: BitcoinTx
   alias BitcoinStream.RPC, as: RPC
 
   @doc """
@@ -12,13 +23,14 @@ defmodule BitcoinStream.Mempool do
   """
   def start_link(opts) do
     IO.puts("Starting mempool agent");
-    case Agent.start_link(fn -> %{count: 0} end, opts) do
+    # cache of all transactions in the node mempool, mapped to {inputs, total_input_value}
-      {:ok, pid} ->
+    :ets.new(:mempool_cache, [:set, :public, :named_table]);
-        sync(pid);
+    # cache of transactions ids in the mempool, but not yet synchronized with the :mempool_cache
-        {:ok, pid}
+    :ets.new(:sync_cache, [:set, :public, :named_table]);
-
+    # cache of transaction ids included in the last block
-      result -> result
+    # used to avoid allowing confirmed transactions back into the mempool if rawtx events arrive late
-    end
+    :ets.new(:block_cache, [:set, :public, :named_table]);
+    Agent.start_link(fn -> %{count: 0, seq: :infinity, queue: [], done: false} end, opts)
   end
 
   def set(pid, n) do
@@ -29,27 +41,223 @@ defmodule BitcoinStream.Mempool do
     Agent.get(pid, &Map.get(&1, :count))
   end
 
-  def increment(pid) do
+  defp increment(pid) do
-    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x + 1 end))
+    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x + 1 end));
   end
 
-  def decrement(pid) do
+  defp decrement(pid) do
-    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x - 1 end))
+    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x - 1 end));
   end
 
-  def add(pid, n) do
+  defp get_seq(pid) do
-    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x + n end))
+    Agent.get(pid, &Map.get(&1, :seq))
   end
 
-  def subtract(pid, n) do
+  defp set_seq(pid, seq) do
-    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x - n end))
+    Agent.update(pid, &Map.put(&1, :seq, seq))
+  end
+
+  defp get_queue(pid) do
+    Agent.get(pid, &Map.get(&1, :queue))
+  end
+
+  defp set_queue(pid, queue) do
+    Agent.update(pid, &Map.put(&1, :queue, queue))
+  end
+
+  defp enqueue(pid, txid) do
+    Agent.update(pid, &Map.update(&1, :queue, [], fn(q) -> [txid | q] end))
+  end
+
+  def is_done(pid) do
+    Agent.get(pid, &Map.get(&1, :done))
+  end
+  defp set_done(pid) do
+    Agent.update(pid, &Map.put(&1, :done, true))
+  end
+
+  def get_tx_status(_pid, txid) do
+    case :ets.lookup(:mempool_cache, txid) do
+      # new transaction, not yet registered
+      [] ->
+        case :ets.lookup(:block_cache, txid) do
+          # not included in the last block
+          [] -> :new
+
+          # already included in the last block
+          [_] -> :block
+        end
+
+      # new transaction, id already registered
+      [{_txid, nil, :ready}] ->
+        :registered
+
+      # already dropped
+      [{_, nil, :drop}] ->
+        :dropped
+
+      # duplicate (ignore)
+      [_] ->
+        :duplicate
+    end
+  end
+
+  def insert(pid, txid, txn) do
+    case get_tx_status(pid, txid) do
+      # new transaction, id already registered
+      :registered ->
+        with [] <- :ets.lookup(:block_cache, txid) do # double check tx isn't included in the last block
+          :ets.insert(:mempool_cache, {txid, { txn.inputs, txn.value + txn.fee }, nil});
+          get(pid)
+        else
+          _ -> false
+        end
+
+      # new transaction, not yet registered
+      :new ->
+        :ets.insert(:mempool_cache, {txid, nil, txn})
+        false
+
+      # new transaction, already included in the last block
+      :block -> false
+
+      # already dropped
+      :dropped ->
+        :ets.delete(:mempool_cache, txid);
+        false
+
+      # duplicate (ignore)
+      :duplicate ->
+        false
+    end
+  end
+
+  def register(pid, txid, sequence, do_count) do
+    cond do
+      # mempool isn't loaded yet - add this tx to the queue
+      (get_seq(pid) == :infinity) ->
+        enqueue(pid, {txid, sequence});
+        false
+
+      ((sequence == nil) or (sequence >= get_seq(pid))) ->
+        case :ets.lookup(:mempool_cache, txid) do
+          # new transaction
+          [] ->
+            with [] <- :ets.lookup(:block_cache, txid) do # double check tx isn't included in the last block
+              :ets.insert(:mempool_cache, {txid, nil, :ready});
+              :ets.delete(:sync_cache, txid);
+              if do_count do
+                increment(pid);
+              end
+            else
+              _ -> false;
+            end
+            false
+
+          # duplicate sequence message (should never happen)
+          [{_txid, _, :ready}] -> false
+
+          # already dropped
+          [{_txid, _, :drop}] -> false
+
+          # data already received, but tx not registered
+          [{_txid, _, txn}] ->
+            :ets.insert(:mempool_cache, {txid, { txn.inputs, txn.value + txn.fee }, nil});
+            :ets.delete(:sync_cache, txid);
+            if do_count do
+              increment(pid);
+            end
+            {txn, get(pid)}
+
+          # some other invalid state (should never happen)
+          [_] -> false
+        end
+
+      true -> false
+    end
+  end
+
+  def drop(pid, txid) do
+    case :ets.lookup(:mempool_cache, txid) do
+      # tx not yet registered
+      [] ->
+        case :ets.lookup(:sync_cache, txid) do
+          [] -> false
+
+          # tx is in the mempool sync cache, mark to be dropped when processed
+          _ ->
+            :ets.insert(:mempool_cache, {txid, nil, :drop});
+            decrement(pid);
+            get(pid)
+        end
+
+      # already marked as dropped (should never happen)
+      [{_txid, nil, :drop}] -> false
+
+      # tx registered but not processed, mark to be dropped
+      [{_txid, nil, :ready}] ->
+        :ets.insert(:mempool_cache, {txid, nil, :drop});
+        decrement(pid);
+        false
+
+      # tx data cached but not registered and not already dropped
+      [{_txid, nil, status}] when status != nil ->
+        case :ets.lookup(:sync_cache, txid) do
+          [] -> true;
+
+          _ -> decrement(pid);
+        end
+        :ets.delete(:mempool_cache, txid);
+        get(pid)
+
+      # tx fully processed and not already dropped
+      [{_txid, data, _status}] when data != nil ->
+        :ets.delete(:mempool_cache, txid);
+        decrement(pid);
+        get(pid)
+
+      _ -> false
+    end
+  end
+
+  defp send_mempool_count(pid) do
+    count = get(pid)
+    case Jason.encode(%{type: "count", count: count}) do
+      {:ok, payload} ->
+        Registry.dispatch(Registry.BitcoinStream, "txs", fn(entries) ->
+          for {pid, _} <- entries do
+            Process.send(pid, payload, []);
+          end
+        end)
+      {:error, reason} -> IO.puts("Error json encoding count: #{reason}");
+    end
+  end
+
+  defp sync_queue(_pid, []) do
+    true
+  end
+
+  defp sync_queue(pid, [{txid, sequence} | tail]) do
+    register(pid, txid, sequence, true);
+    sync_queue(pid, tail)
   end
 
   def sync(pid) do
-    IO.puts("Syncing mempool");
+    IO.puts("Preparing mempool sync");
-    with  {:ok, %{"size" => pool_size}} <- RPC.request(:rpc, "getmempoolinfo", []) do
+    with  {:ok, 200, %{"mempool_sequence" => sequence, "txids" => txns}} <- RPC.request(:rpc, "getrawmempool", [false, true]) do
-      IO.puts("Synced pool: size = #{pool_size}");
+      set_seq(pid, sequence);
-      set(pid, pool_size)
+      count = length(txns);
+      set(pid, count);
+      cache_sync_ids(pid, txns);
+
+      # handle queue accumulated while loading the mempool
+      queue = get_queue(pid);
+      sync_queue(pid, queue);
+      set_queue(pid, []);
+
+      IO.puts("Loaded #{count} mempool transactions");
+      send_mempool_count(pid);
+      do_sync(pid, txns)
     else
       {:error, reason} ->
         IO.puts("Pool sync failed");
@@ -62,4 +270,84 @@ defmodule BitcoinStream.Mempool do
     end
   end
 
+  def do_sync(pid, txns) do
+    IO.puts("Syncing #{length(txns)} mempool transactions");
+    sync_mempool(pid, txns);
+    IO.puts("MEMPOOL SYNC FINISHED");
+    set_done(pid);
+    IO.inspect(is_done(pid))
+  end
+
+  defp sync_mempool(pid, txns) do
+    sync_mempool_txns(pid, txns, 0)
+  end
+
+  defp sync_mempool_txn(pid, txid) do
+    case :ets.lookup(:mempool_cache, txid) do
+      [] ->
+        with  {:ok, 200, hextx} <- RPC.request(:rpc, "getrawtransaction", [txid]),
+              rawtx <- Base.decode16!(hextx, case: :lower),
+              {:ok, txn } <- BitcoinTx.decode(rawtx),
+              inflated_txn <- BitcoinTx.inflate(txn) do
+          register(pid, txid, nil, false);
+          insert(pid, txid, inflated_txn)
+        else
+          _ -> IO.puts("sync_mempool_txn failed #{txid}")
+        end
+
+      [_] -> true
+    end
+  end
+
+  defp sync_mempool_txns(_, [], count) do
+    count
+  end
+
+  defp sync_mempool_txns(pid, [head | tail], count) do
+    IO.puts("Syncing mempool tx #{count}/#{count + length(tail) + 1} | #{head}");
+    sync_mempool_txn(pid, head);
+    sync_mempool_txns(pid, tail, count + 1)
+  end
+
+  defp cache_sync_ids(pid, txns) do
+    :ets.delete_all_objects(:sync_cache);
+    cache_sync_ids(pid, txns, 0)
+  end
+
+  defp cache_sync_ids(pid, [head | tail], cached) do
+    :ets.insert(:sync_cache, {head, true});
+    cache_sync_ids(pid, tail, cached + 1)
+  end
+
+  defp cache_sync_ids(_pid, [], cached) do
+    cached
+  end
+
+  def clear_block_txs(pid, block) do
+    :ets.delete_all_objects(:block_cache)
+    clear_block_txs(pid, block.txns, 0)
+  end
+
+  # clear confirmed transactions
+  # return the total number removed from the mempool
+  # i.e. the amount by which to decrement the mempool counter
+  defp clear_block_txs(pid, [], _cleared) do
+    get(pid)
+  end
+
+  defp clear_block_txs(pid, [head | tail], cleared) do
+    :ets.insert(:block_cache, {head.id, true})
+    if drop(pid, head.id) do # tx was in the mempool
+      clear_block_txs(pid, tail, cleared + 1)
+    else
+      case :ets.lookup(:sync_cache, head.id) do
+        # tx was not in the mempool nor queued for processing
+        [] -> clear_block_txs(pid, tail, cleared)
+
+        # tx was not in the mempool, but is queued for processing
+        _ -> clear_block_txs(pid, tail, cleared + 1)
+      end
+    end
+  end
+
 end

server/lib/protocol/block.ex

@@ -1,3 +1,5 @@
+Application.ensure_all_started(BitcoinStream.RPC)
+
 defmodule BitcoinStream.Protocol.Block do
 @moduledoc """
   Summarised bitcoin block.
@@ -6,8 +8,8 @@ defmodule BitcoinStream.Protocol.Block do
   and condensing transactions into only id, value and version
 """
 
-alias BitcoinStream.Protocol.Block
 alias BitcoinStream.Protocol.Transaction, as: BitcoinTx
+alias BitcoinStream.Mempool, as: Mempool
 
 @derive Jason.Encoder
 defstruct [
@@ -20,6 +22,7 @@ defstruct [
   :nonce,
   :txn_count,
   :txns,
+  :fees,
   :value,
   :id
 ]
@@ -29,7 +32,7 @@ def decode(block_binary) do
         hex <- Base.encode16(block_binary, case: :lower),
         {:ok, raw_block} <- Bitcoinex.Block.decode(hex),
         id <- Bitcoinex.Block.block_id(block_binary),
-        {summarised_txns, total_value} <- summarise_txns(raw_block.txns)
+        {summarised_txns, total_value, total_fees} <- summarise_txns(raw_block.txns)
   do
     {:ok, %__MODULE__{
       version: raw_block.version,
@@ -40,6 +43,7 @@ def decode(block_binary) do
       bytes: bytes,
       txn_count: raw_block.txn_count,
       txns: summarised_txns,
+      fees: total_fees,
       value: total_value,
       id: id
     }}
@@ -54,24 +58,28 @@ def decode(block_binary) do
 end
 
 defp summarise_txns(txns) do
-  summarise_txns(txns, [], 0)
+  # Mempool.is_done returns false while the mempool is still syncing
+  if Mempool.is_done(:mempool) do
+    summarise_txns(txns, [], 0, 0, true)
+  else
+    summarise_txns(txns, [], 0, 0, false)
+  end
 end
 
-defp summarise_txns([], summarised, total) do
+defp summarise_txns([], summarised, total, fees, _do_inflate) do
-  {Enum.reverse(summarised), total}
+  {Enum.reverse(summarised), total, fees}
 end
 
-defp summarise_txns([next | rest], summarised, total) do
+defp summarise_txns([next | rest], summarised, total, fees, do_inflate) do
   extended_txn = BitcoinTx.extend(next)
 
-  summarise_txns(rest, [extended_txn | summarised], total + extended_txn.value)
+  # if the mempool is still syncing, inflating txs will take too long, so skip it
-end
+  if do_inflate do
-
+    inflated_txn = BitcoinTx.inflate(extended_txn)
-def test() do
+    summarise_txns(rest, [inflated_txn | summarised], total + inflated_txn.value, fees + inflated_txn.fee, true)
-  raw_block = File.read!("data/block.dat")
+  else
-  {:ok, block} = Block.decode(raw_block)
+    summarise_txns(rest, [extended_txn | summarised], nil, nil, false)
-
+  end
-  block
 end
 
 end

server/lib/protocol/transaction.ex

@@ -1,3 +1,5 @@
+Application.ensure_all_started(BitcoinStream.RPC)
+
 defmodule BitcoinStream.Protocol.Transaction do
   @moduledoc """
     Extended bitcoin transaction struct
@@ -11,6 +13,8 @@ defmodule BitcoinStream.Protocol.Transaction do
 
   BitcoinStream.Protocol.Transaction
 
+  alias BitcoinStream.RPC, as: RPC
+
   @derive Jason.Encoder
   defstruct [
     :version,
@@ -18,6 +22,7 @@ defmodule BitcoinStream.Protocol.Transaction do
     :inputs,
     :outputs,
     :value,
+    :fee,
     # :witnesses,
     :lock_time,
     :id,
@@ -66,6 +71,22 @@ defmodule BitcoinStream.Protocol.Transaction do
     }
   end
 
+  def inflate(txn) do
+    {inputs, in_value } = inflate_inputs(txn.id, txn.inputs);
+    %__MODULE__{
+      version: txn.version,
+      vbytes: txn.vbytes,
+      inputs: inputs,
+      outputs: txn.outputs,
+      value: txn.value,
+      fee: (in_value - txn.value),
+      # witnesses: txn.witnesses,
+      lock_time: txn.lock_time,
+      id: txn.id,
+      time: txn.time
+    }
+  end
+
   defp count_value([], total) do
     total
   end
@@ -74,6 +95,81 @@ defmodule BitcoinStream.Protocol.Transaction do
     count_value(rest, total + next_output.value)
   end
 
+  defp inflate_input(input) do
+    case input.prev_txid do
+      "0000000000000000000000000000000000000000000000000000000000000000" ->
+        {:ok, %{
+          prev_txid: input.prev_txid,
+          prev_vout: input.prev_vout,
+          script_sig: input.script_sig,
+          sequence_no: input.sequence_no,
+          value: 0,
+          script_pub_key: nil,
+        } }
+      _prev_txid ->
+        with  {:ok, 200, hextx} <- RPC.request(:rpc, "getrawtransaction", [input.prev_txid]),
+              rawtx <- Base.decode16!(hextx, case: :lower),
+              {:ok, txn } <- decode(rawtx),
+              output <- Enum.at(txn.outputs, input.prev_vout) do
+          {:ok, %{
+            prev_txid: input.prev_txid,
+            prev_vout: input.prev_vout,
+            script_sig: input.script_sig,
+            sequence_no: input.sequence_no,
+            value: output.value,
+            script_pub_key: output.script_pub_key,
+          } }
+        else
+          {:ok, 500, reason} ->
+            IO.puts("transaction not found #{input.prev_txid}");
+            IO.inspect(reason)
+          {:error, reason} ->
+            IO.puts("Failed to inflate input:");
+            IO.inspect(reason)
+            :error
+          err ->
+            IO.puts("Failed to inflate input: (unknown reason)");
+            IO.inspect(err);
+            :error
+        end
+    end
+  end
+
+  defp inflate_inputs([], inflated, total) do
+    {inflated, total}
+  end
+  defp inflate_inputs([next_input | rest], inflated, total) do
+    case inflate_input(next_input) do
+      {:ok, inflated_txn} ->
+        inflate_inputs(rest, [inflated_txn | inflated], total + inflated_txn.value)
+      _ ->
+        inflate_inputs(rest, [inflated], total)
+    end
+  end
+  def inflate_inputs([], nil) do
+    { nil, 0 }
+  end
+  def inflate_inputs(txid, inputs) do
+    case :ets.lookup(:mempool_cache, txid) do
+      # cache miss, actually inflate
+      [] ->
+        inflate_inputs(inputs, [], 0)
+
+      # cache hit, but processed inputs not available
+      [{_, nil, _}] ->
+        inflate_inputs(inputs, [], 0)
+
+      # cache hit, just return the cached values
+      [{_, input_state, _}] ->
+        input_state
+
+      other ->
+        IO.puts("unexpected cached value: ")
+        IO.inspect(other);
+        inflate_inputs(inputs, [], 0)
+    end
+  end
+
 end
 
 # defmodule BitcoinStream.Protocol.Transaction.Summary do

server/lib/router.ex

@@ -12,7 +12,27 @@ defmodule BitcoinStream.Router do
     json_decoder: Jason
   plug :dispatch
 
+  match "/api/block/:hash" do
+    case get_block(hash) do
+      {:ok, block} ->
+        put_resp_header(conn, "cache-control", "public, max-age=604800, immutable")
+        |> send_resp(200, block)
+      _ ->
+        IO.puts("Error getting block hash")
+    end
+  end
+
   match _ do
     send_resp(conn, 404, "404")
   end
+
+  defp get_block(last_seen) do
+    last_id = GenServer.call(:block_data, :block_id);
+    cond do
+      (last_seen == last_id) ->
+        payload = GenServer.call(:block_data, :json_block);
+        {:ok, payload}
+      true -> :err
+    end
+  end
 end

server/lib/server.ex

@@ -5,13 +5,14 @@ defmodule BitcoinStream.Server do
     { socket_port, "" } = Integer.parse(System.get_env("PORT"));
     { zmq_tx_port, "" } = Integer.parse(System.get_env("BITCOIN_ZMQ_RAWTX_PORT"));
     { zmq_block_port, "" } = Integer.parse(System.get_env("BITCOIN_ZMQ_RAWBLOCK_PORT"));
+    { zmq_sequence_port, "" } = Integer.parse(System.get_env("BITCOIN_ZMQ_SEQUENCE_PORT"));
     { rpc_port, "" } = Integer.parse(System.get_env("BITCOIN_RPC_PORT"));
     btc_host = System.get_env("BITCOIN_HOST");
 
     children = [
-      { BitcoinStream.BlockData, [name: :block_data] },
       { BitcoinStream.RPC, [host: btc_host, port: rpc_port, name: :rpc] },
       { BitcoinStream.Mempool, [name: :mempool] },
+      { BitcoinStream.BlockData, [name: :block_data] },
       BitcoinStream.Metrics.Probe,
       Plug.Cowboy.child_spec(
         scheme: :http,
@@ -25,7 +26,7 @@ defmodule BitcoinStream.Server do
         keys: :duplicate,
         name: Registry.BitcoinStream
       ),
-      BitcoinStream.Bridge.child_spec(host: btc_host, tx_port: zmq_tx_port, block_port: zmq_block_port)
+      BitcoinStream.Bridge.child_spec(host: btc_host, tx_port: zmq_tx_port, block_port: zmq_block_port, sequence_port: zmq_sequence_port)
     ]
 
     opts = [strategy: :one_for_one, name: BitcoinStream.Application]

server/lib/socket_handler.ex

@@ -18,10 +18,6 @@ defmodule BitcoinStream.SocketHandler do
     {:ok, state}
   end
 
-  # def last_block() do
-  #
-  # end
-
   def get_block(last_seen) do
     IO.puts("getting block with id #{last_seen}")
     last_id = GenServer.call(:block_data, :block_id)
@@ -40,10 +36,14 @@ defmodule BitcoinStream.SocketHandler do
 
   def get_mempool_count_msg() do
     count = Mempool.get(:mempool);
-    IO.puts("Count: #{count}");
     "{ \"type\": \"count\", \"count\": #{count}}"
   end
 
+  def get_block_id_msg() do
+    last_id = GenServer.call(:block_data, :block_id);
+    "{ \"type\": \"block_id\", \"block_id\": \"#{last_id}\"}"
+  end
+
   @timed(key: "timed.function")
   def websocket_handle({:text, msg}, state) do
     # IO.puts("message received: #{msg} | #{inspect self()}");
@@ -52,20 +52,20 @@ defmodule BitcoinStream.SocketHandler do
 
       "block" ->
         IO.puts('block request');
-        {:reply, {:text, "null"}, state};
+        {:reply, {:text, "null"}, state}
 
       "count" ->
         count = get_mempool_count_msg();
-        {:reply, {:text, count}, state};
+        {:reply, {:text, count}, state}
+
+      "block_id" ->
+        last_id = get_block_id_msg();
+        {:reply, {:text, last_id}, state}
 
       json ->
-        IO.puts("attempting to decode msg as json");
         with {:ok, result} <- Jason.decode(json) do
-          IO.puts("decoded ok");
-          IO.inspect(result);
           case result do
             %{"last" => block_id, "method" => "get_block"} ->
-              IO.puts('block request')
               case get_block(block_id) do
                 {:ok, block_msg} ->
                   {:reply, {:text, block_msg}, state};

server/mix.exs

@@ -4,7 +4,7 @@ defmodule BitcoinStream.MixProject do
   def project do
     [
       app: :bitcoin_stream,
-      version: "2.1.3",
+      version: "2.2.0",
       elixir: "~> 1.10",
       start_permanent: Mix.env() == :prod,
       deps: deps(),