Merge pull request #26 from bitfeed-project/fees

API server refactor & fee features
2025-05-12 19:20:46 +02:00 · 2022-02-19 22:10:07 -06:00 · 2022-02-19 22:10:07 -06:00 · d8cc46b137
commit d8cc46b137
parent 0bb7de1e33 7a0c98f7f1
37 changed files with 1103 additions and 485 deletions
--- a/client/package.json
+++ b/client/package.json
@ -1,6 +1,6 @@
 {
  "name": "bitfeed-client",
-  "version": "2.1.5",
+  "version": "2.2.0",
  "scripts": {
    "build": "rollup -c",
    "dev": "rollup -c -w",
--- a/client/src/components/BlockInfo.svelte
+++ b/client/src/components/BlockInfo.svelte
@ -5,7 +5,7 @@
  import { createEventDispatcher } from 'svelte'
  import Icon from '../components/Icon.svelte'
  import closeIcon from '../assets/icon/cil-x-circle.svg'
-  import { shortBtcFormat, longBtcFormat, timeFormat, integerFormat } from '../utils/format.js'
+  import { shortBtcFormat, longBtcFormat, timeFormat, numberFormat } from '../utils/format.js'
  import { exchangeRates, settings } from '../stores.js'
  import { formatCurrency } from '../utils/fx.js'

@ -57,16 +57,22 @@

  function formatBytes (bytes) {
    if (bytes) {
-      return `${integerFormat.format(bytes)} bytes`
+      return `${numberFormat.format(bytes)} bytes`
    } else return `unknown size`
  }

  function formatCount (n) {
    if (n) {
-      return integerFormat.format(n)
+      return numberFormat.format(n)
    } else return '0'
  }

+  function formatFee (n) {
+    if (n) {
+      return numberFormat.format(n.toFixed(2))
+    } else return  '0'
+  }
+
  function hideBlock () {
    analytics.trackEvent('viz', 'block', 'hide')
    dispatch('hideBlock')
@ -108,11 +114,36 @@
      font-size: 4.4vw;
    }

+    .compact {
+      display: none;
+    }
+
+    @media (max-aspect-ratio: 1/1) and (max-height: 760px) {
+      .compact {
+        display: block;
+      }
+      .full-size {
+        display: none;
+      }
+    }
+    @media (aspect-ratio: 1/1) and (max-height: 760px) {
+      .compact {
+        display: none;
+      }
+      .full-size {
+        display: block;
+      }
+    }
+
    .data-row {
      display: flex;
      flex-direction: row;
      flex-wrap: nowrap;
      justify-content: space-between;
+
+      &.spacer {
+        display: none;
+      }
    }

    .data-field {
@ -157,6 +188,10 @@
        flex-direction: column;
        justify-content: flex-start;
        align-items: flex-end;
+
+        &.spacer {
+          display: flex;
+        }
      }

      .data-field {
@ -196,17 +231,46 @@
  {#if block != null && visible }
    <div class="block-info" out:fly="{{ y: -50, duration: 2000, easing: linear }}" in:fly="{{ y: (restoring ? -50 : 50), duration: (restoring ? 500 : 1000), easing: linear, delay: (restoring ? 0 : newBlockDelay) }}">
        <!-- <span class="data-field">Hash: { block.id }</span> -->
-        <div class="data-row">
-          <span class="data-field title-field" title="{block.miner_sig}"><b>Latest Block: </b>{ integerFormat.format(block.height) }</span>
-          <button class="data-field close-button" on:click={hideBlock}><Icon icon={closeIcon} color="var(--palette-x)" /></button>
+        <div class="full-size">
+          <div class="data-row">
+            <span class="data-field title-field" title="{block.miner_sig}"><b>Latest Block: </b>{ numberFormat.format(block.height) }</span>
+            <button class="data-field close-button" on:click={hideBlock}><Icon icon={closeIcon} color="var(--palette-x)" /></button>
+          </div>
+          <div class="data-row">
+            <span class="data-field">Mined { formatTime(block.time) }</span>
+            <span class="data-field">{ formattedBlockValue }</span>
+          </div>
+          <div class="data-row">
+            <span class="data-field">{ formatBytes(block.bytes) }</span>
+            <span class="data-field">{ formatCount(block.txnCount) } transactions</span>
+          </div>
+          <div class="data-row spacer">&nbsp;</div>
+          <div class="data-row">
+            <span class="data-field">Avg Fee Rate</span>
+            {#if block.fees != null}
+              <span class="data-field">{ formatFee(block.avgFeerate) } sats/vbyte</span>
+            {:else}
+              <span class="data-field">unavailable</span>
+            {/if}
+          </div>
        </div>
-        <div class="data-row">
-          <span class="data-field">Mined { formatTime(block.time) }</span>
-          <span class="data-field">{ formattedBlockValue }</span>
-        </div>
-        <div class="data-row">
-          <span class="data-field">{ formatBytes(block.bytes) }</span>
-          <span class="data-field">{ formatCount(block.txnCount) } transactions</span>
+        <div class="compact">
+          <div class="data-row">
+            <span class="data-field title-field" title="{block.miner_sig}"><b>Latest Block: </b>{ numberFormat.format(block.height) }</span>
+            <button class="data-field close-button" on:click={hideBlock}><Icon icon={closeIcon} color="var(--palette-x)" /></button>
+          </div>
+          <div class="data-row">
+            <span class="data-field">Mined { formatTime(block.time) }</span>
+            <span class="data-field">{ formattedBlockValue }</span>
+          </div>
+          <div class="data-row">
+            <span class="data-field">{ formatCount(block.txnCount) } transactions</span>
+            {#if block.fees != null}
+              <span class="data-field">{ formatFee(block.avgFeerate) } sats/vb</span>
+            {:else}
+              <span class="data-field">{ formatBytes(block.bytes) }</span>
+            {/if}
+          </div>
        </div>
    </div>
    <button class="close-button standalone" on:click={hideBlock} out:fly="{{ y: -50, duration: 2000, easing: linear }}" in:fly="{{ y: (restoring ? -50 : 50), duration: (restoring ? 500 : 1000), easing: linear, delay: (restoring ? 0 : newBlockDelay) }}" >
--- a/client/src/components/MempoolLegend.svelte
+++ b/client/src/components/MempoolLegend.svelte
@ -1,10 +1,16 @@
 <script>
 import { onMount } from 'svelte'
-import { settings } from '../stores.js'
-import { integerFormat } from '../utils/format.js'
+import { settings, colorMode } from '../stores.js'
+import { numberFormat } from '../utils/format.js'
 import { logTxSize, byteTxSize } from '../utils/misc.js'
 import { interpolateHcl } from 'd3-interpolate'
-import { color } from 'd3-color'
+import { color, hcl } from 'd3-color'
+import { hlToHex, orange, teal, green, purple } from '../utils/color.js'
+
+const orangeHex = hlToHex(orange)
+const tealHex = hlToHex(teal)
+const greenHex = hlToHex(green)
+const purpleHex = hlToHex(purple)

 const sizes = [
  { value: 1000000, vbytes: 1*256, size: null },
@ -18,8 +24,15 @@ let unitWidth
 let unitPadding
 let gridSize
 let colorScale
+let feeColorScale
 const colorScaleWidth = 200

+let squareColor = orangeHex
+$: {
+  if ($colorMode === 'age') squareColor = orangeHex
+  else squareColor = purpleHex
+}
+
 function resize () {
  unitWidth = Math.floor(Math.max(4, (window.innerWidth - 20) / 250))
  unitPadding = Math.floor(Math.max(1, (window.innerWidth - 20) / 1000))
@ -29,7 +42,8 @@ resize()

 onMount(() => {
  resize()
-  colorScale = generateColorScale('#f7941d', '#00ffc6')
+  colorScale = generateColorScale(orangeHex, tealHex)
+  feeColorScale = generateColorScale(tealHex, purpleHex)
 })

 function calcSizes (gridSize, unitWidth, unitPadding) {
@ -52,7 +66,7 @@ function calcSize ({ vbytes, value }) {
 }

 function formatBytes (bytes) {
-  const str = integerFormat.format(bytes) + ' vbytes'
+  const str = numberFormat.format(bytes) + ' vbytes'
  const padded = str.padStart(13, ' ')
  return padded
 }
@ -98,6 +112,7 @@ function generateColorScale (colorA, colorB) {
    .size-legend {
      display: table;
      margin: auto;
+      margin-bottom: 5px;

      .size-row {
        display: table-row;
@ -166,23 +181,33 @@ function generateColorScale (colorA, colorB) {
    {#if $settings.vbytes }
      {#each sizes as { size, vbytes } }
        <div class="size-row">
-          <span class="square-container"><div class="square" style="width: {size}px; height: {size}px" /></span>
+          <span class="square-container"><div class="square" style="width: {size}px; height: {size}px; background: {squareColor};" /></span>
          <span class="value"><span class="part left">&lt;</span><span class="part center">&nbsp;</span><span class="part right">{ formatBytes(vbytes) }</span></span>
        </div>
      {/each}
    {:else}
      {#each sizes as { size, value } }
        <div class="size-row">
-          <span class="square-container"><div class="square" style="width: {size}px; height: {size}px" /></span>
+          <span class="square-container"><div class="square" style="width: {size}px; height: {size}px; background: {squareColor}" /></span>
          <span class="value"><span class="part left">&lt;</span> <span class="part center">&#8383;</span><span class="part right">{ formatValue(value) }</span></span>
        </div>
      {/each}
    {/if}
  </div>
-  <h3 class="subheading">Age in seconds</h3>
+  {#if $colorMode === 'age'}
+    <h3 class="subheading">Age in seconds</h3>
+  {:else}
+    <h3 class="subheading">Fee rate in sats/vbyte</h3>
+  {/if}
  <div class="color-legend">
-    <span class="value left">0</span>
-    <img src={colorScale} alt="" class="color-scale-img" width="200" height="15">
-    <span class="value right">60+</span>
+    {#if $colorMode === 'age'}
+      <span class="value left">0</span>
+      <img src={colorScale} alt="" class="color-scale-img" width="200" height="15">
+      <span class="value right">60+</span>
+    {:else}
+      <span class="value left">1</span>
+      <img src={feeColorScale} alt="" class="color-scale-img" width="200" height="15">
+      <span class="value right">64+</span>
+    {/if}
  </div>
 </div>
--- a/client/src/components/SearchTab.svelte
+++ b/client/src/components/SearchTab.svelte
@ -10,16 +10,10 @@ import AddressIcon from '../assets/icon/cil-wallet.svg'
 import TxIcon from '../assets/icon/cil-arrow-circle-right.svg'
 import { matchQuery } from '../utils/search.js'
 import { highlight, newHighlightQuery, highlightingFull } from '../stores.js'
-import { hcl } from 'd3-color'
+import { hlToHex, highlightA, highlightB, highlightC, highlightD, highlightE } from '../utils/color.js'

-const highlightColors = [
-  { h: 0.03, l: 0.35 },
-  { h: 0.40, l: 0.35 },
-  { h: 0.65, l: 0.35 },
-  { h: 0.85, l: 0.35 },
-  { h: 0.12, l: 0.35 },
-]
-const highlightHexColors = highlightColors.map(c => hclToHex(c))
+const highlightColors = [highlightA, highlightB, highlightC, highlightD, highlightE]
+const highlightHexColors = highlightColors.map(c => hlToHex(c))
 const usedColors = [false, false, false, false, false]

 const queryIcons = {
@ -36,6 +30,7 @@ export let tab
 let query
 let matchedQuery
 let queryAddress
+let queryColorIndex
 let queryColor
 let queryColorHex
 let watchlist = []
@ -48,8 +43,9 @@ $: {
  if ($newHighlightQuery) {
    matchedQuery = matchQuery($newHighlightQuery)
    if (matchedQuery) {
-      matchedQuery.color = queryColor
-      matchedQuery.colorHex = queryColorHex
+      matchedQuery.colorIndex = queryColorIndex
+      matchedQuery.color = highlightColors[queryColorIndex]
+      matchedQuery.colorHex = highlightHexColors[queryColorIndex]
      add()
      query = null
    }
@ -66,8 +62,9 @@ $: {
  if (query) {
    matchedQuery = matchQuery(query.trim())
    if (matchedQuery) {
-      matchedQuery.color = queryColor
-      matchedQuery.colorHex = queryColorHex
+      matchedQuery.colorIndex = queryColorIndex
+      matchedQuery.color = highlightColors[queryColorIndex]
+      matchedQuery.colorHex = highlightHexColors[queryColorIndex]
    }
  } else matchedQuery = null
 }
@ -81,9 +78,20 @@ function init () {
    try {
      watchlist = JSON.parse(val)
      watchlist.forEach(q => {
-        const i = highlightHexColors.findIndex(c => c === q.colorHex)
-        if (i >= 0) usedColors[i] = q.colorHex
-        else console.log('unknown color')
+        if (q.colorIndex) {
+          usedColors[q.colorIndex] = true
+          q.color = highlightColors[q.colorIndex]
+          q.colorHex = highlightHexColors[q.colorIndex]
+        }
+      })
+      watchlist.forEach(q => {
+        if (!q.colorIndex){
+          const nextIndex = usedColors.findIndex(used => !used)
+          usedColors[nextIndex] = true
+          q.colorIndex = nextIndex
+          q.color = highlightColors[nextIndex]
+          q.colorHex = highlightHexColors[nextIndex]
+        }
      })
    } catch (e) {
      console.log('failed to parse cached highlight queries')
@ -94,18 +102,14 @@ function init () {
 function setNextColor () {
  const nextIndex = usedColors.findIndex(used => !used)
  if (nextIndex >= 0) {
+    queryColorIndex = nextIndex
    queryColor = highlightColors[nextIndex]
    queryColorHex = highlightHexColors[nextIndex]
-    usedColors[nextIndex] = queryColorHex
+    usedColors[nextIndex] = true
  }
 }
-function clearUsedColor (hex) {
-  const clearIndex = usedColors.findIndex(used => used === hex)
-  usedColors[clearIndex] = false
-}
-
-function hclToHex (color) {
-  return hcl(color.h * 360, 78.225, color.l * 150).hex()
+function clearUsedColor (colorIndex) {
+  usedColors[colorIndex] = false
 }

 async function add () {
@ -127,7 +131,7 @@ async function remove (index) {
  const wasFull = $highlightingFull
  const removed = watchlist.splice(index,1)
  if (removed.length) {
-    clearUsedColor(removed[0].colorHex)
+    clearUsedColor(removed[0].colorIndex)
    watchlist = watchlist
    if (tab) {
      await tick()
@ -249,7 +253,7 @@ function searchSubmit (e) {
    </div>
  </div>
  <div class="watchlist">
-    {#each watchlist as watched, index (watched.colorHex)}
+    {#each watchlist as watched, index (watched.colorIndex)}
      <div
        class="watched"
 				transition:fade={{ duration: 200 }}
--- a/client/src/components/Settings.svelte
+++ b/client/src/components/Settings.svelte
@ -46,6 +46,13 @@ let settingConfig = {
    trueLabel: 'vbytes',
    valueType: 'bool'
  },
+  colorByFee: {
+    label: 'Color by',
+    type: 'pill',
+    falseLabel: 'age',
+    trueLabel: 'fee rate',
+    valueType: 'bool'
+  }
 }
 $: {
  if ($nativeAntialias) {
--- a/client/src/components/TxInfo.svelte
+++ b/client/src/components/TxInfo.svelte
@ -1,7 +1,7 @@
 <script>
 import Icon from './Icon.svelte'
 import BookmarkIcon from '../assets/icon/cil-bookmark.svg'
-import { longBtcFormat, integerFormat } from '../utils/format.js'
+import { longBtcFormat, numberFormat, feeRateFormat } from '../utils/format.js'
 import { exchangeRates, settings, sidebarToggle, newHighlightQuery, highlightingFull } from '../stores.js'
 import { formatCurrency } from '../utils/fx.js'

@ -84,6 +84,24 @@ function highlight () {
      word-break: break-all;
    }

+    .inputs {
+      display: flex;
+      flex-direction: row;
+      justify-content: space-between;
+      align-items: baseline;
+
+      span {
+        width: 0;
+        flex-shrink: 1;
+        flex-grow: 1;
+        &.arrow {
+          min-width: 1.5em;
+          flex-shrink: 1;
+          flex-grow: 0;
+        }
+      }
+    }
+
    &:hover {
      .hash {
        white-space: pre-wrap;
@ -119,12 +137,21 @@ function highlight () {
  <p class="field hash">
    TxID: { tx.id }
  </p>
-  {#if tx.inputs && !tx.coinbase }<p class="field inputs">{ tx.inputs.length } input{#if tx.inputs.length != 1}s{/if}</p>
+  {#if tx.inputs && tx.outputs && !tx.coinbase }
+    <p class="field inputs">
+      <span>{ tx.inputs.length } input{#if tx.inputs.length != 1}s{/if}</span>
+      <span class="arrow"> &rarr; </span>
+      <span>{ tx.outputs.length } output{#if tx.outputs.length != 1}s{/if}</span>
+    </p>
  {:else if tx.coinbase }
    <p class="field coinbase">Coinbase: { tx.coinbase.sigAscii }</p>
+    <p class="field inputs">{ tx.outputs.length } output{#if tx.outputs.length != 1}s{/if}</p>
+  {/if}
+  <p class="field vbytes">Size: { numberFormat.format(tx.vbytes) } vbytes</p>
+  {#if !tx.coinbase && tx.fee != null }
+    <p class="field feerate">Fee rate: { numberFormat.format(tx.feerate.toFixed(2)) } sats/vbyte</p>
+    <p class="field fee">Fee: { numberFormat.format(tx.fee) } sats</p>
  {/if}
-  {#if tx.outputs }<p class="field outputs">{ tx.outputs.length } output{#if tx.outputs.length != 1}s{/if}</p>{/if}
-  <p class="field vbytes">{ integerFormat.format(tx.vbytes) } vbytes</p>
  <p class="field value">
    Total value: { formatBTC(tx.value) }
    {#if formattedLocalValue != null }
--- a/client/src/components/TxViz.svelte
+++ b/client/src/components/TxViz.svelte
@ -3,7 +3,7 @@
  import TxController from '../controllers/TxController.js'
  import TxRender from './TxRender.svelte'
  import getTxStream from '../controllers/TxStream.js'
-  import { settings, overlay, serverConnected, serverDelay, txQueueLength, txCount, mempoolCount, mempoolScreenHeight, frameRate, avgFrameRate, blockVisible, currentBlock, selectedTx, blockAreaSize, devEvents, devSettings } from '../stores.js'
+  import { settings, overlay, serverConnected, serverDelay, txCount, mempoolCount, mempoolScreenHeight, frameRate, avgFrameRate, blockVisible, currentBlock, selectedTx, blockAreaSize, devEvents, devSettings } from '../stores.js'
  import BitcoinBlock from '../models/BitcoinBlock.js'
  import BlockInfo from '../components/BlockInfo.svelte'
  import TxInfo from '../components/TxInfo.svelte'
@ -12,7 +12,7 @@
  import DonationOverlay from '../components/DonationOverlay.svelte'
  import SupportersOverlay from '../components/SupportersOverlay.svelte'
  import Alerts from '../components/alert/Alerts.svelte'
-  import { integerFormat } from '../utils/format.js'
+  import { numberFormat } from '../utils/format.js'
  import { exchangeRates, lastBlockId, haveSupporters, sidebarToggle } from '../stores.js'
  import { formatCurrency } from '../utils/fx.js'
  import config from '../config.js'
@ -54,14 +54,16 @@
      txStream.subscribe('tx', tx => {
        txController.addTx(tx)
      })
+      txStream.subscribe('drop_tx', txid => {
+        txController.dropTx(txid)
+      })
    }
    if (!config.noBlockFeed) {
-      txStream.subscribe('block', block => {
+      txStream.subscribe('block', ({block, realtime}) => {
        if (block) {
-          const added = txController.addBlock(block)
+          const added = txController.addBlock(block, realtime)
          if (added && added.id) $lastBlockId = added.id
        }
-        txStream.sendMempoolRequest()
      })
    }
    if (!config.noTxFeed || !config.noBlockFeed) {
@ -398,7 +400,7 @@

    <div class="mempool-height" style="bottom: calc({$mempoolScreenHeight + 20}px)">
      <div class="height-bar" />
-      <span class="mempool-count">Mempool: { integerFormat.format($mempoolCount) } unconfirmed</span>
+      <span class="mempool-count">Mempool: { numberFormat.format(Math.round($mempoolCount)) } unconfirmed</span>
    </div>

    <div class="block-area-wrapper">
--- a/client/src/components/alert/GenericAlertv2.svelte
+++ b/client/src/components/alert/GenericAlertv2.svelte
@ -10,7 +10,7 @@ function chooseImgs () {
  const randomIndex = Math.floor(Math.random() * imgs.length)
  for (let i = 0; i < Math.min(3, imgs.length); i++) {
    const randomImg = imgs[(randomIndex + i) % imgs.length]
-    displayImgs.push(randomImg)
+    if (randomImg && randomImg.img) displayImgs.push(randomImg)
  }
 }

--- a/client/src/config.js
+++ b/client/src/config.js
@ -31,7 +31,7 @@ export default {
  noTxFeed: false,
  noBlockFeed: false,
  // Minimum delay in ms before newly recieved transactions enter the visualization
-  txDelay: 10000,
+  txDelay: 3000,
  donationsEnabled: true,
  // Enables the message bar
  messagesEnabled: true,
--- a/client/src/controllers/TxController.js
+++ b/client/src/controllers/TxController.js
@ -5,7 +5,7 @@ import BitcoinTx from '../models/BitcoinTx.js'
 import BitcoinBlock from '../models/BitcoinBlock.js'
 import TxSprite from '../models/TxSprite.js'
 import { FastVertexArray } from '../utils/memory.js'
-import { txQueueLength, txCount, mempoolCount, mempoolScreenHeight, blockVisible, currentBlock, selectedTx, blockAreaSize, highlight } from '../stores.js'
+import { txCount, mempoolCount, mempoolScreenHeight, blockVisible, currentBlock, selectedTx, blockAreaSize, highlight, colorMode } from '../stores.js'
 import config from "../config.js"

 export default class TxController {
@ -26,17 +26,16 @@ export default class TxController {
    this.selectedTx = null
    this.selectionLocked = false

-    this.pendingTxs = []
-    this.pendingMap = {}
-    this.queueTimeout = null
-    this.queueLength = 0
+    this.lastTxTime = 0
+    this.txDelay = 0

    highlight.subscribe(criteria => {
      this.highlightCriteria = criteria
      this.applyHighlighting()
    })
-
-    this.scheduleQueue(1000)
+    colorMode.subscribe(mode => {
+      this.setColorMode(mode)
+    })
  }

  getVertexData () {
@ -65,6 +64,14 @@ export default class TxController {
    this.redoLayout({ width, height })
  }

+  setColorMode (mode) {
+    this.colorMode = mode
+    this.poolScene.setColorMode(mode)
+    if (this.blockScene) {
+      this.blockScene.setColorMode(mode)
+    }
+  }
+
  applyHighlighting () {
    this.poolScene.applyHighlighting(this.highlightCriteria)
    if (this.blockScene) {
@ -76,71 +83,25 @@ export default class TxController {
    const tx = new BitcoinTx(txData, this.vertexArray)
    tx.applyHighlighting(this.highlightCriteria)
    if (!this.txs[tx.id] && !this.expiredTxs[tx.id]) {
-      this.pendingTxs.push([tx, performance.now()])
-      this.pendingTxs[tx.id] = tx
-      txQueueLength.increment()
+      // smooth near-simultaneous arrivals over up to three seconds
+      const dx = performance.now() - this.lastTxTime
+      this.lastTxTime = performance.now()
+      if (dx <= 250) {
+        this.txDelay = Math.min(3000, this.txDelay + (Math.random() * 250))
+      } else {
+        this.txDelay = Math.max(0, this.txDelay - (dx-250))
+      }
+      this.txs[tx.id] = tx
+      tx.onEnterScene()
+      this.poolScene.insert(this.txs[tx.id], this.txDelay)
    }
  }

-  // Dual-strategy queue processing:
-  // - ensure transactions are queued for at least txDelay
-  // - when queue length exceeds 500, process iteratively to avoid unbounded growth
-  // - while queue is small, use jittered timeouts to evenly distribute arrivals
-  //
-  // transactions tend to arrive in groups, so for smoothest
-  // animation the queue should stay short but never empty.
-  processQueue () {
-    let done
-    let delay
-    while (!done) {
-      if (this.pendingTxs && this.pendingTxs.length) {
-        if (this.txs[this.pendingTxs[0][0].id] || this.expiredTxs[this.pendingTxs[0][0].id]) {
-          // duplicate transaction, skip without delay
-          const tx = this.pendingTxs.shift()[0]
-          delete this.pendingMap[tx.id]
-          txQueueLength.decrement()
-        } else {
-          const timeSince = performance.now() - this.pendingTxs[0][1]
-          if (timeSince > this.txDelay) {
-            //process the next tx in the queue, if it arrived longer ago than txDelay
-            if (this.txDelay < this.maxTxDelay) {
-              // slowly ramp up from 0 to maxTxDelay on start, so there's no wait for the first txs on page load
-              this.txDelay += 50
-            }
-            const tx = this.pendingTxs.shift()[0]
-            delete this.pendingMap[tx.id]
-            txQueueLength.decrement()
-            this.txs[tx.id] = tx
-            this.poolScene.insert(this.txs[tx.id])
-            mempoolCount.increment()
-          } else {
-            // end the loop when the head of the queue arrived more recently than txDelay
-            done = true
-            // schedule to continue processing when head of queue matures
-            delay = this.txDelay - timeSince
-          }
-          if (this.pendingTxs.length < 500) {
-            // or the queue is under 500
-            done = true
-          }
-          // otherwise keep processing the queue
-        }
-      } else done = true
-    }
-    // randomly jitter arrival times so that txs enter more naturally
-    // with jittered delay inversely proportional to size of queue
-    let jitter = Math.random() * Math.max(1, (500 - this.pendingTxs.length))
-    this.scheduleQueue(delay || jitter)
+  dropTx (txid) {
+    // don't actually need to do anything, just let the tx expire
  }

-  scheduleQueue (delay) {
-    if (this.queueTimeout) clearTimeout(this.queueTimeout)
-    this.queueTimeout = setTimeout(() => {
-      this.processQueue()
-    }, delay)
-  }
-
-  addBlock (blockData) {
+  addBlock (blockData, realtime=true) {
    // discard duplicate blocks
    if (!blockData || !blockData.id || this.knownBlocks[blockData.id]) {
      return
@ -148,7 +109,7 @@ export default class TxController {

    this.poolScene.scrollLock = true

-    const block = (blockData && blockData.isBlock) ? blockData : new BitcoinBlock(blockData)
+    const block = new BitcoinBlock(blockData)
    this.knownBlocks[block.id] = true
    if (this.clearBlockTimeout) clearTimeout(this.clearBlockTimeout)

@ -156,39 +117,27 @@ export default class TxController {

    this.clearBlock()

-    this.blockScene = new TxBlockScene({ width: this.blockAreaSize, height: this.blockAreaSize, blockId: block.id, controller: this })
-    let poolCount = 0
+    this.blockScene = new TxBlockScene({ width: this.blockAreaSize, height: this.blockAreaSize, blockId: block.id, controller: this, colorMode: this.colorMode })
    let knownCount = 0
    let unknownCount = 0
    for (let i = 0; i < block.txns.length; i++) {
      if (this.poolScene.remove(block.txns[i].id)) {
-        poolCount++
        knownCount++
-        this.txs[block.txns[i].id].setBlock(block.id)
-        this.blockScene.insert(this.txs[block.txns[i].id], false)
-      } else if (this.pendingMap[block.txns[i].id]) {
-        knownCount++
-        const tx = this.pendingMap[tx.id]
-        const pendingIndex = this.pendingTxs.indexOf(tx)
-        if (pendingIndex >= 0) this.pendingTxs.splice(pendingIndex, 1)
-        delete this.pendingMap[tx.id]
-        tx.setBlock(block.id)
-        this.txs[tx.id] = tx
-        this.blockScene.insert(tx, false)
+        this.txs[block.txns[i].id].setBlock(block)
+        this.blockScene.insert(this.txs[block.txns[i].id], 0, false)
      } else {
        unknownCount++
        const tx = new BitcoinTx({
          ...block.txns[i],
-          block: block.id
+          block: block
        }, this.vertexArray)
        this.txs[tx.id] = tx
        this.txs[tx.id].applyHighlighting(this.highlightCriteria)
-        this.blockScene.insert(tx, false)
+        this.blockScene.insert(tx, 0, false)
      }
      this.expiredTxs[block.txns[i].id] = true
    }
    console.log(`New block with ${knownCount} known transactions and ${unknownCount} unknown transactions`)
-    mempoolCount.subtract(poolCount)
    this.blockScene.initialLayout()
    setTimeout(() => { this.poolScene.scrollLock = false; this.poolScene.layoutAll() }, 4000)

@ -198,64 +147,6 @@ export default class TxController {
    return block
  }

-  simulateBlock () {
-    for (var i = 0; i < 1000; i++) {
-      this.addTx(new BitcoinTx({
-        version: 'simulated',
-        time: Date.now(),
-        id: `simulated_${i}_${Math.random()}`,
-        value: Math.floor(Math.pow(2,(0.1-(Math.log(1 - Math.random()))) * 8))
-      }))
-    }
-    const simulatedTxns = this.pendingTxs.map(pending => {
-      return {
-        version: pending[0].version,
-        id: pending[0].id,
-        time: pending[0].time,
-        value: pending[0].value
-      }
-    })
-    // const simulatedTxns = []
-    Object.values(this.poolScene.txs).forEach(tx => {
-      if (Math.random() < 0.5) {
-        simulatedTxns.push({
-          version: tx.version,
-          id: tx.id,
-          time: tx.time,
-          value: tx.value
-        })
-      }
-    })
-    const block = new BitcoinBlock({
-      version: 'fake',
-      id: 'fake_block' + Math.random(),
-      value: 12345678900,
-      prev_block: 'also_fake',
-      merkle_root: 'merkle',
-      timestamp: Date.now() / 1000,
-      bits: 'none',
-      bytes: 1379334,
-      txn_count: simulatedTxns.length,
-      txns: simulatedTxns
-    })
-    setTimeout(() => {
-      this.addBlock(block)
-    }, 2500)
-    return block
-  }
-
-  simulateDumpTx (n, value) {
-    for (var i = 0; i < n; i++) {
-      this.addTx(new BitcoinTx({
-        version: 'simulated',
-        time: Date.now(),
-        id: `simulated_${i}_${Math.random()}`,
-        value: value || Math.min(10000, Math.floor(Math.pow(2,(0.1-(Math.log(1 - Math.random()))) * 8))) // horrible but plausible distribution of tx values
-        // value: value || Math.pow(10,Math.floor(Math.random() * 5))
-      }, this.vertexArray))
-    }
-  }
-
  hideBlock () {
    if (this.blockScene) {
      this.blockScene.hide()
--- a/client/src/controllers/TxStream.js
+++ b/client/src/controllers/TxStream.js
@ -8,7 +8,9 @@ lastBlockId.subscribe(val => { lastBlockSeen = val })

 class TxStream {
  constructor () {
-    this.websocketUri = `${config.secureSocket ? 'wss://' : 'ws://'}${config.backend ? config.backend : window.location.host }${config.backendPort ? ':' + config.backendPort : ''}/ws/txs`
+    this.apiRoot = `${config.backend ? config.backend : window.location.host }${config.backendPort ? ':' + config.backendPort : ''}`
+    this.websocketUri = `${config.secureSocket ? 'wss://' : 'ws://'}${this.apiRoot}/ws/txs`
+    this.apiUri = `${config.secureSocket ? 'https://' : 'http://'}${this.apiRoot}`
    console.log('connecting to ', this.websocketUri)
    this.reconnectBackoff = 250
    this.websocket = null
@ -91,13 +93,7 @@ class TxStream {
  sendBlockRequest () {
    if (config.noBlockFeed) return
    console.log('Checking for missed blocks...')
-    this.websocket.send(JSON.stringify({method: 'get_block', last: lastBlockSeen }))
-  }
-
-  sendMempoolRequest () {
-    this.websocket.send('count')
-    if (mempoolTimer) clearTimeout(mempoolTimer)
-    mempoolTimer = setTimeout(() => { this.sendMempoolRequest() }, 60000)
+    this.websocket.send("block_id")
  }

  disconnect () {
@ -122,7 +118,25 @@ class TxStream {
    this.reconnectBackoff = 128
    this.sendHeartbeat()
    this.sendBlockRequest()
-    this.sendMempoolRequest()
+  }
+
+  async fetchBlock (id, calledOnLoad) {
+    if (!id) return
+    if (id !== lastBlockSeen) {
+      try {
+        console.log('downloading block', id)
+        const response = await fetch(`${this.apiUri}/api/block/${id}`, {
+          method: 'GET'
+        })
+        let blockData = await response.json()
+        console.log('downloaded block', id)
+        window.dispatchEvent(new CustomEvent('bitcoin_block', { detail: { block: blockData, realtime: !calledOnLoad} }))
+      } catch (err) {
+        console.log("failed to download block ", id)
+      }
+    } else {
+      console.log('already seen block ', lastBlockSeen)
+    }
  }

  onmessage (event) {
@ -134,17 +148,38 @@ class TxStream {
    } else {
      try {
        const msg = JSON.parse(event.data)
-        if (msg && msg.type === 'count') {
-          window.dispatchEvent(new CustomEvent('bitcoin_mempool_count', { detail: msg.count }))
-        } else if (msg && msg.type === 'txn') {
-          window.dispatchEvent(new CustomEvent('bitcoin_tx', { detail: msg.txn }))
-        } else if (msg && msg.type === 'block') {
-          if (msg.block && msg.block.id) window.dispatchEvent(new CustomEvent('bitcoin_block', { detail: msg.block }))
-        } else {
-          // console.log('unknown message from websocket: ', msg)
+
+        if (!msg) throw new Error('null websocket message')
+
+        switch (msg.type) {
+
+          // reply to a last block_id request message
+          case 'block_id':
+            this.fetchBlock(msg.block_id, true)
+            break;
+
+          // notification of tx added to mempool
+          case 'txn':
+            window.dispatchEvent(new CustomEvent('bitcoin_tx', { detail: msg.txn }))
+            break;
+
+          // notification of tx dropped from mempool
+          case 'drop':
+            window.dispatchEvent(new CustomEvent('bitcoin_drop_tx', { detail: msg.txid }))
+            break;
+
+          // notification of a new block
+          case 'block':
+            if (msg.block && msg.block.id) {
+              this.fetchBlock(msg.block.id)
+            }
+            break;
        }
+
+        // all events can include a count field, with the latest mempool size
+        if (msg.count) window.dispatchEvent(new CustomEvent('bitcoin_mempool_count', { detail: msg.count }))
      } catch (err) {
-        // console.log('unknown message from websocket: ', msg)
+        console.log('error parsing msg json: ', err)
      }
    }
  }
--- a/client/src/models/BitcoinBlock.js
+++ b/client/src/models/BitcoinBlock.js
@ -1,7 +1,7 @@
 import BitcoinTx from '../models/BitcoinTx.js'

 export default class BitcoinBlock {
-  constructor ({ version, id, value, prev_block, merkle_root, timestamp, bits, bytes, txn_count, txns }) {
+  constructor ({ version, id, value, prev_block, merkle_root, timestamp, bits, bytes, txn_count, txns, fees }) {
    this.isBlock = true
    this.version = version
    this.id = id
@ -14,7 +14,31 @@ export default class BitcoinBlock {
    this.txnCount = txn_count
    this.txns = txns
    this.coinbase = new BitcoinTx(this.txns[0])
+    if (fees) {
+      this.fees = fees + this.coinbase.value
+    } else {
+      this.fees = null
+    }
    this.height = this.coinbase.coinbase.height
    this.miner_sig = this.coinbase.coinbase.sigAscii
+
+    this.total_vbytes = 0
+
+    if (this.fees != null) {
+      this.maxFeerate = 0
+      this.minFeerate = this.txnCount > 1 ? Infinity : 0
+      this.avgFeerate = 0
+      this.txns.forEach(txn => {
+        if (!BitcoinTx.prototype.isCoinbase(txn)) {
+          if (txn.fee <= 0) console.log(txn)
+          const txFeerate = txn.fee / txn.vbytes
+          this.maxFeerate = Math.max(this.maxFeerate, txFeerate)
+          this.minFeerate = Math.min(this.minFeerate, txFeerate)
+          this.avgFeerate += (txn.feerate / this.txnCount)
+        }
+        this.total_vbytes += txn.vbytes
+      })
+      this.avgFeerate = this.fees / this.total_vbytes
+    }
  }
 }
--- a/client/src/models/BitcoinTx.js
+++ b/client/src/models/BitcoinTx.js
@ -1,17 +1,9 @@
 import TxView from './TxView.js'
 import config from '../config.js'
-
-const highlightColor = {
-  h: 0.03,
-  l: 0.35
-}
-const hoverColor = {
-  h: 0.4,
-  l: 0.42
-}
+import { mixColor, pink, bluegreen, orange, teal, green, purple  } from '../utils/color.js'

 export default class BitcoinTx {
-  constructor ({ version, id, value, vbytes, inputs, outputs, time, block }, vertexArray) {
+  constructor ({ version, id, value, fee, vbytes, inputs, outputs, time, block }, vertexArray) {
    this.version = version
    this.id = id
    this.vertexArray = vertexArray
@ -21,8 +13,11 @@ export default class BitcoinTx {
    this.inputs = inputs
    this.outputs = outputs
    this.value = value
+    this.fee = fee
    this.vbytes = vbytes

+    if (this.fee != null) this.feerate = fee / vbytes
+
    if (inputs && outputs && value == null) {
      this.value = this.calcValue()
    }
@ -31,8 +26,35 @@ export default class BitcoinTx {
    this.highlight = false

    // is a coinbase transaction?
-    if (this.inputs && this.inputs.length === 1 && this.inputs[0].prev_txid === "0000000000000000000000000000000000000000000000000000000000000000") {
-      const cbInfo = this.inputs[0].script_sig
+    this.coinbase = this.isCoinbase(this)
+    if (this.coinbase) {
+      this.fee = null
+      this.feerate = null
+    }
+
+    this.setBlock(block)
+
+    const feeColor = (this.feerate == null
+      ? orange
+      : mixColor(teal, purple, 1, Math.log2(64), Math.log2(this.feerate))
+    )
+    this.colors = {
+      age: {
+        block: { color: orange },
+        pool: { color: orange, endColor: teal, duration: 60000 },
+      },
+      fee: {
+        block: { color: feeColor },
+        pool: { color: feeColor },
+      }
+    }
+
+    this.view = new TxView(this)
+  }
+
+  isCoinbase (txn) {
+    if (txn.inputs && txn.inputs.length === 1 && txn.inputs[0].prev_txid === "0000000000000000000000000000000000000000000000000000000000000000") {
+      const cbInfo = txn.inputs[0].script_sig
      // number of bytes encoding the block height
      const height_bytes = parseInt(cbInfo.substring(0,2), 16)
      // extract the specified number of bytes, reverse the endianness (reverse pairs of hex characters), parse as a hex string
@ -42,15 +64,13 @@ export default class BitcoinTx {
      const sigAscii = sig.match(/../g).reduce((parsed, hexChar) => {
        return parsed + String.fromCharCode(parseInt(hexChar, 16))
      }, "")
-      this.coinbase = {
+
+      return {
        height,
        sig,
        sigAscii
      }
-    }
-
-    this.setBlock(block)
-    this.view = new TxView(this)
+    } else return false
  }

  destroy () {
@ -70,7 +90,15 @@ export default class BitcoinTx {
    this.state = this.block ? 'block' : 'pool'
  }

-  hoverOn (color = hoverColor) {
+  onEnterScene () {
+    this.enteredTime = performance.now()
+  }
+
+  getColor (scene, mode) {
+    return this.colors[mode][scene]
+  }
+
+  hoverOn (color = bluegreen) {
    if (this.view) this.view.setHover(true, color)
  }

@ -78,7 +106,7 @@ export default class BitcoinTx {
    if (this.view) this.view.setHover(false)
  }

-  highlightOn (color = highlightColor) {
+  highlightOn (color = pink) {
    if (this.view) this.view.setHighlight(true, color)
    this.highlight = true
  }
@ -104,6 +132,6 @@ export default class BitcoinTx {
        })
      }
    })
-    this.view.setHighlight(this.highlight, color || highlightColor)
+    this.view.setHighlight(this.highlight, color || pink)
  }
 }
--- a/client/src/models/TxBlockScene.js
+++ b/client/src/models/TxBlockScene.js
@ -1,8 +1,8 @@
 import TxMondrianPoolScene from './TxMondrianPoolScene.js'

 export default class TxBlockScene extends TxMondrianPoolScene {
-  constructor ({ width, height, unit = 4, padding = 1, blockId, controller }) {
-    super({ width, height, unit, padding, controller })
+  constructor ({ width, height, unit = 4, padding = 1, blockId, controller, heightStore, colorMode }) {
+    super({ width, height, unit, padding, controller, heightStore, colorMode })
    this.heightLimit = null
    this.expired = false
    this.laidOut = false
@ -10,6 +10,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
    this.initialised = true
    this.inverted = true
    this.hidden = false
+    this.sceneType = 'block'
  }

  resize ({ width = this.width, height = this.height }) {
@ -55,7 +56,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
            y: -(Math.random() * window.innerWidth) - (this.scene.offset.y * 2) - pixelPosition.r,
            r: pixelPosition.r
          },
-          color: this.defaultColor
+          color: tx.getColor('block', this.colorMode).color
        },
        delay: 0,
        state: 'ready'
@ -67,7 +68,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
      tx.view.update({
        display: {
          position: tx.screenPosition,
-          color: this.defaultColor
+          color: tx.getColor('block', this.colorMode).color
        },
        duration: 0,
        delay: 0,
@ -77,7 +78,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
      tx.view.update({
        display: {
          position: tx.screenPosition,
-          color: this.defaultColor
+          color: tx.getColor('block', this.colorMode).color
        },
        duration: this.laidOut ? 1000 : 2000,
        delay: 0,
@ -96,7 +97,10 @@ export default class TxBlockScene extends TxMondrianPoolScene {
            y: -(Math.random() * window.innerWidth) - (this.scene.offset.y * 2) - tx.pixelPosition.r,
            r: tx.pixelPosition.r
          },
-          color: this.defaultColor
+          color: {
+            ...tx.getColor('block', this.colorMode).color,
+            alpha: 1
+          }
        },
        delay: 0,
        state: 'ready'
@ -104,7 +108,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
    }
    tx.view.update({
      display: {
-        color: this.defaultColor
+      color: tx.getColor('block', this.colorMode).color
      },
      duration: 2000,
      delay: 0
@ -112,7 +116,7 @@ export default class TxBlockScene extends TxMondrianPoolScene {
  }

  prepareTx (tx, sequence) {
-    this.prepareTxOnScreen(tx, this.layoutTx(tx, sequence, false))
+    this.prepareTxOnScreen(tx, this.layoutTx(tx, sequence, 0, false))
  }

  hideTx (tx) {
--- a/client/src/models/TxMondrianPoolScene.js
+++ b/client/src/models/TxMondrianPoolScene.js
@ -10,8 +10,8 @@ settings.subscribe(v => {
 })

 export default class TxMondrianPoolScene extends TxPoolScene {
-  constructor ({ width, height, unit, padding, controller, heightStore }) {
-    super({ width, height, unit, padding, controller, heightStore })
+  constructor ({ width, height, unit, padding, controller, heightStore, colorMode }) {
+    super({ width, height, unit, padding, controller, heightStore, colorMode })
  }

  resize ({ width, height, unit, padding }) {
--- a/client/src/models/TxPackingScene.js
+++ b/client/src/models/TxPackingScene.js
@ -1,50 +0,0 @@
-/*
-  Interface definition for TxPackingScene type classes
-*/
-
-export default class TxPackingScene {
-  constructor ({ width, height }) {
-    this.init({ width, height })
-  }
-
-  init ({ width, height }) {
-    this.width = width
-    this.height = height
-    this.txs = {}
-    this.hiddenTxs = {}
-    this.scene = {
-      width: width,
-      height: height,
-      count: 0
-    }
-  }
-
-  // insert a transaction into the scene
-  // bool 'autoLayout' controls whether to calc & update the tx position immediately
-  insert (tx, autoLayout?) {}
-
-  // remove the transaction with the given id, return false if not present
-  remove (id) {
-    return !!success
-  }
-
-  // return a list of present transaction ids
-  getTxList () {
-    return [
-      ...this.getActiveTxList(),
-      ...this.getHiddenTxList()
-      ...Object.keys(this.hiddenTxs)
-    ]
-  }
-  getActiveTxList () {
-    return Object.keys(this.txs)
-  }
-  getHiddenTxList () {
-    return Object.keys(this.hiddenTxs)
-  }
-
-  // Return a flattened array of all vertex data of active tx sprites
-  getVertexData () {
-    return Object.values(this.txs).slice(-1000).flatMap(tx => tx.view.sprite.getVertexData())
-  }
-}
--- a/client/src/models/TxPackingSquare.js
+++ b/client/src/models/TxPackingSquare.js
@ -1,5 +0,0 @@
-export default class TxPackingSquarez\ {
-  constructor () {
-
-  }
-}
--- a/client/src/models/TxPoolScene.js
+++ b/client/src/models/TxPoolScene.js
@ -1,15 +1,12 @@
 import config from '../config.js'

 export default class TxPoolScene {
-  constructor ({ width, height, unit, padding, controller, heightStore }) {
+  constructor ({ width, height, unit, padding, controller, heightStore, colorMode }) {
+    this.colorMode = colorMode || "age"
    this.maxHeight = 0
    this.heightStore = heightStore
+    this.sceneType = 'pool'
    this.init({ width, height, unit, padding, controller })
-    this.defaultColor = {
-      h: 0.1809005702490606,
-      l: 0.47246995247155193,
-      alpha: 1
-    }
  }

  init ({ width, height, controller }) {
@ -50,9 +47,41 @@ export default class TxPoolScene {
    this.scene.offset.y = (window.innerHeight - (this.blockHeight * this.gridSize)) / 2
  }

-  insert (tx, autoLayout=true) {
+  setColorMode (mode) {
+    this.colorMode = mode
+    Object.values(this.txs).forEach(tx => {
+      const txColor = tx.getColor(this.sceneType, mode)
+      if (txColor.endColor) {
+        tx.view.update({
+          display: {
+            color: txColor.color
+          },
+          duration: 0,
+          delay: 0,
+        })
+        tx.view.update({
+          display: {
+            color: txColor.endColor
+          },
+          start: tx.enteredTime,
+          duration: txColor.duration,
+          delay: 0
+        })
+      } else {
+        tx.view.update({
+          display: {
+            color: txColor.color
+          },
+          duration: 500,
+          delay: 0,
+        })
+      }
+    })
+  }
+
+  insert (tx, insertDelay, autoLayout=true) {
    if (autoLayout) {
-      this.layoutTx(tx, this.scene.count++)
+      this.layoutTx(tx, this.scene.count++, insertDelay)
      this.txs[tx.id] = tx
    } else {
      this.hiddenTxs[tx.id] = tx
@ -132,7 +161,7 @@ export default class TxPoolScene {
    return 1
  }

-  layoutTx (tx, sequence, setOnScreen = true) {
+  layoutTx (tx, sequence, insertDelay, setOnScreen = true) {
    const units = this.txSize(tx)
    this.place(tx, sequence, units)
    this.saveGridToPixelPosition(tx)
@ -148,11 +177,12 @@ export default class TxPoolScene {
      if (this.heightStore) this.heightStore.set(this.maxHeight)
      this.saveGridToPixelPosition(tx)
    }
-    if (setOnScreen) this.setTxOnScreen(tx)
+    if (setOnScreen) this.setTxOnScreen(tx, insertDelay)
  }

-  setTxOnScreen (tx) {
+  setTxOnScreen (tx, insertDelay=0) {
    if (!tx.view.initialised) {
+      const txColor = tx.getColor(this.sceneType, this.colorMode)
      tx.view.update({
        display: {
          position: this.pixelsToScreen({
@ -160,7 +190,10 @@ export default class TxPoolScene {
            y: window.innerHeight + 10,
            r: this.unitWidth / 2
          }),
-          color: this.defaultColor
+          color: {
+            ...txColor.color,
+            alpha: 1
+          },
        },
        delay: 0,
        state: 'ready'
@ -168,22 +201,22 @@ export default class TxPoolScene {
      tx.view.update({
        display: {
          position: this.pixelsToScreen(tx.pixelPosition),
-          color: this.defaultColor
+          color: txColor.color
        },
        duration: 2500,
-        delay: 0,
+        delay: insertDelay,
        state: 'pool'
      })
-      tx.view.update({
-        display: {
-          color: {
-            h: 0.4751474394406347,
-            l: 0.5970385691107944
-          }
-        },
-        duration: 60000,
-        delay: 0
-      })
+      if (txColor.endColor) {
+        tx.view.update({
+          display: {
+            color: txColor.endColor
+          },
+          start: tx.enteredTime,
+          duration: txColor.duration,
+          delay: 0
+        })
+      }
    } else {
      tx.view.update({
        display: {
--- a/client/src/stores.js
+++ b/client/src/stores.js
@ -1,4 +1,5 @@
 import { writable, derived } from 'svelte/store'
+import { tweened } from 'svelte/motion';
 import { makePollStore } from './utils/pollStore.js'
 import LocaleCurrency from 'locale-currency'
 import { currencies } from './utils/fx.js'
@ -79,10 +80,9 @@ export const devEvents = writable({
 	addBlockCallback: null
 })

-export const txQueueLength = createCounter()
 export const txCount = createCounter()
 export const lastBlockId = writable(null)
-export const mempoolCount = createCounter()
+export const mempoolCount = tweened(0)
 export const mempoolScreenHeight = writable(0)
 export const frameRate = writable(null)
 export const avgFrameRate = writable(null)
@ -102,10 +102,14 @@ export const settings = createCachedDict('settings', {
 	currency: localeCurrencyCode,
 	showFX: true,
 	vbytes: false,
+	colorByFee: false,
 	fancyGraphics: true,
 	showMessages: true,
 	noTrack: false
 })
+export const colorMode = derived([settings], ([$settings]) => {
+	return $settings.colorByFee ? "fee" : "age"
+})

 export const devSettings = (config.dev && config.debug) ? createCachedDict('dev-settings', {
 	guides: false,
--- a/client/src/utils/color.js
+++ b/client/src/utils/color.js
@ -0,0 +1,26 @@
+import { hcl } from 'd3-color'
+
+export function hlToHex ({h, l}) {
+  return hcl(h * 360, 78.225, l * 150).hex()
+}
+
+export function mixColor (startColor, endColor, min, max, value) {
+  const dx = Math.max(0, Math.min(1, (value - min) / (max - min)))
+  return {
+    h: startColor.h + (dx *(endColor.h - startColor.h)),
+    l: startColor.l + (dx *(endColor.l - startColor.l))
+  }
+}
+
+export const pink = { h: 0.03, l: 0.35 }
+export const bluegreen = { h: 0.45, l: 0.4 }
+export const orange = { h: 0.181, l: 0.472 }
+export const teal = { h: 0.475, l: 0.55 }
+export const green = { h: 0.37, l: 0.35 }
+export const purple = { h: 0.95, l: 0.35 }
+
+export const highlightA = { h: 0.93, l: 0.5 } //pink
+export const highlightB = { h: 0.214, l: 0.62 } // green
+export const highlightC = { h: 0.30, l: 1.0 } // white
+export const highlightD = { h: 0.42, l: 0.35 } // blue
+export const highlightE = { h: 0.12, l: 0.375 } // red
--- a/client/src/utils/format.js
+++ b/client/src/utils/format.js
@ -10,13 +10,18 @@ export const longBtcFormat = (Intl && Intl.NumberFormat) ? new Intl.NumberFormat
    return Number(number).toFixed(8)
  }
 }
+export const feeRateFormat = (Intl && Intl.NumberFormat) ? new Intl.NumberFormat(undefined, { maximumFractionDigits: 2, maximumSignificantDigits: 2 }) : {
+  format (number) {
+    return Number(number).toPrecision(2).slice(0,3)
+  }
+}
 export const timeFormat = (Intl && Intl.DateTimeFormat) ? new Intl.DateTimeFormat(undefined, { timeStyle: "short"}) : {
  format (date) {
    const d = new Date(date)
    return `${('' + d.getHours()).padStart(2, '0')}:${('' + d.getMinutes()).padStart(2, '0')}`
  }
 }
-export const integerFormat = (Intl && Intl.NumberFormat) ? new Intl.NumberFormat(undefined) : {
+export const numberFormat = (Intl && Intl.NumberFormat) ? new Intl.NumberFormat(undefined) : {
  format (number) {
    return Number(number).toLocaleString()
  }
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -24,6 +24,7 @@ services:
      BITCOIN_HOST: "172.17.0.1"
      BITCOIN_ZMQ_RAWBLOCK_PORT: "29000"
      BITCOIN_ZMQ_RAWTX_PORT: "29001"
+      BITCOIN_ZMQ_SEQUENCE_PORT: "29002"
      BITCOIN_RPC_PORT: "8332"
      BITCOIN_RPC_USER: "bitcoin"
      BITCOIN_RPC_PASS: "correcthorsebatterystaple"
--- a/server/README.md
+++ b/server/README.md
@ -27,6 +27,7 @@ The API server expects the following environment variables to be set:
 | BITCOIN_HOST | Bitcoin node host address |
 | BITCOIN_ZMQ_RAWBLOCK_PORT | Bitcoin node ZMQ port for block events (to match `zmqpubrawblock` in bitcoin.conf) |
 | BITCOIN_ZMQ_RAWTX_PORT | Bitcoin node ZMQ port for transaction events (to match `zmqpubrawtx` in bitcoin.conf) |
+| BITCOIN_ZMQ_SEQUENCE_PORT | Bitcoin node ZMQ port for sequence events (to match `zmqpubsequence` in bitcoin.conf) |
 | BITCOIN_RPC_PORT | Bitcoin node RPC port |
 | either | |
 | BITCOIN_RPC_USER | Bitcoin node RPC user |
--- a/server/bitfeed.service
+++ b/server/bitfeed.service
@ -14,6 +14,12 @@ Environment=LANG=en_US.UTF-8
 Environment=PORT=<port>
 Environment=BITCOIN_RPC_USER=<rpc user>
 Environment=BITCOIN_RPC_PASS=<rpc password>
+Environment=BITCOIN_HOST=<rpc host>
+Environment=BITCOIN_RPC_PORT=<rpc port>
+Environment=BITCOIN_ZMQ_RAWBLOCK_PORT=<zmq rawblock port>
+Environment=BITCOIN_ZMQ_RAWTX_PORT=<zmq rawtx port>
+Environment=BITCOIN_ZMQ_SEQUENCE_PORT=<zmq sequence port>
+

 WorkingDirectory=<installation root>/server

--- a/server/data/.gitkeep
+++ b/server/data/.gitkeep
--- a/server/data/block.dat
+++ b/server/data/block.dat
--- a/server/lib/bitcoin_rpc.ex
+++ b/server/lib/bitcoin_rpc.ex
@ -10,14 +10,16 @@ defmodule BitcoinStream.RPC do
    {port, opts} = Keyword.pop(opts, :port);
    {host, opts} = Keyword.pop(opts, :host);
    IO.puts("Starting Bitcoin RPC server on #{host} port #{port}")
-    GenServer.start_link(__MODULE__, {host, port, nil}, opts)
+    GenServer.start_link(__MODULE__, {host, port, nil, nil}, opts)
  end

  @impl true
-  def init(state) do
+  def init({host, port, status, _}) do
    # start node monitoring loop
-    send(self(), :check_status)
-    {:ok, state}
+    creds = rpc_creds();
+
+    send(self(), :check_status);
+    {:ok, {host, port, status, creds}}
  end

  def handle_info(:check_status, state) do
@ -28,28 +30,28 @@ defmodule BitcoinStream.RPC do
  end

  @impl true
-  def handle_call({:request, method, params}, _from, {host, port, status}) do
-    case make_request(host, port, method, params) do
-      {:ok, info} ->
-        {:reply, {:ok, info}, {host, port, status}}
+  def handle_call({:request, method, params}, _from, {host, port, status, creds}) do
+    case make_request(host, port, creds, method, params) do
+      {:ok, code, info} ->
+        {:reply, {:ok, code, info}, {host, port, status, creds}}

      {:error, reason} ->
-        {:reply, {:error, reason}, {host, port, status}}
+        {:reply, {:error, reason}, {host, port, status, creds}}
    end
  end

  @impl true
-  def handle_call({:get_node_status}, _from, {host, port, status}) do
-    {:reply, {:ok, status}, {host, port, status}}
+  def handle_call({:get_node_status}, _from, {host, port, status, creds}) do
+    {:reply, {:ok, status}, {host, port, status, creds}}
  end

-  defp make_request(host, port, method, params) do
-    with  { user, pw } <- rpc_creds(),
+  defp make_request(host, port, creds, method, params) do
+    with  { user, pw } <- creds,
          {:ok, rpc_request} <- Jason.encode(%{method: method, params: params}),
-          {:ok, 200, _headers, body_ref} <- :hackney.request(:post, "http://#{host}:#{port}", [{"content-type", "application/json"}], rpc_request, [basic_auth: { user, pw }]),
+          {:ok, code, _headers, body_ref} <- :hackney.request(:post, "http://#{host}:#{port}", [{"content-type", "application/json"}], rpc_request, [basic_auth: { user, pw }]),
          {:ok, body} <- :hackney.body(body_ref),
          {:ok, %{"result" => info}} <- Jason.decode(body) do
-      {:ok, info}
+      {:ok, code, info}
    else
      {:ok, code, _} ->
        IO.puts("RPC request #{method} failed with HTTP code #{code}")
@ -66,7 +68,6 @@ defmodule BitcoinStream.RPC do
  end

  def request(pid, method, params) do
-    IO.inspect({pid, method, params});
    GenServer.call(pid, {:request, method, params}, 60000)
  catch
    :exit, reason ->
@ -78,18 +79,18 @@ defmodule BitcoinStream.RPC do
    GenServer.call(pid, {:get_node_status})
  end

-  def check_status({host, port, status}) do
-    with  {:ok, info} <- make_request(host, port, "getblockchaininfo", []) do
-      {host, port, info}
+  def check_status({host, port, status, creds}) do
+    with  {:ok, 200, info} <- make_request(host, port, creds, "getblockchaininfo", []) do
+      {host, port, info, creds}
    else
      {:error, reason} ->
        IO.puts("node status check failed");
        IO.inspect(reason)
-        {host, port, status}
+        {host, port, status, creds}
      err ->
        IO.puts("node status check failed: (unknown reason)");
        IO.inspect(err);
-        {host, port, status}
+        {host, port, status, creds}
    end
  end

--- a/server/lib/block_data.ex
+++ b/server/lib/block_data.ex
@ -1,3 +1,5 @@
+Application.ensure_all_started(BitcoinStream.RPC)
+
 defmodule BitcoinStream.BlockData do
  @moduledoc """
  Block data module.
@ -10,12 +12,11 @@ defmodule BitcoinStream.BlockData do

  def start_link(opts) do
    IO.puts("Starting block data link")
-    # load block.dat
+    # load block

-    with {:ok, block_data} <- File.read("data/block.dat"),
-         {:ok, block} <- BitcoinBlock.decode(block_data),
-         {:ok, payload} <- Jason.encode(%{type: "block", block: block}) do
-      GenServer.start_link(__MODULE__, {block, payload}, opts)
+    with {:ok, json} <- File.read("data/last_block.json"),
+         {:ok, %{"id" => id}} <- Jason.decode(json) do
+      GenServer.start_link(__MODULE__, {id, json}, opts)
    else
      _ -> GenServer.start_link(__MODULE__, {nil, "null"}, opts)
    end
@ -27,41 +28,17 @@ defmodule BitcoinStream.BlockData do
  end

  @impl true
-  def handle_call(:block_id, _from, {block, json}) do
-    case block do
-      %{id: id} ->
-        {:reply, id, {block, json}}
-      _ -> {:reply, nil, {block, json}}
-    end
+  def handle_call(:block_id, _from, {id, json}) do
+    {:reply, id, {id, json}}
  end

  @impl true
-  def handle_call(:block, _from, {block, json}) do
-    {:reply, block, {block, json}}
+  def handle_call(:json_block, _from, {id, json}) do
+    {:reply, json, {id, json}}
  end

  @impl true
-  def handle_call(:json_block, _from, {block, json}) do
-    {:reply, json, {block, json}}
+  def handle_cast({:json, {id, json}}, _state) do
+      {:noreply, {id, json}}
  end
-
-  @impl true
-  def handle_cast({:block, block}, state) do
-    with {:ok, json } <- Jason.encode(%{type: "block", block: block}) do
-      IO.puts("Storing block data");
-      {:noreply, {block, json}}
-    else
-      {:err, reason} ->
-        IO.puts("Failed to json encode block data");
-        IO.inspect(reason);
-        {:noreply, state}
-      _ ->
-        IO.puts("Failed to json encode block data");
-        {:noreply, state}
-    end
-  end
-
-  def get_block_number() do
-  end
-
 end
--- a/server/lib/bridge.ex
+++ b/server/lib/bridge.ex
@ -12,17 +12,21 @@ defmodule BitcoinStream.Bridge do
  alias BitcoinStream.Mempool, as: Mempool
  alias BitcoinStream.RPC, as: RPC

-  def child_spec(host: host, tx_port: tx_port, block_port: block_port) do
+  def child_spec(host: host, tx_port: tx_port, block_port: block_port, sequence_port: sequence_port) do
    %{
      id: BitcoinStream.Bridge,
-      start: {BitcoinStream.Bridge, :start_link, [host, tx_port, block_port]}
+      start: {BitcoinStream.Bridge, :start_link, [host, tx_port, block_port, sequence_port]}
    }
  end

-  def start_link(host, tx_port, block_port) do
-    IO.puts("Starting Bitcoin bridge on #{host} ports #{tx_port}, #{block_port}")
+  def start_link(host, tx_port, block_port, sequence_port) do
+    IO.puts("Starting Bitcoin bridge on #{host} ports #{tx_port}, #{block_port}, #{sequence_port}")
+    IO.puts("Mempool loaded, ready to syncronize");
    Task.start(fn -> connect_tx(host, tx_port) end);
    Task.start(fn -> connect_block(host, block_port) end);
+    Task.start(fn -> connect_sequence(host, sequence_port) end);
+    :timer.sleep(2000);
+    Task.start(fn -> Mempool.sync(:mempool) end);
    GenServer.start_link(__MODULE__, %{})
  end

@ -48,7 +52,7 @@ defmodule BitcoinStream.Bridge do
    end

    # start tx loop
-    tx_loop(socket)
+    tx_loop(socket, 0)
  end

  defp connect_block(host, port) do
@ -57,9 +61,6 @@ defmodule BitcoinStream.Bridge do
    wait_for_ibd();
    IO.puts("Node is fully synced, connecting to block socket");

-    # sync mempool
-    Mempool.sync(:mempool);
-
    # connect to socket
    {:ok, socket} = :chumak.socket(:sub);
    IO.puts("Connected block zmq socket on #{host} port #{port}");
@ -72,7 +73,28 @@ defmodule BitcoinStream.Bridge do
    end

    # start block loop
-    block_loop(socket)
+    block_loop(socket, 0)
+  end
+
+  defp connect_sequence(host, port) do
+    # check rpc online & synced
+    IO.puts("Waiting for node to come online and fully sync before connecting to sequence socket");
+    wait_for_ibd();
+    IO.puts("Node is fully synced, connecting to sequence socket");
+
+    # connect to socket
+    {:ok, socket} = :chumak.socket(:sub);
+    IO.puts("Connected sequence zmq socket on #{host} port #{port}");
+    :chumak.subscribe(socket, 'sequence')
+    IO.puts("Subscribed to sequence events")
+    case :chumak.connect(socket, :tcp, String.to_charlist(host), port) do
+      {:ok, pid} -> IO.puts("Binding ok to sequence socket pid #{inspect pid}");
+      {:error, reason} -> IO.puts("Binding sequence socket failed: #{reason}");
+      _ -> IO.puts("???");
+    end
+
+    # start tx loop
+    sequence_loop(socket)
  end

  defp wait_for_ibd() do
@ -84,9 +106,9 @@ defmodule BitcoinStream.Bridge do
    end
  end

-  defp sendTxn(txn) do
+  defp send_txn(txn, count) do
    # IO.puts("Forwarding transaction to websocket clients")
-    case Jason.encode(%{type: "txn", txn: txn}) do
+    case Jason.encode(%{type: "txn", txn: txn, count: count}) do
      {:ok, payload} ->
        Registry.dispatch(Registry.BitcoinStream, "txs", fn(entries) ->
          for {pid, _} <- entries do
@ -97,12 +119,8 @@ defmodule BitcoinStream.Bridge do
    end
  end

-  defp incrementMempool() do
-    Mempool.increment(:mempool)
-  end
-
-  defp sendBlock(block) do
-    case Jason.encode(%{type: "block", block: block}) do
+  defp send_block(block, count) do
+    case Jason.encode(%{type: "block", block: %{id: block.id}, drop: count}) do
      {:ok, payload} ->
        Registry.dispatch(Registry.BitcoinStream, "txs", fn(entries) ->
          for {pid, _} <- entries do
@ -114,37 +132,112 @@ defmodule BitcoinStream.Bridge do
    end
  end

-  defp tx_loop(socket) do
-    # IO.puts("client tx loop");
-    with  {:ok, message} <- :chumak.recv_multipart(socket),
-          [_topic, payload, _size] <- message,
-          {:ok, txn} <- BitcoinTx.decode(payload) do
-      sendTxn(txn);
-      incrementMempool();
-    else
-      {:error, reason} -> IO.puts("Bitcoin node transaction feed bridge error: #{reason}");
-      _ -> IO.puts("Bitcoin node transaction feed bridge error (unknown reason)");
+  defp send_drop_tx(txid, count) do
+    case Jason.encode(%{type: "drop", txid: txid, count: count}) do
+      {:ok, payload} ->
+        Registry.dispatch(Registry.BitcoinStream, "txs", fn(entries) ->
+          for {pid, _} <- entries do
+            Process.send(pid, payload, []);
+          end
+        end)
+      {:error, reason} -> IO.puts("Error json encoding drop message: #{reason}");
    end
-
-    tx_loop(socket)
  end

-  defp block_loop(socket) do
-    IO.puts("client block loop");
-    with  {:ok, message} <- :chumak.recv_multipart(socket),
-          [_topic, payload, _size] <- message,
-          :ok <- File.write("data/block.dat", payload, [:binary]),
-          {:ok, block} <- BitcoinBlock.decode(payload) do
-      GenServer.cast(:block_data, {:block, block})
-      sendBlock(block);
-      Mempool.sync(:mempool);
-      IO.puts("new block")
-    else
-      {:error, reason} -> IO.puts("Bitcoin node block feed bridge error: #{reason}");
-      _ -> IO.puts("Bitcoin node block feed bridge error (unknown reason)");
-    end
+  defp tx_process(payload) do
+    case BitcoinTx.decode(payload) do
+      {:ok, txn} ->
+        case Mempool.get_tx_status(:mempool, txn.id) do
+          # :registered and :new transactions are inflated and inserted into the mempool
+          status when (status in [:registered, :new]) ->
+            inflated_txn = BitcoinTx.inflate(txn);
+            case Mempool.insert(:mempool, txn.id, inflated_txn) do
+              # Mempool.insert returns the size of the mempool if insertion was successful
+              # Forward tx to clients in this case
+              count when is_integer(count) -> send_txn(inflated_txn, count)

-    block_loop(socket)
+              _ -> false
+            end
+
+          # other statuses indicate duplicate or dropped transaction
+          _ -> false
+        end
+
+      {:err, reason} ->
+        IO.puts("Error decoding tx: #{reason}");
+        false
+
+      error ->
+        IO.puts("Error decoding tx");
+        IO.inspect(error);
+        false
+    end
+  end
+
+  defp tx_loop(socket, seq) do
+    with  {:ok, message} <- :chumak.recv_multipart(socket),
+          [_topic, payload, <<sequence::little-size(32)>>] <- message,
+          true <- (seq  != sequence) do
+      Task.start(fn -> tx_process(payload) end);
+      tx_loop(socket, sequence)
+    else
+      _ -> tx_loop(socket, seq)
+    end
+  end
+
+  defp block_loop(socket, seq) do
+    IO.puts("client block loop");
+    with  {:ok, message} <- :chumak.recv_multipart(socket), # wait for the next zmq message in the queue
+          [_topic, payload, <<sequence::little-size(32)>>] <- message,
+          true <- (seq != sequence), # discard contiguous duplicate messages
+          _ <- IO.puts("block received"),
+          {:ok, block} <- BitcoinBlock.decode(payload),
+          count <- Mempool.clear_block_txs(:mempool, block),
+          {:ok, json} <- Jason.encode(block),
+          :ok <- File.write("data/last_block.json", json) do
+      IO.puts("processed block #{block.id}");
+      GenServer.cast(:block_data, {:json, { block.id, json }});
+      send_block(block, count);
+      block_loop(socket, sequence)
+    else
+      _ -> block_loop(socket, seq)
+    end
+  end
+
+  defp sequence_loop(socket) do
+    with  {:ok, message} <- :chumak.recv_multipart(socket),
+          [_topic, <<hash::binary-size(32), type::binary-size(1), seq::little-size(64)>>, <<_sequence::little-size(32)>>] <- message,
+          txid <- Base.encode16(hash, case: :lower),
+          event <- to_charlist(type) do
+      # IO.puts("loop #{sequence}");
+      case event do
+        # Transaction added to mempool
+        'A' ->
+          case Mempool.register(:mempool, txid, seq, true) do
+            false -> false
+
+            { txn, count } ->
+              # IO.puts("*SEQ* #{txid}");
+              send_txn(txn, count)
+          end
+
+        # Transaction removed from mempool for non block inclusion reason
+        'R' ->
+          case Mempool.drop(:mempool, txid) do
+            count when is_integer(count) ->
+              send_drop_tx(txid, count);
+
+            _ ->
+              true
+          end
+
+        # Don't care about other events
+        _ -> true
+      end
+    else
+      _ -> false
+    end
+    sequence_loop(socket)
  end

 end
--- a/server/lib/exometer_report_dash.ex
+++ b/server/lib/exometer_report_dash.ex
@ -6,6 +6,8 @@ defmodule BitcoinStream.ExometerReportDash do
  @impl true
  def exometer_init(opts) do
    IO.puts("Initialising dashboard exometer reporter")
+    Registry.BitcoinStream
+    |> Registry.register("metrics", {})
    {:ok, opts}
  end

--- a/server/lib/mempool.ex
+++ b/server/lib/mempool.ex
@ -1,9 +1,20 @@
 defmodule BitcoinStream.Mempool do
  @moduledoc """
-  Agent for retrieving and maintaining mempool info (primarily tx count)
+  Agent for retrieving and maintaining mempool info
+  Used for tracking mempool count, and maintaining an :ets cache of transaction prevouts
+
+  Transaction lifecycle:
+    Register -> a ZMQ sequence 'A' message received with this txid
+    Insert -> a ZMQ rawtx message is received
+    Drop -> EITHER a ZMQ sequence 'R' message is received
+         -> OR the transaction is included in a ZMQ rawblock message
+
+    ZMQ 'A' and 'R' messages are guaranteed to arrive in order relative to each other
+    but rawtx and rawblock messages may arrive in any order
  """
  use Agent

+  alias BitcoinStream.Protocol.Transaction, as: BitcoinTx
  alias BitcoinStream.RPC, as: RPC

  @doc """
@ -12,13 +23,14 @@ defmodule BitcoinStream.Mempool do
  """
  def start_link(opts) do
    IO.puts("Starting mempool agent");
-    case Agent.start_link(fn -> %{count: 0} end, opts) do
-      {:ok, pid} ->
-        sync(pid);
-        {:ok, pid}
-
-      result -> result
-    end
+    # cache of all transactions in the node mempool, mapped to {inputs, total_input_value}
+    :ets.new(:mempool_cache, [:set, :public, :named_table]);
+    # cache of transactions ids in the mempool, but not yet synchronized with the :mempool_cache
+    :ets.new(:sync_cache, [:set, :public, :named_table]);
+    # cache of transaction ids included in the last block
+    # used to avoid allowing confirmed transactions back into the mempool if rawtx events arrive late
+    :ets.new(:block_cache, [:set, :public, :named_table]);
+    Agent.start_link(fn -> %{count: 0, seq: :infinity, queue: [], done: false} end, opts)
  end

  def set(pid, n) do
@ -29,27 +41,223 @@ defmodule BitcoinStream.Mempool do
    Agent.get(pid, &Map.get(&1, :count))
  end

-  def increment(pid) do
-    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x + 1 end))
+  defp increment(pid) do
+    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x + 1 end));
  end

-  def decrement(pid) do
-    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x - 1 end))
+  defp decrement(pid) do
+    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x - 1 end));
  end

-  def add(pid, n) do
-    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x + n end))
+  defp get_seq(pid) do
+    Agent.get(pid, &Map.get(&1, :seq))
  end

-  def subtract(pid, n) do
-    Agent.update(pid, &Map.update(&1, :count, 0, fn(x) -> x - n end))
+  defp set_seq(pid, seq) do
+    Agent.update(pid, &Map.put(&1, :seq, seq))
+  end
+
+  defp get_queue(pid) do
+    Agent.get(pid, &Map.get(&1, :queue))
+  end
+
+  defp set_queue(pid, queue) do
+    Agent.update(pid, &Map.put(&1, :queue, queue))
+  end
+
+  defp enqueue(pid, txid) do
+    Agent.update(pid, &Map.update(&1, :queue, [], fn(q) -> [txid | q] end))
+  end
+
+  def is_done(pid) do
+    Agent.get(pid, &Map.get(&1, :done))
+  end
+  defp set_done(pid) do
+    Agent.update(pid, &Map.put(&1, :done, true))
+  end
+
+  def get_tx_status(_pid, txid) do
+    case :ets.lookup(:mempool_cache, txid) do
+      # new transaction, not yet registered
+      [] ->
+        case :ets.lookup(:block_cache, txid) do
+          # not included in the last block
+          [] -> :new
+
+          # already included in the last block
+          [_] -> :block
+        end
+
+      # new transaction, id already registered
+      [{_txid, nil, :ready}] ->
+        :registered
+
+      # already dropped
+      [{_, nil, :drop}] ->
+        :dropped
+
+      # duplicate (ignore)
+      [_] ->
+        :duplicate
+    end
+  end
+
+  def insert(pid, txid, txn) do
+    case get_tx_status(pid, txid) do
+      # new transaction, id already registered
+      :registered ->
+        with [] <- :ets.lookup(:block_cache, txid) do # double check tx isn't included in the last block
+          :ets.insert(:mempool_cache, {txid, { txn.inputs, txn.value + txn.fee }, nil});
+          get(pid)
+        else
+          _ -> false
+        end
+
+      # new transaction, not yet registered
+      :new ->
+        :ets.insert(:mempool_cache, {txid, nil, txn})
+        false
+
+      # new transaction, already included in the last block
+      :block -> false
+
+      # already dropped
+      :dropped ->
+        :ets.delete(:mempool_cache, txid);
+        false
+
+      # duplicate (ignore)
+      :duplicate ->
+        false
+    end
+  end
+
+  def register(pid, txid, sequence, do_count) do
+    cond do
+      # mempool isn't loaded yet - add this tx to the queue
+      (get_seq(pid) == :infinity) ->
+        enqueue(pid, {txid, sequence});
+        false
+
+      ((sequence == nil) or (sequence >= get_seq(pid))) ->
+        case :ets.lookup(:mempool_cache, txid) do
+          # new transaction
+          [] ->
+            with [] <- :ets.lookup(:block_cache, txid) do # double check tx isn't included in the last block
+              :ets.insert(:mempool_cache, {txid, nil, :ready});
+              :ets.delete(:sync_cache, txid);
+              if do_count do
+                increment(pid);
+              end
+            else
+              _ -> false;
+            end
+            false
+
+          # duplicate sequence message (should never happen)
+          [{_txid, _, :ready}] -> false
+
+          # already dropped
+          [{_txid, _, :drop}] -> false
+
+          # data already received, but tx not registered
+          [{_txid, _, txn}] ->
+            :ets.insert(:mempool_cache, {txid, { txn.inputs, txn.value + txn.fee }, nil});
+            :ets.delete(:sync_cache, txid);
+            if do_count do
+              increment(pid);
+            end
+            {txn, get(pid)}
+
+          # some other invalid state (should never happen)
+          [_] -> false
+        end
+
+      true -> false
+    end
+  end
+
+  def drop(pid, txid) do
+    case :ets.lookup(:mempool_cache, txid) do
+      # tx not yet registered
+      [] ->
+        case :ets.lookup(:sync_cache, txid) do
+          [] -> false
+
+          # tx is in the mempool sync cache, mark to be dropped when processed
+          _ ->
+            :ets.insert(:mempool_cache, {txid, nil, :drop});
+            decrement(pid);
+            get(pid)
+        end
+
+      # already marked as dropped (should never happen)
+      [{_txid, nil, :drop}] -> false
+
+      # tx registered but not processed, mark to be dropped
+      [{_txid, nil, :ready}] ->
+        :ets.insert(:mempool_cache, {txid, nil, :drop});
+        decrement(pid);
+        false
+
+      # tx data cached but not registered and not already dropped
+      [{_txid, nil, status}] when status != nil ->
+        case :ets.lookup(:sync_cache, txid) do
+          [] -> true;
+
+          _ -> decrement(pid);
+        end
+        :ets.delete(:mempool_cache, txid);
+        get(pid)
+
+      # tx fully processed and not already dropped
+      [{_txid, data, _status}] when data != nil ->
+        :ets.delete(:mempool_cache, txid);
+        decrement(pid);
+        get(pid)
+
+      _ -> false
+    end
+  end
+
+  defp send_mempool_count(pid) do
+    count = get(pid)
+    case Jason.encode(%{type: "count", count: count}) do
+      {:ok, payload} ->
+        Registry.dispatch(Registry.BitcoinStream, "txs", fn(entries) ->
+          for {pid, _} <- entries do
+            Process.send(pid, payload, []);
+          end
+        end)
+      {:error, reason} -> IO.puts("Error json encoding count: #{reason}");
+    end
+  end
+
+  defp sync_queue(_pid, []) do
+    true
+  end
+
+  defp sync_queue(pid, [{txid, sequence} | tail]) do
+    register(pid, txid, sequence, true);
+    sync_queue(pid, tail)
  end

  def sync(pid) do
-    IO.puts("Syncing mempool");
-    with  {:ok, %{"size" => pool_size}} <- RPC.request(:rpc, "getmempoolinfo", []) do
-      IO.puts("Synced pool: size = #{pool_size}");
-      set(pid, pool_size)
+    IO.puts("Preparing mempool sync");
+    with  {:ok, 200, %{"mempool_sequence" => sequence, "txids" => txns}} <- RPC.request(:rpc, "getrawmempool", [false, true]) do
+      set_seq(pid, sequence);
+      count = length(txns);
+      set(pid, count);
+      cache_sync_ids(pid, txns);
+
+      # handle queue accumulated while loading the mempool
+      queue = get_queue(pid);
+      sync_queue(pid, queue);
+      set_queue(pid, []);
+
+      IO.puts("Loaded #{count} mempool transactions");
+      send_mempool_count(pid);
+      do_sync(pid, txns)
    else
      {:error, reason} ->
        IO.puts("Pool sync failed");
@ -62,4 +270,84 @@ defmodule BitcoinStream.Mempool do
    end
  end

+  def do_sync(pid, txns) do
+    IO.puts("Syncing #{length(txns)} mempool transactions");
+    sync_mempool(pid, txns);
+    IO.puts("MEMPOOL SYNC FINISHED");
+    set_done(pid);
+    IO.inspect(is_done(pid))
+  end
+
+  defp sync_mempool(pid, txns) do
+    sync_mempool_txns(pid, txns, 0)
+  end
+
+  defp sync_mempool_txn(pid, txid) do
+    case :ets.lookup(:mempool_cache, txid) do
+      [] ->
+        with  {:ok, 200, hextx} <- RPC.request(:rpc, "getrawtransaction", [txid]),
+              rawtx <- Base.decode16!(hextx, case: :lower),
+              {:ok, txn } <- BitcoinTx.decode(rawtx),
+              inflated_txn <- BitcoinTx.inflate(txn) do
+          register(pid, txid, nil, false);
+          insert(pid, txid, inflated_txn)
+        else
+          _ -> IO.puts("sync_mempool_txn failed #{txid}")
+        end
+
+      [_] -> true
+    end
+  end
+
+  defp sync_mempool_txns(_, [], count) do
+    count
+  end
+
+  defp sync_mempool_txns(pid, [head | tail], count) do
+    IO.puts("Syncing mempool tx #{count}/#{count + length(tail) + 1} | #{head}");
+    sync_mempool_txn(pid, head);
+    sync_mempool_txns(pid, tail, count + 1)
+  end
+
+  defp cache_sync_ids(pid, txns) do
+    :ets.delete_all_objects(:sync_cache);
+    cache_sync_ids(pid, txns, 0)
+  end
+
+  defp cache_sync_ids(pid, [head | tail], cached) do
+    :ets.insert(:sync_cache, {head, true});
+    cache_sync_ids(pid, tail, cached + 1)
+  end
+
+  defp cache_sync_ids(_pid, [], cached) do
+    cached
+  end
+
+  def clear_block_txs(pid, block) do
+    :ets.delete_all_objects(:block_cache)
+    clear_block_txs(pid, block.txns, 0)
+  end
+
+  # clear confirmed transactions
+  # return the total number removed from the mempool
+  # i.e. the amount by which to decrement the mempool counter
+  defp clear_block_txs(pid, [], _cleared) do
+    get(pid)
+  end
+
+  defp clear_block_txs(pid, [head | tail], cleared) do
+    :ets.insert(:block_cache, {head.id, true})
+    if drop(pid, head.id) do # tx was in the mempool
+      clear_block_txs(pid, tail, cleared + 1)
+    else
+      case :ets.lookup(:sync_cache, head.id) do
+        # tx was not in the mempool nor queued for processing
+        [] -> clear_block_txs(pid, tail, cleared)
+
+        # tx was not in the mempool, but is queued for processing
+        _ -> clear_block_txs(pid, tail, cleared + 1)
+      end
+    end
+  end
+
 end
--- a/server/lib/protocol/block.ex
+++ b/server/lib/protocol/block.ex
@ -1,3 +1,5 @@
+Application.ensure_all_started(BitcoinStream.RPC)
+
 defmodule BitcoinStream.Protocol.Block do
@moduledoc """
  Summarised bitcoin block.
@ -6,8 +8,8 @@ defmodule BitcoinStream.Protocol.Block do
  and condensing transactions into only id, value and version
 """

-alias BitcoinStream.Protocol.Block
 alias BitcoinStream.Protocol.Transaction, as: BitcoinTx
+alias BitcoinStream.Mempool, as: Mempool

@derive Jason.Encoder
 defstruct [
@ -20,6 +22,7 @@ defstruct [
  :nonce,
  :txn_count,
  :txns,
+  :fees,
  :value,
  :id
 ]
@ -29,7 +32,7 @@ def decode(block_binary) do
        hex <- Base.encode16(block_binary, case: :lower),
        {:ok, raw_block} <- Bitcoinex.Block.decode(hex),
        id <- Bitcoinex.Block.block_id(block_binary),
-        {summarised_txns, total_value} <- summarise_txns(raw_block.txns)
+        {summarised_txns, total_value, total_fees} <- summarise_txns(raw_block.txns)
  do
    {:ok, %__MODULE__{
      version: raw_block.version,
@ -40,6 +43,7 @@ def decode(block_binary) do
      bytes: bytes,
      txn_count: raw_block.txn_count,
      txns: summarised_txns,
+      fees: total_fees,
      value: total_value,
      id: id
    }}
@ -54,24 +58,28 @@ def decode(block_binary) do
 end

 defp summarise_txns(txns) do
-  summarise_txns(txns, [], 0)
+  # Mempool.is_done returns false while the mempool is still syncing
+  if Mempool.is_done(:mempool) do
+    summarise_txns(txns, [], 0, 0, true)
+  else
+    summarise_txns(txns, [], 0, 0, false)
+  end
 end

-defp summarise_txns([], summarised, total) do
-  {Enum.reverse(summarised), total}
+defp summarise_txns([], summarised, total, fees, _do_inflate) do
+  {Enum.reverse(summarised), total, fees}
 end

-defp summarise_txns([next | rest], summarised, total) do
+defp summarise_txns([next | rest], summarised, total, fees, do_inflate) do
  extended_txn = BitcoinTx.extend(next)

-  summarise_txns(rest, [extended_txn | summarised], total + extended_txn.value)
-end
-
-def test() do
-  raw_block = File.read!("data/block.dat")
-  {:ok, block} = Block.decode(raw_block)
-
-  block
+  # if the mempool is still syncing, inflating txs will take too long, so skip it
+  if do_inflate do
+    inflated_txn = BitcoinTx.inflate(extended_txn)
+    summarise_txns(rest, [inflated_txn | summarised], total + inflated_txn.value, fees + inflated_txn.fee, true)
+  else
+    summarise_txns(rest, [extended_txn | summarised], nil, nil, false)
+  end
 end

 end
--- a/server/lib/protocol/transaction.ex
+++ b/server/lib/protocol/transaction.ex
@ -1,3 +1,5 @@
+Application.ensure_all_started(BitcoinStream.RPC)
+
 defmodule BitcoinStream.Protocol.Transaction do
  @moduledoc """
    Extended bitcoin transaction struct
@ -11,6 +13,8 @@ defmodule BitcoinStream.Protocol.Transaction do

  BitcoinStream.Protocol.Transaction

+  alias BitcoinStream.RPC, as: RPC
+
  @derive Jason.Encoder
  defstruct [
    :version,
@ -18,6 +22,7 @@ defmodule BitcoinStream.Protocol.Transaction do
    :inputs,
    :outputs,
    :value,
+    :fee,
    # :witnesses,
    :lock_time,
    :id,
@ -66,6 +71,22 @@ defmodule BitcoinStream.Protocol.Transaction do
    }
  end

+  def inflate(txn) do
+    {inputs, in_value } = inflate_inputs(txn.id, txn.inputs);
+    %__MODULE__{
+      version: txn.version,
+      vbytes: txn.vbytes,
+      inputs: inputs,
+      outputs: txn.outputs,
+      value: txn.value,
+      fee: (in_value - txn.value),
+      # witnesses: txn.witnesses,
+      lock_time: txn.lock_time,
+      id: txn.id,
+      time: txn.time
+    }
+  end
+
  defp count_value([], total) do
    total
  end
@ -74,6 +95,81 @@ defmodule BitcoinStream.Protocol.Transaction do
    count_value(rest, total + next_output.value)
  end

+  defp inflate_input(input) do
+    case input.prev_txid do
+      "0000000000000000000000000000000000000000000000000000000000000000" ->
+        {:ok, %{
+          prev_txid: input.prev_txid,
+          prev_vout: input.prev_vout,
+          script_sig: input.script_sig,
+          sequence_no: input.sequence_no,
+          value: 0,
+          script_pub_key: nil,
+        } }
+      _prev_txid ->
+        with  {:ok, 200, hextx} <- RPC.request(:rpc, "getrawtransaction", [input.prev_txid]),
+              rawtx <- Base.decode16!(hextx, case: :lower),
+              {:ok, txn } <- decode(rawtx),
+              output <- Enum.at(txn.outputs, input.prev_vout) do
+          {:ok, %{
+            prev_txid: input.prev_txid,
+            prev_vout: input.prev_vout,
+            script_sig: input.script_sig,
+            sequence_no: input.sequence_no,
+            value: output.value,
+            script_pub_key: output.script_pub_key,
+          } }
+        else
+          {:ok, 500, reason} ->
+            IO.puts("transaction not found #{input.prev_txid}");
+            IO.inspect(reason)
+          {:error, reason} ->
+            IO.puts("Failed to inflate input:");
+            IO.inspect(reason)
+            :error
+          err ->
+            IO.puts("Failed to inflate input: (unknown reason)");
+            IO.inspect(err);
+            :error
+        end
+    end
+  end
+
+  defp inflate_inputs([], inflated, total) do
+    {inflated, total}
+  end
+  defp inflate_inputs([next_input | rest], inflated, total) do
+    case inflate_input(next_input) do
+      {:ok, inflated_txn} ->
+        inflate_inputs(rest, [inflated_txn | inflated], total + inflated_txn.value)
+      _ ->
+        inflate_inputs(rest, [inflated], total)
+    end
+  end
+  def inflate_inputs([], nil) do
+    { nil, 0 }
+  end
+  def inflate_inputs(txid, inputs) do
+    case :ets.lookup(:mempool_cache, txid) do
+      # cache miss, actually inflate
+      [] ->
+        inflate_inputs(inputs, [], 0)
+
+      # cache hit, but processed inputs not available
+      [{_, nil, _}] ->
+        inflate_inputs(inputs, [], 0)
+
+      # cache hit, just return the cached values
+      [{_, input_state, _}] ->
+        input_state
+
+      other ->
+        IO.puts("unexpected cached value: ")
+        IO.inspect(other);
+        inflate_inputs(inputs, [], 0)
+    end
+  end
+
 end

 # defmodule BitcoinStream.Protocol.Transaction.Summary do
--- a/server/lib/router.ex
+++ b/server/lib/router.ex
@ -12,7 +12,27 @@ defmodule BitcoinStream.Router do
    json_decoder: Jason
  plug :dispatch

+  match "/api/block/:hash" do
+    case get_block(hash) do
+      {:ok, block} ->
+        put_resp_header(conn, "cache-control", "public, max-age=604800, immutable")
+        |> send_resp(200, block)
+      _ ->
+        IO.puts("Error getting block hash")
+    end
+  end
+
  match _ do
    send_resp(conn, 404, "404")
  end
+
+  defp get_block(last_seen) do
+    last_id = GenServer.call(:block_data, :block_id);
+    cond do
+      (last_seen == last_id) ->
+        payload = GenServer.call(:block_data, :json_block);
+        {:ok, payload}
+      true -> :err
+    end
+  end
 end
--- a/server/lib/server.ex
+++ b/server/lib/server.ex
@ -5,13 +5,14 @@ defmodule BitcoinStream.Server do
    { socket_port, "" } = Integer.parse(System.get_env("PORT"));
    { zmq_tx_port, "" } = Integer.parse(System.get_env("BITCOIN_ZMQ_RAWTX_PORT"));
    { zmq_block_port, "" } = Integer.parse(System.get_env("BITCOIN_ZMQ_RAWBLOCK_PORT"));
+    { zmq_sequence_port, "" } = Integer.parse(System.get_env("BITCOIN_ZMQ_SEQUENCE_PORT"));
    { rpc_port, "" } = Integer.parse(System.get_env("BITCOIN_RPC_PORT"));
    btc_host = System.get_env("BITCOIN_HOST");

    children = [
-      { BitcoinStream.BlockData, [name: :block_data] },
      { BitcoinStream.RPC, [host: btc_host, port: rpc_port, name: :rpc] },
      { BitcoinStream.Mempool, [name: :mempool] },
+      { BitcoinStream.BlockData, [name: :block_data] },
      BitcoinStream.Metrics.Probe,
      Plug.Cowboy.child_spec(
        scheme: :http,
@ -25,7 +26,7 @@ defmodule BitcoinStream.Server do
        keys: :duplicate,
        name: Registry.BitcoinStream
      ),
-      BitcoinStream.Bridge.child_spec(host: btc_host, tx_port: zmq_tx_port, block_port: zmq_block_port)
+      BitcoinStream.Bridge.child_spec(host: btc_host, tx_port: zmq_tx_port, block_port: zmq_block_port, sequence_port: zmq_sequence_port)
    ]

    opts = [strategy: :one_for_one, name: BitcoinStream.Application]
--- a/server/lib/socket_handler.ex
+++ b/server/lib/socket_handler.ex
@ -18,10 +18,6 @@ defmodule BitcoinStream.SocketHandler do
    {:ok, state}
  end

-  # def last_block() do
-  #
-  # end
-
  def get_block(last_seen) do
    IO.puts("getting block with id #{last_seen}")
    last_id = GenServer.call(:block_data, :block_id)
@ -40,10 +36,14 @@ defmodule BitcoinStream.SocketHandler do

  def get_mempool_count_msg() do
    count = Mempool.get(:mempool);
-    IO.puts("Count: #{count}");
    "{ \"type\": \"count\", \"count\": #{count}}"
  end

+  def get_block_id_msg() do
+    last_id = GenServer.call(:block_data, :block_id);
+    "{ \"type\": \"block_id\", \"block_id\": \"#{last_id}\"}"
+  end
+
  @timed(key: "timed.function")
  def websocket_handle({:text, msg}, state) do
    # IO.puts("message received: #{msg} | #{inspect self()}");
@ -52,20 +52,20 @@ defmodule BitcoinStream.SocketHandler do

      "block" ->
        IO.puts('block request');
-        {:reply, {:text, "null"}, state};
+        {:reply, {:text, "null"}, state}

      "count" ->
        count = get_mempool_count_msg();
-        {:reply, {:text, count}, state};
+        {:reply, {:text, count}, state}
+
+      "block_id" ->
+        last_id = get_block_id_msg();
+        {:reply, {:text, last_id}, state}

      json ->
-        IO.puts("attempting to decode msg as json");
        with {:ok, result} <- Jason.decode(json) do
-          IO.puts("decoded ok");
-          IO.inspect(result);
          case result do
            %{"last" => block_id, "method" => "get_block"} ->
-              IO.puts('block request')
              case get_block(block_id) do
                {:ok, block_msg} ->
                  {:reply, {:text, block_msg}, state};
--- a/server/mix.exs
+++ b/server/mix.exs
@ -4,7 +4,7 @@ defmodule BitcoinStream.MixProject do
  def project do
    [
      app: :bitcoin_stream,
-      version: "2.1.3",
+      version: "2.2.0",
      elixir: "~> 1.10",
      start_permanent: Mix.env() == :prod,
      deps: deps(),