diff --git a/BloomFilter/App.config b/BloomFilter/App.config new file mode 100644 index 0000000..d40514d --- /dev/null +++ b/BloomFilter/App.config @@ -0,0 +1,32 @@ + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/BloomFilter/BloomFilterTest.csproj b/BloomFilter/BloomFilterTest.csproj new file mode 100644 index 0000000..719059b --- /dev/null +++ b/BloomFilter/BloomFilterTest.csproj @@ -0,0 +1,79 @@ + + + + + Debug + AnyCPU + {771AE5E2-149F-40C3-8FE8-D83E5D6F282D} + Exe + BitCoinTest + BitCoinTest + v4.6.2 + 512 + true + + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + packages\log4net.2.0.8\lib\net45-full\log4net.dll + + + packages\NBitcoin.4.0.0.22\lib\net461\NBitcoin.dll + + + packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll + + + + + packages\System.Net.Http.4.3.2\lib\net46\System.Net.Http.dll + + + packages\System.Security.Cryptography.Algorithms.4.3.0\lib\net461\System.Security.Cryptography.Algorithms.dll + + + packages\System.Security.Cryptography.Encoding.4.3.0\lib\net46\System.Security.Cryptography.Encoding.dll + + + packages\System.Security.Cryptography.Primitives.4.3.0\lib\net46\System.Security.Cryptography.Primitives.dll + + + packages\System.Security.Cryptography.X509Certificates.4.3.0\lib\net461\System.Security.Cryptography.X509Certificates.dll + + + + + + + + + + + + + + Designer + + + + + \ No newline at end of file diff --git a/BloomFilter/BloomFilterTest.sln b/BloomFilter/BloomFilterTest.sln new file mode 100644 index 0000000..ce9577e --- /dev/null +++ b/BloomFilter/BloomFilterTest.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26403.7 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BloomFilterTest", "BloomFilterTest.csproj", "{771AE5E2-149F-40C3-8FE8-D83E5D6F282D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {771AE5E2-149F-40C3-8FE8-D83E5D6F282D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {771AE5E2-149F-40C3-8FE8-D83E5D6F282D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {771AE5E2-149F-40C3-8FE8-D83E5D6F282D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {771AE5E2-149F-40C3-8FE8-D83E5D6F282D}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/BloomFilter/Program.cs b/BloomFilter/Program.cs new file mode 100644 index 0000000..dbf594b --- /dev/null +++ b/BloomFilter/Program.cs @@ -0,0 +1,342 @@ +// ============================================================================ +// FileName: Program.cs +// +// Description: +// A minimal BitCoin client that searches for all transactions related to a single +// address using a bloom filter. The steps are: +// +// 1. Load blockchain headers from disk (if not available will be requested from +// the full node but that takes longer), +// 2. Connect to a full BitCoin node (this sample is hard coded to use the loopback +// address so the full node will need to be on the same machine), +// 3. Keep the blockchain headers synchronised with the full node and periodically +// save them to disk, +// 4. Once the blockchain headers are synchronised use set a bloom filter and get +// all blocks within a certain date range to check for relevant transactions. +// +// NOTE: This sample does not do any block verification and is NOT suitable for +// any kind of use on the main BitCoin network. +// +// Dependencies: +// The program relies on NBitcoin (https://github.com/MetacoSA/NBitcoin) for the +// underlying BitCoin primitives. +// +// Hints: +// The original purpose for this sample was to gain an understanding of the BitCoin +// protocol. An invaluable tool for anyone attempting the same thing is WireShark +// (https://www.wireshark.org/) which has a built in BitCoin protocol decoder. To +// use WireShark with the loopback adapter on Windows install Npcap (https://nmap.org/npcap/). +// +// The command line used for the local bitcoin full node: +// "C:\Program Files\Bitcoin\daemon\bitcoind" -printtoconsole -datadir=f:\temp\bitcoind -server -testnet -debug=1 -bind=[::1]:18333 +// +// Author(s): +// Aaron Clauson (https://github.com/sipsorcery) +// +// History: +// 14 Aug 2017 Aaron Clauson Created. +// +// License: +// Public Domain +// ============================================================================= + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NBitcoin; +using NBitcoin.Protocol; +using log4net; + +namespace BitCoinTest +{ + class Program + { + static ILog logger = log4net.LogManager.GetLogger("default"); + static Network _network = Network.TestNet; + static string _chainFile = "chaintest.data"; + + // Adjust the values below based on the BitCoin address that needs to be searched + // and the dates that the address had some transactions (check with https://testnet.blockexplorer.com/). + static string _addressPrivateKey = "cR7X4Nd5WqA5mNwgX67th4Jo3K9vTTm28w8njLL9JT8hHPdbstL8"; + static DateTimeOffset _startSearchTime = DateTimeOffset.Parse("14 Jul 2017"); + static DateTimeOffset _endSearchTime = DateTimeOffset.Parse("29 Jul 2017"); + + // Bloom filter parameters. + static int _nElements = 64; + static double _falsePositiveRate = 0.0001; + static uint _nTweakIn = 50; + + static void Main(string[] args) + { + Console.WriteLine("Press q at any time to quit."); + + log4net.Config.XmlConfigurator.Configure(); + + var tokenSource = new CancellationTokenSource(); + CancellationToken ct = tokenSource.Token; + + Key key = Key.Parse(_addressPrivateKey, _network); + BitcoinPubKeyAddress addr = key.PubKey.GetAddress(_network); + + var chain = new ConcurrentChain(_network); + Node node = null; + + LoadChain(chain, ct).ContinueWith(t => + { + Task.Run(() => { PersistChain(chain, ct); }); + ConnectNodeAndSyncHeaders(chain, ct).ContinueWith(async (nodeTask) => + { + node = nodeTask.Result; + var txs = await GetTransactions(chain, node, addr, _startSearchTime, _endSearchTime, ct); + + logger.DebugFormat("Number of matching transactions {0}.", txs.Count); + }); + }, ct); + + while (true) + { + var keyPress = Console.ReadKey(); + if (keyPress.KeyChar == 'q') + { + break; + } + } + + logger.DebugFormat("Exiting..."); + + if (node != null) + { + node.DisconnectAsync(); + } + + tokenSource.Cancel(); + } + + /// + /// Attempts to load the persisted blockchain headers from persistent storage. This is + /// a lot quicker than loading from a peer node. + /// + private async static Task LoadChain(ConcurrentChain chain, CancellationToken ct) + { + logger.DebugFormat("Commencing blockchain headers load from disk..."); + + ct.ThrowIfCancellationRequested(); + + Stopwatch sw = new Stopwatch(); + sw.Start(); + + if (File.Exists(_chainFile)) + { + await Task.Run(() => { chain.Load(File.ReadAllBytes(_chainFile)); }); + } + + sw.Stop(); + + logger.DebugFormat("Block headers load from disk, chain height {0} in {1}s.", chain.Height, sw.Elapsed.Seconds); + } + + /// + /// Peridically persists any updated blockchain headers to disk. + /// + private static void PersistChain(ConcurrentChain chain, CancellationToken ct) + { + logger.DebugFormat("Starting persist blockchain task."); + + ct.ThrowIfCancellationRequested(); + + int chainHeight = (chain != null) ? chain.Height : 0; + + while (ct.IsCancellationRequested == false) + { + if (chain.Height != chainHeight) + { + using (var fs = File.Open(_chainFile, FileMode.Create)) + { + chain.WriteTo(fs); + } + + logger.DebugFormat("Chain height increased to {0} ({1})", chain.Height, DateTime.Now.ToString("HH:mm:ss")); + chainHeight = chain.Height; + } + + Task.Delay(5000, ct).Wait(); + } + } + + /// + /// Attempts to connect to a full BitCoin node and request any missing block headers. + /// + private static async Task ConnectNodeAndSyncHeaders(ConcurrentChain chain, CancellationToken ct) + { + logger.DebugFormat("Connecting to full node..."); + + ct.ThrowIfCancellationRequested(); + + ManualResetEventSlim headersSyncedSignal = new ManualResetEventSlim(); + var parameters = new NodeConnectionParameters(); + parameters.IsRelay = false; + + var scanLocation = new BlockLocator(); + scanLocation.Blocks.Add(chain.Tip != null ? chain.Tip.HashBlock : _network.GetGenesis().GetHash()); + + var node = Node.ConnectToLocal(_network, parameters); + + logger.DebugFormat("Connected to node " + node.RemoteSocketEndpoint + "."); + + node.MessageReceived += (node1, message) => + { + ct.ThrowIfCancellationRequested(); + + switch (message.Message.Payload) + { + case HeadersPayload hdr: + + if (hdr.Headers != null && hdr.Headers.Count > 0) + { + logger.DebugFormat("Received {0} blocks start {1} to {2} height {3}.", hdr.Headers.Count, hdr.Headers.First().BlockTime, hdr.Headers.Last().BlockTime, chain.Height); + + scanLocation.Blocks.Clear(); + scanLocation.Blocks.Add(hdr.Headers.Last().GetHash()); + + if (hdr != null) + { + var tip = chain.Tip; + foreach (var header in hdr.Headers) + { + var prev = tip.FindAncestorOrSelf(header.HashPrevBlock); + if (prev == null) + { + break; + } + tip = new ChainedBlock(header, header.GetHash(), prev); + chain.SetTip(tip); + + ct.ThrowIfCancellationRequested(); + } + } + + var getHeadersPayload = new GetHeadersPayload(scanLocation); + node.SendMessageAsync(getHeadersPayload); + } + else + { + // Headers synchronised. + logger.DebugFormat("Block headers synchronised."); + headersSyncedSignal.Set(); + } + + break; + + case InvPayload inv: + logger.DebugFormat("Inventory items {0}, first type {1}.", inv.Count(), inv.First().Type); + + if (inv.Any(x => x.Type == InventoryType.MSG_BLOCK)) + { + // New block available. + var getHeadersPayload = new GetHeadersPayload(scanLocation); + node.SendMessage(getHeadersPayload); + } + + break; + + case MerkleBlockPayload merkleBlk: + break; + + case TxPayload tx: + break; + + default: + logger.DebugFormat(message.Message.Command); + break; + } + }; + + node.Disconnected += n => + { + logger.DebugFormat("Node disconnected, chain height " + chain.Height + "."); + }; + + node.VersionHandshake(ct); + node.PingPong(ct); + + logger.DebugFormat("Requesting block headers greater than height {0}.", chain.Height); + node.SendMessage(new GetHeadersPayload(scanLocation)); + + logger.DebugFormat("Bitcoin node connected."); + + await Task.Run(() => { + headersSyncedSignal.Wait(ct); + }); + + return node; + } + + /// + /// Once the blockchain headers have been synchronised this method will attempt to find all transactions relevant to a single address. + /// To find the transactions there are two options: first option the full blocks can be completely downloaded and searched which is what a full node + /// would do; second option is to set a bloom filter and then request the desired blocks from a connected full node. + /// + private static async Task> GetTransactions(ConcurrentChain chain, Node node, BitcoinPubKeyAddress addr, DateTimeOffset start, DateTimeOffset end, CancellationToken ct) + { + logger.DebugFormat("Transaction search task commencing..."); + + ct.ThrowIfCancellationRequested(); + + ManualResetEventSlim searchCompleteSignal = new ManualResetEventSlim(); + BloomFilter filter = new BloomFilter(_nElements, _falsePositiveRate, _nTweakIn, BloomFlags.UPDATE_NONE); + logger.DebugFormat("Setting bloom for address " + addr.Hash + "."); + filter.Insert(addr.Hash.ToBytes()); + + List txs = new List(); + + var searchBlocks = chain.ToEnumerable(true).Where(x => x.Header.BlockTime > start && x.Header.BlockTime < end).ToList(); + int searchBlocksIndex = 0; + + node.MessageReceived += (node1, message) => + { + switch (message.Message.Payload) + { + case MerkleBlockPayload merkleBlk: + foreach (var tx in merkleBlk.Object.PartialMerkleTree.GetMatchedTransactions()) + { + logger.DebugFormat("Matched merkle block TX ID {0}.", tx); + txs.Add(tx); + } + + if(searchBlocksIndex < searchBlocks.Count()) + { + var dp = new GetDataPayload(new InventoryVector(InventoryType.MSG_FILTERED_BLOCK, searchBlocks[searchBlocksIndex++].HashBlock)); + node.SendMessage(dp); + } + else + { + searchCompleteSignal.Set(); + } + + break; + + case TxPayload tx: + logger.DebugFormat("TX ID {0}.", tx.Object.GetHash()); + break; + } + }; + + node.SendMessage(new FilterLoadPayload(filter)); + + var dataPayload = new GetDataPayload(new InventoryVector(InventoryType.MSG_FILTERED_BLOCK, searchBlocks[searchBlocksIndex++].HashBlock)); + node.SendMessage(dataPayload); + + await Task.Run(() => + { + searchCompleteSignal.Wait(ct); + logger.DebugFormat("Block search task completed."); + }); + + return txs; + } + } +} diff --git a/BloomFilter/Properties/AssemblyInfo.cs b/BloomFilter/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..de69b29 --- /dev/null +++ b/BloomFilter/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("BloomFilterTest")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("BloomFilterTest")] +[assembly: AssemblyCopyright("Copyright © 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("771ae5e2-149f-40c3-8fe8-d83e5d6f282d")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/BloomFilter/packages.config b/BloomFilter/packages.config new file mode 100644 index 0000000..d482a2f --- /dev/null +++ b/BloomFilter/packages.config @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file