From e0fef50732948d442f21040aeb1c6c8e17980431 Mon Sep 17 00:00:00 2001 From: Anton Date: Mon, 15 Jun 2026 09:54:12 +0300 Subject: [PATCH 1/4] Implement Merkle functionality --- lib/types/hashes.rs | 28 ++++++++++ lib/types/mod.rs | 123 ++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 148 insertions(+), 3 deletions(-) diff --git a/lib/types/hashes.rs b/lib/types/hashes.rs index 18b87b38..6ce7ae08 100644 --- a/lib/types/hashes.rs +++ b/lib/types/hashes.rs @@ -147,6 +147,34 @@ impl utoipa::ToSchema for MerkleRoot { } } +#[derive(Clone, Debug)] +pub struct MerkleProof { + pub leaf_index: usize, + pub siblings: Vec, +} + +#[derive(Clone, Copy, Debug)] +pub struct MerkleProofNode { + pub hash: Hash, + pub is_left: bool, +} + +impl MerkleProof { + pub fn verify(&self, leaf: Hash, root: MerkleRoot) -> bool { + let mut hash = leaf; + + for sibling in &self.siblings { + hash = if sibling.is_left { + hash_with_scratch_buffer(&(sibling.hash, hash)) + } else { + hash_with_scratch_buffer(&(hash, sibling.hash)) + }; + } + + hash == Hash::from(root) + } +} + #[derive( BorshDeserialize, BorshSerialize, diff --git a/lib/types/mod.rs b/lib/types/mod.rs index dbf6b905..51831c38 100644 --- a/lib/types/mod.rs +++ b/lib/types/mod.rs @@ -25,7 +25,7 @@ pub use address::Address; pub use bitasset_data::{BitAssetData, BitAssetDataUpdates, Update}; pub use hashes::{ AssetId, BitAssetId, BlockHash, DutchAuctionId, Hash, M6id, MerkleRoot, - Txid, + MerkleProof, MerkleProofNode, Txid, }; pub use keys::{EncryptionPubKey, VerifyingKey}; pub use transaction::{ @@ -365,8 +365,62 @@ impl Body { coinbase: &[Output], txs: &[Transaction], ) -> MerkleRoot { - // FIXME: Compute actual merkle root instead of just a hash. - hashes::hash_with_scratch_buffer(&(coinbase, txs)).into() + let mut leaves = Vec::with_capacity(txs.len() + 1); + leaves.push(hashes::hash_with_scratch_buffer(&coinbase)); + leaves.extend(txs.iter().map(hashes::hash_with_scratch_buffer)); + Body::compute_cbmt_tree(&leaves)[0].into() + } + + // https://github.com/nervosnetwork/merkle-tree/blob/5d1898263e7167560fdaa62f09e8d52991a1c712/README.md#tree-struct + fn compute_cbmt_tree(leaves: &[Hash]) -> Vec { + let n = leaves.len(); + let mut nodes = vec![Hash::default(); 2 * n - 1]; + + nodes[n - 1..].copy_from_slice(leaves); + + for idx in (0..n - 1).rev() { + nodes[idx] = hashes::hash_with_scratch_buffer(&( + nodes[2 * idx + 1], + nodes[2 * idx + 2], + )); + } + + nodes + } + + pub fn compute_tx_merkle_proof( + coinbase: &[Output], + txs: &[Transaction], + tx_idx: usize, + ) -> MerkleProof { + let mut leaves = Vec::with_capacity(txs.len() + 1); + leaves.push(hashes::hash_with_scratch_buffer(&coinbase)); + leaves.extend(txs.iter().map(hashes::hash_with_scratch_buffer)); + Body::compute_cbmt_proof(&leaves, tx_idx + 1) + } + + fn compute_cbmt_proof(leaves: &[Hash], leaf_index: usize) -> MerkleProof { + let n = leaves.len(); + let nodes = Body::compute_cbmt_tree(leaves); + let mut idx = leaf_index + n - 1; + let mut siblings = Vec::new(); + + while idx > 0 { + let sibling_idx = (idx + 1) ^ 1; + let sibling_idx = sibling_idx - 1; + + siblings.push(MerkleProofNode { + hash: nodes[sibling_idx], + is_left: sibling_idx < idx, + }); + + idx = (idx - 1) / 2; + } + + MerkleProof { + leaf_index, + siblings, + } } pub fn get_inputs(&self) -> Vec { @@ -558,3 +612,66 @@ pub(crate) static VERSION: LazyLock = LazyLock::new(|| { const VERSION_STR: &str = env!("CARGO_PKG_VERSION"); semver::Version::parse(VERSION_STR).unwrap().into() }); + +#[cfg(test)] +mod merkle_tests { + use super::*; + + fn tx(memo: &[u8]) -> Transaction { + let mut tx = Transaction::default(); + tx.memo = memo.to_vec(); + tx + } + + #[test] + fn tx_merkle_proof_verifies_membership() { + let coinbase = Vec::new(); + let txs = vec![ + tx(b"tx-0"), + tx(b"tx-1"), + tx(b"tx-2"), + tx(b"tx-3"), + ]; + + let root = Body::compute_merkle_root(&coinbase, &txs); + let proof = Body::compute_tx_merkle_proof(&coinbase, &txs, 2); + let leaf = hashes::hash_with_scratch_buffer(&txs[2]); + + assert!(proof.verify(leaf, root)); + } + + #[test] + fn tx_merkle_proof_rejects_non_member() { + let coinbase = Vec::new(); + let txs = vec![ + tx(b"tx-0"), + tx(b"tx-1"), + tx(b"tx-2"), + tx(b"tx-3"), + ]; + let other = tx(b"other"); + + let root = Body::compute_merkle_root(&coinbase, &txs); + let proof = Body::compute_tx_merkle_proof(&coinbase, &txs, 2); + let leaf = hashes::hash_with_scratch_buffer(&other); + + assert!(!proof.verify(leaf, root)); + } + + #[test] + fn tx_merkle_proof_rejects_wrong_position() { + let coinbase = Vec::new(); + let txs = vec![ + tx(b"tx-0"), + tx(b"tx-1"), + tx(b"tx-2"), + tx(b"tx-3"), + ]; + + let root = Body::compute_merkle_root(&coinbase, &txs); + let proof = Body::compute_tx_merkle_proof(&coinbase, &txs, 2); + let leaf = hashes::hash_with_scratch_buffer(&txs[1]); + + assert!(!proof.verify(leaf, root)); + } +} From 9b95f45d89762b6d179422f381a87c8228e1646d Mon Sep 17 00:00:00 2001 From: Anton Date: Tue, 16 Jun 2026 16:46:33 +0300 Subject: [PATCH 2/4] Index UTXOs by address with creation height Store UTXO creation height in LMDB so lite wallets can request delta updates. Add utxos_by_address to serve those wallet queries directly and speed up existing fn get_utxos_by_addresses (now using LMDB range queries). Route UTXO put/delete/spend/unspend paths through helpers that keep the primary outpoint index, address index, and STXO table in sync. --- app/app.rs | 2 +- lib/node/mod.rs | 9 ++- lib/state/block.rs | 65 ++++++++--------- lib/state/mod.rs | 133 +++++++++++++++++++++++++++++----- lib/state/two_way_peg_data.rs | 112 ++++++++++++++-------------- lib/types/mod.rs | 37 +++------- lib/types/transaction/mod.rs | 91 ++++++++++++++++++++++- 7 files changed, 311 insertions(+), 138 deletions(-) diff --git a/app/app.rs b/app/app.rs index a9f9be65..a47477dd 100644 --- a/app/app.rs +++ b/app/app.rs @@ -61,7 +61,7 @@ fn update_wallet(node: &Node, wallet: &Wallet) -> Result<(), Error> { let addresses = wallet.get_addresses()?; let unconfirmed_utxos = node.get_unconfirmed_utxos_by_addresses(&addresses)?; - let utxos = node.get_utxos_by_addresses(&addresses)?; + let utxos = node.get_utxos_by_addresses(&addresses, 0)?; let confirmed_outpoints: Vec<_> = wallet.get_utxos()?.into_keys().collect(); let confirmed_spent = node .get_spent_utxos(&confirmed_outpoints)? diff --git a/lib/node/mod.rs b/lib/node/mod.rs index 0365d205..51deedc0 100644 --- a/lib/node/mod.rs +++ b/lib/node/mod.rs @@ -519,7 +519,7 @@ where .try_get(&rotxn, &outpoint_key) .map_err(state::Error::from)? { - spent.push((*outpoint, output)); + spent.push((*outpoint, output.output)); } } Ok(spent) @@ -564,9 +564,14 @@ where pub fn get_utxos_by_addresses( &self, addresses: &HashSet
, + height_threshold: u32, ) -> Result, Error> { let rotxn = self.env.read_txn()?; - let utxos = self.state.get_utxos_by_addresses(&rotxn, addresses)?; + let utxos = self.state.get_utxos_by_addresses( + &rotxn, + addresses, + height_threshold, + )?; Ok(utxos) } diff --git a/lib/state/block.rs b/lib/state/block.rs index f99fcc4c..7d40a190 100644 --- a/lib/state/block.rs +++ b/lib/state/block.rs @@ -6,8 +6,8 @@ use sneed::{RoTxn, RwTxn}; use crate::{ state::{Error, PrevalidatedBlock, State, amm, dutch_auction, error}, types::{ - AmountOverflowError, Authorization, BitAssetId, BlockHash, Body, - FilledOutput, FilledOutputContent, GetAddress as _, + AddressOutPointKey, AmountOverflowError, Authorization, BitAssetId, + BlockHash, Body, FilledOutput, FilledOutputContent, GetAddress as _, GetBitcoinValue as _, Hash, Header, InPoint, OutPoint, OutPointKey, OutputContent, SpentOutput, TxData, Verify as _, }, @@ -207,8 +207,6 @@ pub fn connect_prevalidated( .sum::() + body.coinbase.len(); - // Use Vec + sort_unstable instead of individual DB operations for better performance - let mut utxo_deletes: Vec = Vec::with_capacity(total_inputs); let mut stxo_puts: Vec<(OutPointKey, SpentOutput)> = Vec::with_capacity(total_inputs); let mut utxo_puts: Vec<(OutPointKey, FilledOutput)> = @@ -263,7 +261,6 @@ pub fn connect_prevalidated( vin: vin as u32, }, }; - utxo_deletes.push(key); stxo_puts.push((key, spent_output)); } @@ -358,21 +355,20 @@ pub fn connect_prevalidated( } // Sort all vectors in parallel for optimal cursor access - utxo_deletes.par_sort_unstable(); stxo_puts.par_sort_unstable_by_key(|(key, _)| *key); utxo_puts.par_sort_unstable_by_key(|(key, _)| *key); - // Apply all database operations using pre-sorted keys for optimal B-tree access - for key in &utxo_deletes { - state.utxos.delete(rwtxn, key)?; - } - - for (key, spent_output) in &stxo_puts { - state.stxos.put(rwtxn, key, spent_output)?; + for (key, output) in stxo_puts { + if !state.spend_utxo(rwtxn, key, output)? { + return Err(error::NoUtxo { + outpoint: key.to_outpoint(), + } + .into()); + } } - for (key, filled_output) in &utxo_puts { - state.utxos.put(rwtxn, key, filled_output)?; + for (key, filled_output) in utxo_puts { + state.put_utxo(rwtxn, key, filled_output, prevalidated.next_height)?; } // Update tip and height using precomputed values @@ -433,7 +429,7 @@ pub fn connect( content: filled_content, memo: output.memo.clone(), }; - state.utxos.put(rwtxn, &outpoint_key, &filled_output)?; + state.put_utxo(rwtxn, outpoint_key, filled_output, height)?; } for transaction in &body.transactions { let filled_tx = state.fill_transaction(rwtxn, transaction)?; @@ -451,20 +447,21 @@ pub fn connect( vin: vin as u32, }, }; - state.utxos.delete(rwtxn, &input_key)?; - state.stxos.put(rwtxn, &input_key, &spent_output)?; + if !state.spend_utxo(rwtxn, input_key, spent_output)? { + return Err(error::NoUtxo { outpoint: *input }.into()); + } } let Some(filled_outputs) = filled_tx.filled_outputs() else { let err = error::FillTxOutputContents(Box::new(filled_tx)); return Err(err.into()); }; - for (vout, filled_output) in filled_outputs.iter().enumerate() { + for (vout, filled_output) in filled_outputs.into_iter().enumerate() { let outpoint = OutPoint::Regular { txid, vout: vout as u32, }; let outpoint_key = OutPointKey::from_outpoint(&outpoint); - state.utxos.put(rwtxn, &outpoint_key, filled_output)?; + state.put_utxo(rwtxn, outpoint_key, filled_output, height)?; } match &transaction.data { None => (), @@ -646,13 +643,15 @@ pub fn disconnect_tip( } // delete UTXOs, last-to-first tx.outputs.iter().enumerate().rev().try_for_each( - |(vout, _output)| { + |(vout, output)| { let outpoint = OutPoint::Regular { txid, vout: vout as u32, }; let outpoint_key = OutPointKey::from_outpoint(&outpoint); - if state.utxos.delete(rwtxn, &outpoint_key)? { + let address_key = + AddressOutPointKey::new(output.address, outpoint_key); + if state.delete_utxo(rwtxn, &address_key)? { Ok::<_, Error>(()) } else { Err(error::NoUtxo { outpoint }.into()) @@ -662,13 +661,7 @@ pub fn disconnect_tip( // unspend STXOs, last-to-first tx.inputs.iter().rev().try_for_each(|outpoint| { let outpoint_key = OutPointKey::from_outpoint(outpoint); - if let Some(spent_output) = - state.stxos.try_get(rwtxn, &outpoint_key)? - { - state.stxos.delete(rwtxn, &outpoint_key)?; - state - .utxos - .put(rwtxn, &outpoint_key, &spent_output.output)?; + if state.unspend_utxo(rwtxn, &outpoint_key)? { Ok(()) } else { Err(Error::NoStxo { @@ -678,20 +671,24 @@ pub fn disconnect_tip( }) })?; // delete coinbase UTXOs, last-to-first - body.coinbase.iter().enumerate().rev().try_for_each( - |(vout, _output)| { + body.coinbase + .iter() + .enumerate() + .rev() + .try_for_each(|(vout, output)| { let outpoint = OutPoint::Coinbase { merkle_root: header.merkle_root, vout: vout as u32, }; let outpoint_key = OutPointKey::from_outpoint(&outpoint); - if state.utxos.delete(rwtxn, &outpoint_key)? { + let address_key = + AddressOutPointKey::new(output.address, outpoint_key); + if state.delete_utxo(rwtxn, &address_key)? { Ok::<_, Error>(()) } else { Err(error::NoUtxo { outpoint }.into()) } - }, - )?; + })?; match (header.prev_side_hash, height) { (None, 0) => { state.tip.delete(rwtxn, &())?; diff --git a/lib/state/mod.rs b/lib/state/mod.rs index d5a71e2b..751369c5 100644 --- a/lib/state/mod.rs +++ b/lib/state/mod.rs @@ -10,12 +10,12 @@ use sneed::{DatabaseUnique, RoDatabaseUnique, RoTxn, RwTxn, UnitKey}; use crate::{ authorization::Authorization, types::{ - Address, AmountOverflowError, Authorized, AuthorizedTransaction, - BitAssetId, BlockHash, Body, FilledOutput, FilledTransaction, - GetAddress as _, GetBitcoinValue as _, Header, InPoint, M6id, OutPoint, - OutPointKey, SpentOutput, Transaction, TxData, VERSION, Verify as _, - Version, WithdrawalBundle, WithdrawalBundleStatus, - proto::mainchain::TwoWayPegData, + Address, AddressOutPointKey, AmountOverflowError, Authorized, + AuthorizedTransaction, BitAssetId, BlockHash, Body, FilledOutput, + FilledTransaction, GetAddress as _, GetBitcoinValue as _, Header, + InPoint, M6id, OutPoint, OutPointKey, SpentOutput, Transaction, TxData, + VERSION, Verify as _, Version, WithdrawalBundle, + WithdrawalBundleStatus, proto::mainchain::TwoWayPegData, }, util::Watchable, }; @@ -68,6 +68,18 @@ type WithdrawalBundlesDb = DatabaseUnique< )>, >; +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct UtxoEntry { + pub created_height: u32, + pub output: FilledOutput, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct SpentUtxoEntry { + pub created_height: u32, + pub output: SpentOutput, +} + #[derive(Clone)] pub struct State { /// Current tip @@ -80,7 +92,9 @@ pub struct State { /// Associates Dutch auction sequence numbers with auction state dutch_auctions: dutch_auction::Db, utxos: DatabaseUnique>, - stxos: DatabaseUnique>, + utxos_by_address: + DatabaseUnique>, + stxos: DatabaseUnique>, /// Pending withdrawal bundle. MUST exist in withdrawal_bundles pending_withdrawal_bundle: DatabaseUnique>, /// Latest failed (known) withdrawal bundle @@ -104,7 +118,7 @@ pub struct State { } impl State { - pub const NUM_DBS: u32 = bitassets::Dbs::NUM_DBS + 12; + pub const NUM_DBS: u32 = bitassets::Dbs::NUM_DBS + 12 + 1; pub fn new(env: &sneed::Env) -> Result { let mut rwtxn = env.write_txn()?; @@ -115,6 +129,8 @@ impl State { let dutch_auctions = DatabaseUnique::create(env, &mut rwtxn, "dutch_auctions")?; let utxos = DatabaseUnique::create(env, &mut rwtxn, "utxos")?; + let utxos_by_address = + DatabaseUnique::create(env, &mut rwtxn, "utxos_by_address")?; let stxos = DatabaseUnique::create(env, &mut rwtxn, "stxos")?; let pending_withdrawal_bundle = DatabaseUnique::create( env, @@ -147,6 +163,7 @@ impl State { bitassets, dutch_auctions, utxos, + utxos_by_address, stxos, pending_withdrawal_bundle, latest_failed_withdrawal_bundle, @@ -180,10 +197,79 @@ impl State { pub fn stxos( &self, - ) -> &RoDatabaseUnique> { + ) -> &RoDatabaseUnique> { &self.stxos } + fn put_utxo( + &self, + rwtxn: &mut RwTxn, + outpoint_key: OutPointKey, + output: FilledOutput, + created_height: u32, + ) -> Result { + let entry = UtxoEntry { + created_height, + output, + }; + let address_key = + AddressOutPointKey::new(entry.output.address, outpoint_key); + self.utxos_by_address.put(rwtxn, &address_key, &entry)?; + self.utxos.put(rwtxn, &outpoint_key, &entry.output)?; + Ok(address_key) + } + + fn delete_utxo( + &self, + rwtxn: &mut RwTxn, + address_key: &AddressOutPointKey, + ) -> Result { + self.utxos_by_address.delete(rwtxn, &address_key)?; + Ok(self.utxos.delete(rwtxn, &address_key.outpoint_key())?) + } + + fn spend_utxo( + &self, + rwtxn: &mut RwTxn, + outpoint_key: OutPointKey, + spent_output: SpentOutput, + ) -> Result { + let address_key = + AddressOutPointKey::new(spent_output.output.address, outpoint_key); + + let Some(entry) = self.utxos_by_address.try_get(rwtxn, &address_key)? + else { + return Ok(false); + }; + + let entry = SpentUtxoEntry { + created_height: entry.created_height, + output: spent_output, + }; + + self.stxos.put(rwtxn, &outpoint_key, &entry)?; + self.delete_utxo(rwtxn, &address_key) + } + + fn unspend_utxo( + &self, + rwtxn: &mut RwTxn, + outpoint_key: &OutPointKey, + ) -> Result { + let Some(entry) = self.stxos.try_get(rwtxn, outpoint_key)? else { + return Ok(false); + }; + + self.put_utxo( + rwtxn, + *outpoint_key, + entry.output.output, + entry.created_height, + )?; + + Ok(self.stxos.delete(rwtxn, outpoint_key)?) + } + pub fn withdrawal_bundle_event_blocks( &self, ) -> &RoDatabaseUnique< @@ -222,13 +308,23 @@ impl State { &self, rotxn: &RoTxn, addresses: &HashSet
, + height_threshold: u32, ) -> Result, Error> { - let utxos: HashMap = self - .utxos - .iter(rotxn)? - .filter(|(_, output)| Ok(addresses.contains(&output.address))) - .map(|(key, output)| Ok((key.to_outpoint(), output))) - .collect()?; + let mut utxos = HashMap::new(); + for address in addresses { + let start = AddressOutPointKey::start(*address); + let end = AddressOutPointKey::end(*address); + let mut iter = self + .utxos_by_address + .range(rotxn, &(start..=end)) + .map_err(sneed::db::Error::from)?; + while let Some((key, entry)) = iter.next()? { + if entry.created_height >= height_threshold { + let outpoint = key.outpoint_key().to_outpoint(); + utxos.insert(outpoint, entry.output); + } + } + } Ok(utxos) } @@ -300,13 +396,13 @@ impl State { .try_get(rotxn, &key)? .ok_or(Error::NoStxo { outpoint: *input })?; assert_eq!( - stxo.inpoint, + stxo.output.inpoint, InPoint::Regular { txid, vin: vin as u32 } ); - spent_utxos.push(stxo.output); + spent_utxos.push(stxo.output.output); } spent_utxos.reverse(); Ok(FilledTransaction { @@ -656,7 +752,8 @@ impl State { let mut total_deposit_stxo_value = bitcoin::Amount::ZERO; let mut total_withdrawal_stxo_value = bitcoin::Amount::ZERO; self.stxos.iter(rotxn)?.map_err(Error::from).for_each( - |(outpoint_key, spent_output)| { + |(outpoint_key, spent_entry)| { + let spent_output = spent_entry.output; let outpoint = outpoint_key.to_outpoint(); if let OutPoint::Deposit(_) = outpoint { total_deposit_stxo_value = total_deposit_stxo_value diff --git a/lib/state/two_way_peg_data.rs b/lib/state/two_way_peg_data.rs index c68035a9..8bdb8107 100644 --- a/lib/state/two_way_peg_data.rs +++ b/lib/state/two_way_peg_data.rs @@ -12,10 +12,11 @@ use crate::{ rollback::{HeightStamped, RollBack}, }, types::{ - AggregatedWithdrawal, AmountOverflowError, FilledOutput, - FilledOutputContent, InPoint, M6id, OutPoint, OutPointKey, SpentOutput, - WithdrawalBundle, WithdrawalBundleEvent, WithdrawalBundleEventStatus, - WithdrawalBundleStatus, WithdrawalOutputContent, + AddressOutPointKey, AggregatedWithdrawal, AmountOverflowError, + FilledOutput, FilledOutputContent, InPoint, M6id, OutPoint, + OutPointKey, SpentOutput, WithdrawalBundle, WithdrawalBundleEvent, + WithdrawalBundleEventStatus, WithdrawalBundleStatus, + WithdrawalOutputContent, proto::mainchain::{BlockEvent, TwoWayPegData}, }, }; @@ -139,19 +140,13 @@ fn connect_withdrawal_bundle_submitted( return Err(err.into()); } }; - for (outpoint, spend_output) in bundle.spend_utxos() { - let outpoint_key = OutPointKey::from_outpoint(outpoint); - if !state.utxos.delete(rwtxn, &outpoint_key)? { + for (outpoint, output) in bundle.spend_utxos() { + if !spend_withdrawal_utxo(state, rwtxn, m6id, outpoint, output)? { return Err(error::NoUtxo { outpoint: *outpoint, } .into()); - }; - let spent_output = SpentOutput { - output: spend_output.clone(), - inpoint: InPoint::Withdrawal { m6id }, - }; - state.stxos.put(rwtxn, &outpoint_key, &spent_output)?; + } } assert_eq!( bundle_status.latest().value, @@ -320,17 +315,15 @@ fn connect_withdrawal_bundle_confirmed( .map(|(key, output)| Ok((key.into(), output))) .collect()?; for (outpoint, output) in &utxos { - let spent_output = SpentOutput { - output: output.clone(), - inpoint: InPoint::Withdrawal { m6id }, - }; - state.stxos.put( - rwtxn, - &OutPointKey::from(outpoint), - &spent_output, - )?; + if !spend_withdrawal_utxo( + state, rwtxn, m6id, outpoint, output, + )? { + return Err(error::NoUtxo { + outpoint: *outpoint, + } + .into()); + } } - state.utxos.clear(rwtxn)?; bundle = WithdrawalBundleInfo::UnknownConfirmed { spend_utxos: utxos, }; @@ -355,8 +348,9 @@ fn connect_withdrawal_bundle_confirmed( "Unexpected withdrawal bundle confirmed, marking bundle UTXOs as spent" ); for (outpoint, output) in bundle.spend_utxos() { - let outpoint_key = OutPointKey::from(outpoint); - if !state.utxos.delete(rwtxn, &outpoint_key)? { + if !spend_withdrawal_utxo( + state, rwtxn, m6id, outpoint, output, + )? { return Err( Error::UnexpectedWithdrawalBundleInsolvency { event_block_hash: *event_block_hash, @@ -365,11 +359,6 @@ fn connect_withdrawal_bundle_confirmed( }, ); } - let spent_output = SpentOutput { - output: output.clone(), - inpoint: InPoint::Withdrawal { m6id }, - }; - state.stxos.put(rwtxn, &outpoint_key, &spent_output)?; } } } @@ -416,10 +405,13 @@ fn connect_withdrawal_bundle_failed( ) { break 'known; } - for (outpoint, output) in bundle.spend_utxos() { + for outpoint in bundle.spend_utxos().keys() { let outpoint_key = OutPointKey::from_outpoint(outpoint); - state.stxos.delete(rwtxn, &outpoint_key)?; - state.utxos.put(rwtxn, &outpoint_key, output)?; + if !state.unspend_utxo(rwtxn, &outpoint_key)? { + return Err(Error::NoStxo { + outpoint: *outpoint, + }); + }; } let latest_failed_m6id = if let Some(mut latest_failed_m6id) = state.latest_failed_withdrawal_bundle.try_get(rwtxn, &())? @@ -499,7 +491,7 @@ fn connect_2wpd_event( let outpoint = OutPoint::Deposit(deposit.outpoint); let output = deposit.output.clone(); let outpoint_key = OutPointKey::from_outpoint(&outpoint); - state.utxos.put(rwtxn, &outpoint_key, &output)?; + state.put_utxo(rwtxn, outpoint_key, output, block_height)?; *latest_deposit_block_hash = Some(event_block_hash); } BlockEvent::WithdrawalBundle(withdrawal_bundle_event) => { @@ -641,14 +633,13 @@ fn disconnect_withdrawal_bundle_submitted( && bundle_status.latest().value == WithdrawalBundleStatus::Pending { - for (outpoint, output) in bundle.spend_utxos().iter().rev() { + for outpoint in bundle.spend_utxos().keys().rev() { let outpoint_key = OutPointKey::from_outpoint(outpoint); - if !state.stxos.delete(rwtxn, &outpoint_key)? { + if !state.unspend_utxo(rwtxn, &outpoint_key)? { return Err(Error::NoStxo { outpoint: *outpoint, }); - }; - state.utxos.put(rwtxn, &outpoint_key, output)?; + } } state.pending_withdrawal_bundle.put(rwtxn, &(), &m6id)?; } @@ -701,26 +692,24 @@ fn disconnect_withdrawal_bundle_confirmed( prev_bundle_status.latest().value, WithdrawalBundleStatus::SubmittedUnexpected ) { - for (outpoint, output) in bundle.spend_utxos() { + for outpoint in bundle.spend_utxos().keys() { let outpoint_key = OutPointKey::from(outpoint); - state.utxos.put(rwtxn, &outpoint_key, output)?; - if !state.stxos.delete(rwtxn, &outpoint_key)? { + if !state.unspend_utxo(rwtxn, &outpoint_key)? { return Err(Error::NoStxo { outpoint: *outpoint, }); - }; + } } } } WithdrawalBundleInfo::UnknownConfirmed { spend_utxos } => { - for (outpoint, output) in spend_utxos { + for outpoint in spend_utxos.keys() { let outpoint_key = OutPointKey::from_outpoint(outpoint); - state.utxos.put(rwtxn, &outpoint_key, output)?; - if !state.stxos.delete(rwtxn, &outpoint_key)? { + if !state.unspend_utxo(rwtxn, &outpoint_key)? { return Err(Error::NoStxo { outpoint: *outpoint, }); - }; + } } bundle = WithdrawalBundleInfo::Unknown; } @@ -770,18 +759,13 @@ fn disconnect_withdrawal_bundle_failed( break 'known; } for (outpoint, output) in bundle.spend_utxos().iter().rev() { - let outpoint_key = OutPointKey::from_outpoint(outpoint); - let spent_output = SpentOutput { - output: output.clone(), - inpoint: InPoint::Withdrawal { m6id }, - }; - state.stxos.put(rwtxn, &outpoint_key, &spent_output)?; - if state.utxos.delete(rwtxn, &outpoint_key)? { + if !spend_withdrawal_utxo(state, rwtxn, m6id, outpoint, output)? + { return Err(error::NoUtxo { outpoint: *outpoint, } .into()); - }; + } } let (prev_latest_failed_m6id, latest_failed_m6id) = state .latest_failed_withdrawal_bundle @@ -809,6 +793,22 @@ fn disconnect_withdrawal_bundle_failed( Ok(()) } +fn spend_withdrawal_utxo( + state: &State, + rwtxn: &mut RwTxn, + m6id: M6id, + outpoint: &OutPoint, + output: &FilledOutput, +) -> Result { + let inpoint = InPoint::Withdrawal { m6id }; + let outpoint_key = OutPointKey::from_outpoint(outpoint); + let spent_output = SpentOutput { + output: output.clone(), + inpoint, + }; + state.spend_utxo(rwtxn, outpoint_key, spent_output) +} + fn disconnect_withdrawal_bundle_event( state: &State, rwtxn: &mut RwTxn, @@ -856,7 +856,9 @@ fn disconnect_event( BlockEvent::Deposit(deposit) => { let outpoint = OutPoint::Deposit(deposit.outpoint); let outpoint_key = OutPointKey::from_outpoint(&outpoint); - if !state.utxos.delete(rwtxn, &outpoint_key)? { + let address_key = + AddressOutPointKey::new(deposit.output.address, outpoint_key); + if !state.delete_utxo(rwtxn, &address_key)? { return Err(error::NoUtxo { outpoint }.into()); } *latest_deposit_block_hash = Some(event_block_hash); diff --git a/lib/types/mod.rs b/lib/types/mod.rs index 51831c38..1678813f 100644 --- a/lib/types/mod.rs +++ b/lib/types/mod.rs @@ -24,17 +24,17 @@ mod transaction; pub use address::Address; pub use bitasset_data::{BitAssetData, BitAssetDataUpdates, Update}; pub use hashes::{ - AssetId, BitAssetId, BlockHash, DutchAuctionId, Hash, M6id, MerkleRoot, - MerkleProof, MerkleProofNode, Txid, + AssetId, BitAssetId, BlockHash, DutchAuctionId, Hash, M6id, MerkleProof, + MerkleProofNode, MerkleRoot, Txid, }; pub use keys::{EncryptionPubKey, VerifyingKey}; pub use transaction::{ - AmmBurn, AmmMint, AmmSwap, AssetOutput, AssetOutputContent, Authorized, - AuthorizedTransaction, BitcoinOutput, BitcoinOutputContent, - DutchAuctionBid, DutchAuctionCollect, DutchAuctionParams, FilledOutput, - FilledOutputContent, FilledTransaction, InPoint, OutPoint, OutPointKey, - Output, OutputContent, PointedOutput, SpentOutput, Transaction, TxData, - TxInputs, WithdrawalOutputContent, + AddressOutPointKey, AmmBurn, AmmMint, AmmSwap, AssetOutput, + AssetOutputContent, Authorized, AuthorizedTransaction, BitcoinOutput, + BitcoinOutputContent, DutchAuctionBid, DutchAuctionCollect, + DutchAuctionParams, FilledOutput, FilledOutputContent, FilledTransaction, + InPoint, OutPoint, OutPointKey, Output, OutputContent, PointedOutput, + SpentOutput, Transaction, TxData, TxInputs, WithdrawalOutputContent, }; pub const THIS_SIDECHAIN: u8 = 4; @@ -626,12 +626,7 @@ mod merkle_tests { #[test] fn tx_merkle_proof_verifies_membership() { let coinbase = Vec::new(); - let txs = vec![ - tx(b"tx-0"), - tx(b"tx-1"), - tx(b"tx-2"), - tx(b"tx-3"), - ]; + let txs = vec![tx(b"tx-0"), tx(b"tx-1"), tx(b"tx-2"), tx(b"tx-3")]; let root = Body::compute_merkle_root(&coinbase, &txs); let proof = Body::compute_tx_merkle_proof(&coinbase, &txs, 2); @@ -643,12 +638,7 @@ mod merkle_tests { #[test] fn tx_merkle_proof_rejects_non_member() { let coinbase = Vec::new(); - let txs = vec![ - tx(b"tx-0"), - tx(b"tx-1"), - tx(b"tx-2"), - tx(b"tx-3"), - ]; + let txs = vec![tx(b"tx-0"), tx(b"tx-1"), tx(b"tx-2"), tx(b"tx-3")]; let other = tx(b"other"); let root = Body::compute_merkle_root(&coinbase, &txs); @@ -661,12 +651,7 @@ mod merkle_tests { #[test] fn tx_merkle_proof_rejects_wrong_position() { let coinbase = Vec::new(); - let txs = vec![ - tx(b"tx-0"), - tx(b"tx-1"), - tx(b"tx-2"), - tx(b"tx-3"), - ]; + let txs = vec![tx(b"tx-0"), tx(b"tx-1"), tx(b"tx-2"), tx(b"tx-3")]; let root = Body::compute_merkle_root(&coinbase, &txs); let proof = Body::compute_tx_merkle_proof(&coinbase, &txs, 2); diff --git a/lib/types/transaction/mod.rs b/lib/types/transaction/mod.rs index dad7bfc0..d37d0e11 100644 --- a/lib/types/transaction/mod.rs +++ b/lib/types/transaction/mod.rs @@ -1,5 +1,6 @@ use std::{ borrow::Borrow, + borrow::Cow, cmp::Ordering, collections::{HashMap, HashSet}, io::Cursor, @@ -113,7 +114,9 @@ impl std::fmt::Display for OutPoint { } } +const ADDRESS_SIZE: usize = 20; const OUTPOINT_KEY_SIZE: usize = 37; +const ADDRESS_OUTPOINT_KEY_SIZE: usize = ADDRESS_SIZE + OUTPOINT_KEY_SIZE; /// Fixed-width key for OutPoint based on its canonical Borsh encoding. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -202,8 +205,8 @@ impl<'a> BytesEncode<'a> for OutPointKey { #[inline] fn bytes_encode( item: &'a Self::EItem, - ) -> Result, BoxedError> { - Ok(std::borrow::Cow::Borrowed(item.as_ref())) + ) -> Result, BoxedError> { + Ok(Cow::Borrowed(item.as_ref())) } } @@ -224,6 +227,90 @@ impl<'a> BytesDecode<'a> for OutPointKey { } } +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct AddressOutPointKey([u8; ADDRESS_OUTPOINT_KEY_SIZE]); + +impl AddressOutPointKey { + #[inline] + pub fn new(address: Address, outpoint: OutPointKey) -> Self { + let mut bytes = [0u8; ADDRESS_OUTPOINT_KEY_SIZE]; + bytes[..ADDRESS_SIZE].copy_from_slice(&address.0); + bytes[ADDRESS_SIZE..].copy_from_slice(outpoint.as_bytes()); + Self(bytes) + } + + #[inline] + pub fn start(address: Address) -> Self { + let mut bytes = [0u8; ADDRESS_OUTPOINT_KEY_SIZE]; + bytes[..ADDRESS_SIZE].copy_from_slice(&address.0); + Self(bytes) + } + + #[inline] + pub fn end(address: Address) -> Self { + let mut bytes = [0xffu8; ADDRESS_OUTPOINT_KEY_SIZE]; + bytes[..ADDRESS_SIZE].copy_from_slice(&address.0); + Self(bytes) + } + + #[inline] + pub fn address(&self) -> Address { + let mut address = [0u8; ADDRESS_SIZE]; + address.copy_from_slice(&self.0[..ADDRESS_SIZE]); + Address(address) + } + + #[inline] + pub fn outpoint_key(&self) -> OutPointKey { + ::bytes_decode(&self.0[ADDRESS_SIZE..]) + .expect("AddressOutPointKey should contain a valid OutPointKey") + } +} + +impl Ord for AddressOutPointKey { + #[inline] + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0.cmp(&other.0) + } +} + +impl PartialOrd for AddressOutPointKey { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl AsRef<[u8]> for AddressOutPointKey { + #[inline] + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl<'a> BytesEncode<'a> for AddressOutPointKey { + type EItem = AddressOutPointKey; + + #[inline] + fn bytes_encode( + item: &'a Self::EItem, + ) -> Result, BoxedError> { + Ok(Cow::Borrowed(item.as_ref())) + } +} + +impl<'a> BytesDecode<'a> for AddressOutPointKey { + type DItem = AddressOutPointKey; + + #[inline] + fn bytes_decode(bytes: &'a [u8]) -> Result { + ::bytes_decode(&bytes[ADDRESS_SIZE..])?; + let mut key = [0u8; ADDRESS_OUTPOINT_KEY_SIZE]; + key.copy_from_slice(bytes); + Ok(Self(key)) + } +} + #[cfg(test)] mod tests { use super::{OUTPOINT_KEY_SIZE, OutPoint, OutPointKey}; From 74323cdc839047a1c6169f414ca1f03793ba97d7 Mon Sep 17 00:00:00 2001 From: Anton Date: Wed, 17 Jun 2026 07:59:53 +0300 Subject: [PATCH 3/4] Move ADDRESS_SIZE definition to address.rs --- lib/authorization.rs | 6 +++--- lib/types/address.rs | 15 +++++++++------ lib/types/mod.rs | 2 +- lib/types/transaction/mod.rs | 3 +-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/authorization.rs b/lib/authorization.rs index 8467c43b..1b7fedae 100644 --- a/lib/authorization.rs +++ b/lib/authorization.rs @@ -7,8 +7,8 @@ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use crate::types::{ - Address, AuthorizedTransaction, Body, GetAddress, Transaction, Verify, - VerifyingKey, + ADDRESS_SIZE, Address, AuthorizedTransaction, Body, GetAddress, + Transaction, Verify, VerifyingKey, }; pub use ed25519_dalek::{SignatureError, Signer, SigningKey, Verifier}; @@ -147,7 +147,7 @@ impl Verify for Authorization { pub fn get_address(verifying_key: &VerifyingKey) -> Address { let mut hasher = blake3::Hasher::new(); let mut reader = hasher.update(&verifying_key.to_bytes()).finalize_xof(); - let mut output: [u8; 20] = [0; 20]; + let mut output: [u8; ADDRESS_SIZE] = [0; ADDRESS_SIZE]; reader.fill(&mut output); Address(output) } diff --git a/lib/types/address.rs b/lib/types/address.rs index b43c901b..baa069e8 100644 --- a/lib/types/address.rs +++ b/lib/types/address.rs @@ -7,11 +7,13 @@ use utoipa::ToSchema; use crate::types::THIS_SIDECHAIN; +pub const ADDRESS_SIZE: usize = 20; + #[derive(Debug, Error)] pub enum AddressParseError { #[error("bs58 error")] Bs58(#[from] bitcoin::base58::InvalidCharacterError), - #[error("wrong address length {0} != 20")] + #[error("wrong address length {0} != {ADDRESS_SIZE}")] WrongLength(usize), } @@ -20,10 +22,10 @@ pub enum AddressParseError { )] #[repr(transparent)] #[schema(value_type = String)] -pub struct Address(pub [u8; 20]); +pub struct Address(pub [u8; ADDRESS_SIZE]); impl Address { - pub const ALL_ZEROS: Self = Self([0; 20]); + pub const ALL_ZEROS: Self = Self([0; ADDRESS_SIZE]); pub fn as_base58(&self) -> String { bitcoin::base58::encode(&self.0) @@ -50,8 +52,8 @@ impl std::fmt::Debug for Address { } } -impl From<[u8; 20]> for Address { - fn from(other: [u8; 20]) -> Self { +impl From<[u8; ADDRESS_SIZE]> for Address { + fn from(other: [u8; ADDRESS_SIZE]) -> Self { Self(other) } } @@ -74,7 +76,8 @@ impl<'de> Deserialize<'de> for Address { if deserializer.is_human_readable() { DisplayFromStr::deserialize_as(deserializer) } else { - <[u8; 20] as Deserialize>::deserialize(deserializer).map(Self) + <[u8; ADDRESS_SIZE] as Deserialize>::deserialize(deserializer) + .map(Self) } } } diff --git a/lib/types/mod.rs b/lib/types/mod.rs index 1678813f..f091ff00 100644 --- a/lib/types/mod.rs +++ b/lib/types/mod.rs @@ -21,7 +21,7 @@ pub mod proto; pub mod schema; mod transaction; -pub use address::Address; +pub use address::{ADDRESS_SIZE, Address}; pub use bitasset_data::{BitAssetData, BitAssetDataUpdates, Update}; pub use hashes::{ AssetId, BitAssetId, BlockHash, DutchAuctionId, Hash, M6id, MerkleProof, diff --git a/lib/types/transaction/mod.rs b/lib/types/transaction/mod.rs index d37d0e11..2ff84a25 100644 --- a/lib/types/transaction/mod.rs +++ b/lib/types/transaction/mod.rs @@ -18,7 +18,7 @@ use crate::{ types::{ AmountOverflowError, BitAssetData, BitAssetDataUpdates, GetAddress, GetBitcoinValue, - address::Address, + address::{ADDRESS_SIZE, Address}, hashes::{ self, AssetId, BitAssetId, DutchAuctionId, Hash, M6id, MerkleRoot, Txid, @@ -114,7 +114,6 @@ impl std::fmt::Display for OutPoint { } } -const ADDRESS_SIZE: usize = 20; const OUTPOINT_KEY_SIZE: usize = 37; const ADDRESS_OUTPOINT_KEY_SIZE: usize = ADDRESS_SIZE + OUTPOINT_KEY_SIZE; From 6043514183de52e16ee80b6a7b912e26ddf6933a Mon Sep 17 00:00:00 2001 From: Anton Date: Wed, 17 Jun 2026 10:02:54 +0300 Subject: [PATCH 4/4] Add time-capped txdb to Archive - Add AddressTxidKey keyed as address || txid - Store filled tx, its Merkle proof, block hash/height, and Unix timestamp - Update txdb with new confirmed txs when blocks connect to the active chain - Query txdb by address set and minimum block height - Prune txdb by Unix timestamp every 144 * 7 blocks (~weekly) - Prune txdb records for disconnected blocks during reorgs - Make Merkle proof types serializable General rationale: lite wallet is UTXO-based so strictly speaking it does not need to know tx history, however, it is still nice to show users actual tx history so they can figure out what was going on. However again, storing full tx history is too demanding for a full node so it is capped. --- lib/archive.rs | 145 ++++++++++++++++++++++++++++++++++- lib/node/net_task.rs | 38 +++++++-- lib/types/hashes.rs | 4 +- lib/types/mod.rs | 2 +- lib/types/transaction/mod.rs | 74 ++++++++++++++++++ 5 files changed, 250 insertions(+), 13 deletions(-) diff --git a/lib/archive.rs b/lib/archive.rs index 0033d574..d06987d0 100644 --- a/lib/archive.rs +++ b/lib/archive.rs @@ -7,6 +7,7 @@ use std::{ use bitcoin::{self, hashes::Hash as _}; use fallible_iterator::{FallibleIterator, IteratorExt}; use heed::types::SerdeBincode; +use serde::{Deserialize, Serialize}; use sneed::{ DatabaseUnique, EnvError, RoTxn, RwTxn, RwTxnError, UnitKey, db::{self, error::Error as DbError}, @@ -14,12 +15,16 @@ use sneed::{ }; use crate::types::{ - Block, BlockHash, BmmResult, Body, Header, Tip, Txid, VERSION, Version, + Address, AddressTxidKey, Block, BlockHash, BmmResult, Body, + FilledTransaction, Header, MerkleProof, Tip, Txid, VERSION, Version, proto::mainchain, }; #[allow(clippy::duplicated_attributes)] #[derive(thiserror::Error, transitive::Transitive, Debug)] +#[transitive(from(db::error::Delete, DbError))] +#[transitive(from(db::error::IterInit, DbError))] +#[transitive(from(db::error::IterItem, DbError))] #[transitive(from(db::error::Put, DbError))] #[transitive(from(db::error::TryGet, DbError))] #[transitive(from(env::error::CreateDb, EnvError))] @@ -73,6 +78,19 @@ pub enum Error { NoMainHeight(bitcoin::BlockHash), #[error("no tx with txid {0}")] NoTx(Txid), + #[error( + "txdb filled transaction length mismatch: expected {expected}, actual {actual}" + )] + TxDbFilledTxLenMismatch { expected: usize, actual: usize }, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct FilledTxEntry { + pub tx: FilledTransaction, + pub merkle_proof: MerkleProof, + pub block_hash: BlockHash, + pub block_height: u32, + pub unix_stamp: u64, } #[derive(Clone)] @@ -148,11 +166,13 @@ pub struct Archive { SerdeBincode, SerdeBincode>, >, + /// Capped tx cache, keyed by (address, txid). + txdb: DatabaseUnique>, _version: DatabaseUnique>, } impl Archive { - pub const NUM_DBS: u32 = 14; + pub const NUM_DBS: u32 = 15; pub fn new(env: &sneed::Env) -> Result { let mut rwtxn = env.write_txn()?; @@ -224,6 +244,7 @@ impl Archive { let total_work = DatabaseUnique::create(env, &mut rwtxn, "total_work")?; let txid_to_inclusions = DatabaseUnique::create(env, &mut rwtxn, "txid_to_inclusions")?; + let txdb = DatabaseUnique::create(env, &mut rwtxn, "txdb")?; rwtxn.commit()?; Ok(Self { block_hash_to_height, @@ -239,6 +260,7 @@ impl Archive { successors, total_work, txid_to_inclusions, + txdb, _version: version, }) } @@ -672,6 +694,125 @@ impl Archive { Ok(inclusions) } + pub fn prune_txdb_block( + &self, + rwtxn: &mut RwTxn, + block_hash: BlockHash, + ) -> Result { + let keys = { + let mut keys = Vec::new(); + let mut iter = self.txdb.iter(rwtxn).map_err(DbError::from)?; + while let Some((key, entry)) = iter.next().map_err(DbError::from)? { + if entry.block_hash == block_hash { + keys.push(key); + } + } + keys + }; + for key in &keys { + let _ = self.txdb.delete(rwtxn, key)?; + } + Ok(keys.len()) + } + + pub fn prune_txdb_older_than( + &self, + rwtxn: &mut RwTxn, + cutoff_unix: u64, + ) -> Result { + let keys = { + let mut keys = Vec::new(); + let mut iter = self.txdb.iter(rwtxn).map_err(DbError::from)?; + while let Some((key, entry)) = iter.next().map_err(DbError::from)? { + if entry.unix_stamp < cutoff_unix { + keys.push(key); + } + } + keys + }; + for key in &keys { + let _ = self.txdb.delete(rwtxn, key)?; + } + Ok(keys.len()) + } + + pub fn put_txdb_for_connected_block( + &self, + rwtxn: &mut RwTxn, + block_hash: BlockHash, + block_height: u32, + body: &Body, + filled_txs: &[FilledTransaction], + unix_stamp: u64, + ) -> Result<(), Error> { + if body.transactions.len() != filled_txs.len() { + return Err(Error::TxDbFilledTxLenMismatch { + expected: body.transactions.len(), + actual: filled_txs.len(), + }); + } + for (tx_index, filled_tx) in filled_txs.iter().enumerate() { + let txid = filled_tx.txid(); + debug_assert_eq!(body.transactions[tx_index].txid(), txid); + let merkle_proof = Body::compute_tx_merkle_proof( + &body.coinbase, + &body.transactions, + tx_index, + ); + debug_assert_eq!(merkle_proof.leaf_index, tx_index + 1); + + let mut addresses = HashSet::new(); + addresses.extend( + filled_tx.spent_utxos.iter().map(|output| output.address), + ); + addresses.extend( + filled_tx + .transaction + .outputs + .iter() + .map(|output| output.address), + ); + + for address in addresses { + let key = AddressTxidKey::new(address, txid); + let entry = FilledTxEntry { + tx: filled_tx.clone(), + merkle_proof: merkle_proof.clone(), + block_hash, + block_height, + unix_stamp, + }; + self.txdb.put(rwtxn, &key, &entry)?; + } + } + Ok(()) + } + + pub fn get_txdb_entries_by_address( + &self, + rotxn: &RoTxn, + addresses: &HashSet
, + height_threshold: u32, + ) -> Result, Error> { + let mut entries = Vec::new(); + for address in addresses { + let start = AddressTxidKey::start(*address); + let end = AddressTxidKey::end(*address); + let mut iter = self + .txdb + .range(rotxn, &(start..=end)) + .map_err(DbError::from)?; + while let Some((_key, entry)) = + iter.next().map_err(DbError::from)? + { + if entry.block_height >= height_threshold { + entries.push(entry); + } + } + } + Ok(entries) + } + /// Store a block body. The header must already exist. pub fn put_body( &self, diff --git a/lib/node/net_task.rs b/lib/node/net_task.rs index 37b3920f..89414f07 100644 --- a/lib/node/net_task.rs +++ b/lib/node/net_task.rs @@ -5,7 +5,7 @@ use std::{ collections::{HashMap, HashSet}, net::SocketAddr, sync::Arc, - time::Duration, + time::{Duration, SystemTime, UNIX_EPOCH}, }; use error_fatality::{Nested as _, Split}; @@ -129,6 +129,15 @@ impl From for Error { } } +const TXDB_PRUNE_INTERVAL_BLOCKS: u32 = 144 * 7; +const TXDB_RETENTION_SECS: u64 = 28 * 24 * 60 * 60; + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| duration.as_secs()) +} + fn connect_tip_( rwtxn: &mut RwTxn<'_>, archive: &Archive, @@ -139,18 +148,30 @@ fn connect_tip_( two_way_peg_data: &mainchain::TwoWayPegData, ) -> Result<(), Error> { let block_hash = header.hash(); + let prevalidated = state.prevalidate_block(rwtxn, header, body)?; + let block_height = prevalidated.next_height; + let merkle_root = prevalidated.computed_merkle_root; + let filled_txs = prevalidated.filled_transactions.clone(); + state.connect_prevalidated_block(rwtxn, header, body, prevalidated)?; if tracing::enabled!(tracing::Level::DEBUG) { - let merkle_root = - Body::compute_merkle_root(&body.coinbase, &body.transactions); - let height = state.try_get_height(rwtxn)?; - state.apply_block(rwtxn, header, body)?; - tracing::debug!(?height, %merkle_root, %block_hash, "connected body") - } else { - state.apply_block(rwtxn, header, body)?; + tracing::debug!(height = block_height, %merkle_root, %block_hash, "connected body") } let () = state.connect_two_way_peg_data(rwtxn, two_way_peg_data)?; let () = archive.put_header(rwtxn, header)?; let () = archive.put_body(rwtxn, block_hash, body)?; + let unix_stamp = unix_now(); + let () = archive.put_txdb_for_connected_block( + rwtxn, + block_hash, + block_height, + body, + &filled_txs, + unix_stamp, + )?; + if block_height > 0 && block_height % TXDB_PRUNE_INTERVAL_BLOCKS == 0 { + let cutoff_unix = unix_stamp.saturating_sub(TXDB_RETENTION_SECS); + let _ = archive.prune_txdb_older_than(rwtxn, cutoff_unix)?; + } for transaction in &body.transactions { let () = mempool.delete(rwtxn, transaction.txid())?; } @@ -250,6 +271,7 @@ fn disconnect_tip_( } }; let () = state.disconnect_two_way_peg_data(rwtxn, &two_way_peg_data)?; + let _ = archive.prune_txdb_block(rwtxn, tip_block_hash)?; let () = state.disconnect_tip(rwtxn, &tip_header, &tip_body)?; for transaction in tip_body.authorized_transactions().iter().rev() { mempool.put(rwtxn, transaction)?; diff --git a/lib/types/hashes.rs b/lib/types/hashes.rs index 6ce7ae08..8b1aa0dc 100644 --- a/lib/types/hashes.rs +++ b/lib/types/hashes.rs @@ -147,13 +147,13 @@ impl utoipa::ToSchema for MerkleRoot { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct MerkleProof { pub leaf_index: usize, pub siblings: Vec, } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] pub struct MerkleProofNode { pub hash: Hash, pub is_left: bool, diff --git a/lib/types/mod.rs b/lib/types/mod.rs index f091ff00..3c5622ce 100644 --- a/lib/types/mod.rs +++ b/lib/types/mod.rs @@ -29,7 +29,7 @@ pub use hashes::{ }; pub use keys::{EncryptionPubKey, VerifyingKey}; pub use transaction::{ - AddressOutPointKey, AmmBurn, AmmMint, AmmSwap, AssetOutput, + AddressOutPointKey, AddressTxidKey, AmmBurn, AmmMint, AmmSwap, AssetOutput, AssetOutputContent, Authorized, AuthorizedTransaction, BitcoinOutput, BitcoinOutputContent, DutchAuctionBid, DutchAuctionCollect, DutchAuctionParams, FilledOutput, FilledOutputContent, FilledTransaction, diff --git a/lib/types/transaction/mod.rs b/lib/types/transaction/mod.rs index 2ff84a25..0af1b0c7 100644 --- a/lib/types/transaction/mod.rs +++ b/lib/types/transaction/mod.rs @@ -310,6 +310,80 @@ impl<'a> BytesDecode<'a> for AddressOutPointKey { } } +const TXID_SIZE: usize = blake3::OUT_LEN; +const ADDR_TXID_DB_KEY_SIZE: usize = ADDRESS_SIZE + TXID_SIZE; + +/// Fixed-width txdb key: address || txid. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct AddressTxidKey([u8; ADDR_TXID_DB_KEY_SIZE]); + +impl AddressTxidKey { + #[inline] + pub fn new(address: Address, txid: Txid) -> Self { + let mut key = [0u8; ADDR_TXID_DB_KEY_SIZE]; + key[..ADDRESS_SIZE].copy_from_slice(&address.0); + key[ADDRESS_SIZE..].copy_from_slice(txid.as_slice()); + Self(key) + } + + #[inline] + pub fn start(address: Address) -> Self { + let mut key = [0u8; ADDR_TXID_DB_KEY_SIZE]; + key[..ADDRESS_SIZE].copy_from_slice(&address.0); + Self(key) + } + + #[inline] + pub fn end(address: Address) -> Self { + let mut key = [0xffu8; ADDR_TXID_DB_KEY_SIZE]; + key[..ADDRESS_SIZE].copy_from_slice(&address.0); + Self(key) + } + + #[inline] + pub fn address(&self) -> Address { + let mut address = [0u8; ADDRESS_SIZE]; + address.copy_from_slice(&self.0[..ADDRESS_SIZE]); + Address(address) + } + + #[inline] + pub fn txid(&self) -> Txid { + let mut txid = [0u8; TXID_SIZE]; + txid.copy_from_slice(&self.0[ADDRESS_SIZE..]); + Txid(txid) + } +} + +impl AsRef<[u8]> for AddressTxidKey { + #[inline] + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl<'a> BytesEncode<'a> for AddressTxidKey { + type EItem = AddressTxidKey; + + #[inline] + fn bytes_encode( + item: &'a Self::EItem, + ) -> Result, BoxedError> { + Ok(Cow::Borrowed(item.as_ref())) + } +} + +impl<'a> BytesDecode<'a> for AddressTxidKey { + type DItem = AddressTxidKey; + + #[inline] + fn bytes_decode(bytes: &'a [u8]) -> Result { + let mut key = [0u8; ADDR_TXID_DB_KEY_SIZE]; + key.copy_from_slice(bytes); + Ok(Self(key)) + } +} + #[cfg(test)] mod tests { use super::{OUTPOINT_KEY_SIZE, OutPoint, OutPointKey};