configurable thresholds for utxo merge policies

bitcoin/secp256k1_transaction.py

@@ -433,20 +433,3 @@ def mktx(*args):
         txobj["outs"].append(outobj)
 
     return serialize(txobj)
-
-
-def select(unspent, value):
-    value = int(value)
-    high = [u for u in unspent if u["value"] >= value]
-    high.sort(key=lambda u: u["value"])
-    low = [u for u in unspent if u["value"] < value]
-    low.sort(key=lambda u: -u["value"])
-    if len(high):
-        return [high[0]]
-    i, tv = 0, 0
-    while tv < value and i < len(low):
-        tv += low[i]["value"]
-        i += 1
-    if tv < value:
-        raise Exception("Not enough funds")
-    return low[:i]

joinmarket/configure.py

@@ -121,16 +121,15 @@ def jm_single():
 confirm_timeout_hours = 6
 
 [POLICY]
-# for dust sweeping, try merge_algorithm = gradual
-# for more rapid dust sweeping, try merge_algorithm = greedy
-# for most rapid dust sweeping, try merge_algorithm = greediest
-# but don't forget to bump your miner fees!
-merge_algorithm = default
 # For takers: the minimum number of makers you allow in a transaction
 # to complete, accounting for the fact that some makers might not be
 # responsive. Should be an integer >=2 for privacy, or set to 0 if you
 # want to disallow any reduction from your chosen number of makers.
 minimum_makers = 2
+# this knob is a list of ints, being each the amount of utxos in a mixdepth
+# sufficient for kicking in the next-mergiest utxo selection algorithm. the
+# default errs on the side of privacy; lower values make tracking easier.
+merge_algorithm = [42, 59, 72]
 # the fee estimate is based on a projection of how many satoshis
 # per kB are needed to get in one of the next N blocks, N set here
 # as the value of 'tx_fees'. This estimate is high if you set N=2,

joinmarket/maker.py

@@ -439,20 +439,8 @@ def create_my_orders(self):
         # Maker code so im adding utxo and mixdepth here
         return orderlist
 
-        # has to return a list of utxos and mixing depth the cj address will
-        # be in the change address will be in mixing_depth-1
-
+    # has to return a list of utxos, and addresses for cjout and change
     def oid_to_order(self, cjorder, oid, amount):
-        """
-		unspent = []
-		for utxo, addrvalue in self.wallet.unspent.iteritems():
-			unspent.append({'value': addrvalue['value'], 'utxo': utxo})
-		inputs = btc.select(unspent, amount)
-		#TODO this raises an exception if you dont have enough money, id rather it just returned None
-		mixing_depth = 1
-		return [i['utxo'] for i in inputs], mixing_depth
-		"""
-
         order = [o for o in self.orderlist if o['oid'] == oid][0]
         cj_addr = self.wallet.get_internal_addr(order['mixdepth'] + 1)
         change_addr = self.wallet.get_internal_addr(order['mixdepth'])

joinmarket/support.py

@@ -98,6 +98,21 @@ def rand_weighted_choice(n, p_arr):
 def chunks(d, n):
     return [d[x:x + n] for x in xrange(0, len(d), n)]
 
+def select_default(unspent, value):
+    value = int(value)
+    high = [u for u in unspent if u["value"] >= value]
+    high.sort(key=lambda u: u["value"])
+    low = [u for u in unspent if u["value"] < value]
+    low.sort(key=lambda u: -u["value"])
+    if len(high):
+        return [high[0]]
+    i, tv = 0, 0
+    while tv < value and i < len(low):
+        tv += low[i]["value"]
+        i += 1
+    if tv < value:
+        raise Exception("Not enough funds")
+    return low[:i]
 
 def select_gradual(unspent, value):
     """
@@ -174,6 +189,21 @@ def select_greediest(unspent, value):
             end += 1
         return low[0:end]
 
+# ordered from most dusty (arguably, most private) to most mergiest (cheaper!)
+selectors = [select_default, select_gradual, select_greedy, select_greediest]
+
+def utxo_selector(configured_levels):
+    def select(unspent, value):
+        length = len(unspent)   # NB - counted only within each mixdepth
+        try:
+            for i in xrange(len(configured_levels)):
+                if length < configured_levels[i]:
+                    return selectors[i](unspent, value)
+            return selectors[-1](unspent, value) # greedy fallback
+        except IndexError:      # luser configured more levels than algos
+            log.debug("I'm sorry, Dave, but I can't let you merge that!")
+        return selectors[0](unspent, value) # default to improve privacy
+    return select
 
 def calc_cj_fee(ordertype, cjfee, cj_amount):
     if ordertype == 'absoffer':

joinmarket/wallet.py

@@ -3,6 +3,7 @@
 import os
 import pprint
 import sys
+import ast
 from decimal import Decimal
 
 from ConfigParser import NoSectionError
@@ -13,8 +14,7 @@
 from joinmarket.blockchaininterface import BitcoinCoreInterface, RegtestBitcoinCoreInterface
 from joinmarket.configure import jm_single, get_network, get_p2pk_vbyte
 
-from joinmarket.support import get_log, select_gradual, select_greedy, \
-    select_greediest
+from joinmarket.support import get_log, utxo_selector
 
 log = get_log()
 
@@ -46,19 +46,27 @@ def __init__(self):
         #some consumer scripts don't use an unspent, this marks it
         #as specifically absent (rather than just empty).
         self.unspent = None
-        self.utxo_selector = btc.select  # default fallback: upstream
         try:
-            config = jm_single().config
-            if config.get("POLICY", "merge_algorithm") == "gradual":
-                self.utxo_selector = select_gradual
-            elif config.get("POLICY", "merge_algorithm") == "greedy":
-                self.utxo_selector = select_greedy
-            elif config.get("POLICY", "merge_algorithm") == "greediest":
-                self.utxo_selector = select_greediest
-            elif config.get("POLICY", "merge_algorithm") != "default":
-                raise Exception("Unknown merge algorithm")
+            policy = jm_single().config.get("POLICY", "merge_algorithm")
         except NoSectionError:
-            pass
+            policy = "default"  # maintain backwards compatibility!
+        if policy == "default":
+            self.merge_policy = [42] # well, almost (python lacks infinites)
+        elif policy == "gradual":
+            self.merge_policy = [60] # never goes beyond gradual
+        elif policy == "greedy":
+            self.merge_policy = [70, 70] # skip gradual, go greedy
+        elif policy == "greediest":
+            self.merge_policy = [80, 80, 80] # straight to greediest
+        else:
+            try:                # stop supporting word configs, someday...
+                self.merge_policy = ast.literal_eval(policy)
+                if ((type(self.merge_policy) is not list) or
+                    any(type(level) is not int for level in self.merge_policy)):
+                    raise Exception("Merge policy must be a list of ints")
+            except ValueError:
+                raise Exception("Unparseable merge policy: "+policy)
+        self.utxo_selector = utxo_selector(self.merge_policy)
 
     def get_key_from_addr(self, addr):
         return None

test/commontest.py

@@ -149,7 +149,7 @@ def make_wallets(n,
             for k in range(wallet_structures[i][j]):
                 deviation = sdev_amt * random.random()
                 amt = mean_amt - sdev_amt / 2.0 + deviation
-                if amt < 0: amt = 0.001
+                if amt < 0.001: amt = 0.001
                 amt = float(Decimal(amt).quantize(Decimal(10)**-8))
                 jm_single().bc_interface.grab_coins(
                     wallets[i + start_index]['wallet'].get_external_addr(j),

test/regtest_joinmarket.cfg

@@ -21,11 +21,8 @@ socks5_port = 9150, 9150
 [LOGGING]
 console_log_level = DEBUG
 [POLICY]
-# for dust sweeping, try merge_algorithm = gradual
-# for more rapid dust sweeping, try merge_algorithm = greedy
-# for most rapid dust sweeping, try merge_algorithm = greediest
-# but don't forget to bump your miner fees!
-merge_algorithm = default
+# tiers for testing
+merge_algorithm = [6, 10, 20]
 # the fee estimate is based on a projection of how many satoshis
 # per kB are needed to get in one of the next N blocks, N set here
 # as the value of 'tx_fees'. This estimate can be extremely high

test/test_wallets.py

@@ -350,6 +350,44 @@ def test_utxo_selection(setup_wallets, nw, wallet_structures, mean_amt,
                        "failed to select sufficient coins, total: " + \
                        str(total_selected) + ", should be: " + str(amount)
 
+# the following test mainly checks that the algo switching based on
+# number of utxos occurs correctly; it also provides some rough sanity
+# testing on the general behavior of the merge algorithms.
+# TODO: separate & deterministic tests for each merge algo!
+
+def test_merge_algo_switching(setup_wallets):
+    # FIXME: is wallet generation a little broken?
+    wallet = make_wallets(1, [[5, 10, 25, 0, 0]], 1, 3)[0]['wallet']
+    sync_wallet(wallet)
+    sync_wallet(wallet)
+    all_utxos = wallet.get_utxos_by_mixdepth()
+    assert (len(all_utxos[2]) == 25), "Not enough utxos!"
+    # test merge-avoiding with small utxo sets - select_default
+    utxos = sorted(map(lambda x:x['value'], all_utxos[0].values()))
+    for i in range(4):
+        amount = (utxos[i]+utxos[i+1])/2
+        selected = wallet.select_utxos(0, amount)
+        assert (1 == len(selected)), "Default selection misbehaved! " + \
+            "Selected " + str(selected) + " to reach sum " + str(amount) + \
+            " from utxos " + str(utxos) + " (should pick SINGLE utxo)"
+    # test merging with larger utxo sets - select_gradual
+    utxos = sorted(map(lambda x:x['value'], all_utxos[1].values()))
+    for i in range(5):
+        amount = (utxos[i+3]+utxos[i+4])/2
+        selected = wallet.select_utxos(1, amount)
+        assert (1 < len(selected)), "Default selection misbehaved! "+\
+            "Selected " + str(selected) + " to reach sum " + str(amount) + \
+            " from utxos " + str(utxos) + " (should pick MULTIPLE utxos)"
+    # TODO: test merging with intermediate sets - select_greedy
+    # test merging with even larger utxo sets - select_greediest
+    utxos = sorted(map(lambda x:x['value'], all_utxos[2].values()))
+    for i in range(20):
+        amount = sum(utxos[0:i+2])
+        selected = wallet.select_utxos(2, amount)
+        assert (i+2 <= len(selected)), "Default selection misbehaved! "+\
+            "Selected " + str(selected) + " to reach sum " + str(amount) + \
+            " from utxos " + str(utxos) + " (expected " + str(i+2) + ")"
+
 
 class TestWalletCreation(unittest.TestCase):