Extract with or without balance

.gitignore

@@ -0,0 +1,7 @@
+requirements.txt
+*.csv
+__pycache__
+*.pyo
+*.pyc
+*.txt
+*.csvtmp

Makefile

@@ -0,0 +1,32 @@
+QUIET ?= -OO  # run with QUIET= to see debugging messages
+CHAINSTATE ?= $(HOME)/.bitcoin/chainstate
+BALANCES ?= balances.csv
+PYTHON ?= python
+ifeq ($(PYTHON),python3)
+ PIP := pip3
+else
+ PIP := pip
+endif
+export
+$(BALANCES): requirements.txt btcposbal2csv.py
+	# stop bitcoind if running, to avoid corrupting chainstate
+	$(PIP) install --user --requirement $<
+	if pidof bitcoind >/dev/null; then \
+	 bitcoin-cli stop; \
+	 while pidof bitcoind >/dev/null; do sleep 1; done; \
+	 $(PYTHON) $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES)tmp; \
+	 bitcoind; \
+	else \
+	 $(PYTHON) $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES)tmp; \
+	fi
+	if [ "$$(wc -c $(BALANCES)tmp | awk '{print 1}')" -gt 1 ]; then \
+	 mv $(BALANCES)tmp $(BALANCES); \
+	else \
+	 echo No balances found in chainstate >&2; \
+	fi
+requirements.txt:
+	# WARNING: may be necessary to `sudo apt install libleveldb-dev`
+	for requirement in plyvel base58 sqlite3 hashlib; do \
+	 $(PYTHON) -c "import $$requirement" 2>/dev/null || \
+	  (echo $$requirement >> $@; true); \
+	done

btcposbal2csv.py

@@ -105,7 +105,7 @@ def in_mem(in_args):
         else:
             add_dict[add] = [val, height]
 
-    for key in add_dict.iterkeys():
+    for key in add_dict:
         ll = add_dict[key]
         yield key, ll[0], ll[1]
 
@@ -201,8 +201,10 @@ def low_mem(in_args):
             for address, sat_val, block_height in add_iter:
                 if sat_val == 0:
                     continue
+                # make this work the same on Python2 and Python3
+                decoded = str(address.decode())
                 w.append(
-                    address + ',' + str(sat_val) + ',' + str(block_height)
+                    ','.join((decoded, str(sat_val), str(block_height)))
                 )
                 c += 1
                 if c == 1000:
@@ -212,4 +214,4 @@ def low_mem(in_args):
             if c > 0:
                 f.write('\n'.join(w) + '\n')
             f.write('\n')
-        print('writen to %s' % args.out)
+        print('written to %s' % args.out)

readme.md

@@ -7,50 +7,62 @@ python 2.7
 pip  
 
 #### To install:  
-run pip install -r requirements.txt
+`make requirements.txt`
+`pip install -r requirements.txt`
 
 or install following packages with pip manualy
-* hashlib
+* hashlib (already included in modern Python installations)
 * plyvel
 * base58
-* sqlite3
+* sqlite3 (already included in modern Python installations)
 
 #### Usage
-To use you will need copy of chainstate database as created by [bitcoin core](https://bitcoin.org/en/bitcoin-core/)
- client. I've not tried different clients.
+To use you will need copy of chainstate database as created by 
+[bitcoin core](https://bitcoin.org/en/bitcoin-core/) client. I've not 
+tried different clients.
 
-To get current addresses with positive balance, let the full node client sync with the network. 
-**Stop** the bitcoin-core client before running this utility. If you not stop the client, the database might get corrupted.  
-Then run this program with path to chainstate directory (usualy $HOME/.bitcoin/chainstate).
+To get current addresses with positive balance, let the full node client 
+sync with the network.  **Stop** the bitcoin-core client before running this 
+utility. If you not stop the client, the database might get corrupted.  
+Then run this program with path to chainstate directory 
+(usually $HOME/.bitcoin/chainstate).
 
 Show help
 ```
 python btcposbal2csv.py -h
 ```
 #### Example:  
-Following will read from `/home/USER/.bitcoin/chainstate`, and write result to `/home/USER/addresses_with_balance.csv`.
-```
-python btcposbal2csv.py /home/USER/.bitcoin/chainstate /home/USER/addresses_with_balance.csv
-```
+Following will read from `$HOME/.bitcoin/chainstate`, and write result to `$HOME/addresses_with_balance.csv`.
+
+`python btcposbal2csv.py $HOME/.bitcoin/chainstate $HOME/addresses_with_balance.csv`
+
+Or you can just accept the defaults in the Makefile, and run: `make`
 
 ##### Notice
-* That the output may not be complete as there are some transactions which are not understood by the decoding lib, or that which do not have "address" at all. Such transactions are not processed. Number of them and the total ammount is displayed after the analysis.  
-* The output csv file only reflects the chainstate leveldb at your disk. So it will always be few blocks behind the network as you need to stop the bitcoin-core client.
+* The output may not be complete, as there are some transactions which are
+  not understood by the decoding lib, or which do not have "address" at all.
+  Such transactions are not processed. The number of them and the total amount
+  is displayed after the analysis.  
+* The output csv file only reflects the chainstate leveldb on your disk. So
+  it will always be few blocks behind the network, as you need to stop the
+  bitcoin-core client.
 
 #### Converting to RIPEMD160
 Per request, I'm adding script which is able to convert BTC address to RIPEMD160 representation.
 BTC address must be in fist column, RIPEMD160 is added to csv. Output goes to stdout.
 
 Example:
 ```
-python convert2ripemd160.py /home/USER/addresses_with_balance.csv
+python convert2ripemd160.py $HOME/addresses_with_balance.csv
 ```
 
 #### Acknowledgement
-This utility builds on very nice [bitcoin_tools](https://github.com/sr-gi/bitcoin_tools/) lib,
- which does the UTXO parsing.
+This utility builds on the very nice 
+[bitcoin_tools](https://github.com/sr-gi/bitcoin_tools/) lib,
+which does the UTXO parsing.
 #### Support
-If you like this utility, please consider supporting the bitcoin_tools library which does all the heavy lifting.
+If you like this utility, please consider supporting the bitcoin_tools library, 
+which does all the heavy lifting.
 
-If this particular functionality made your life easier you can support coffee consumption in BTC 
-1FxC1mgJkad63beJcECfZMRaFSf4PBLr2f.
+If this particular functionality made your life easier, you can support coffee
+consumption in BTC, 1FxC1mgJkad63beJcECfZMRaFSf4PBLr2f.

utils.py

@@ -11,6 +11,11 @@
 # Fee per byte range
 NSPECIALSCRIPTS = 6
 
+# python3 compatibility
+if chr(189) == b'\xbd':
+    BYTECHR = chr
+else:
+    BYTECHR = lambda c: bytes((c,))
 
 def txout_decompress(x):
     """ Decompresses the Satoshi amount of a UTXO stored in the LevelDB. Code is a port from the Bitcoin Core C++
@@ -27,10 +32,10 @@ def txout_decompress(x):
         return 0
     x -= 1
     e = x % 10
-    x /= 10
+    x //= 10
     if e < 9:
         d = (x % 9) + 1
-        x /= 9
+        x //= 9
         n = x * 10 + d
     else:
         n = x + 1
@@ -134,7 +139,11 @@ def decode_utxo(coin, outpoint, version=0.15):
     else:
         # First we will parse all the data encoded in the outpoint, that is, the transaction id and index of the utxo.
         # Check that the input data corresponds to a transaction.
-        assert outpoint[:2] == '43'
+        try:
+            assert outpoint[:2] == b'43'
+        except AssertionError:
+            raise AssertionError('First two characters of %r are not "43"' %
+                                 outpoint)
         # Check the provided outpoint has at least the minimum length (1 byte of key code, 32 bytes tx id, 1 byte index)
         assert len(outpoint) >= 68
         # Get the transaction id (LE) by parsing the next 32 bytes of the outpoint.
@@ -314,7 +323,7 @@ def parse_ldb(fin_name, version=0.15, types=(0, 1)):
     db = plyvel.DB(fin_name, compression=None)  # Change with path to chainstate
 
     # Load obfuscation key (if it exists)
-    o_key = db.get((unhexlify("0e00") + "obfuscate_key"))
+    o_key = db.get((unhexlify("0e00") + b"obfuscate_key"))
 
     # If the key exists, the leading byte indicates the length of the key (8 byte by default). If there is no key,
     # 8-byte zeros are used (since the key will be XORed with the given values).
@@ -394,7 +403,7 @@ def deobfuscate_value(obfuscation_key, value):
     # Get the extended obfuscation key by concatenating the obfuscation key with itself until it is as large as the
     # value to be de-obfuscated.
     if l_obf < l_value:
-        extended_key = (obfuscation_key * ((l_value / l_obf) + 1))[:l_value]
+        extended_key = (obfuscation_key * ((l_value // l_obf) + 1))[:l_value]
     else:
         extended_key = obfuscation_key[:l_value]
 
@@ -448,7 +457,7 @@ def hash_160_to_btc_address(h160, v):
         h160 = unhexlify(h160)
 
     # Add the network version leading the previously calculated RIPEMD-160 hash.
-    vh160 = chr(v) + h160
+    vh160 = BYTECHR(v) + h160
     # Double sha256.
     h = sha256(sha256(vh160).digest()).digest()
     # Add the two first bytes of the result as a checksum tailing the RIPEMD-160 hash.