Merge pull request ZeroNetX#219 from caryoscelus/bootstrap

Bootstrap 0net trackers on first launch
marek22k · Jul 26, 2023 · 615e155 · 615e155
2 parents a8c2117 + 70f2eea
commit 615e155
Show file tree

Hide file tree

Showing 8 changed files with 99 additions and 191 deletions.
diff --git a/bootstrap.url b/bootstrap.url
@@ -0,0 +1 @@
+https://github.com/zeronet-conservancy/zeronet-conservancy/releases/download/v0.7.9/data-default-2023-07-23.zip
diff --git a/requirements.txt b/requirements.txt
@@ -14,3 +14,4 @@ maxminddb
 rich
 defusedxml>=0.7
 pyaes
+requests
diff --git a/src/Config.py b/src/Config.py
@@ -9,178 +9,6 @@
 import stat
 import time
 
-trackers = [
-    'zero://188.242.242.224:26474',
-    'zero://2001:19f0:8001:1d2f:5400:2ff:fe83:5bf7:23141',
-    'zero://200:1e7a:5100:ef7c:6fa4:d8ae:b91c:a74:15441',
-    'zero://23.184.48.134:15441',
-    'zero://57hzgtu62yzxqgbvgxs7g3lfck3za4zrda7qkskar3tlak5recxcebyd.onion:15445',
-    'zero://6i54dd5th73oelv636ivix6sjnwfgk2qsltnyvswagwphub375t3xcad.onion:15441',
-    'zero://f2hnjbggc3c2u2apvxdugirnk6bral54ibdoul3hhvu7pd4fso5fq3yd.onion:15441',
-    'zero://gugt43coc5tkyrhrc3esf6t6aeycvcqzw7qafxrjpqbwt4ssz5czgzyd.onion:15441',
-    'zero://k5w77dozo3hy5zualyhni6vrh73iwfkaofa64abbilwyhhd3wgenbjqd.onion:15441',
-    'zero://ow7in4ftwsix5klcbdfqvfqjvimqshbm2o75rhtpdnsderrcbx74wbad.onion:15441',
-    'zero://pn4q2zzt2pw4nk7yidxvsxmydko7dfibuzxdswi6gu6ninjpofvqs2id.onion:15441',
-    'zero://skdeywpgm5xncpxbbr4cuiip6ey4dkambpanog6nruvmef4f3e7o47qd.onion:15441',
-    'zero://wlxav3szbrdhest4j7dib2vgbrd7uj7u7rnuzg22cxbih7yxyg2hsmid.onion:15441',
-    'zero://zy7wttvjtsijt5uwmlar4yguvjc2gppzbdj4v6bujng6xwjmkdg7uvqd.onion:15441',
-    'http://bt.okmp3.ru:2710/announce',
-    'http://fxtt.ru:80/announce',
-    'http://incine.ru:6969/announce',
-    'http://moeweb.pw:6969/announce',
-    'http://open.acgnxtracker.com:80/announce',
-    'http://t.acg.rip:6699/announce',
-    'http://t.nyaatracker.com:80/announce',
-    'http://t.overflow.biz:6969/announce',
-    'http://tracker.files.fm:6969/announce',
-    'http://tracker.mywaifu.best:6969/announce',
-    'http://tracker.vrpnet.org:6969/announce',
-    'http://vps02.net.orel.ru:80/announce',
-    'udp://960303.xyz:6969/announce',
-    'udp://aarsen.me:6969/announce',
-    'udp://astrr.ru:6969/announce',
-    'udp://ben.kerbertools.xyz:6969/announce',
-    'udp://bt1.archive.org:6969/announce',
-    'udp://bt2.archive.org:6969/announce',
-    'udp://bt.ktrackers.com:6666/announce',
-    'udp://bubu.mapfactor.com:6969/announce',
-    'udp://c.ns.cluefone.com:6969/announce',
-    'udp://cutscloud.duckdns.org:6969/announce',
-    'udp://download.nerocloud.me:6969/announce',
-    'udp://epider.me:6969/announce',
-    'udp://exodus.desync.com:6969/announce',
-    'udp://htz3.noho.st:6969/announce',
-    'udp://ipv4.tracker.harry.lu:80/announce',
-    'udp://laze.cc:6969/announce',
-    'udp://mail.artixlinux.org:6969/announce',
-    'udp://mirror.aptus.co.tz:6969/announce',
-    'udp://moonburrow.club:6969/announce',
-    'udp://movies.zsw.ca:6969/announce',
-    'udp://mts.tvbit.co:6969/announce',
-    'udp://new-line.net:6969/announce',
-    'udp://open.demonii.com:1337/announce',
-    'udp://open.stealth.si:80/announce',
-    'udp://opentracker.i2p.rocks:6969/announce',
-    'udp://p4p.arenabg.com:1337/announce',
-    'udp://psyco.fr:6969/announce',
-    'udp://public.publictracker.xyz:6969/announce',
-    'udp://rep-art.ynh.fr:6969/announce',
-    'udp://run.publictracker.xyz:6969/announce',
-    'udp://sanincode.com:6969/announce',
-    'udp://slicie.icon256.com:8000/announce',
-    'udp://tamas3.ynh.fr:6969/announce',
-    'udp://thouvenin.cloud:6969/announce',
-    'udp://torrentclub.space:6969/announce',
-    'udp://tracker.0x.tf:6969/announce',
-    'udp://tracker1.bt.moack.co.kr:80/announce',
-    'udp://tracker.4.babico.name.tr:3131/announce',
-    'udp://tracker.altrosky.nl:6969/announce',
-    'udp://tracker.artixlinux.org:6969/announce',
-    'udp://tracker.farted.net:6969/announce',
-    'udp://tracker.jonaslsa.com:6969/announce',
-    'udp://tracker.joybomb.tw:6969/announce',
-    'udp://tracker.monitorit4.me:6969/announce',
-    'udp://tracker.opentrackr.org:1337/announce',
-    'udp://tracker.pomf.se:80/announce',
-    'udp://tracker.publictracker.xyz:6969/announce',
-    'udp://tracker.srv00.com:6969/announce',
-    'udp://tracker.tcp.exchange:6969/announce',
-    'udp://tracker.theoks.net:6969/announce',
-    'udp://transkaroo.joustasie.net:6969/announce',
-    'udp://uploads.gamecoast.net:6969/announce',
-    'udp://v2.iperson.xyz:6969/announce',
-    'udp://vibe.sleepyinternetfun.xyz:1738/announce',
-    'udp://www.skynetcenter.me:6969/announce',
-    'udp://www.torrent.eu.org:451/announce',
-    'zero://194.5.98.39:15441',
-    'zero://145.239.95.38:15441',
-    'zero://178.128.34.249:26117',
-    'zero://217.18.217.143:39288',
-    'zero://83.246.141.203:22207',
-    'zero://syncronite.loki:15441',
-    'zero://2a05:dfc1:4000:1e00::a:15441',
-    'zero://2400:6180:100:d0::8fd:8001:21697',
-    'zero://2001:19f0:8001:1d2f:5400:2ff:fe83:5bf7:30530',
-    'zero://73pyhfwfwsrhfw76knkjfnw6o3lk53zfo7hlxdmxbj75sjcnol5cioad.onion:15442',
-    'zero://fzlzmxuz2bust72cuy5g4w6d62tx624xcjaupf2kp7ffuitbiniy2hqd.onion:15441',
-    'zero://rlcjomszyitxpwv7kzopmqgzk3bdpsxeull4c3s6goszkk6h2sotfoad.onion:15441',
-    'zero://tqmo2nffqo4qc5jgmz3me5eri3zpgf3v2zciufzmhnvznjve5c3argad.onion:15441',
-    'http://107.189.31.134:6969/announce',
-    'http://119.28.71.45:8080/announce',
-    'http://129.146.193.240:6699/announce',
-    'http://159.69.65.157:6969/announce',
-    'http://163.172.29.130:80/announce',
-    'http://185.130.47.2:6969/announce',
-    'http://45.67.35.111:6969/announce',
-    'http://61.222.178.254:6969/announce',
-    'http://83.31.30.182:6969/announce',
-    'http://93.158.213.92:1337/announce',
-    'http://95.217.167.10:6969/announce',
-    'udp://102.223.180.235:6969/announce',
-    'udp://103.122.21.50:6969/announce',
-    'udp://104.131.98.232:6969/announce',
-    'udp://104.244.77.87:6969/announce',
-    'udp://107.189.11.58:6969/announce',
-    'udp://107.189.31.134:6969/announce',
-    'udp://139.144.68.88:6969/announce',
-    'udp://149.28.239.70:6969/announce',
-    'udp://15.204.205.14:6969/announce',
-    'udp://156.234.201.18:80/announce',
-    'udp://158.101.161.60:3131/announce',
-    'udp://163.172.29.130:80/announce',
-    'udp://167.99.185.219:6969/announce',
-    'udp://176.31.250.174:6969/announce',
-    'udp://176.56.4.238:6969/announce',
-    'udp://178.32.222.98:3391/announce',
-    'udp://184.105.151.166:6969/announce',
-    'udp://185.102.219.163:6969/announce',
-    'udp://185.181.60.155:80/announce',
-    'udp://185.217.199.21:6969/announce',
-    'udp://185.44.82.25:1337/announce',
-    'udp://185.68.21.244:6969/announce',
-    'udp://192.3.165.191:6969/announce',
-    'udp://192.3.165.198:6969/announce',
-    'udp://192.95.46.115:6969/announce',
-    'udp://193.176.158.162:6969/announce',
-    'udp://193.37.214.12:6969/announce',
-    'udp://193.42.111.57:9337/announce',
-    'udp://198.100.149.66:6969/announce',
-    'udp://20.100.205.229:6969/announce',
-    'udp://207.241.226.111:6969/announce',
-    'udp://207.241.231.226:6969/announce',
-    'udp://209.141.59.16:6969/announce',
-    'udp://212.237.53.230:6969/announce',
-    'udp://23.153.248.2:6969/announce',
-    'udp://23.254.228.89:6969/announce',
-    'udp://37.187.111.136:6969/announce',
-    'udp://37.27.4.53:6969/announce',
-    'udp://38.7.201.142:6969/announce',
-    'udp://45.154.253.6:6969/announce',
-    'udp://45.63.30.114:6969/announce',
-    'udp://45.9.60.30:6969/announce',
-    'udp://46.38.238.105:6969/announce',
-    'udp://49.12.76.8:8080/announce',
-    'udp://5.102.159.190:6969/announce',
-    'udp://5.196.89.204:6969/announce',
-    'udp://51.15.79.209:6969/announce',
-    'udp://51.159.54.68:6666/announce',
-    'udp://51.68.174.87:6969/announce',
-    'udp://51.81.222.188:6969/announce',
-    'udp://52.58.128.163:6969/announce',
-    'udp://61.222.178.254:6969/announce',
-    'udp://77.73.69.230:6969/announce',
-    'udp://83.102.180.21:80/announce',
-    'udp://83.31.30.182:6969/announce',
-    'udp://85.206.172.159:6969/announce',
-    'udp://85.239.33.28:6969/announce',
-    'udp://86.57.161.157:6969/announce',
-    'udp://91.216.110.52:451/announce',
-    'udp://93.158.213.92:1337/announce',
-    'udp://94.103.87.87:6969/announce',
-    'udp://95.216.74.39:6969/announce',
-    'udp://95.31.11.224:6969/announce',
-]
-
 class Config(object):
 
     def __init__(self, argv):
@@ -322,6 +150,10 @@ def createArguments(self):
         action.add_argument('cmd', help='API command name')
         action.add_argument('parameters', help='Parameters of the command', nargs='?')
 
+        # Import bundled sites
+        action = self.subparsers.add_parser("importBundle", help='Import sites from a .zip bundle')
+        action.add_argument('bundle', help='Path to a data bundle')
+
         # dbRebuild
         action = self.subparsers.add_parser("dbRebuild", help='Rebuild site database cache')
         action.add_argument('address', help='Site to rebuild')
@@ -428,8 +260,10 @@ def createArguments(self):
         self.parser.add_argument('--disable_udp', help='Disable UDP connections', action='store_true')
         self.parser.add_argument('--proxy', help='Socks proxy address', metavar='ip:port')
         self.parser.add_argument('--bind', help='Bind outgoing sockets to this address', metavar='ip')
-        self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=trackers, metavar='protocol://address', nargs='*')
-        self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', metavar='path', nargs='*')
+        self.parser.add_argument('--bootstrap_url', help='URL of file with link to bootstrap bundle', default='https://raw.githubusercontent.com/caryoscelus/zeronet-conservancy/bootstrap/bootstrap.url', type=str)
+        self.parser.add_argument('--disable_bootstrap', help='Disable downloading bootstrap information from clearnet', action='store_true')
+        self.parser.add_argument('--trackers', help='Bootstraping torrent trackers', default=[], metavar='protocol://address', nargs='*')
+        self.parser.add_argument('--trackers_file', help='Load torrent trackers dynamically from a file', default=['{data_dir}/15CEFKBRHFfAP9rmL6hhLmHoXrrgmw4B5o/cache/1/Syncronite.html'], metavar='path', nargs='*')
         self.parser.add_argument('--trackers_proxy', help='Force use proxy to connect to trackers (disable, tor, ip:port)', default="disable")
         self.parser.add_argument('--use_libsecp256k1', help='Use Libsecp256k1 liblary for speedup', type='bool', choices=[True, False], default=True)
         self.parser.add_argument('--use_openssl', help='Use OpenSSL liblary for speedup', type='bool', choices=[True, False], default=True)
@@ -799,10 +633,6 @@ def initLogging(self, console_logging=None, file_logging=None):
             except Exception as err:
                 print("Can't change permission of %s: %s" % (self.log_dir, err))
 
-        # Make warning hidden from console
-        logging.WARNING = 15  # Don't display warnings if not in debug mode
-        logging.addLevelName(15, "WARNING")
-
         logging.getLogger('').name = "-"  # Remove root prefix
 
         self.error_logger = ErrorLogHandler()

diff --git a/src/Crypt/CryptBitcoin.py b/src/Crypt/CryptBitcoin.py
@@ -99,3 +99,16 @@ def verify(data, valid_address, sign, lib_verify=None):  # Verify data using add
         return sign_address in valid_address
     else:  # One possible address
         return sign_address == valid_address
+
+def isValidAddress(addr):
+    '''Check if provided address is valid bitcoin address'''
+    if addr[0] != '1':
+        # no support for new-style addrs
+        return False
+    from base58 import b58decode
+    bs = b58decode(addr)
+    main = bs[:-4]
+    checksum = bs[-4:]
+    h1 = hashlib.sha256(main).digest()
+    h2 = hashlib.sha256(h1).digest()
+    return h2[:4] == checksum
diff --git a/src/Site/Site.py b/src/Site/Site.py
@@ -87,7 +87,11 @@ def __repr__(self):
     # Load site settings from data/sites.json
     def loadSettings(self, settings=None):
         if not settings:
-            settings = json.load(open("%s/sites.json" % config.data_dir)).get(self.address)
+            try:
+                settings = json.load(open(f'{config.data_dir}/sites.json')).get(self.address)
+            except Exception as err:
+                logging.error(f'Error loading {config.data_dir}/sites.json: {err}')
+                settings = {}
         if settings:
             self.settings = settings
             if "cache" not in settings:

diff --git a/src/Site/SiteAnnouncer.py b/src/Site/SiteAnnouncer.py
@@ -202,7 +202,7 @@ def announceTracker(self, tracker, mode="start", num_want=10):
             else:
                 raise AnnounceError("Unknown protocol: %s" % address_parts["protocol"])
         except Exception as err:
-            self.site.log.warning("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode))
+            self.site.log.debug("Tracker %s announce failed: %s in mode %s" % (tracker, Debug.formatException(err), mode))
             error = err
 
         if error:

diff --git a/src/Site/SiteManager.py b/src/Site/SiteManager.py
@@ -31,17 +31,18 @@ def __init__(self):
     @util.Noparallel()
     def load(self, cleanup=True, startup=False):
         from .Site import Site
-        self.log.info("Loading sites... (cleanup: %s, startup: %s)" % (cleanup, startup))
+        self.log.info(f'Loading sites... ({cleanup=}, {startup=})')
         self.loaded = False
         address_found = []
         added = 0
         load_s = time.time()
         # Load new adresses
         try:
-            json_path = "%s/sites.json" % config.data_dir
+            json_path = f"{config.data_dir}/sites.json"
             data = json.load(open(json_path))
         except Exception as err:
-            raise Exception("Unable to load %s: %s" % (json_path, err))
+            self.log.error(f"Unable to load {json_path}: {err}")
+            data = {}
 
         sites_need = []
 

diff --git a/src/main.py b/src/main.py
@@ -34,24 +34,79 @@ def load_config():
 
 load_config()
 
+def importBundle(bundle):
+    from zipfile import ZipFile
+    from Crypt.CryptBitcoin import isValidAddress
+    import json
+
+    sites_json_path = f"{config.data_dir}/sites.json"
+    try:
+        with open(sites_json_path) as f:
+            sites = json.load(f)
+    except Exception as err:
+        sites = {}
+
+    with ZipFile(bundle) as zf:
+        all_files = zf.namelist()
+        top_files = list(set(map(lambda f: f.split('/')[0], all_files)))
+        if len(top_files) == 1 and not isValidAddress(top_files[0]):
+            prefix = top_files[0]+'/'
+        else:
+            prefix = ''
+        top_2 = list(set(filter(lambda f: len(f)>0,
+                                map(lambda f: f.removeprefix(prefix).split('/')[0], all_files))))
+        for d in top_2:
+            if isValidAddress(d):
+                logging.info(f'unpack {d} into {config.data_dir}')
+                for fname in filter(lambda f: f.startswith(prefix+d) and not f.endswith('/'), all_files):
+                    tgt = config.data_dir + '/' + fname.removeprefix(prefix)
+                    logging.info(f'-- {fname} --> {tgt}')
+                    info = zf.getinfo(fname)
+                    info.filename = tgt
+                    zf.extract(info)
+                logging.info(f'add site {d}')
+                sites[d] = {}
+            else:
+                logging.info(f'Warning: unknown file in a bundle: {prefix+d}')
+    with open(sites_json_path, 'w') as f:
+        json.dump(sites, f)
+
 def init_dirs():
-    if not os.path.isdir(config.data_dir):
-        os.mkdir(config.data_dir)
+    data_dir = config.data_dir
+    if not os.path.isdir(data_dir):
+        os.mkdir(data_dir)
         try:
-            os.chmod(config.data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+            os.chmod(data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
         except Exception as err:
-            startupError("Can't change permission of %s: %s" % (config.data_dir, err))
-
-    sites_json = f"{config.data_dir}/sites.json"
+            startupError(f"Can't change permission of {data_dir}: {err}")
+
+        # download latest bootstrap bundle
+        if not config.disable_bootstrap and not config.offline:
+            import requests
+            from io import BytesIO
+
+            print(f'fetching {config.bootstrap_url}')
+            response = requests.get(config.bootstrap_url)
+            if response.status_code != 200:
+                startupError(f"Cannot load bootstrap bundle (response status: {response.status_code})")
+            url = response.text
+            print(f'got {url}')
+            response = requests.get(url)
+            if response.status_code < 200 or response.status_code >= 300:
+                startupError(f"Cannot load boostrap bundle (response status: {response.status_code})")
+            importBundle(BytesIO(response.content))
+
+    sites_json = f"{data_dir}/sites.json"
     if not os.path.isfile(sites_json):
         with open(sites_json, "w") as f:
             f.write("{}")
-    users_json = f"{config.data_dir}/users.json"
+    users_json = f"{data_dir}/users.json"
     if not os.path.isfile(users_json):
         with open(users_json, "w") as f:
             f.write("{}")
 
 # TODO: GET RID OF TOP-LEVEL CODE!!!
+config.initConsoleLogger()
 
 try:
     init_dirs()
@@ -73,7 +128,7 @@ def init_dirs():
         r = proc.wait()
         sys.exit(r)
 
-config.initLogging()
+config.initLogging(console_logging=False)
 
 # Debug dependent configuration
 from Debug import DebugHook
@@ -415,6 +470,9 @@ def siteCmd(self, address, cmd, parameters):
         else:
             return res
 
+    def importBundle(self, bundle):
+        importBundle(bundle)
+
     def getWebsocket(self, site):
         import websocket