From ed78c20c1fd1eafa1a542057a67679325fbab2fc Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Thu, 9 May 2024 09:22:42 +0200 Subject: [PATCH 01/58] feat: chain config feat: strategy overlap --- .env.example | 1 + fastlane_bot/config/network.py | 41 ++++- fastlane_bot/config/selectors.py | 1 + .../sei/solidly_v2_event_mappings.csv | 1 + .../blockchain_data/sei/static_pool_data.csv | 4 + .../data/blockchain_data/sei/tokens.csv | 9 + .../sei/uniswap_v2_event_mappings.csv | 5 + .../sei/uniswap_v3_event_mappings.csv | 1 + fastlane_bot/data/multichain_addresses.csv | 4 + fastlane_bot/helpers/poolandtokens.py | 67 +++++++ fastlane_bot/tests/test_039_TestMultiMode.py | 3 + fastlane_bot/utils.py | 19 +- main.py | 2 +- .../NBTest/NBTest_002_CPCandOptimizer.ipynb | 12 ++ .../NBTest/NBTest_002_CPCandOptimizer.py | 4 + .../NBTest/NBTest_003_Serialization.ipynb | 163 ++++++++++++------ resources/NBTest/NBTest_003_Serialization.py | 22 ++- run_blockchain_terraformer.py | 9 +- 18 files changed, 295 insertions(+), 73 deletions(-) create mode 100644 fastlane_bot/data/blockchain_data/sei/solidly_v2_event_mappings.csv create mode 100644 fastlane_bot/data/blockchain_data/sei/static_pool_data.csv create mode 100644 fastlane_bot/data/blockchain_data/sei/tokens.csv create mode 100644 fastlane_bot/data/blockchain_data/sei/uniswap_v2_event_mappings.csv create mode 100644 fastlane_bot/data/blockchain_data/sei/uniswap_v3_event_mappings.csv diff --git a/.env.example b/.env.example index 5f1f4a5ae..5cb8f2b48 100644 --- a/.env.example +++ b/.env.example @@ -3,6 +3,7 @@ export ETH_PRIVATE_KEY_BE_CAREFUL="0x123-USE-YOUR-OWN-PRIVATE-KEY-HERE" export WEB3_FANTOM="FANTOM-API-KEY-HERE" // "public" can be used in place of a paid API key export WEB3_MANTLE="MANTLE-API-KEY-HERE" export WEB3_LINEA="LINEA-API-KEY-HERE" // +export WEB3_SEI="SEI-API-KEY-HERE" // #******** For Development - not required to run bot ********# export ETHERSCAN_TOKEN="ONLY_REQUIRED_IN_DEV" diff --git a/fastlane_bot/config/network.py b/fastlane_bot/config/network.py index ca18856fe..0a5593cf6 100644 --- a/fastlane_bot/config/network.py +++ b/fastlane_bot/config/network.py @@ -276,6 +276,7 @@ class ConfigNetwork(ConfigBase): NETWORK_FANTOM = S.NETWORK_FANTOM NETWORK_MANTLE = S.NETWORK_MANTLE NETWORK_LINEA = S.NETWORK_LINEA + NETWORK_SEI = S.NETWORK_SEI # FLAGS ####################################################################################### @@ -317,7 +318,9 @@ def new(cls, network=None): elif network == cls.NETWORK_MANTLE: return _ConfigNetworkMantle(_direct=False) elif network == cls.NETWORK_LINEA: - return _ConfigNetworkLinea(_direct=False) + return _ConfigNetworkLinea(_direct=False) + elif network == cls.NETWORK_SEI: + return _ConfigNetworkSei(_direct=False) elif network == cls.NETWORK_TENDERLY: return _ConfigNetworkTenderly(_direct=False) else: @@ -783,6 +786,42 @@ class _ConfigNetworkLinea(ConfigNetwork): # Add any exchanges unique to the chain here CHAIN_SPECIFIC_EXCHANGES = [] +class _ConfigNetworkSei(ConfigNetwork): + """ + Fastlane bot config -- network [Base Mainnet] + """ + + NETWORK = S.NETWORK_SEI + NETWORK_ID = "1" # TODO + NETWORK_NAME = "sei" + DEFAULT_PROVIDER = S.PROVIDER_ALCHEMY + RPC_ENDPOINT = "https://evm-rpc.arctic-1.seinetwork.io/" # TODO currently Sei devnet + WEB3_ALCHEMY_PROJECT_ID = os.environ.get("WEB3_SEI") + + network_df = get_multichain_addresses(network=NETWORK_NAME) + FASTLANE_CONTRACT_ADDRESS = "0xC7Dd38e64822108446872c5C2105308058c5C55C" #TODO - UPDATE WITH Mainnet + MULTICALL_CONTRACT_ADDRESS = "0xcA11bde05977b3631167028862bE2a173976CA11" + + CARBON_CONTROLLER_ADDRESS = "0x59f21012B2E9BA67ce6a7605E74F945D0D4C84EA" #TODO - UPDATE WITH Mainnet + CARBON_CONTROLLER_VOUCHER = "0xe4816658ad10bF215053C533cceAe3f59e1f1087" #TODO - UPDATE WITH Mainnet + + NATIVE_GAS_TOKEN_ADDRESS = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" + WRAPPED_GAS_TOKEN_ADDRESS = "0x57eE725BEeB991c70c53f9642f36755EC6eb2139" # TODO confirm for Mainnet + NATIVE_GAS_TOKEN_SYMBOL = "SEI" + WRAPPED_GAS_TOKEN_SYMBOL = "WSEI" + STABLECOIN_ADDRESS = "0xace5f7Ea93439Af39b46d2748fA1aC19951c8d7C" #TODO USDC on devnet + + IS_INJECT_POA_MIDDLEWARE = False + # Balancer + BALANCER_VAULT_ADDRESS = "0x7ccBebeb88696f9c8b061f1112Bb970158e29cA5" # # TODO Jellyswap on devnet + + CHAIN_FLASHLOAN_TOKENS = { + "0x57eE725BEeB991c70c53f9642f36755EC6eb2139": "WSEI", #TODO confirm for Mainnet + "0xace5f7Ea93439Af39b46d2748fA1aC19951c8d7C": "USDC", #TODO confirm for Mainnet + } + # Add any exchanges unique to the chain here + CHAIN_SPECIFIC_EXCHANGES = [] + class _ConfigNetworkTenderly(ConfigNetwork): """ Fastlane bot config -- network [Ethereum Tenderly] diff --git a/fastlane_bot/config/selectors.py b/fastlane_bot/config/selectors.py index d910f52be..791810fea 100644 --- a/fastlane_bot/config/selectors.py +++ b/fastlane_bot/config/selectors.py @@ -18,6 +18,7 @@ NETWORK_CANTO = "canto" NETWORK_FANTOM = "fantom" NETWORK_LINEA = "linea" +NETWORK_SEI = "sei" NETWORK_MANTLE = "mantle" NETWORK_SCROLL = "scroll" NETWORK_BSC = "binance_smart_chain" diff --git a/fastlane_bot/data/blockchain_data/sei/solidly_v2_event_mappings.csv b/fastlane_bot/data/blockchain_data/sei/solidly_v2_event_mappings.csv new file mode 100644 index 000000000..2785f2805 --- /dev/null +++ b/fastlane_bot/data/blockchain_data/sei/solidly_v2_event_mappings.csv @@ -0,0 +1 @@ +exchange,address diff --git a/fastlane_bot/data/blockchain_data/sei/static_pool_data.csv b/fastlane_bot/data/blockchain_data/sei/static_pool_data.csv new file mode 100644 index 000000000..54e89f03e --- /dev/null +++ b/fastlane_bot/data/blockchain_data/sei/static_pool_data.csv @@ -0,0 +1,4 @@ +cid,strategy_id,last_updated,last_updated_block,descr,pair_name,exchange_name,fee,fee_float,address,anchor,tkn0_address,tkn1_address,tkn0_decimals,tkn1_decimals,exchange_id,tkn0_symbol,tkn1_symbol,timestamp,tkn0_balance,tkn1_balance,liquidity,sqrt_price_q96,tick,tick_spacing,exchange,pool_type,tkn0_weight,tkn1_weight,tkn2_address,tkn2_decimals,tkn2_symbol,tkn2_balance,tkn2_weight,tkn3_address,tkn3_decimals,tkn3_symbol,tkn3_balance,tkn3_weight,tkn4_address,tkn4_decimals,tkn4_symbol,tkn4_balance,tkn4_weight,tkn5_address,tkn5_decimals,tkn5_symbol,tkn5_balance,tkn5_weight,tkn6_address,tkn6_decimals,tkn6_symbol,tkn6_balance,tkn6_weight,tkn7_address,tkn7_decimals,tkn7_symbol,tkn7_balance,tkn7_weight +0x1422169ab760ea6994358267b7d3783e8e7fa55c6a74b365b3fd3d17cbf4c6f1,0,,2354,dragonswap 0x027D2E627209f1cebA52ADc8A5aFE9318459b44B/0x7b75109369ACb528d9fa989E227812a6589712b9,0x027D2E627209f1cebA52ADc8A5aFE9318459b44B/0x7b75109369ACb528d9fa989E227812a6589712b9,dragonswap,0.003,0.003,0x01A34Dfa104F020FEE739268679338169945D5B1,,0x027D2E627209f1cebA52ADc8A5aFE9318459b44B,0x7b75109369ACb528d9fa989E227812a6589712b9,18,18,3,WSEI,DSWAP,0,0,0,,,,,dragonswap,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +0xbfd9612b2cb8035908dff18c040f64de75999cefd1020b5ce8a2e533c2ecd5dc,0,,2354,dragonswap 0x027D2E627209f1cebA52ADc8A5aFE9318459b44B/0xace5f7Ea93439Af39b46d2748fA1aC19951c8d7C,0x027D2E627209f1cebA52ADc8A5aFE9318459b44B/0xace5f7Ea93439Af39b46d2748fA1aC19951c8d7C,dragonswap,0.003,0.003,0x85CB6BFd781e1f42f4E79Efb6bf1F1fEfE4E9732,,0x027D2E627209f1cebA52ADc8A5aFE9318459b44B,0xace5f7Ea93439Af39b46d2748fA1aC19951c8d7C,18,6,3,WSEI,USDC,0,0,0,,,,,dragonswap,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, +0xe3aead757d877a15316e4896d5c5ab7639adbcba1ff76e3434b4e0af90f6225e,0,,2354,dragonswap 0xace5f7Ea93439Af39b46d2748fA1aC19951c8d7C/0xF983afa393199D6902a1Dd04f8E93465915ffD8B,0xace5f7Ea93439Af39b46d2748fA1aC19951c8d7C/0xF983afa393199D6902a1Dd04f8E93465915ffD8B,dragonswap,0.003,0.003,0x72A788B0A83e18ce1757171321E82c03e4351498,,0xace5f7Ea93439Af39b46d2748fA1aC19951c8d7C,0xF983afa393199D6902a1Dd04f8E93465915ffD8B,6,6,3,USDC,USDT,0,0,0,,,,,dragonswap,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/fastlane_bot/data/blockchain_data/sei/tokens.csv b/fastlane_bot/data/blockchain_data/sei/tokens.csv new file mode 100644 index 000000000..e79bf6c3b --- /dev/null +++ b/fastlane_bot/data/blockchain_data/sei/tokens.csv @@ -0,0 +1,9 @@ +address,decimals,symbol +0x26841a0A5D958B128209F4ea9a1DD7E61558c330,18,WSEI +0xace5f7Ea93439Af39b46d2748fA1aC19951c8d7C,6,USDC +0x027D2E627209f1cebA52ADc8A5aFE9318459b44B,18,WSEI +0x7b75109369ACb528d9fa989E227812a6589712b9,18,DSWAP +0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE,18,SEI +0x9e7A8e558Ce582511f4104465a886b7bEfBC146b,18,JLY +0x57eE725BEeB991c70c53f9642f36755EC6eb2139,18,WSEI +0xF983afa393199D6902a1Dd04f8E93465915ffD8B,6,USDT diff --git a/fastlane_bot/data/blockchain_data/sei/uniswap_v2_event_mappings.csv b/fastlane_bot/data/blockchain_data/sei/uniswap_v2_event_mappings.csv new file mode 100644 index 000000000..c23f7b0da --- /dev/null +++ b/fastlane_bot/data/blockchain_data/sei/uniswap_v2_event_mappings.csv @@ -0,0 +1,5 @@ +exchange,address +dragonswap,0x01A34Dfa104F020FEE739268679338169945D5B1 +dragonswap,0x85CB6BFd781e1f42f4E79Efb6bf1F1fEfE4E9732 +dragonswap,0x38BcEBb9A3fbF05B0Ab7ce9b485c9669578409fE +dragonswap,0x72A788B0A83e18ce1757171321E82c03e4351498 diff --git a/fastlane_bot/data/blockchain_data/sei/uniswap_v3_event_mappings.csv b/fastlane_bot/data/blockchain_data/sei/uniswap_v3_event_mappings.csv new file mode 100644 index 000000000..2785f2805 --- /dev/null +++ b/fastlane_bot/data/blockchain_data/sei/uniswap_v3_event_mappings.csv @@ -0,0 +1 @@ +exchange,address diff --git a/fastlane_bot/data/multichain_addresses.csv b/fastlane_bot/data/multichain_addresses.csv index c1ba30702..0650447ea 100644 --- a/fastlane_bot/data/multichain_addresses.csv +++ b/fastlane_bot/data/multichain_addresses.csv @@ -135,3 +135,7 @@ sushiswap_v3,thundercore,uniswap_v3,0xc35DADB65012eC5796536bD9864eD8773aBc74C4,0 pancakeswap_v3,zkevm,uniswap_v3,0x0BFbCF9fa4f9C56B0F40a671Ad40E0805A091865,0x1b81D678ffb9C0263b24A97847620C99d213eB14,,, pancakeswap_v3,zksync,uniswap_v3,0x1BB72E0CbbEA93c08f535fc7856E0338D7F7a8aB,0xD70C70AD87aa8D45b8D59600342FB3AEe76E3c68,,, xfai_v0,linea,solidly_v2,0xa5136eAd459F0E61C99Cec70fe8F5C24cF3ecA26,0xD538be6e9026C13D130C9e17d509E69C8Bb0eF33,,222864, +carbon_v1,sei,carbon_v1,0x59f21012B2E9BA67ce6a7605E74F945D0D4C84EA,0x59f21012B2E9BA67ce6a7605E74F945D0D4C84EA,,17658678, +dragonswap,sei,uniswap_v2,0x5D370a6189F89603FaB67e9C68383e63F7B6A262,0x2346d3A6fb18Ff3ae590Ea31d9e41E6AB8c9f5EB,0.003,1008775, +jellyswap,sei,balancer,BALANCER_VAULT_ADDRESS,0x7ccBebeb88696f9c8b061f1112Bb970158e29cA5,0,222832, +uniswap_v3,sei,uniswap_v3,0x0000000000000000000000000000000000000000,0x0000000000000000000000000000000000000000,,1, diff --git a/fastlane_bot/helpers/poolandtokens.py b/fastlane_bot/helpers/poolandtokens.py index d4f10bc87..08f82506d 100644 --- a/fastlane_bot/helpers/poolandtokens.py +++ b/fastlane_bot/helpers/poolandtokens.py @@ -403,10 +403,25 @@ def _carbon_to_cpc(self) -> ConstantProductCurve: allow to omit yint (in which case it is set to y, but this does not make a difference for the result) """ + def calculate_parameters(y: Decimal, pa: Decimal, pb: Decimal, pm: Decimal, n: Decimal): + H = pa.sqrt() ** n + L = pb.sqrt() ** n + M = pm.sqrt() ** n + A = H - L + B = L + z = y * (H - L) / (M - L) if M > L else y + return z + + def check_overlap(pa0, pb0, pa1, pb1): + min0, max0 = sorted([pa0, pb0]) + min1, max1 = sorted([1 / pa1, 1 / pb1]) + prices_overlap = max(min0, min1) < min(max0, max1) + return prices_overlap # if idx == 0, use the first curve, otherwise use the second curve. change the numerical values to Decimal lst = [] errors = [] + strategy_typed_args = [] for i in [0, 1]: S = Decimal(self.A_1) if i == 0 else Decimal(self.A_0) @@ -450,6 +465,7 @@ def decimal_converter(idx): decimal_converter = decimal_converter(i) p_start = Decimal(encoded_order.p_start) * decimal_converter + p_marg = Decimal(encoded_order.p_marg) * decimal_converter p_end = Decimal(encoded_order.p_end) * decimal_converter yint = Decimal(yint) / ( Decimal("10") ** [self.tkn1_decimals, self.tkn0_decimals][i] @@ -457,6 +473,7 @@ def decimal_converter(idx): y = Decimal(y) / ( Decimal("10") ** [self.tkn1_decimals, self.tkn0_decimals][i] ) + is_limit_order = p_start==p_end tkny = 1 if i == 0 else 0 typed_args = { @@ -466,7 +483,9 @@ def decimal_converter(idx): "yint": yint, "y": y, "pb": p_end, + "p_marg": p_marg, # deleted later since not supported by from_carbon() "pa": p_start, + "is_limit_order": is_limit_order, # deleted later since not supported by from_carbon() "tkny": self.pair_name.split("/")[tkny].replace( self.ConfigObj.NATIVE_GAS_TOKEN_ADDRESS, self.ConfigObj.WRAPPED_GAS_TOKEN_ADDRESS ), @@ -476,6 +495,54 @@ def decimal_converter(idx): "descr": self.descr, "params": self._params, } + + strategy_typed_args += [typed_args] + + # Only overlapping strategies are selected for modification + if len(strategy_typed_args) == 2: + + is_overlapping = False + pmarg_threshold = Decimal("0.01") # 1% # WARNING using this condition alone can included stable/stable pairs incidently + + # evaluate that the marginal prices are within the pmarg_threshold + pmarg0, pmarg1 = [x['p_marg'] for x in strategy_typed_args] + pmarg0_inv = 1/pmarg0 # one of the orders must be flipped since prices are always dy/dx - but must flip same geomean_pmarg later + percent_component = pmarg_threshold * max(pmarg0_inv, pmarg1) + percent_component_met = abs(pmarg0_inv - pmarg1) <= percent_component + + # overlapping strategies by defintion cannot have A=0 i.e. there must be no limit orders + no_limit_orders = (strategy_typed_args[0]['is_limit_order'] == False) and (strategy_typed_args[1]['is_limit_order'] == False) + + # evaluate if the price boundaries pa/pb overlap at one end # TODO check logic and remove duplicate logic if necessary + prices_overlap = check_overlap(strategy_typed_args[0]['pa'], strategy_typed_args[0]['pb'], strategy_typed_args[1]['pa'], strategy_typed_args[1]['pb']) + + # if the threshold is met and neither is a limit order and prices overlap then likely to be overlapping + is_overlapping = percent_component_met and no_limit_orders and prices_overlap + + if is_overlapping: + # calculate the geometric mean + geomean_p_marg = Decimal.sqrt(pmarg0_inv * pmarg1) + + # modify the y_int based on the new geomean to the limit of y #TODO check that this math is correct + typed_args0 = strategy_typed_args[0] + new_yint0 = calculate_parameters(y=typed_args0['y'], pa=typed_args0['pa'], pb=typed_args0['pb'], pm=(1/geomean_p_marg), n=1) + if new_yint0 < typed_args0['y']: + new_yint0 = typed_args0['y'] + typed_args0['yint'] = new_yint0 + + typed_args1 = strategy_typed_args[1] + new_yint1 = calculate_parameters(y=typed_args1['y'], pa=typed_args1['pa'], pb=typed_args1['pb'], pm=(geomean_p_marg), n=1) + if new_yint1 < typed_args1['y']: + new_yint1 = typed_args1['y'] + typed_args1['yint'] = new_yint1 + + # repack the strateg_typed_args + strategy_typed_args = [typed_args0, typed_args1] + + for typed_args in strategy_typed_args: + # delete new args that arent supported by from_carbon() + del typed_args["p_marg"] + del typed_args["is_limit_order"] try: if typed_args["y"] > 0: lst.append( diff --git a/fastlane_bot/tests/test_039_TestMultiMode.py b/fastlane_bot/tests/test_039_TestMultiMode.py index dc47761d1..b3c875d70 100644 --- a/fastlane_bot/tests/test_039_TestMultiMode.py +++ b/fastlane_bot/tests/test_039_TestMultiMode.py @@ -140,6 +140,7 @@ def test_test_tax_tokens(): # ------------------------------------------------------------ assert any(token.address in cfg.TAX_TOKENS for token in tokens), f"[TestMultiMode], DB does not include any tax tokens" + assert len(CCm) == 516, f"[NBTest 039 TestMultiMode] Expected 516 curves, found {len(CCm)}" for curve in CCm: for token in cfg.TAX_TOKENS: @@ -178,6 +179,7 @@ def test_test_combos_and_tokens(): # ------------------------------------------------------------ # + + assert len(CCm) == 516, f"[NBTest 039 TestMultiMode] Expected 516 curves, found {len(CCm)}" arb_finder = bot._get_arb_finder("multi") finder = arb_finder( flashloan_tokens=flashloan_tokens, @@ -205,6 +207,7 @@ def test_test_expected_output(): # ------------------------------------------------------------ # + + assert len(CCm) == 516, f"[NBTest 039 TestMultiMode] Expected 516 curves, found {len(CCm)}" arb_finder = bot._get_arb_finder("multi") finder = arb_finder( flashloan_tokens=flashloan_tokens, diff --git a/fastlane_bot/utils.py b/fastlane_bot/utils.py index e449c6dd8..bfae0fe7b 100644 --- a/fastlane_bot/utils.py +++ b/fastlane_bot/utils.py @@ -68,6 +68,10 @@ def __getitem__(self, item): def decodeFloat(cls, value): """undoes the mantisse/exponent encoding in A,B""" return (value % cls.ONE) << (value // cls.ONE) + + @classmethod + def decodeRate(cls, value): + return (value / cls.ONE) ** 2 @classmethod def decode(cls, value): @@ -178,17 +182,12 @@ def p_start(self): @property def p_marg(self): + A = self.decodeFloat(int(self.A)) + B = self.decodeFloat(int(self.B)) if self.y == self.z: - return self.p_start - elif self.y == 0: - return self.p_end - raise NotImplementedError("p_marg not implemented for non-full / empty orders") - A = self.decodeFloat(self.A) - B = self.decodeFloat(self.B) - return self.decode(B + A * self.y / self.z) ** 2 - # https://github.com/bancorprotocol/carbon-simulator/blob/beta/benchmark/core/trade/impl.py - # 'marginalRate' : decodeRate(B + A if y == z else B + A * y / z), - + return self.decodeRate(B + A) + else: + return self.decodeRate(B + A * self.y/self.z) def find_latest_timestamped_folder(logging_path=None): """ diff --git a/main.py b/main.py index 69a64dd57..6ba36805d 100644 --- a/main.py +++ b/main.py @@ -671,7 +671,7 @@ def run(mgr, args, tenderly_uri=None) -> None: "--blockchain", default="ethereum", help="A blockchain from the list. Blockchains not in this list do not have a deployed Fast Lane contract and are not supported.", - choices=["ethereum", "coinbase_base", "fantom", "mantle", "linea"], + choices=["ethereum", "coinbase_base", "fantom", "mantle", "linea", "sei"], ) parser.add_argument( "--pool_data_update_frequency", diff --git a/resources/NBTest/NBTest_002_CPCandOptimizer.ipynb b/resources/NBTest/NBTest_002_CPCandOptimizer.ipynb index 099e10b48..0f16cffb0 100644 --- a/resources/NBTest/NBTest_002_CPCandOptimizer.ipynb +++ b/resources/NBTest/NBTest_002_CPCandOptimizer.ipynb @@ -2732,6 +2732,8 @@ "r = O.margp_optimizer(\"WETH\", result=O.MO_DEBUG)\n", "assert isinstance(r, dict)\n", "prices0 = r[\"price_estimates_t\"]\n", + "dtknfromp_f = r[\"dtknfromp_f\"]\n", + "assert np.linalg.norm(dtknfromp_f(np.log10(prices0))) < 1e-6\n", "assert not prices0 is None, f\"prices0 must not be None [{prices0}]\"\n", "r1 = O.arb(\"WETH\")\n", "r2 = O.SelfFinancingConstraints.arb(\"WETH\")\n", @@ -2792,6 +2794,16 @@ "prices0" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "30424c63", + "metadata": {}, + "outputs": [], + "source": [ + "dtknfromp_f(np.log10(prices0))" + ] + }, { "cell_type": "code", "execution_count": 115, diff --git a/resources/NBTest/NBTest_002_CPCandOptimizer.py b/resources/NBTest/NBTest_002_CPCandOptimizer.py index b985dacf7..4e34b9a1e 100644 --- a/resources/NBTest/NBTest_002_CPCandOptimizer.py +++ b/resources/NBTest/NBTest_002_CPCandOptimizer.py @@ -1278,6 +1278,8 @@ r = O.margp_optimizer("WETH", result=O.MO_DEBUG) assert isinstance(r, dict) prices0 = r["price_estimates_t"] +dtknfromp_f = r["dtknfromp_f"] +assert np.linalg.norm(dtknfromp_f(np.log10(prices0))) < 1e-6 assert not prices0 is None, f"prices0 must not be None [{prices0}]" r1 = O.arb("WETH") r2 = O.SelfFinancingConstraints.arb("WETH") @@ -1291,6 +1293,8 @@ prices0 +dtknfromp_f(np.log10(prices0)) + f = O.optimize("WETH", result=O.MO_DTKNFROMPF, params=dict(verbose=True, debug=False)) r3 = f(prices0, islog10=False) assert np.all(r3 == (0,0)) diff --git a/resources/NBTest/NBTest_003_Serialization.ipynb b/resources/NBTest/NBTest_003_Serialization.ipynb index d5b5680f6..fb042d8c0 100644 --- a/resources/NBTest/NBTest_003_Serialization.ipynb +++ b/resources/NBTest/NBTest_003_Serialization.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 58, "id": "be65f3d2-769a-449f-90cd-2633a11478d0", "metadata": {}, "outputs": [ @@ -10,8 +10,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "imported m, np, pd, plt, os, sys, decimal; defined iseq, raises, require, Timer\n", - "ConstantProductCurve v3.4 (23/Jan/2024)\n", + "ConstantProductCurve v3.5 (22/Apr/2023)\n", "CPCArbOptimizer v5.1 (15/Sep/2023)\n" ] } @@ -21,7 +20,6 @@ " from fastlane_bot.tools.cpc import ConstantProductCurve as CPC, CPCContainer\n", " from fastlane_bot.tools.optimizer import CPCArbOptimizer, cp, time\n", " from fastlane_bot.testing import *\n", - "\n", "except:\n", " from tools.cpc import ConstantProductCurve as CPC, CPCContainer\n", " from tools.optimizer import CPCArbOptimizer, cp, time\n", @@ -55,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 59, "id": "4030cea3-3e03-4e0f-8d80-7a2bcca05fcf", "metadata": {}, "outputs": [], @@ -65,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 60, "id": "8cb4f9bc-2f31-4eae-b77f-533aa188e49b", "metadata": {}, "outputs": [], @@ -84,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 61, "id": "a5ed0075-5ee5-4592-a192-e06d2b5af454", "metadata": {}, "outputs": [], @@ -95,7 +93,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 62, "id": "1bf13d91-2bc0-4819-96b9-2712ef89b6f1", "metadata": {}, "outputs": [], @@ -105,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 63, "id": "ce05c578-5060-498e-b4eb-f55617d10cdd", "metadata": {}, "outputs": [], @@ -140,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 64, "id": "41a5cdfe-fb7b-4c8b-a270-1a52f0765e94", "metadata": {}, "outputs": [ @@ -150,7 +148,7 @@ "ConstantProductCurve(k=10000, x=100, x_act=100, y_act=100, alpha=0.5, pair='TKNB/TKNQ', cid='1', fee=0, descr='UniV2', constr='uv2', params={})" ] }, - "execution_count": 7, + "execution_count": 64, "metadata": {}, "output_type": "execute_result" } @@ -174,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 65, "id": "ea3cdfbc-8edd-41f1-9703-0ae0d72fdb9a", "metadata": {}, "outputs": [ @@ -194,7 +192,7 @@ " 'params': {}}" ] }, - "execution_count": 8, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } @@ -205,7 +203,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 66, "id": "595de023-5c66-40fc-928f-eca5fe6a50c9", "metadata": {}, "outputs": [], @@ -227,7 +225,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 67, "id": "215b5105-08d9-4077-a51a-7658cafcffa9", "metadata": {}, "outputs": [], @@ -261,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 68, "id": "0963034a-b36c-4cfb-84da-ccb3c88c4389", "metadata": {}, "outputs": [], @@ -279,7 +277,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 69, "id": "eb5dd380-dd90-4a3b-b88a-5a697bdbc3a0", "metadata": {}, "outputs": [], @@ -310,7 +308,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 70, "id": "624b80f1-c811-483b-ba24-b76c72fe3e0c", "metadata": {}, "outputs": [], @@ -325,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 71, "id": "34d52402-18d6-4485-8e5c-6cb4f8af2ab2", "metadata": {}, "outputs": [ @@ -349,7 +347,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 72, "id": "85175836-0fa9-4f64-a42f-b5b787e622f0", "metadata": {}, "outputs": [], @@ -364,7 +362,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 73, "id": "9753798a-b154-4865-a845-a1f5f1eb8e4b", "metadata": {}, "outputs": [ @@ -388,17 +386,17 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 74, "id": "5f683913-1799-4f3a-9473-a663d803448a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "ConstantProductCurve(k=0.01, x=0.0015438708879488485, x_act=0, y_act=1, alpha=0.5, pair='ETH/USDC', cid='4', fee=0, descr='Carbon', constr='carb', params={'y': 1, 'yint': 1, 'A': 10, 'B': 54.772255750516614, 'pa': 4195.445115010333, 'pb': 3000.0000000000005})" + "ConstantProductCurve(k=0.01, x=0.0015438708879488485, x_act=0, y_act=1, alpha=0.5, pair='ETH/USDC', cid='4', fee=0, descr='Carbon', constr='carb', params={'y': 1, 'yint': 1, 'A': 10, 'B': 54.772255750516614, 'pa': 4195.445115010333, 'pb': 3000.0000000000005, 'minrw': 1e-06})" ] }, - "execution_count": 17, + "execution_count": 74, "metadata": {}, "output_type": "execute_result" } @@ -412,7 +410,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 75, "id": "cffdcaa4-f221-4bd7-bf2d-5418a33e3592", "metadata": {}, "outputs": [], @@ -431,12 +429,35 @@ "assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, A=100, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)\n", "assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, B=100, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)\n", "assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, A=100, B=100, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)\n", - "assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)" + "#assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)" + ] + }, + { + "cell_type": "markdown", + "id": "6d4698a1-5df9-4c5d-a1c9-7e48fd9aa580", + "metadata": {}, + "source": [ + "TODO" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 76, + "id": "c1b70bbc-2531-458a-a507-24d89559bf41", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair=\"ETH/USDC\", tkny=\"ETH\", cid=\"1\", descr=\"Carbon\", isdydx=False)\n", + "#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, descr=\"Carbon\", isdydx=False)\n", + "#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", isdydx=False)\n", + "#assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 77, "id": "f66fc490-97e0-4c5e-958d-1e9014934d5c", "metadata": {}, "outputs": [], @@ -450,13 +471,31 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 78, "id": "465ff937-2382-4215-8e11-ec8096e1ea3d", "metadata": {}, "outputs": [], "source": [ "assert not raises(CPC.from_carbon, yint=1, y=1, pa=3100, pb=2900, pair=\"ETH/USDC\", tkny=\"USDC\", fee=0, cid=\"2\", descr=\"Carbon\", isdydx=True)\n", - "assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair=\"ETH/USDC\", tkny=\"USDC\", fee=0, cid=\"2\", descr=\"Carbon\", isdydx=True)" + "#assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair=\"ETH/USDC\", tkny=\"USDC\", fee=0, cid=\"2\", descr=\"Carbon\", isdydx=True)" + ] + }, + { + "cell_type": "markdown", + "id": "b0da3d2e-9b91-4c7a-89b8-8aa140901e32", + "metadata": {}, + "source": [ + "TODO" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "id": "d30a97ad-0188-4388-a3f8-3efa1151aa4a", + "metadata": {}, + "outputs": [], + "source": [ + "#assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair=\"ETH/USDC\", tkny=\"USDC\", fee=0, cid=\"2\", descr=\"Carbon\", isdydx=True)" ] }, { @@ -469,7 +508,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 80, "id": "c5c8d6c3-0d15-4c3d-8852-b2870a7b4caa", "metadata": {}, "outputs": [], @@ -485,7 +524,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 81, "id": "8296d087-d5a5-4b77-825a-dd53ed60d4bd", "metadata": {}, "outputs": [], @@ -503,7 +542,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 82, "id": "e72d0162-dd59-489c-8efb-dbb8327ff553", "metadata": {}, "outputs": [ @@ -567,7 +606,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 83, "id": "c2d5dc97-05e8-4eca-abc7-66eee6e7d706", "metadata": {}, "outputs": [], @@ -581,7 +620,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 84, "id": "9f467a32-370b-4634-bec8-3c28be84a0a0", "metadata": {}, "outputs": [], @@ -593,7 +632,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 85, "id": "d7563934-5381-476d-b9cb-99b909691049", "metadata": {}, "outputs": [ @@ -603,7 +642,7 @@ "CPCContainer(curves=[ConstantProductCurve(k=2000, x=1, x_act=1, y_act=2000, alpha=0.5, pair='ETH/USDC', cid='1', fee=0.001, descr='UniV2', constr='uv2', params={'meh': 1}), ConstantProductCurve(k=8040, x=2, x_act=2, y_act=4020, alpha=0.5, pair='ETH/USDC', cid='2', fee=0.001, descr='UniV2', constr='uv2', params={}), ConstantProductCurve(k=1970, x=1, x_act=1, y_act=1970, alpha=0.5, pair='ETH/USDC', cid='3', fee=0.001, descr='UniV2', constr='uv2', params={})])" ] }, - "execution_count": 26, + "execution_count": 85, "metadata": {}, "output_type": "execute_result" } @@ -621,7 +660,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 86, "id": "131928b8-f927-4799-97c6-ec50631c7959", "metadata": {}, "outputs": [ @@ -723,7 +762,7 @@ "3 1970 1 1 1970 0.5 ETH/USDC 0.001 UniV2 uv2 {}" ] }, - "execution_count": 27, + "execution_count": 86, "metadata": {}, "output_type": "execute_result" } @@ -750,7 +789,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 87, "id": "6cd062ae-c465-4102-a57c-587874023de5", "metadata": {}, "outputs": [], @@ -779,7 +818,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 88, "id": "8c046e70-ef8a-4de8-bd17-726afb617ea1", "metadata": {}, "outputs": [ @@ -788,7 +827,7 @@ "output_type": "stream", "text": [ "len 2355000\n", - "elapsed time: 0.29s\n" + "elapsed time: 0.34s\n" ] } ], @@ -814,7 +853,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 89, "id": "e892dc06-329d-477f-adcb-40a87eb7a009", "metadata": {}, "outputs": [ @@ -822,7 +861,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "elapsed time: 0.21s\n" + "elapsed time: 0.22s\n" ] }, { @@ -913,7 +952,7 @@ "2 3 1970 1 1 1970 0.5 ETH/USDC 0.001 UniV2 uv2 {}" ] }, - "execution_count": 30, + "execution_count": 89, "metadata": {}, "output_type": "execute_result" } @@ -939,7 +978,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 90, "id": "a2976017-2a84-4fba-885d-7680d9f61c3a", "metadata": {}, "outputs": [ @@ -947,7 +986,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "elapsed time: 0.17s\n" + "elapsed time: 0.16s\n" ] } ], @@ -971,7 +1010,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 91, "id": "ed5aaa2c-2f5a-4863-87cf-a77240826a85", "metadata": { "lines_to_next_cell": 2 @@ -981,7 +1020,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "elapsed time: 0.21s\n" + "elapsed time: 0.16s\n" ] } ], @@ -1005,7 +1044,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 92, "id": "f1507cc7-96ba-4342-bf1e-955b248bd8b4", "metadata": {}, "outputs": [], @@ -1030,7 +1069,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 93, "id": "a1c75dfe-ce14-4840-9c62-39a8d5cfc3ad", "metadata": {}, "outputs": [ @@ -1139,7 +1178,7 @@ "3 1970 1 1 1970 0.5 ETH/USDC 0.001 UniV2 uv2 {}" ] }, - "execution_count": 34, + "execution_count": 93, "metadata": {}, "output_type": "execute_result" } @@ -1156,7 +1195,9 @@ { "cell_type": "markdown", "id": "3cfc2ff5-bf9d-4684-9b8c-2aff57937a46", - "metadata": {}, + "metadata": { + "tags": [] + }, "source": [ "### Benchmarking\n", "\n", @@ -1174,7 +1215,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 94, "id": "c43b9431-603d-49af-b5fd-1975e9f59e2f", "metadata": {}, "outputs": [ @@ -1183,10 +1224,10 @@ "output_type": "stream", "text": [ " 2355000 .curves.json\n", - "-rw-r--r-- 1 skl staff 720055 1 May 07:51 .curves.csv\n", - "-rw-r--r-- 1 skl staff 2965 1 May 07:51 .curves.csv.gz\n", - "-rw-r--r-- 1 skl staff 961219 1 May 07:51 .curves.pkl\n", - "-rw-r--r-- 1 skl staff 720055 1 May 07:51 .curves.tsv\n" + "-rw-r--r-- 1 skl staff 720055 1 May 15:40 .curves.csv\n", + "-rw-r--r-- 1 skl staff 2965 1 May 15:40 .curves.csv.gz\n", + "-rw-r--r-- 1 skl staff 961219 1 May 15:40 .curves.pkl\n", + "-rw-r--r-- 1 skl staff 720055 1 May 15:40 .curves.tsv\n" ] } ], @@ -1227,6 +1268,14 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73a341c5-36e5-47c2-9fb0-0a63b589b98b", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/resources/NBTest/NBTest_003_Serialization.py b/resources/NBTest/NBTest_003_Serialization.py index 95f7a43db..d1530d580 100644 --- a/resources/NBTest/NBTest_003_Serialization.py +++ b/resources/NBTest/NBTest_003_Serialization.py @@ -19,7 +19,6 @@ from fastlane_bot.tools.cpc import ConstantProductCurve as CPC, CPCContainer from fastlane_bot.tools.optimizer import CPCArbOptimizer, cp, time from fastlane_bot.testing import * - except: from tools.cpc import ConstantProductCurve as CPC, CPCContainer from tools.optimizer import CPCArbOptimizer, cp, time @@ -205,7 +204,16 @@ assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, A=100, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, B=100, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, A=100, B=100, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) -assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) +#assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) + +# TODO + +# + +#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair="ETH/USDC", tkny="ETH", cid="1", descr="Carbon", isdydx=False) +#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair="ETH/USDC", tkny="ETH", fee=0, descr="Carbon", isdydx=False) +#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", isdydx=False) +#assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) +# - assert not raises(CPC.from_carbon, yint=1, y=1, A=1/10, B=m.sqrt(1/2000), pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) assert raises(CPC.from_carbon, yint=1, y=1, A=1/10, B=m.sqrt(1/2000), pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=False) @@ -214,7 +222,13 @@ assert raises(CPC.from_carbon, yint=1, y=1, A=-1/10, B=m.sqrt(1/2000), pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) assert not raises(CPC.from_carbon, yint=1, y=1, pa=3100, pb=2900, pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) -assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) +#assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) + +# TODO + +# + +#assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) +# - # ## Charts [NOTEST] @@ -386,3 +400,5 @@ + + diff --git a/run_blockchain_terraformer.py b/run_blockchain_terraformer.py index 62d7f2840..96269107f 100644 --- a/run_blockchain_terraformer.py +++ b/run_blockchain_terraformer.py @@ -49,6 +49,7 @@ async def gather(): FANTOM = "fantom" MANTLE = "mantle" LINEA = "linea" +SEI = "sei" coingecko_network_map = { "ethereum": "ethereum", @@ -67,6 +68,7 @@ async def gather(): "cosmos": "cosmos", "kava": "kava", "mantle": "mantle", + "sei": "sei", } BLOCK_CHUNK_SIZE_MAP = { @@ -78,7 +80,8 @@ async def gather(): "coinbase_base": 0, "fantom": 5000, "mantle": 0, - "linea": 0 + "linea": 0, + "sei": 0, } ALCHEMY_KEY_DICT = { @@ -91,6 +94,7 @@ async def gather(): "fantom": "WEB3_FANTOM", "mantle": "WEB3_MANTLE", "linea": "WEB3_LINEA", + "sei": "WEB3_SEI", } ALCHEMY_RPC_LIST = { @@ -103,6 +107,7 @@ async def gather(): "fantom": "https://fantom.blockpi.network/v1/rpc/", "mantle": "https://rpc.mantle.xyz/", "linea": "https://rpc.linea.build/", + "sei": "https://evm-rpc.arctic-1.seinetwork.io/", # TODO update with mainnet } BALANCER_SUBGRAPH_CHAIN_URL = { @@ -114,6 +119,7 @@ async def gather(): "coinbase_base": "https://api.studio.thegraph.com/query/24660/balancer-base-v2/version/latest", "avalanche": "https://api.thegraph.com/subgraphs/name/balancer-labs/balancer-avalanche-v2", "fantom": "https://api.thegraph.com/subgraphs/name/beethovenxfi/beethovenx", + "sei": "https://thegraph.dev.mvpworkshop.co/subgraphs/name/jelly" # TODO verify this for mainnet } @@ -1073,3 +1079,4 @@ def terraform_blockchain(network_name: str): #terraform_blockchain(network_name=FANTOM) #terraform_blockchain(network_name=MANTLE) #terraform_blockchain(network_name=LINEA) +#terraform_blockchain(network_name=SEI) From 3aabd33501e0ec8f1bf8035439b9e79b033933c9 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Thu, 9 May 2024 10:14:19 +0200 Subject: [PATCH 02/58] revert: optimizer tests --- .../NBTest/NBTest_002_CPCandOptimizer.ipynb | 12 -- .../NBTest/NBTest_002_CPCandOptimizer.py | 4 - .../NBTest/NBTest_003_Serialization.ipynb | 163 ++++++------------ resources/NBTest/NBTest_003_Serialization.py | 22 +-- 4 files changed, 60 insertions(+), 141 deletions(-) diff --git a/resources/NBTest/NBTest_002_CPCandOptimizer.ipynb b/resources/NBTest/NBTest_002_CPCandOptimizer.ipynb index 0f16cffb0..099e10b48 100644 --- a/resources/NBTest/NBTest_002_CPCandOptimizer.ipynb +++ b/resources/NBTest/NBTest_002_CPCandOptimizer.ipynb @@ -2732,8 +2732,6 @@ "r = O.margp_optimizer(\"WETH\", result=O.MO_DEBUG)\n", "assert isinstance(r, dict)\n", "prices0 = r[\"price_estimates_t\"]\n", - "dtknfromp_f = r[\"dtknfromp_f\"]\n", - "assert np.linalg.norm(dtknfromp_f(np.log10(prices0))) < 1e-6\n", "assert not prices0 is None, f\"prices0 must not be None [{prices0}]\"\n", "r1 = O.arb(\"WETH\")\n", "r2 = O.SelfFinancingConstraints.arb(\"WETH\")\n", @@ -2794,16 +2792,6 @@ "prices0" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "30424c63", - "metadata": {}, - "outputs": [], - "source": [ - "dtknfromp_f(np.log10(prices0))" - ] - }, { "cell_type": "code", "execution_count": 115, diff --git a/resources/NBTest/NBTest_002_CPCandOptimizer.py b/resources/NBTest/NBTest_002_CPCandOptimizer.py index 4e34b9a1e..b985dacf7 100644 --- a/resources/NBTest/NBTest_002_CPCandOptimizer.py +++ b/resources/NBTest/NBTest_002_CPCandOptimizer.py @@ -1278,8 +1278,6 @@ r = O.margp_optimizer("WETH", result=O.MO_DEBUG) assert isinstance(r, dict) prices0 = r["price_estimates_t"] -dtknfromp_f = r["dtknfromp_f"] -assert np.linalg.norm(dtknfromp_f(np.log10(prices0))) < 1e-6 assert not prices0 is None, f"prices0 must not be None [{prices0}]" r1 = O.arb("WETH") r2 = O.SelfFinancingConstraints.arb("WETH") @@ -1293,8 +1291,6 @@ prices0 -dtknfromp_f(np.log10(prices0)) - f = O.optimize("WETH", result=O.MO_DTKNFROMPF, params=dict(verbose=True, debug=False)) r3 = f(prices0, islog10=False) assert np.all(r3 == (0,0)) diff --git a/resources/NBTest/NBTest_003_Serialization.ipynb b/resources/NBTest/NBTest_003_Serialization.ipynb index fb042d8c0..d5b5680f6 100644 --- a/resources/NBTest/NBTest_003_Serialization.ipynb +++ b/resources/NBTest/NBTest_003_Serialization.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 58, + "execution_count": 1, "id": "be65f3d2-769a-449f-90cd-2633a11478d0", "metadata": {}, "outputs": [ @@ -10,7 +10,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "ConstantProductCurve v3.5 (22/Apr/2023)\n", + "imported m, np, pd, plt, os, sys, decimal; defined iseq, raises, require, Timer\n", + "ConstantProductCurve v3.4 (23/Jan/2024)\n", "CPCArbOptimizer v5.1 (15/Sep/2023)\n" ] } @@ -20,6 +21,7 @@ " from fastlane_bot.tools.cpc import ConstantProductCurve as CPC, CPCContainer\n", " from fastlane_bot.tools.optimizer import CPCArbOptimizer, cp, time\n", " from fastlane_bot.testing import *\n", + "\n", "except:\n", " from tools.cpc import ConstantProductCurve as CPC, CPCContainer\n", " from tools.optimizer import CPCArbOptimizer, cp, time\n", @@ -53,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 2, "id": "4030cea3-3e03-4e0f-8d80-7a2bcca05fcf", "metadata": {}, "outputs": [], @@ -63,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 3, "id": "8cb4f9bc-2f31-4eae-b77f-533aa188e49b", "metadata": {}, "outputs": [], @@ -82,7 +84,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 4, "id": "a5ed0075-5ee5-4592-a192-e06d2b5af454", "metadata": {}, "outputs": [], @@ -93,7 +95,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 5, "id": "1bf13d91-2bc0-4819-96b9-2712ef89b6f1", "metadata": {}, "outputs": [], @@ -103,7 +105,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 6, "id": "ce05c578-5060-498e-b4eb-f55617d10cdd", "metadata": {}, "outputs": [], @@ -138,7 +140,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 7, "id": "41a5cdfe-fb7b-4c8b-a270-1a52f0765e94", "metadata": {}, "outputs": [ @@ -148,7 +150,7 @@ "ConstantProductCurve(k=10000, x=100, x_act=100, y_act=100, alpha=0.5, pair='TKNB/TKNQ', cid='1', fee=0, descr='UniV2', constr='uv2', params={})" ] }, - "execution_count": 64, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -172,7 +174,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 8, "id": "ea3cdfbc-8edd-41f1-9703-0ae0d72fdb9a", "metadata": {}, "outputs": [ @@ -192,7 +194,7 @@ " 'params': {}}" ] }, - "execution_count": 65, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -203,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 9, "id": "595de023-5c66-40fc-928f-eca5fe6a50c9", "metadata": {}, "outputs": [], @@ -225,7 +227,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 10, "id": "215b5105-08d9-4077-a51a-7658cafcffa9", "metadata": {}, "outputs": [], @@ -259,7 +261,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 11, "id": "0963034a-b36c-4cfb-84da-ccb3c88c4389", "metadata": {}, "outputs": [], @@ -277,7 +279,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 12, "id": "eb5dd380-dd90-4a3b-b88a-5a697bdbc3a0", "metadata": {}, "outputs": [], @@ -308,7 +310,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 13, "id": "624b80f1-c811-483b-ba24-b76c72fe3e0c", "metadata": {}, "outputs": [], @@ -323,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 14, "id": "34d52402-18d6-4485-8e5c-6cb4f8af2ab2", "metadata": {}, "outputs": [ @@ -347,7 +349,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 15, "id": "85175836-0fa9-4f64-a42f-b5b787e622f0", "metadata": {}, "outputs": [], @@ -362,7 +364,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 16, "id": "9753798a-b154-4865-a845-a1f5f1eb8e4b", "metadata": {}, "outputs": [ @@ -386,17 +388,17 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 17, "id": "5f683913-1799-4f3a-9473-a663d803448a", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "ConstantProductCurve(k=0.01, x=0.0015438708879488485, x_act=0, y_act=1, alpha=0.5, pair='ETH/USDC', cid='4', fee=0, descr='Carbon', constr='carb', params={'y': 1, 'yint': 1, 'A': 10, 'B': 54.772255750516614, 'pa': 4195.445115010333, 'pb': 3000.0000000000005, 'minrw': 1e-06})" + "ConstantProductCurve(k=0.01, x=0.0015438708879488485, x_act=0, y_act=1, alpha=0.5, pair='ETH/USDC', cid='4', fee=0, descr='Carbon', constr='carb', params={'y': 1, 'yint': 1, 'A': 10, 'B': 54.772255750516614, 'pa': 4195.445115010333, 'pb': 3000.0000000000005})" ] }, - "execution_count": 74, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -410,7 +412,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 18, "id": "cffdcaa4-f221-4bd7-bf2d-5418a33e3592", "metadata": {}, "outputs": [], @@ -429,35 +431,12 @@ "assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, A=100, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)\n", "assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, B=100, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)\n", "assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, A=100, B=100, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)\n", - "#assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)" - ] - }, - { - "cell_type": "markdown", - "id": "6d4698a1-5df9-4c5d-a1c9-7e48fd9aa580", - "metadata": {}, - "source": [ - "TODO" + "assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)" ] }, { "cell_type": "code", - "execution_count": 76, - "id": "c1b70bbc-2531-458a-a507-24d89559bf41", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair=\"ETH/USDC\", tkny=\"ETH\", cid=\"1\", descr=\"Carbon\", isdydx=False)\n", - "#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, descr=\"Carbon\", isdydx=False)\n", - "#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", isdydx=False)\n", - "#assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair=\"ETH/USDC\", tkny=\"ETH\", fee=0, cid=\"1\", descr=\"Carbon\", isdydx=False)" - ] - }, - { - "cell_type": "code", - "execution_count": 77, + "execution_count": 19, "id": "f66fc490-97e0-4c5e-958d-1e9014934d5c", "metadata": {}, "outputs": [], @@ -471,31 +450,13 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 20, "id": "465ff937-2382-4215-8e11-ec8096e1ea3d", "metadata": {}, "outputs": [], "source": [ "assert not raises(CPC.from_carbon, yint=1, y=1, pa=3100, pb=2900, pair=\"ETH/USDC\", tkny=\"USDC\", fee=0, cid=\"2\", descr=\"Carbon\", isdydx=True)\n", - "#assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair=\"ETH/USDC\", tkny=\"USDC\", fee=0, cid=\"2\", descr=\"Carbon\", isdydx=True)" - ] - }, - { - "cell_type": "markdown", - "id": "b0da3d2e-9b91-4c7a-89b8-8aa140901e32", - "metadata": {}, - "source": [ - "TODO" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "id": "d30a97ad-0188-4388-a3f8-3efa1151aa4a", - "metadata": {}, - "outputs": [], - "source": [ - "#assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair=\"ETH/USDC\", tkny=\"USDC\", fee=0, cid=\"2\", descr=\"Carbon\", isdydx=True)" + "assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair=\"ETH/USDC\", tkny=\"USDC\", fee=0, cid=\"2\", descr=\"Carbon\", isdydx=True)" ] }, { @@ -508,7 +469,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 21, "id": "c5c8d6c3-0d15-4c3d-8852-b2870a7b4caa", "metadata": {}, "outputs": [], @@ -524,7 +485,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 22, "id": "8296d087-d5a5-4b77-825a-dd53ed60d4bd", "metadata": {}, "outputs": [], @@ -542,7 +503,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 23, "id": "e72d0162-dd59-489c-8efb-dbb8327ff553", "metadata": {}, "outputs": [ @@ -606,7 +567,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 24, "id": "c2d5dc97-05e8-4eca-abc7-66eee6e7d706", "metadata": {}, "outputs": [], @@ -620,7 +581,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 25, "id": "9f467a32-370b-4634-bec8-3c28be84a0a0", "metadata": {}, "outputs": [], @@ -632,7 +593,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 26, "id": "d7563934-5381-476d-b9cb-99b909691049", "metadata": {}, "outputs": [ @@ -642,7 +603,7 @@ "CPCContainer(curves=[ConstantProductCurve(k=2000, x=1, x_act=1, y_act=2000, alpha=0.5, pair='ETH/USDC', cid='1', fee=0.001, descr='UniV2', constr='uv2', params={'meh': 1}), ConstantProductCurve(k=8040, x=2, x_act=2, y_act=4020, alpha=0.5, pair='ETH/USDC', cid='2', fee=0.001, descr='UniV2', constr='uv2', params={}), ConstantProductCurve(k=1970, x=1, x_act=1, y_act=1970, alpha=0.5, pair='ETH/USDC', cid='3', fee=0.001, descr='UniV2', constr='uv2', params={})])" ] }, - "execution_count": 85, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -660,7 +621,7 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 27, "id": "131928b8-f927-4799-97c6-ec50631c7959", "metadata": {}, "outputs": [ @@ -762,7 +723,7 @@ "3 1970 1 1 1970 0.5 ETH/USDC 0.001 UniV2 uv2 {}" ] }, - "execution_count": 86, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -789,7 +750,7 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 28, "id": "6cd062ae-c465-4102-a57c-587874023de5", "metadata": {}, "outputs": [], @@ -818,7 +779,7 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 29, "id": "8c046e70-ef8a-4de8-bd17-726afb617ea1", "metadata": {}, "outputs": [ @@ -827,7 +788,7 @@ "output_type": "stream", "text": [ "len 2355000\n", - "elapsed time: 0.34s\n" + "elapsed time: 0.29s\n" ] } ], @@ -853,7 +814,7 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 30, "id": "e892dc06-329d-477f-adcb-40a87eb7a009", "metadata": {}, "outputs": [ @@ -861,7 +822,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "elapsed time: 0.22s\n" + "elapsed time: 0.21s\n" ] }, { @@ -952,7 +913,7 @@ "2 3 1970 1 1 1970 0.5 ETH/USDC 0.001 UniV2 uv2 {}" ] }, - "execution_count": 89, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -978,7 +939,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 31, "id": "a2976017-2a84-4fba-885d-7680d9f61c3a", "metadata": {}, "outputs": [ @@ -986,7 +947,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "elapsed time: 0.16s\n" + "elapsed time: 0.17s\n" ] } ], @@ -1010,7 +971,7 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 32, "id": "ed5aaa2c-2f5a-4863-87cf-a77240826a85", "metadata": { "lines_to_next_cell": 2 @@ -1020,7 +981,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "elapsed time: 0.16s\n" + "elapsed time: 0.21s\n" ] } ], @@ -1044,7 +1005,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 33, "id": "f1507cc7-96ba-4342-bf1e-955b248bd8b4", "metadata": {}, "outputs": [], @@ -1069,7 +1030,7 @@ }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 34, "id": "a1c75dfe-ce14-4840-9c62-39a8d5cfc3ad", "metadata": {}, "outputs": [ @@ -1178,7 +1139,7 @@ "3 1970 1 1 1970 0.5 ETH/USDC 0.001 UniV2 uv2 {}" ] }, - "execution_count": 93, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -1195,9 +1156,7 @@ { "cell_type": "markdown", "id": "3cfc2ff5-bf9d-4684-9b8c-2aff57937a46", - "metadata": { - "tags": [] - }, + "metadata": {}, "source": [ "### Benchmarking\n", "\n", @@ -1215,7 +1174,7 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 35, "id": "c43b9431-603d-49af-b5fd-1975e9f59e2f", "metadata": {}, "outputs": [ @@ -1224,10 +1183,10 @@ "output_type": "stream", "text": [ " 2355000 .curves.json\n", - "-rw-r--r-- 1 skl staff 720055 1 May 15:40 .curves.csv\n", - "-rw-r--r-- 1 skl staff 2965 1 May 15:40 .curves.csv.gz\n", - "-rw-r--r-- 1 skl staff 961219 1 May 15:40 .curves.pkl\n", - "-rw-r--r-- 1 skl staff 720055 1 May 15:40 .curves.tsv\n" + "-rw-r--r-- 1 skl staff 720055 1 May 07:51 .curves.csv\n", + "-rw-r--r-- 1 skl staff 2965 1 May 07:51 .curves.csv.gz\n", + "-rw-r--r-- 1 skl staff 961219 1 May 07:51 .curves.pkl\n", + "-rw-r--r-- 1 skl staff 720055 1 May 07:51 .curves.tsv\n" ] } ], @@ -1268,14 +1227,6 @@ "metadata": {}, "outputs": [], "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73a341c5-36e5-47c2-9fb0-0a63b589b98b", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/resources/NBTest/NBTest_003_Serialization.py b/resources/NBTest/NBTest_003_Serialization.py index d1530d580..95f7a43db 100644 --- a/resources/NBTest/NBTest_003_Serialization.py +++ b/resources/NBTest/NBTest_003_Serialization.py @@ -19,6 +19,7 @@ from fastlane_bot.tools.cpc import ConstantProductCurve as CPC, CPCContainer from fastlane_bot.tools.optimizer import CPCArbOptimizer, cp, time from fastlane_bot.testing import * + except: from tools.cpc import ConstantProductCurve as CPC, CPCContainer from tools.optimizer import CPCArbOptimizer, cp, time @@ -204,16 +205,7 @@ assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, A=100, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, B=100, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, A=100, B=100, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) -#assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) - -# TODO - -# + -#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair="ETH/USDC", tkny="ETH", cid="1", descr="Carbon", isdydx=False) -#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair="ETH/USDC", tkny="ETH", fee=0, descr="Carbon", isdydx=False) -#assert raises(CPC.from_carbon, yint=1, y=1, pa=1800, pb=2200, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", isdydx=False) -#assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) -# - +assert raises(CPC.from_carbon, yint=1, y=1, pb=1800, pa=2200, pair="ETH/USDC", tkny="ETH", fee=0, cid="1", descr="Carbon", isdydx=False) assert not raises(CPC.from_carbon, yint=1, y=1, A=1/10, B=m.sqrt(1/2000), pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) assert raises(CPC.from_carbon, yint=1, y=1, A=1/10, B=m.sqrt(1/2000), pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=False) @@ -222,13 +214,7 @@ assert raises(CPC.from_carbon, yint=1, y=1, A=-1/10, B=m.sqrt(1/2000), pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) assert not raises(CPC.from_carbon, yint=1, y=1, pa=3100, pb=2900, pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) -#assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) - -# TODO - -# + -#assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) -# - +assert raises(CPC.from_carbon, yint=1, y=1, pb=3100, pa=2900, pair="ETH/USDC", tkny="USDC", fee=0, cid="2", descr="Carbon", isdydx=True) # ## Charts [NOTEST] @@ -400,5 +386,3 @@ - - From 268b6aca6b004e8bc4369f73e9026a8b1e53a66f Mon Sep 17 00:00:00 2001 From: Nicholas Welch Date: Wed, 8 May 2024 10:41:08 +1000 Subject: [PATCH 03/58] remove handle_target_token_addresses --- fastlane_bot/events/utils.py | 38 ------------------------------------ main.py | 7 +------ 2 files changed, 1 insertion(+), 44 deletions(-) diff --git a/fastlane_bot/events/utils.py b/fastlane_bot/events/utils.py index 8f2e66618..154683b69 100644 --- a/fastlane_bot/events/utils.py +++ b/fastlane_bot/events/utils.py @@ -1678,44 +1678,6 @@ def delete_tenderly_forks(forks_to_cleanup: List[str], mgr: Any) -> List[str]: return forks_to_keep - -def handle_target_token_addresses(static_pool_data: pd.DataFrame, target_tokens: List): - """ - Get the addresses of the target tokens. - - Parameters - ---------- - static_pool_data : pd.DataFrame - The static pool data. - target_tokens : List - The target tokens. - - Returns - ------- - List - The addresses of the target tokens. - - """ - # Get the addresses of the target tokens - target_token_addresses = [] - if target_tokens: - for token in target_tokens: - target_token_addresses = ( - target_token_addresses - + static_pool_data[static_pool_data["tkn0_address"] == token][ - "tkn0_address" - ].tolist() - ) - target_token_addresses = ( - target_token_addresses - + static_pool_data[static_pool_data["tkn1_address"] == token][ - "tkn1_address" - ].tolist() - ) - target_token_addresses = list(set(target_token_addresses)) - return target_token_addresses - - def get_current_block( last_block: int, mgr: Any, diff --git a/main.py b/main.py index 6ba36805d..8a4e42e5d 100644 --- a/main.py +++ b/main.py @@ -47,7 +47,6 @@ get_start_block, set_network_to_mainnet_if_replay, set_network_to_tenderly_if_replay, - handle_target_token_addresses, get_current_block, handle_tenderly_event_exchanges, handle_static_pools_update, @@ -255,10 +254,6 @@ def main(args: argparse.Namespace) -> None: cfg, exchanges, args.blockchain, args.static_pool_data_filename, args.read_only ) - target_token_addresses = handle_target_token_addresses( - static_pool_data, args.target_tokens - ) - # Break if timeout is hit to test the bot flags if args.timeout == 1: cfg.logger.info("Timeout to test the bot flags") @@ -284,7 +279,7 @@ def main(args: argparse.Namespace) -> None: solidly_v2_event_mappings=solidly_v2_event_mappings, tokens=tokens.to_dict(orient="records"), replay_from_block=args.replay_from_block, - target_tokens=target_token_addresses, + target_tokens=args.target_tokens, tenderly_fork_id=args.tenderly_fork_id, tenderly_event_exchanges=tenderly_event_exchanges, w3_tenderly=w3_tenderly, From d2d8779a69b03884bdfa8507c601393cf53d695e Mon Sep 17 00:00:00 2001 From: Nicholas Welch Date: Wed, 8 May 2024 10:41:38 +1000 Subject: [PATCH 04/58] bugfix always add native wrapped gas infor to the token objects --- fastlane_bot/events/interface.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/fastlane_bot/events/interface.py b/fastlane_bot/events/interface.py index cc7ab2888..a0f200db3 100644 --- a/fastlane_bot/events/interface.py +++ b/fastlane_bot/events/interface.py @@ -509,11 +509,11 @@ def populate_tokens(self): self.token_list[token.address] = token except AttributeError: pass - if self.ConfigObj.GAS_TKN_IN_FLASHLOAN_TOKENS: - native_gas_tkn = Token(symbol=self.ConfigObj.NATIVE_GAS_TOKEN_SYMBOL, address=self.ConfigObj.NATIVE_GAS_TOKEN_ADDRESS, decimals=18) - wrapped_gas_tkn = Token(symbol=self.ConfigObj.WRAPPED_GAS_TOKEN_SYMBOL, address=self.ConfigObj.WRAPPED_GAS_TOKEN_ADDRESS, decimals=18) - self.token_list[native_gas_tkn.address] = native_gas_tkn - self.token_list[wrapped_gas_tkn.address] = wrapped_gas_tkn + # native and wrapped gas token info populated everytime + native_gas_tkn = Token(symbol=self.ConfigObj.NATIVE_GAS_TOKEN_SYMBOL, address=self.ConfigObj.NATIVE_GAS_TOKEN_ADDRESS, decimals=18) + wrapped_gas_tkn = Token(symbol=self.ConfigObj.WRAPPED_GAS_TOKEN_SYMBOL, address=self.ConfigObj.WRAPPED_GAS_TOKEN_ADDRESS, decimals=18) + self.token_list[native_gas_tkn.address] = native_gas_tkn + self.token_list[wrapped_gas_tkn.address] = wrapped_gas_tkn def create_token(self, record: Dict[str, Any], prefix: str) -> Token: """ From 5ba114b5ee0b46feaf4bede66fe6aac62204eeee Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Thu, 9 May 2024 15:19:41 +0300 Subject: [PATCH 05/58] Remove the `GAS_TKN_IN_FLASHLOAN_TOKENS` flag --- fastlane_bot/config/network.py | 1 - fastlane_bot/events/interface.py | 5 ++--- fastlane_bot/events/utils.py | 4 ---- fastlane_bot/tests/test_034_Interface.py | 1 - 4 files changed, 2 insertions(+), 9 deletions(-) diff --git a/fastlane_bot/config/network.py b/fastlane_bot/config/network.py index 0a5593cf6..e144ffdac 100644 --- a/fastlane_bot/config/network.py +++ b/fastlane_bot/config/network.py @@ -280,7 +280,6 @@ class ConfigNetwork(ConfigBase): # FLAGS ####################################################################################### - GAS_TKN_IN_FLASHLOAN_TOKENS = None IS_NO_FLASHLOAN_AVAILABLE = False # HOOKS diff --git a/fastlane_bot/events/interface.py b/fastlane_bot/events/interface.py index a0f200db3..1672cfafc 100644 --- a/fastlane_bot/events/interface.py +++ b/fastlane_bot/events/interface.py @@ -492,9 +492,8 @@ def get_tokens(self) -> List[Token]: token_set.add(self.create_token(record, f"tkn{str(idx)}_")) except AttributeError: pass - if self.ConfigObj.GAS_TKN_IN_FLASHLOAN_TOKENS: - token_set.add(Token(symbol=self.ConfigObj.NATIVE_GAS_TOKEN_SYMBOL, address=self.ConfigObj.NATIVE_GAS_TOKEN_ADDRESS, decimals=18)) - token_set.add(Token(symbol=self.ConfigObj.WRAPPED_GAS_TOKEN_SYMBOL, address=self.ConfigObj.WRAPPED_GAS_TOKEN_ADDRESS, decimals=18)) + token_set.add(Token(symbol=self.ConfigObj.NATIVE_GAS_TOKEN_SYMBOL, address=self.ConfigObj.NATIVE_GAS_TOKEN_ADDRESS, decimals=18)) + token_set.add(Token(symbol=self.ConfigObj.WRAPPED_GAS_TOKEN_SYMBOL, address=self.ConfigObj.WRAPPED_GAS_TOKEN_ADDRESS, decimals=18)) return list(token_set) def populate_tokens(self): diff --git a/fastlane_bot/events/utils.py b/fastlane_bot/events/utils.py index 154683b69..719ebf09b 100644 --- a/fastlane_bot/events/utils.py +++ b/fastlane_bot/events/utils.py @@ -637,10 +637,6 @@ def get_config( cfg.LIMIT_BANCOR3_FLASHLOAN_TOKENS = limit_bancor3_flashloan_tokens cfg.DEFAULT_MIN_PROFIT_GAS_TOKEN = Decimal(default_min_profit_gas_token) - cfg.GAS_TKN_IN_FLASHLOAN_TOKENS = ( - cfg.NATIVE_GAS_TOKEN_ADDRESS in flashloan_tokens - or cfg.WRAPPED_GAS_TOKEN_ADDRESS in flashloan_tokens - ) return cfg diff --git a/fastlane_bot/tests/test_034_Interface.py b/fastlane_bot/tests/test_034_Interface.py index 8b7f3dea8..82a98ee87 100644 --- a/fastlane_bot/tests/test_034_Interface.py +++ b/fastlane_bot/tests/test_034_Interface.py @@ -35,7 +35,6 @@ cfg_mock = Mock() cfg_mock.logger = MagicMock() -cfg_mock.GAS_TKN_IN_FLASHLOAN_TOKENS = False qi = QueryInterface(mgr=None, ConfigObj=cfg_mock) qi.state = [{'exchange_name': 'uniswap_v2', 'address': '0x123', 'tkn0_key': 'TKN-0x123', 'tkn1_key': 'TKN-0x456', 'pair_name': 'Pair-0x789', 'liquidity': 10}, {'exchange_name': 'sushiswap_v2', 'address': '0xabc', 'tkn0_key': 'TKN-0xabc', 'tkn1_key': 'TKN-0xdef', 'pair_name': 'Pair-0xghi', 'liquidity': 0}] From 99b702aabfab76d0e6c9a06552f03f575eed9868 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Tue, 7 May 2024 23:32:47 +0200 Subject: [PATCH 06/58] refactor: events --- .gitignore | 2 + .../events/async_event_update_utils.py | 9 +-- fastlane_bot/events/exchanges/balancer.py | 8 ++- fastlane_bot/events/exchanges/bancor_pol.py | 20 ++++-- fastlane_bot/events/exchanges/bancor_v2.py | 22 +++--- fastlane_bot/events/exchanges/bancor_v3.py | 22 ++++-- fastlane_bot/events/exchanges/base.py | 7 +- fastlane_bot/events/exchanges/carbon_v1.py | 31 +++++++-- fastlane_bot/events/exchanges/solidly_v2.py | 9 ++- fastlane_bot/events/exchanges/uniswap_v2.py | 8 ++- fastlane_bot/events/exchanges/uniswap_v3.py | 8 ++- fastlane_bot/events/interfaces/event.py | 27 ++++++++ .../events/interfaces/subscription.py | 46 +++++++++++++ fastlane_bot/events/managers/base.py | 49 ++++++------- fastlane_bot/events/managers/contracts.py | 7 +- fastlane_bot/events/managers/manager.py | 46 +++++++++---- fastlane_bot/events/pools/balancer.py | 4 +- fastlane_bot/events/pools/bancor_pol.py | 15 ++-- fastlane_bot/events/pools/bancor_v2.py | 17 ++--- fastlane_bot/events/pools/bancor_v3.py | 16 ++--- fastlane_bot/events/pools/base.py | 14 ++-- fastlane_bot/events/pools/carbon_v1.py | 38 ++++------ fastlane_bot/events/pools/solidly_v2.py | 12 ++-- fastlane_bot/events/pools/uniswap_v2.py | 14 ++-- fastlane_bot/events/pools/uniswap_v3.py | 16 ++--- fastlane_bot/events/utils.py | 69 ++++++++----------- 26 files changed, 337 insertions(+), 199 deletions(-) create mode 100644 fastlane_bot/events/interfaces/event.py create mode 100644 fastlane_bot/events/interfaces/subscription.py diff --git a/.gitignore b/.gitignore index 5e3d05b03..a35a7e42a 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,5 @@ logs/* missing_tokens_df.csv tokens_and_fee_df.csv fastlane_bot/tests/nbtest/* + +.python-version diff --git a/fastlane_bot/events/async_event_update_utils.py b/fastlane_bot/events/async_event_update_utils.py index bbfa99273..a847599d0 100644 --- a/fastlane_bot/events/async_event_update_utils.py +++ b/fastlane_bot/events/async_event_update_utils.py @@ -27,6 +27,7 @@ from fastlane_bot.events.async_utils import get_contract_chunks from fastlane_bot.events.utils import update_pools_from_events from fastlane_bot.events.pools.utils import get_pool_cid +from .interfaces.event import Event nest_asyncio.apply() @@ -89,7 +90,7 @@ async def _get_missing_tkns(mgr: Any, c: List[Dict[str, Any]]) -> pd.DataFrame: return pd.concat(vals) -async def _get_token_and_fee(mgr: Any, exchange_name: str, ex: Any, address: str, contract: AsyncContract, event: Any): +async def _get_token_and_fee(mgr: Any, exchange_name: str, ex: Any, address: str, contract: AsyncContract, event: Event): """ This function uses the exchange object to get the tokens and fee for a given pool. @@ -97,7 +98,7 @@ async def _get_token_and_fee(mgr: Any, exchange_name: str, ex: Any, address: str ex(Any): The exchange object address(str): The pool address contract(AsyncContract): The contract object - event(Any): The event object + event(Event): The event object Returns: The tokens and fee info for the pool @@ -119,7 +120,7 @@ async def _get_token_and_fee(mgr: Any, exchange_name: str, ex: Any, address: str elif tkn1 == mgr.cfg.BNT_ADDRESS: tkn0 = connector_token - strategy_id = 0 if not ex.is_carbon_v1_fork else str(event["args"]["id"]) + strategy_id = 0 if not ex.is_carbon_v1_fork else str(event.args["id"]) pool_info = { "exchange_name": exchange_name, "address": address, @@ -319,7 +320,7 @@ def _get_pool_contracts(mgr: Any) -> List[Dict[str, Any]]: exchange_name = mgr.exchange_name_from_event(event) ex = mgr.exchanges[exchange_name] abi = ex.get_abi() - address = event["address"] + address = event.address contracts.append( { "exchange_name": exchange_name, diff --git a/fastlane_bot/events/exchanges/balancer.py b/fastlane_bot/events/exchanges/balancer.py index 01383cc51..791180f0c 100644 --- a/fastlane_bot/events/exchanges/balancer.py +++ b/fastlane_bot/events/exchanges/balancer.py @@ -17,8 +17,9 @@ from web3.contract import Contract from fastlane_bot.data.abi import BALANCER_VAULT_ABI, BALANCER_POOL_ABI_V1 -from fastlane_bot.events.exchanges.base import Exchange -from fastlane_bot.events.pools.base import Pool +from ..exchanges.base import Exchange +from ..pools.base import Pool +from ..interfaces.subscription import Subscription @dataclass @@ -47,6 +48,9 @@ def get_pool_abi(self): def get_events(self, contract: Contract) -> List[Type[Contract]]: return [contract.events.AuthorizerChanged] + def get_subscriptions(self, contract: Contract) -> List[Subscription]: + return [] + async def get_fee(self, pool_id: str, contract: Contract) -> Tuple[str, float]: pool = self.get_pool(pool_id) if pool: diff --git a/fastlane_bot/events/exchanges/bancor_pol.py b/fastlane_bot/events/exchanges/bancor_pol.py index 1211cc5cc..4e40c7397 100644 --- a/fastlane_bot/events/exchanges/bancor_pol.py +++ b/fastlane_bot/events/exchanges/bancor_pol.py @@ -17,9 +17,11 @@ from web3.contract import Contract from fastlane_bot.data.abi import BANCOR_POL_ABI -from fastlane_bot.events.exchanges.base import Exchange -from fastlane_bot.events.pools.base import Pool from fastlane_bot import Config +from ..exchanges.base import Exchange +from ..pools.base import Pool +from ..interfaces.event import Event +from ..interfaces.subscription import Subscription @dataclass @@ -46,14 +48,20 @@ def get_factory_abi(self): def get_events(self, contract: Contract) -> List[Type[Contract]]: return [contract.events.TokenTraded, contract.events.TradingEnabled] + def get_subscriptions(self, contract: Contract) -> List[Subscription]: + return [ + Subscription(contract.events.TokenTraded), + Subscription(contract.events.TradingEnabled), + ] + async def get_fee(self, address: str, contract: Contract) -> Tuple[str, float]: return "0.000", 0.000 - async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: - return event["args"]["token"] + async def get_tkn0(self, address: str, contract: Contract, event: Event) -> str: + return event.args["token"] - async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: - return self.ETH_ADDRESS if event["args"]["token"] not in self.ETH_ADDRESS else self.BNT_ADDRESS + async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: + return self.ETH_ADDRESS if event.args["token"] not in self.ETH_ADDRESS else self.BNT_ADDRESS def save_strategy( self, diff --git a/fastlane_bot/events/exchanges/bancor_v2.py b/fastlane_bot/events/exchanges/bancor_v2.py index 217b1297a..d1b4b149e 100644 --- a/fastlane_bot/events/exchanges/bancor_v2.py +++ b/fastlane_bot/events/exchanges/bancor_v2.py @@ -14,11 +14,14 @@ from dataclasses import dataclass from typing import List, Type, Tuple, Any +from web3 import AsyncWeb3 from web3.contract import Contract, AsyncContract from fastlane_bot.data.abi import BANCOR_V2_CONVERTER_ABI -from fastlane_bot.events.exchanges.base import Exchange -from fastlane_bot.events.pools.base import Pool +from ..exchanges.base import Exchange +from ..pools.base import Pool +from ..interfaces.event import Event +from ..interfaces.subscription import Subscription @dataclass @@ -44,6 +47,9 @@ def get_factory_abi(self): def get_events(self, contract: Contract) -> List[Type[Contract]]: return [contract.events.TokenRateUpdate] + def get_subscriptions(self, contract: Contract) -> List[Subscription]: + return [Subscription(contract.events.TokenRateUpdate)] + # def async convert_address(self, address: str, contract: Contract) -> str: # return @@ -59,15 +65,15 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee_float = float(fee) / 1e6 return fee, fee_float - async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: + async def get_tkn0(self, address: str, contract: Contract, event: Event) -> str: if event: - return event["args"]["_token1"] - return await contract.caller.reserveTokens()[0] + return event.args["_token1"] + return await contract.functions.reserveTokens()[0] - async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: + async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: if event: - return event["args"]["_token2"] - return await contract.caller.reserveTokens()[1] + return event.args["_token2"] + return await contract.functions.reserveTokens()[1] async def get_anchor(self, contract: Contract) -> str: return await contract.caller.anchor() diff --git a/fastlane_bot/events/exchanges/bancor_v3.py b/fastlane_bot/events/exchanges/bancor_v3.py index 7e0e5b20d..da2c76733 100644 --- a/fastlane_bot/events/exchanges/bancor_v3.py +++ b/fastlane_bot/events/exchanges/bancor_v3.py @@ -17,8 +17,13 @@ from web3.contract import Contract from fastlane_bot.data.abi import BANCOR_V3_POOL_COLLECTION_ABI -from fastlane_bot.events.exchanges.base import Exchange -from fastlane_bot.events.pools.base import Pool +from ..exchanges.base import Exchange +from ..pools.base import Pool +from ..interfaces.event import Event +from ..interfaces.subscription import Subscription + + +LIQUIDITY_UPDATED_TOPIC = "0x6e96dc5343d067ec486a9920e0304c3610ed05c65e45cc029d9b9fe7ecfa7620" @dataclass @@ -44,15 +49,18 @@ def get_factory_abi(self): def get_events(self, contract: Contract) -> List[Type[Contract]]: return [contract.events.TradingLiquidityUpdated] + def get_subscriptions(self, contract: Contract) -> List[Subscription]: + return [Subscription(contract.events.TradingLiquidityUpdated, LIQUIDITY_UPDATED_TOPIC)] + async def get_fee(self, address: str, contract: Contract) -> Tuple[str, float]: return "0.000", 0.000 - async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: + async def get_tkn0(self, address: str, contract: Contract, event: Event) -> str: return self.BNT_ADDRESS - async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: + async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: return ( - event["args"]["pool"] - if event["args"]["pool"] != self.BNT_ADDRESS - else event["args"]["tkn_address"] + event.args["pool"] + if event.args["pool"] != self.BNT_ADDRESS + else event.args["tkn_address"] ) diff --git a/fastlane_bot/events/exchanges/base.py b/fastlane_bot/events/exchanges/base.py index a63c7ed29..ed641d974 100644 --- a/fastlane_bot/events/exchanges/base.py +++ b/fastlane_bot/events/exchanges/base.py @@ -17,7 +17,8 @@ from web3.contract import Contract, AsyncContract from fastlane_bot.config.constants import CARBON_V1_NAME -from fastlane_bot.events.pools.base import Pool +from ..pools.base import Pool +from ..interfaces.subscription import Subscription @dataclass @@ -96,6 +97,10 @@ def get_events(self, contract: Contract) -> List[Type[Contract]]: """ pass + @abstractmethod + def get_subscriptions(self, contract: Contract) -> List[Subscription]: + ... + @staticmethod @abstractmethod async def get_fee(address: str, contract: AsyncContract) -> float: diff --git a/fastlane_bot/events/exchanges/carbon_v1.py b/fastlane_bot/events/exchanges/carbon_v1.py index 3dcf93a98..2957e04cf 100644 --- a/fastlane_bot/events/exchanges/carbon_v1.py +++ b/fastlane_bot/events/exchanges/carbon_v1.py @@ -18,9 +18,16 @@ from web3.contract import Contract from fastlane_bot.data.abi import CARBON_CONTROLLER_ABI -from fastlane_bot.events.exchanges.base import Exchange -from fastlane_bot.events.pools.base import Pool -from fastlane_bot.events.pools.utils import get_pool_cid +from ..exchanges.base import Exchange +from ..pools.base import Pool +from ..interfaces.event import Event +from ..interfaces.subscription import Subscription +from ..pools.utils import get_pool_cid + + +STRATEGY_CREATED_TOPIC = "0xff24554f8ccfe540435cfc8854831f8dcf1cf2068708cfaf46e8b52a4ccc4c8d" +STRATEGY_UPDATED_TOPIC = "0x720da23a5c920b1d8827ec83c4d3c4d90d9419eadb0036b88cb4c2ffa91aef7d" +STRATEGY_DELETED_TOPIC = "0x4d5b6e0627ea711d8e9312b6ba56f50e0b51d41816fd6fd38643495ac81d38b6" @dataclass @@ -73,6 +80,16 @@ def get_events(self, contract: Contract) -> List[Type[Contract]]: contract.events.PairCreated, ] if self.exchange_initialized else [] + def get_subscriptions(self, contract: Contract) -> List[Subscription]: + return [ + Subscription(contract.events.StrategyCreated, STRATEGY_CREATED_TOPIC), + Subscription(contract.events.StrategyUpdated, STRATEGY_UPDATED_TOPIC), + Subscription(contract.events.StrategyDeleted, STRATEGY_DELETED_TOPIC), + Subscription(contract.events.PairTradingFeePPMUpdated), + Subscription(contract.events.TradingFeePPMUpdated), + Subscription(contract.events.PairCreated), + ] + async def get_fee( self, address: str, contract: Contract ) -> Tuple[str, float]: @@ -94,7 +111,7 @@ async def get_fee( fee = await contract.tradingFeePPM() return f"{fee}", fee / 1e6 - async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: + async def get_tkn0(self, address: str, contract: Contract, event: Event) -> str: """ Get the token0 address from the contract or event. @@ -116,9 +133,9 @@ async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: if event is None: return await contract.caller.token0() else: - return event["args"]["token0"] + return event.args["token0"] - async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: + async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: """ Get the token1 address from the contract or event. @@ -140,7 +157,7 @@ async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: if event is None: return await contract.caller.token1() else: - return event["args"]["token1"] + return event.args["token1"] def delete_strategy(self, id: str): """ diff --git a/fastlane_bot/events/exchanges/solidly_v2.py b/fastlane_bot/events/exchanges/solidly_v2.py index 0197f5315..d6f5aa02a 100644 --- a/fastlane_bot/events/exchanges/solidly_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2.py @@ -19,8 +19,10 @@ from fastlane_bot.data.abi import SOLIDLY_V2_POOL_ABI, VELOCIMETER_V2_FACTORY_ABI, SOLIDLY_V2_FACTORY_ABI, \ SCALE_V2_FACTORY_ABI, CLEOPATRA_V2_FACTORY_ABI, LYNEX_V2_FACTORY_ABI, NILE_V2_FACTORY_ABI, \ XFAI_V0_FACTORY_ABI, XFAI_V0_CORE_ABI, XFAI_V0_POOL_ABI -from fastlane_bot.events.exchanges.base import Exchange -from fastlane_bot.events.pools.base import Pool +from ..exchanges.base import Exchange +from ..pools.base import Pool +from ..interfaces.subscription import Subscription + async def _get_fee_1(address: str, contract: Contract, factory_contract: Contract) -> int: return await factory_contract.caller.getFee(address) @@ -104,6 +106,9 @@ def get_factory_abi(self): def get_events(self, contract: Contract) -> List[Type[Contract]]: return [contract.events.Sync] if self.exchange_initialized else [] + def get_subscriptions(self, contract: Contract) -> List[Subscription]: + return [Subscription(contract.events.Sync)] + async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: exchange_info = EXCHANGE_INFO[self.exchange_name] fee = await exchange_info["get_fee"](address, contract, self.factory_contract) diff --git a/fastlane_bot/events/exchanges/uniswap_v2.py b/fastlane_bot/events/exchanges/uniswap_v2.py index 198ce4a2f..870b938bf 100644 --- a/fastlane_bot/events/exchanges/uniswap_v2.py +++ b/fastlane_bot/events/exchanges/uniswap_v2.py @@ -17,8 +17,9 @@ from web3.contract import Contract, AsyncContract from fastlane_bot.data.abi import UNISWAP_V2_POOL_ABI, UNISWAP_V2_FACTORY_ABI -from fastlane_bot.events.exchanges.base import Exchange -from fastlane_bot.events.pools.base import Pool +from ..exchanges.base import Exchange +from ..pools.base import Pool +from ..interfaces.subscription import Subscription @dataclass @@ -49,6 +50,9 @@ def get_abi(self): def get_events(self, contract: Contract) -> List[Type[Contract]]: return [contract.events.Sync] if self.exchange_initialized else [] + def get_subscriptions(self, contract: Contract) -> List[Subscription]: + return [Subscription(contract.events.Sync)] + async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: return self.fee, self.fee_float diff --git a/fastlane_bot/events/exchanges/uniswap_v3.py b/fastlane_bot/events/exchanges/uniswap_v3.py index 97870bf68..2136a32a2 100644 --- a/fastlane_bot/events/exchanges/uniswap_v3.py +++ b/fastlane_bot/events/exchanges/uniswap_v3.py @@ -18,8 +18,9 @@ from fastlane_bot.config.constants import AGNI_V3_NAME, PANCAKESWAP_V3_NAME, FUSIONX_V3_NAME, ECHODEX_V3_NAME, SECTA_V3_NAME from fastlane_bot.data.abi import UNISWAP_V3_POOL_ABI, UNISWAP_V3_FACTORY_ABI, PANCAKESWAP_V3_POOL_ABI -from fastlane_bot.events.exchanges.base import Exchange -from fastlane_bot.events.pools.base import Pool +from ..exchanges.base import Exchange +from ..pools.base import Pool +from ..interfaces.subscription import Subscription @dataclass @@ -45,6 +46,9 @@ def get_factory_abi(self): def get_events(self, contract: Contract) -> List[Type[Contract]]: return [contract.events.Swap] if self.exchange_initialized else [] + def get_subscriptions(self, contract: Contract) -> List[Subscription]: + return [Subscription(contract.events.Swap)] + async def get_fee(self, address: str, contract: Contract) -> Tuple[str, float]: fee = await contract.caller.fee() fee_float = float(fee) / 1e6 diff --git a/fastlane_bot/events/interfaces/event.py b/fastlane_bot/events/interfaces/event.py new file mode 100644 index 000000000..98bb5c2fc --- /dev/null +++ b/fastlane_bot/events/interfaces/event.py @@ -0,0 +1,27 @@ +from dataclasses import dataclass +from typing import Any, Dict + + +@dataclass +class Event: + args: Dict[str, Any] + event: str + log_index: int + transaction_index: int + transaction_hash: str + address: str + block_hash: str + block_number: int + + @classmethod + def from_dict(cls, data): + return cls( + args=data["args"], + event=data["event"], + log_index=data["logIndex"], + transaction_index=data["transactionIndex"], + transaction_hash=data["transactionHash"], + address=data["address"], + block_hash=data["blockHash"], + block_number=data["blockNumber"], + ) diff --git a/fastlane_bot/events/interfaces/subscription.py b/fastlane_bot/events/interfaces/subscription.py new file mode 100644 index 000000000..5d78bf8ab --- /dev/null +++ b/fastlane_bot/events/interfaces/subscription.py @@ -0,0 +1,46 @@ +from typing import Optional + +from web3 import AsyncWeb3, Web3 +from web3.contract.contract import ContractEvent + +from ..utils import complex_handler +from .event import Event + + +def _get_event_topic(event): + abi = event().abi + topic = Web3.keccak(text=f'{abi["name"]}({",".join([arg["type"] for arg in abi["inputs"]])})') + return topic.hex() + + +class Subscription: + def __init__(self, event: ContractEvent, topic: Optional[str] = None): + self._event = event + self._topic = _get_event_topic(event) if topic is None else topic + self._subscription_id = None + self._latest_event_index = (-1, -1) # (block_number, block_index) + + async def subscribe(self, w3: AsyncWeb3): + self._subscription_id = await w3.eth.subscribe("logs", {"topics": [self._topic]}) + + @property + def subscription_id(self): + return self._subscription_id + + def process_log(self, log) -> Optional[Event]: + if self._is_event_latest(log): + self._latest_event_index = (log["blockNumber"], log["transactionIndex"]) + return self._parse_log(log) + else: + return None + + def _parse_log(self, log) -> Event: + try: + event_data = complex_handler(self._event().process_log(log)) + except: + print(log) + raise + return Event.from_dict(event_data) + + def _is_event_latest(self, event) -> bool: + return (event["blockNumber"], event["transactionIndex"]) > self._latest_event_index diff --git a/fastlane_bot/events/managers/base.py b/fastlane_bot/events/managers/base.py index becf27d89..cf85d906a 100644 --- a/fastlane_bot/events/managers/base.py +++ b/fastlane_bot/events/managers/base.py @@ -23,6 +23,7 @@ from fastlane_bot.events.exchanges.base import Exchange from fastlane_bot.events.pools.utils import get_pool_cid from fastlane_bot.events.pools import pool_factory +from ..interfaces.event import Event @dataclass @@ -241,13 +242,13 @@ def get_fee_pairs( ) return fee_pairs - def exchange_name_from_event(self, event: Dict[str, Any]) -> str: + def exchange_name_from_event(self, event: Event) -> str: """ Get the exchange name from the event. Parameters ---------- - event : Dict[str, Any] + event : Event The event. Returns @@ -255,8 +256,8 @@ def exchange_name_from_event(self, event: Dict[str, Any]) -> str: str The exchange name. """ - if 'id' in event['args']: - carbon_controller_address = event['address'] + if 'id' in event.args: + carbon_controller_address = event.address for ex in self.cfg.CARBON_CONTROLLER_MAPPING: if self.cfg.CARBON_CONTROLLER_MAPPING[ex] == carbon_controller_address: return ex @@ -270,7 +271,7 @@ def exchange_name_from_event(self, event: Dict[str, Any]) -> str: return None def check_forked_exchange_names( - self, exchange_name_default: str = None, address: str = None, event: Any = None + self, exchange_name_default: str = None, address: str = None, event: Event = None ) -> str: """ Check the forked exchange names. If the exchange name is forked (Sushiswap from UniswapV2, etc) return the @@ -282,7 +283,7 @@ def check_forked_exchange_names( The default exchange name. address : str, optional The address. - event : Any, optional + event : Event, optional The event. Returns @@ -778,11 +779,11 @@ def validate_pool_info( if key != "strategy_id" and (pool_info is None or not pool_info): # Uses method in ContractsManager.add_pool_info_from_contract class to get pool info from contract pool_info = self.add_pool_info_from_contract( - address=addr, event=event, block_number=event["blockNumber"] + address=addr, event=event, block_number=event.block_number ) # if addr in self.cfg.CARBON_CONTROLLER_MAPPING: - # cid = event["args"]["id"] if event is not None else pool_info["strategy_id"] + # cid = event.args["id"] if event is not None else pool_info["strategy_id"] # # for pool in self.pool_data: # if pool["cid"] == cid: @@ -795,14 +796,14 @@ def validate_pool_info( return pool_info def get_key_and_value( - self, event: Dict[str, Any], addr: str, ex_name: str + self, event: Event, addr: str, ex_name: str ) -> Tuple[str, Any]: """ Get the key and value. Parameters ---------- - event : Dict[str, Any] + event : Event The event. addr : str The address. @@ -816,38 +817,38 @@ def get_key_and_value( """ if ex_name == "bancor_pol": - return "token", event["args"]["token"] + return "token", event.args["token"] if ex_name in self.cfg.CARBON_V1_FORKS: - info = {'exchange_name': ex_name, 'strategy_id': event["args"]["id"]} + info = {'exchange_name': ex_name, 'strategy_id': event.args["id"]} return "cid", get_pool_cid(info, self.cfg.CARBON_V1_FORKS) if ex_name in self.cfg.ALL_FORK_NAMES_WITHOUT_CARBON: return "address", addr if ex_name == "bancor_v2": return ("tkn0_address", "tkn1_address"), ( - event["args"]["_token1"], - event["args"]["_token2"], + event.args["_token1"], + event.args["_token2"], ) if ex_name == "bancor_v3": value = ( - event["args"]["tkn_address"] - if event["args"]["tkn_address"] != self.cfg.BNT_ADDRESS - else event["args"]["pool"] + event.args["tkn_address"] + if event.args["tkn_address"] != self.cfg.BNT_ADDRESS + else event.args["pool"] ) return "tkn1_address", value raise ValueError( f"[managers.base.get_key_and_value] Exchange {ex_name} not supported" ) - def handle_strategy_deleted(self, event: Dict[str, Any]) -> None: + def handle_strategy_deleted(self, event: Event) -> None: """ Handle the strategy deleted event. Parameters ---------- - event : Dict[str, Any] + event : Event The event. """ - strategy_id = event["args"]["id"] + strategy_id = event.args["id"] exchange_name = self.exchange_name_from_event(event) cids = [p["cid"] for p in self.pool_data if p["strategy_id"] == strategy_id and p["exchange_name"] == exchange_name] @@ -886,13 +887,13 @@ def pool_key_value_from_event(key: str, event: Dict[str, Any]) -> Any: The pool key value. """ if key == "cid": - return event["args"]["id"] + return event.args["id"] elif key == "address": - return event["address"] + return event.address elif key == "tkn0_address": - return event["args"]["token0"] + return event.args["token0"] elif key == "tkn1_address": - return event["args"]["token1"] + return event.args["token1"] print_events = [] diff --git a/fastlane_bot/events/managers/contracts.py b/fastlane_bot/events/managers/contracts.py index 03a4cd0c1..1355d4ba9 100644 --- a/fastlane_bot/events/managers/contracts.py +++ b/fastlane_bot/events/managers/contracts.py @@ -28,6 +28,7 @@ ) from fastlane_bot.events.managers.base import BaseManager from fastlane_bot.events.pools.utils import get_pool_cid +from ..interfaces.event import Event class ContractsManager(BaseManager): @@ -203,7 +204,7 @@ def add_pool_info_from_contract( self, exchange_name: str = None, address: str = None, - event: Any = None, + event: Event = None, tenderly_exchanges: List[str] = None, ) -> Dict[str, Any]: """ @@ -244,8 +245,8 @@ def add_pool_info_from_contract( t0_addr = self.exchanges[exchange_name].get_tkn0(address, pool_contract, event) t1_addr = self.exchanges[exchange_name].get_tkn1(address, pool_contract, event) - block_number = event["blockNumber"] - strategy_id = event["args"]["id"] if exchange_name in self.cfg.CARBON_V1_FORKS else None + block_number = event.block_number + strategy_id = event.args["id"] if exchange_name in self.cfg.CARBON_V1_FORKS else None temp_pool_info = { "exchange_name": exchange_name, "fee": f"{fee}", diff --git a/fastlane_bot/events/managers/manager.py b/fastlane_bot/events/managers/manager.py index 998356c53..7b80e66f6 100644 --- a/fastlane_bot/events/managers/manager.py +++ b/fastlane_bot/events/managers/manager.py @@ -18,33 +18,33 @@ from fastlane_bot.events.managers.contracts import ContractsManager from fastlane_bot.events.managers.events import EventManager from fastlane_bot.events.managers.pools import PoolManager -from fastlane_bot.events.pools.utils import get_pool_cid +from ..interfaces.event import Event class Manager(PoolManager, EventManager, ContractsManager): - def update_from_event(self, event: Dict[str, Any]): + def update_from_event(self, event: Event): """ Updates the state of the pool data from an event. StrategyCreated and StrategyUpdated events are handled as the "default" event types to process. Args: - event (Dict[str, Any]): The event to process. + event (Event): The event to process. """ ex_name = self.exchange_name_from_event(event) - if event["event"] in ["TradingFeePPMUpdated", "PairTradingFeePPMUpdated"]: + if event.event in ["TradingFeePPMUpdated", "PairTradingFeePPMUpdated"]: self.handle_trading_fee_updated() return - if event["event"] == "PairCreated": + if event.event == "PairCreated": self.set_carbon_v1_fee_pairs() return - if event["event"] == "StrategyDeleted": + if event.event == "StrategyDeleted": self.handle_strategy_deleted(event) return - addr = self.web3.to_checksum_address(event["address"]) + addr = self.web3.to_checksum_address(event.address) if not ex_name: return @@ -63,9 +63,7 @@ def update_from_event(self, event: Dict[str, Any]): pool_info["descr"] = self.pool_descr_from_info(pool_info) pool = self.get_or_init_pool(pool_info) - data = pool.update_from_event( - event or {}, pool.get_common_data(event, pool_info) - ) + data = pool.update_from_event(event, pool.get_common_data(event, pool_info)) self.update_pool_data(pool_info, data) def update_from_pool_info( @@ -166,18 +164,18 @@ def update( def handle_pair_trading_fee_updated( self, - event: Dict[str, Any] = None, + event: Event = None, ): """ Handle the pair trading fee updated event by updating the fee pairs and pool info for the given pair. Parameters ---------- - event : Dict[str, Any], optional + event : Event, optional The event, by default None. """ - tkn0_address = event["args"]["token0"] - tkn1_address = event["args"]["token1"] + tkn0_address = event.args["token0"] + tkn1_address = event.args["token1"] for exchange_name in self.cfg.CARBON_V1_FORKS: if exchange_name in self.exchanges: @@ -244,3 +242,23 @@ def handle_trading_fee_updated(self): pool["fee_float"] = pool["fee"] / 1e6 pool["descr"] = self.pool_descr_from_info(pool) self.pool_data[idx] = pool + + + def update_remaining_pools(self): + remaining_pools = [] + all_events = [pool[2] for pool in self.pools_to_add_from_contracts] + for event in all_events: + addr = self.web3.to_checksum_address(event.address) + ex_name = self.exchange_name_from_event(event) + if not ex_name: + self.cfg.logger.warning("[update_remaining_pools] ex_name not found from event") + continue + + key, key_value = self.get_key_and_value(event, addr, ex_name) + pool_info = self.get_pool_info(key, key_value, ex_name) + + if not pool_info: + remaining_pools.append((addr, ex_name, event, key, key_value)) + + random.shuffle(remaining_pools) + self.pools_to_add_from_contracts = remaining_pools diff --git a/fastlane_bot/events/pools/balancer.py b/fastlane_bot/events/pools/balancer.py index 80f894aea..6f8cd4b2b 100644 --- a/fastlane_bot/events/pools/balancer.py +++ b/fastlane_bot/events/pools/balancer.py @@ -15,7 +15,7 @@ from web3.contract import Contract from .base import Pool -from fastlane_bot import Config +from ..interfaces.event import Event @dataclass @@ -46,7 +46,7 @@ def event_matches_format( return False def update_from_event( - self, event_args: Dict[str, Any], data: Dict[str, Any] + self, event: Event, data: Dict[str, Any] ) -> Dict[str, Any]: """ See base class. diff --git a/fastlane_bot/events/pools/bancor_pol.py b/fastlane_bot/events/pools/bancor_pol.py index 58e58bc83..d0f1a6065 100644 --- a/fastlane_bot/events/pools/bancor_pol.py +++ b/fastlane_bot/events/pools/bancor_pol.py @@ -18,6 +18,7 @@ from fastlane_bot.data.abi import ERC20_ABI, BANCOR_POL_ABI from fastlane_bot.events.pools.base import Pool +from ..interfaces.event import Event from _decimal import Decimal @@ -41,14 +42,14 @@ def unique_key() -> str: @classmethod def event_matches_format( - cls, event: Dict[str, Any], static_pools: Dict[str, Any], exchange_name: str = None + cls, event: Event, static_pools: Dict[str, Any], exchange_name: str = None ) -> bool: """ Check if an event matches the format of a Bancor pol event. Parameters ---------- - event : Dict[str, Any] + event : Event The event arguments. Returns @@ -61,7 +62,7 @@ def event_matches_format( return ("token" in event_args) and ("token0" not in event_args) and (event_args['token'] == "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE") def update_from_event( - self, event_args: Dict[str, Any], data: Dict[str, Any] + self, event: Event, data: Dict[str, Any] ) -> Dict[str, Any]: """ This updates the pool balance from TokenTraded events. @@ -69,12 +70,12 @@ def update_from_event( See base class. """ - event_type = event_args["event"] + event_type = event.event if event_type in "TradingEnabled": - data["tkn0_address"] = event_args["args"]["token"] - data["tkn1_address"] = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" if event_args["args"]["token"] not in "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" else "0x1F573D6Fb3F13d689FF844B4cE37794d79a7FF1C" + data["tkn0_address"] = event.args["token"] + data["tkn1_address"] = "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" if event.args["token"] not in "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" else "0x1F573D6Fb3F13d689FF844B4cE37794d79a7FF1C" - if event_args["args"]["token"] == self.state["tkn0_address"] and event_type in [ + if event.args["token"] == self.state["tkn0_address"] and event_type in [ "TokenTraded" ]: # *** Balance now updated from multicall *** diff --git a/fastlane_bot/events/pools/bancor_v2.py b/fastlane_bot/events/pools/bancor_v2.py index 68068bea7..b0f993418 100644 --- a/fastlane_bot/events/pools/bancor_v2.py +++ b/fastlane_bot/events/pools/bancor_v2.py @@ -15,6 +15,7 @@ from web3.contract import Contract from fastlane_bot.events.pools.base import Pool +from ..interfaces.event import Event @dataclass @@ -34,14 +35,14 @@ def unique_key() -> str: @classmethod def event_matches_format( - cls, event: Dict[str, Any], static_pools: Dict[str, Any], exchange_name: str = None + cls, event: Event, static_pools: Dict[str, Any], exchange_name: str = None ) -> bool: """ Check if an event matches the format of a Bancor v2 event. Parameters ---------- - event : Dict[str, Any] + event : Event The event arguments. Returns @@ -50,21 +51,21 @@ def event_matches_format( True if the event matches the format of a Bancor v3 event, False otherwise. """ - event_args = event["args"] + event_args = event.args return "_rateN" in event_args def update_from_event( - self, event_args: Dict[str, Any], data: Dict[str, Any] + self, event: Event, data: Dict[str, Any] ) -> Dict[str, Any]: """ **** IMPORTANT **** Bancor V2 pools emit 3 events per trade. Only one of them contains the new token balances we want. The one we want is the one where _token1 and _token2 match the token addresses of the pool. """ - data["tkn0_address"] = event_args["args"]["_token1"] - data["tkn1_address"] = event_args["args"]["_token2"] - data["tkn0_balance"] = event_args["args"]["_rateD"] - data["tkn1_balance"] = event_args["args"]["_rateN"] + data["tkn0_address"] = event.args["_token1"] + data["tkn1_address"] = event.args["_token2"] + data["tkn0_balance"] = event.args["_rateD"] + data["tkn1_balance"] = event.args["_rateN"] for key, value in data.items(): self.state[key] = value diff --git a/fastlane_bot/events/pools/bancor_v3.py b/fastlane_bot/events/pools/bancor_v3.py index 1b2281662..b266a819d 100644 --- a/fastlane_bot/events/pools/bancor_v3.py +++ b/fastlane_bot/events/pools/bancor_v3.py @@ -15,6 +15,7 @@ from web3.contract import Contract from .base import Pool +from ..interfaces.event import Event @dataclass @@ -34,14 +35,14 @@ def unique_key() -> str: @classmethod def event_matches_format( - cls, event: Dict[str, Any], static_pools: Dict[str, Any], exchange_name: str = None + cls, event: Event, static_pools: Dict[str, Any], exchange_name: str = None ) -> bool: """ Check if an event matches the format of a Bancor v3 event. Parameters ---------- - event : Dict[str, Any] + event : Event The event arguments. Returns @@ -50,20 +51,19 @@ def event_matches_format( True if the event matches the format of a Bancor v3 event, False otherwise. """ - event_args = event["args"] + event_args = event.args return "pool" in event_args def update_from_event( - self, event_args: Dict[str, Any], data: Dict[str, Any] + self, event: Event, data: Dict[str, Any] ) -> Dict[str, Any]: """ See base class. """ - event_args = event_args["args"] - if event_args["tkn_address"] == "0x1F573D6Fb3F13d689FF844B4cE37794d79a7FF1C": - data["tkn0_balance"] = event_args["newLiquidity"] + if event.args["tkn_address"] == "0x1F573D6Fb3F13d689FF844B4cE37794d79a7FF1C": + data["tkn0_balance"] = event.args["newLiquidity"] else: - data["tkn1_balance"] = event_args["newLiquidity"] + data["tkn1_balance"] = event.args["newLiquidity"] for key, value in data.items(): self.state[key] = value diff --git a/fastlane_bot/events/pools/base.py b/fastlane_bot/events/pools/base.py index e7f5870bb..4aa92999a 100644 --- a/fastlane_bot/events/pools/base.py +++ b/fastlane_bot/events/pools/base.py @@ -18,6 +18,8 @@ from web3 import Web3 from web3.contract import Contract +from ..interfaces.event import Event + @dataclass class Pool(ABC): @@ -61,37 +63,37 @@ def event_matches_format( @staticmethod def get_common_data( - event: Dict[str, Any], pool_info: Dict[str, Any] + event: Event, pool_info: Dict[str, Any] ) -> Dict[str, Any]: """ Get common (common to all Pool child classes) data from an event and pool info. Args: - event (Dict[str, Any]): The event data. + event (Event): The event data. pool_info (Dict[str, Any]): The pool information. Returns: Dict[str, Any]: A dictionary containing common data extracted from the event and pool info. """ return { - "last_updated_block": event["blockNumber"], + "last_updated_block": event.block_number, "timestamp": datetime.now().strftime("%H:%M:%S"), "pair_name": pool_info["pair_name"], "descr": pool_info["descr"], - "address": event["address"], + "address": event.address, } @staticmethod @abstractmethod def update_from_event( - event_args: Dict[str, Any], data: Dict[str, Any] + event: Event, data: Dict[str, Any] ) -> Dict[str, Any]: """ Update the pool state from an event. Parameters ---------- - event_args : Dict[str, Any] + event : Event The event arguments. data : Dict[str, Any] The pool data. diff --git a/fastlane_bot/events/pools/carbon_v1.py b/fastlane_bot/events/pools/carbon_v1.py index 4d62da546..184b13e83 100644 --- a/fastlane_bot/events/pools/carbon_v1.py +++ b/fastlane_bot/events/pools/carbon_v1.py @@ -15,6 +15,7 @@ from web3.contract import Contract from .base import Pool +from ..interfaces.event import Event from fastlane_bot.events.pools.utils import get_pool_cid @@ -36,34 +37,31 @@ def unique_key() -> str: @classmethod def event_matches_format( - cls, event: Dict[str, Any], static_pools: Dict[str, Any], exchange_name: str = None + cls, event: Event, static_pools: Dict[str, Any], exchange_name: str = None ) -> bool: """ see base class. """ - event_args = event["args"] + event_args = event.args return "order0" in event_args def update_from_event( - self, event_args: Dict[str, Any], data: Dict[str, Any] + self, event: Event, data: Dict[str, Any] ) -> Dict[str, Any]: """ See base class. """ - event_type = event_args["event"] - assert event_type not in ["TradingFeePPMUpdated", "PairTradingFeePPMUpdated"], ( + assert event.event not in ["TradingFeePPMUpdated", "PairTradingFeePPMUpdated"], ( "This event should not be " "handled by this class." ) - data = CarbonV1Pool.parse_event(data, event_args, event_type) + data = CarbonV1Pool.parse_event(data, event) data["router"] = self.router_address for key, value in data.items(): self.state[key] = value return data @staticmethod - def parse_event( - data: Dict[str, Any], event_args: Dict[str, Any], event_type: str - ) -> Dict[str, Any]: + def parse_event(data: Dict[str, Any], event: Event) -> Dict[str, Any]: """ Parse the event args into a dict. @@ -71,18 +69,16 @@ def parse_event( ---------- data : Dict[str, Any] The data to update. - event_args : Dict[str, Any] - The event arguments. - event_type : str - The event type. + event_args : Event + The event object. Returns ------- Dict[str, Any] The updated data. """ - order0, order1 = CarbonV1Pool.parse_orders(event_args, event_type) - data["strategy_id"] = event_args["args"].get("id") + order0, order1 = CarbonV1Pool.parse_orders(event) + data["strategy_id"] = event.args.get("id") if isinstance(order0, list) and isinstance(order1, list): data["y_0"] = order0[0] data["z_0"] = order0[1] @@ -105,9 +101,7 @@ def parse_event( return data @staticmethod - def parse_orders( - event_args: Dict[str, Any], event_type: str - ) -> Tuple[List[int], List[int]]: + def parse_orders(event: Event) -> Tuple[List[int], List[int]]: """ Parse the orders from the event args. If the event type is StrategyDeleted, then the orders are set to 0. @@ -115,17 +109,15 @@ def parse_orders( ---------- event_args : Dict[str, Any] The event arguments. - event_type : str - The event type. Returns ------- Tuple[List[int], List[int]] The parsed orders. """ - if event_type not in ["StrategyDeleted"]: - order0 = event_args["args"].get("order0") - order1 = event_args["args"].get("order1") + if event.event not in ["StrategyDeleted"]: + order0 = event.args.get("order0") + order1 = event.args.get("order1") else: order0 = [0, 0, 0, 0] order1 = [0, 0, 0, 0] diff --git a/fastlane_bot/events/pools/solidly_v2.py b/fastlane_bot/events/pools/solidly_v2.py index 61a1acf03..8ca967a7f 100644 --- a/fastlane_bot/events/pools/solidly_v2.py +++ b/fastlane_bot/events/pools/solidly_v2.py @@ -15,6 +15,7 @@ from web3.contract import Contract from fastlane_bot.events.pools.base import Pool +from ..interfaces.event import Event def _balances_A(contract: Contract) -> List[int]: return contract.caller.getReserves() @@ -88,21 +89,20 @@ def event_matches_format( """ Check if an event matches the format of a Uniswap v2 event. """ - event_args = event["args"] + event_args = event.args return ( "reserve0" in event_args - and event["address"] in static_pools[f"{exchange_name}_pools"] + and event.address in static_pools[f"{exchange_name}_pools"] ) def update_from_event( - self, event_args: Dict[str, Any], data: Dict[str, Any] + self, event: Event, data: Dict[str, Any] ) -> Dict[str, Any]: """ See base class. """ - event_args = event_args["args"] - data["tkn0_balance"] = event_args["reserve0"] - data["tkn1_balance"] = event_args["reserve1"] + data["tkn0_balance"] = event.args["reserve0"] + data["tkn1_balance"] = event.args["reserve1"] for key, value in data.items(): self.state[key] = value diff --git a/fastlane_bot/events/pools/uniswap_v2.py b/fastlane_bot/events/pools/uniswap_v2.py index 55549e89a..c48639f59 100644 --- a/fastlane_bot/events/pools/uniswap_v2.py +++ b/fastlane_bot/events/pools/uniswap_v2.py @@ -15,6 +15,7 @@ from web3.contract import Contract from fastlane_bot.events.pools.base import Pool +from ..interfaces.event import Event @dataclass @@ -41,26 +42,25 @@ def unique_key() -> str: @classmethod def event_matches_format( - cls, event: Dict[str, Any], static_pools: Dict[str, Any], exchange_name: str = None + cls, event: Event, static_pools: Dict[str, Any], exchange_name: str = None ) -> bool: """ Check if an event matches the format of a Uniswap v2 event. """ - event_args = event["args"] + event_args = event.args return ( "reserve0" in event_args - and event["address"] in static_pools[f"{exchange_name}_pools"] + and event.address in static_pools[f"{exchange_name}_pools"] ) def update_from_event( - self, event_args: Dict[str, Any], data: Dict[str, Any] + self, event: Event, data: Dict[str, Any] ) -> Dict[str, Any]: """ See base class. """ - event_args = event_args["args"] - data["tkn0_balance"] = event_args["reserve0"] - data["tkn1_balance"] = event_args["reserve1"] + data["tkn0_balance"] = event.args["reserve0"] + data["tkn1_balance"] = event.args["reserve1"] for key, value in data.items(): self.state[key] = value diff --git a/fastlane_bot/events/pools/uniswap_v3.py b/fastlane_bot/events/pools/uniswap_v3.py index 265c72296..06e67f65f 100644 --- a/fastlane_bot/events/pools/uniswap_v3.py +++ b/fastlane_bot/events/pools/uniswap_v3.py @@ -14,6 +14,7 @@ from web3.contract import Contract from fastlane_bot.events.pools.base import Pool +from ..interfaces.event import Event @dataclass @@ -34,27 +35,26 @@ def unique_key() -> str: @classmethod def event_matches_format( - cls, event: Dict[str, Any], static_pools: Dict[str, Any], exchange_name: str = None + cls, event: Event, static_pools: Dict[str, Any], exchange_name: str = None ) -> bool: """ Check if an event matches the format of a Uniswap v3 event. """ - event_args = event["args"] + event_args = event.args return ( "sqrtPriceX96" in event_args - and event["address"] in static_pools[f"{exchange_name}_pools"] + and event.address in static_pools[f"{exchange_name}_pools"] ) def update_from_event( - self, event_args: Dict[str, Any], data: Dict[str, Any] + self, event: Event, data: Dict[str, Any] ) -> Dict[str, Any]: """ See base class. """ - event_args = event_args["args"] - data["liquidity"] = event_args["liquidity"] - data["sqrt_price_q96"] = event_args["sqrtPriceX96"] - data["tick"] = event_args["tick"] + data["liquidity"] = event.args["liquidity"] + data["sqrt_price_q96"] = event.args["sqrtPriceX96"] + data["tick"] = event.args["tick"] for key, value in data.items(): self.state[key] = value diff --git a/fastlane_bot/events/utils.py b/fastlane_bot/events/utils.py index 719ebf09b..4f4f7aa92 100644 --- a/fastlane_bot/events/utils.py +++ b/fastlane_bot/events/utils.py @@ -31,15 +31,15 @@ from fastlane_bot.data.abi import FAST_LANE_CONTRACT_ABI from fastlane_bot.exceptions import ReadOnlyException from fastlane_bot.events.interface import QueryInterface -from fastlane_bot.events.managers.manager import Manager from fastlane_bot.helpers import TxHelpers from fastlane_bot.utils import safe_int +from .interfaces.event import Event def filter_latest_events( - mgr: Manager, events: List[List[AttributeDict]] -) -> List[AttributeDict]: + mgr: Any, events: List[Event] +) -> List[Event]: """ This function filters out the latest events for each pool. Given a nested list of events, it iterates through all events and keeps track of the latest event (i.e., with the highest block number) for each pool. The key used to identify each pool @@ -54,19 +54,16 @@ def filter_latest_events( List[AttributeDict]: A list of events, each representing the latest event for its corresponding pool. """ latest_entry_per_pool = {} - all_events = [event for event_list in events for event in event_list] # Handles the case where multiple pools are created in the same block - all_events.reverse() + events.reverse() bancor_v2_anchor_addresses = { pool["anchor"] for pool in mgr.pool_data if pool["exchange_name"] == "bancor_v2" } - for event in all_events: - pool_type = mgr.pool_type_from_exchange_name( - mgr.exchange_name_from_event(event) - ) + for event in events: + pool_type = mgr.pool_type_from_exchange_name(mgr.exchange_name_from_event(event)) if pool_type: key = pool_type.unique_key() else: @@ -74,43 +71,33 @@ def filter_latest_events( if key == "cid": key = "id" elif key == "tkn1_address": - if event["args"]["pool"] != mgr.cfg.BNT_ADDRESS: + if event.args["pool"] != mgr.cfg.BNT_ADDRESS: key = "pool" else: key = "tkn_address" - unique_key = event[key] if key in event else event["args"][key] + unique_key = event.address if key == "address" else event.args[key] + # unique_key = event.args[key] # Skip events for Bancor v2 anchors if ( key == "address" - and "_token1" in event["args"] + and "_token1" in event.args and ( - event["args"]["_token1"] in bancor_v2_anchor_addresses - or event["args"]["_token2"] in bancor_v2_anchor_addresses + event.args["_token1"] in bancor_v2_anchor_addresses + or event.args["_token2"] in bancor_v2_anchor_addresses ) ): continue if unique_key in latest_entry_per_pool: - if event["blockNumber"] > latest_entry_per_pool[unique_key]["blockNumber"]: + if event.block_number > latest_entry_per_pool[unique_key].block_number: latest_entry_per_pool[unique_key] = event - elif ( - event["blockNumber"] == latest_entry_per_pool[unique_key]["blockNumber"] - ): - if ( - event["transactionIndex"] - == latest_entry_per_pool[unique_key]["transactionIndex"] - ): - if ( - event["logIndex"] - > latest_entry_per_pool[unique_key]["logIndex"] - ): + elif event.block_number == latest_entry_per_pool[unique_key].block_number: + if event.transaction_index == latest_entry_per_pool[unique_key].transaction_index: + if event.log_index > latest_entry_per_pool[unique_key].log_index: latest_entry_per_pool[unique_key] = event - elif ( - event["transactionIndex"] - > latest_entry_per_pool[unique_key]["transactionIndex"] - ): + elif event.transaction_index > latest_entry_per_pool[unique_key].transaction_index: latest_entry_per_pool[unique_key] = event else: continue @@ -138,7 +125,7 @@ def complex_handler(obj: Any) -> Union[Dict, str, List, Set, Any]: If the input object does not match any of the specified types, it is returned as is. """ if isinstance(obj, AttributeDict): - return dict(obj) + return complex_handler(dict(obj)) elif isinstance(obj, HexBytes): return obj.hex() elif isinstance(obj, bytes): @@ -148,7 +135,7 @@ def complex_handler(obj: Any) -> Union[Dict, str, List, Set, Any]: elif isinstance(obj, (list, tuple)): return [complex_handler(i) for i in obj] elif isinstance(obj, set): - return list(obj) + return complex_handler(list(obj)) else: return obj @@ -806,7 +793,7 @@ def save_events_to_json( mgr.cfg.logger.debug(f"[events.utils.save_events_to_json] Saved events to {path}") -def update_pools_from_events(n_jobs: int, mgr: Any, latest_events: List[Any]): +def update_pools_from_events(n_jobs: int, mgr: Any, latest_events: List[Event]): """ Updates the pools with the given events. @@ -1216,14 +1203,12 @@ def get_latest_events( # Get all event filters, events, and flatten them events = [ - complex_handler(event) - for event in [ - complex_handler(event) - for event in get_all_events( - n_jobs, - get_event_filters(n_jobs, mgr, start_block, current_block), - ) - ] + Event.from_dict(complex_handler(event)) + for events in get_all_events( + n_jobs, + get_event_filters(n_jobs, mgr, start_block, current_block), + ) + for event in events ] # Filter out the latest events per pool, save them to disk, and update the pools @@ -1242,7 +1227,7 @@ def get_latest_events( if not pool_type.event_matches_format(event) ] - carbon_pol_events = [event for event in latest_events if "token" in event["args"]] + carbon_pol_events = [event for event in latest_events if "token" in event.args] mgr.cfg.logger.info( f"[events.utils.get_latest_events] Found {len(latest_events)} new events, {len(carbon_pol_events)} carbon_pol_events" ) From aff59023c9ff9c290a27ae78e9d553e0e22a8d3e Mon Sep 17 00:00:00 2001 From: Lesigh-3100 Date: Wed, 8 May 2024 16:09:33 +0300 Subject: [PATCH 07/58] Change event gathering from filters to get_logs --- fastlane_bot/events/event_gatherer.py | 50 +++++++++++++++++++ .../events/interfaces/subscription.py | 16 +++--- fastlane_bot/events/utils.py | 13 ++--- main.py | 6 ++- 4 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 fastlane_bot/events/event_gatherer.py diff --git a/fastlane_bot/events/event_gatherer.py b/fastlane_bot/events/event_gatherer.py new file mode 100644 index 000000000..257482c59 --- /dev/null +++ b/fastlane_bot/events/event_gatherer.py @@ -0,0 +1,50 @@ +import time +from itertools import chain + +import asyncio +import nest_asyncio + +from typing import List, Any +from web3 import AsyncWeb3 + + +nest_asyncio.apply() + +class EventGatherer: + """ + The EventGatherer manages event gathering using eth.get_logs. + """ + + def __init__( + self, + w3: AsyncWeb3, + exchanges: Any, + event_contracts: Any, + ): + """ Initializes the EventManager. + Args: + manager: The Manager object + w3: The connected AsyncWeb3 object. + """ + self._w3 = w3 + self._subscriptions = [] + unique_topics = [] + + for exchange_name, exchange in exchanges.items(): + subscriptions = exchange.get_subscriptions(event_contracts[exchange_name]) + _unique_subscriptions = [] + for sub in subscriptions: + if sub.get_topic not in unique_topics: + unique_topics.append(sub.get_topic) + _unique_subscriptions.append(sub) + self._subscriptions += _unique_subscriptions + + def get_all_events(self, from_block: int, to_block: int): + results = asyncio.get_event_loop().run_until_complete(asyncio.gather(*[self._get_events_for_topic(from_block, to_block, sub) for sub in self._subscriptions])) + return list(chain.from_iterable(results)) + + async def _get_events_for_topic(self, from_block: int, to_block: int or str, subscription: Any): + events = await self._w3.eth.get_logs(filter_params={"fromBlock": from_block, "toBlock": to_block, "topics": [subscription.get_topic]}) + return [subscription.parse_log(event) for event in events] + + diff --git a/fastlane_bot/events/interfaces/subscription.py b/fastlane_bot/events/interfaces/subscription.py index 5d78bf8ab..be60aed02 100644 --- a/fastlane_bot/events/interfaces/subscription.py +++ b/fastlane_bot/events/interfaces/subscription.py @@ -27,14 +27,11 @@ async def subscribe(self, w3: AsyncWeb3): def subscription_id(self): return self._subscription_id - def process_log(self, log) -> Optional[Event]: - if self._is_event_latest(log): - self._latest_event_index = (log["blockNumber"], log["transactionIndex"]) - return self._parse_log(log) - else: - return None - - def _parse_log(self, log) -> Event: + @property + def get_topic(self): + return self._topic + + def parse_log(self, log) -> Event: try: event_data = complex_handler(self._event().process_log(log)) except: @@ -42,5 +39,4 @@ def _parse_log(self, log) -> Event: raise return Event.from_dict(event_data) - def _is_event_latest(self, event) -> bool: - return (event["blockNumber"], event["transactionIndex"]) > self._latest_event_index + diff --git a/fastlane_bot/events/utils.py b/fastlane_bot/events/utils.py index 4f4f7aa92..68f4c7921 100644 --- a/fastlane_bot/events/utils.py +++ b/fastlane_bot/events/utils.py @@ -34,6 +34,7 @@ from fastlane_bot.helpers import TxHelpers from fastlane_bot.utils import safe_int +from .event_gatherer import EventGatherer from .interfaces.event import Event @@ -1164,6 +1165,7 @@ def get_latest_events( start_block: int, cache_latest_only: bool, logging_path: str, + event_gatherer: EventGatherer ) -> List[Any]: """ Gets the latest events. @@ -1201,15 +1203,8 @@ def get_latest_events( f"[events.utils.get_latest_events] tenderly_events: {len(tenderly_events)}" ) - # Get all event filters, events, and flatten them - events = [ - Event.from_dict(complex_handler(event)) - for events in get_all_events( - n_jobs, - get_event_filters(n_jobs, mgr, start_block, current_block), - ) - for event in events - ] + # Get all events + events = event_gatherer.get_all_events(from_block=start_block, to_block=current_block) # Filter out the latest events per pool, save them to disk, and update the pools latest_events = filter_latest_events(mgr, events) diff --git a/main.py b/main.py index 8a4e42e5d..f2dce7d9c 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ (c) Copyright Bprotocol foundation 2023. Licensed under MIT """ - +from fastlane_bot.events.event_gatherer import EventGatherer from fastlane_bot.exceptions import ReadOnlyException, FlashloanUnavailableException from fastlane_bot.events.version_utils import check_version_requirements from fastlane_bot.tools.cpc import T @@ -301,6 +301,9 @@ def run(mgr, args, tenderly_uri=None) -> None: start_timeout = time.time() mainnet_uri = mgr.cfg.w3.provider.endpoint_uri handle_static_pools_update(mgr) + + event_gatherer = EventGatherer(w3=mgr.w3_async, exchanges=mgr.exchanges, event_contracts=mgr.event_contracts) + while True: try: # ensure 'last_updated_block' is in pool_data for all pools @@ -352,6 +355,7 @@ def run(mgr, args, tenderly_uri=None) -> None: start_block, args.cache_latest_only, args.logging_path, + event_gatherer ) ) iteration_start_time = time.time() From 87fc3d99057b05123964eb9d215f8944c0140b5a Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Thu, 9 May 2024 19:51:18 +0200 Subject: [PATCH 08/58] refactor --- fastlane_bot/events/event_gatherer.py | 27 +++++++++++-------- .../events/interfaces/subscription.py | 2 +- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/fastlane_bot/events/event_gatherer.py b/fastlane_bot/events/event_gatherer.py index 257482c59..989793716 100644 --- a/fastlane_bot/events/event_gatherer.py +++ b/fastlane_bot/events/event_gatherer.py @@ -1,15 +1,18 @@ -import time from itertools import chain import asyncio import nest_asyncio -from typing import List, Any from web3 import AsyncWeb3 +from web3.contract import Contract + +from .interfaces.subscription import Subscription +from .exchanges.base import Exchange nest_asyncio.apply() + class EventGatherer: """ The EventGatherer manages event gathering using eth.get_logs. @@ -18,8 +21,8 @@ class EventGatherer: def __init__( self, w3: AsyncWeb3, - exchanges: Any, - event_contracts: Any, + exchanges: dict[str, Exchange], + event_contracts: dict[str, Contract], ): """ Initializes the EventManager. Args: @@ -28,23 +31,25 @@ def __init__( """ self._w3 = w3 self._subscriptions = [] - unique_topics = [] + unique_topics = set() for exchange_name, exchange in exchanges.items(): subscriptions = exchange.get_subscriptions(event_contracts[exchange_name]) - _unique_subscriptions = [] for sub in subscriptions: if sub.get_topic not in unique_topics: - unique_topics.append(sub.get_topic) - _unique_subscriptions.append(sub) - self._subscriptions += _unique_subscriptions + unique_topics.add(sub.topic) + self._subscriptions.append(sub) def get_all_events(self, from_block: int, to_block: int): results = asyncio.get_event_loop().run_until_complete(asyncio.gather(*[self._get_events_for_topic(from_block, to_block, sub) for sub in self._subscriptions])) return list(chain.from_iterable(results)) - async def _get_events_for_topic(self, from_block: int, to_block: int or str, subscription: Any): - events = await self._w3.eth.get_logs(filter_params={"fromBlock": from_block, "toBlock": to_block, "topics": [subscription.get_topic]}) + async def _get_events_for_topic(self, from_block: int, to_block: int, subscription: Subscription): + events = await self._w3.eth.get_logs(filter_params={ + "fromBlock": from_block, + "toBlock": to_block, + "topics": [subscription.topic] + }) return [subscription.parse_log(event) for event in events] diff --git a/fastlane_bot/events/interfaces/subscription.py b/fastlane_bot/events/interfaces/subscription.py index be60aed02..660f45e0c 100644 --- a/fastlane_bot/events/interfaces/subscription.py +++ b/fastlane_bot/events/interfaces/subscription.py @@ -28,7 +28,7 @@ def subscription_id(self): return self._subscription_id @property - def get_topic(self): + def topic(self): return self._topic def parse_log(self, log) -> Event: From 18c1e701c410d37ee861bfb1502ccd5644e07b82 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Fri, 10 May 2024 11:32:33 +0300 Subject: [PATCH 09/58] fix: circular import --- fastlane_bot/events/interfaces/subscription.py | 2 -- fastlane_bot/events/utils.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/fastlane_bot/events/interfaces/subscription.py b/fastlane_bot/events/interfaces/subscription.py index 660f45e0c..b1165f757 100644 --- a/fastlane_bot/events/interfaces/subscription.py +++ b/fastlane_bot/events/interfaces/subscription.py @@ -38,5 +38,3 @@ def parse_log(self, log) -> Event: print(log) raise return Event.from_dict(event_data) - - diff --git a/fastlane_bot/events/utils.py b/fastlane_bot/events/utils.py index 68f4c7921..2577ba460 100644 --- a/fastlane_bot/events/utils.py +++ b/fastlane_bot/events/utils.py @@ -34,7 +34,6 @@ from fastlane_bot.helpers import TxHelpers from fastlane_bot.utils import safe_int -from .event_gatherer import EventGatherer from .interfaces.event import Event @@ -1165,7 +1164,7 @@ def get_latest_events( start_block: int, cache_latest_only: bool, logging_path: str, - event_gatherer: EventGatherer + event_gatherer: "EventGatherer" ) -> List[Any]: """ Gets the latest events. From 2b9dd4a8cd677b51cc67d68cc38b38e51e04f6d5 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Fri, 10 May 2024 12:39:07 +0300 Subject: [PATCH 10/58] fix: tests --- fastlane_bot/tests/test_033_Pools.py | 27 ++++++++++++------------ fastlane_bot/tests/test_035_Utils.py | 15 +++++++------ fastlane_bot/tests/test_036_Manager.py | 15 +++++++------ fastlane_bot/tests/test_037_Exchanges.py | 9 ++++---- 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/fastlane_bot/tests/test_033_Pools.py b/fastlane_bot/tests/test_033_Pools.py index d40a20074..26959204e 100644 --- a/fastlane_bot/tests/test_033_Pools.py +++ b/fastlane_bot/tests/test_033_Pools.py @@ -11,6 +11,7 @@ import json from fastlane_bot import Bot +from fastlane_bot.events.interfaces.event import Event from fastlane_bot.events.pools import BancorPolPool, BancorV2Pool, BancorV3Pool, CarbonV1Pool, SolidlyV2Pool, \ UniswapV2Pool, UniswapV3Pool from fastlane_bot.tools.cpc import ConstantProductCurve as CPC @@ -46,7 +47,7 @@ def test_test_uniswap_v2_pool(): # ------------------------------------------------------------ uniswap_v2_pool = UniswapV2Pool() - uniswap_v2_pool.update_from_event(setup_data['uniswap_v2_event'], {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'sushiswap_v2', 'reserve0': setup_data['uniswap_v2_event']['args']['reserve0'], 'reserve1': setup_data['uniswap_v2_event']['args']['reserve1'], 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) + uniswap_v2_pool.update_from_event(Event.from_dict(setup_data['uniswap_v2_event']), {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'sushiswap_v2', 'reserve0': setup_data['uniswap_v2_event']['args']['reserve0'], 'reserve1': setup_data['uniswap_v2_event']['args']['reserve1'], 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) assert (uniswap_v2_pool.state['tkn0_balance'] == setup_data['uniswap_v2_event']['args']['reserve0']) @@ -59,7 +60,7 @@ def test_test_solidly_v2_pool(): # ------------------------------------------------------------ solidly_v2_pool = SolidlyV2Pool() - solidly_v2_pool.update_from_event(setup_data['solidly_v2_event'], {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'velocimeter_v2', 'reserve0': setup_data['solidly_v2_event']['args']['reserve0'], 'reserve1': setup_data['solidly_v2_event']['args']['reserve1'], 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) + solidly_v2_pool.update_from_event(Event.from_dict(setup_data['solidly_v2_event']), {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'velocimeter_v2', 'reserve0': setup_data['solidly_v2_event']['args']['reserve0'], 'reserve1': setup_data['solidly_v2_event']['args']['reserve1'], 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) assert (solidly_v2_pool.state['tkn0_balance'] == setup_data['solidly_v2_event']['args']['reserve0']) @@ -75,7 +76,7 @@ def test_test_bancor_v2_pool(): bancor_v2_pool.state['tkn0_address'] = '0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE' bancor_v2_pool.state['tkn1_address'] = '0x1F573D6Fb3F13d689FF844B4cE37794d79a7FF1C' bancor_v2_pool.state['anchor']= '0xb1CD6e4153B2a390Cf00A6556b0fC1458C4A5533' - bancor_v2_pool.update_from_event(setup_data['bancor_v2_event'], {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'bancor_v2'}) + bancor_v2_pool.update_from_event(Event.from_dict(setup_data['bancor_v2_event']), {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'bancor_v2'}) assert (5698079648237338312679700 == setup_data['bancor_v2_event']['args']['_rateN']), f"expected {bancor_v2_pool.state['tkn0_balance']}, found {setup_data['bancor_v2_event']['args']['_rateN']}" assert (1404376232459809237924 == setup_data['bancor_v2_event']['args']['_rateD']), f"expected {bancor_v2_pool.state['tkn1_balance']}, found {setup_data['bancor_v2_event']['args']['_rateD']}" @@ -89,7 +90,7 @@ def test_test_pancakeswap_v2_pool(): # ------------------------------------------------------------ pancakeswap_v2_pool = UniswapV2Pool() - pancakeswap_v2_pool.update_from_event(setup_data['pancakeswap_v2_event'], {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'pancakeswap_v2', 'reserve0': setup_data['pancakeswap_v2_event']['args']['reserve0'], 'reserve1': setup_data['pancakeswap_v2_event']['args']['reserve1'], 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) + pancakeswap_v2_pool.update_from_event(Event.from_dict(setup_data['pancakeswap_v2_event']), {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'pancakeswap_v2', 'reserve0': setup_data['pancakeswap_v2_event']['args']['reserve0'], 'reserve1': setup_data['pancakeswap_v2_event']['args']['reserve1'], 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) assert (pancakeswap_v2_pool.state['tkn0_balance'] == setup_data['pancakeswap_v2_event']['args']['reserve0']) @@ -102,7 +103,7 @@ def test_test_uniswap_v3_pool(): # ------------------------------------------------------------ uniswap_v3_pool = UniswapV3Pool() - uniswap_v3_pool.update_from_event(setup_data['uniswap_v3_event'], {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'uniswap_v3', 'liquidity': setup_data['uniswap_v3_event']['args']['liquidity'], 'sqrtPriceX96': setup_data['uniswap_v3_event']['args']['sqrtPriceX96'], 'tick': setup_data['uniswap_v3_event']['args']['tick'], 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) + uniswap_v3_pool.update_from_event(Event.from_dict(setup_data['uniswap_v3_event']), {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'uniswap_v3', 'liquidity': setup_data['uniswap_v3_event']['args']['liquidity'], 'sqrtPriceX96': setup_data['uniswap_v3_event']['args']['sqrtPriceX96'], 'tick': setup_data['uniswap_v3_event']['args']['tick'], 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) assert (uniswap_v3_pool.state['liquidity'] == setup_data['uniswap_v3_event']['args']['liquidity']) assert (uniswap_v3_pool.state['sqrt_price_q96'] == setup_data['uniswap_v3_event']['args']['sqrtPriceX96']) @@ -116,7 +117,7 @@ def test_test_pancakeswap_v3_pool(): # ------------------------------------------------------------ pancakeswap_v3_pool = UniswapV3Pool() - pancakeswap_v3_pool.update_from_event(setup_data['pancakeswap_v3_event'], {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'pancakeswap_v3', 'liquidity': setup_data['pancakeswap_v3_event']['args']['liquidity'], 'sqrtPriceX96': setup_data['pancakeswap_v3_event']['args']['sqrtPriceX96'], 'tick': setup_data['pancakeswap_v3_event']['args']['tick'], 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) + pancakeswap_v3_pool.update_from_event(Event.from_dict(setup_data['pancakeswap_v3_event']), {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'pancakeswap_v3', 'liquidity': setup_data['pancakeswap_v3_event']['args']['liquidity'], 'sqrtPriceX96': setup_data['pancakeswap_v3_event']['args']['sqrtPriceX96'], 'tick': setup_data['pancakeswap_v3_event']['args']['tick'], 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) assert (pancakeswap_v3_pool.state['liquidity'] == setup_data['pancakeswap_v3_event']['args']['liquidity']) assert (pancakeswap_v3_pool.state['sqrt_price_q96'] == setup_data['pancakeswap_v3_event']['args']['sqrtPriceX96']) @@ -130,7 +131,7 @@ def test_test_bancor_v3_pool(): # ------------------------------------------------------------ bancor_v3_pool = BancorV3Pool() - bancor_v3_pool.update_from_event(setup_data['bancor_v3_event'], {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'bancor_v3', 'tkn0_balance': setup_data['bancor_v3_event']['args']['newLiquidity'], 'tkn1_balance': 0, 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) + bancor_v3_pool.update_from_event(Event.from_dict(setup_data['bancor_v3_event']), {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, 'exchange_name': 'bancor_v3', 'tkn0_balance': setup_data['bancor_v3_event']['args']['newLiquidity'], 'tkn1_balance': 0, 'tkn0_symbol': 'tkn0', 'tkn1_symbol': 'tkn1'}) # ------------------------------------------------------------ @@ -142,7 +143,7 @@ def test_test_carbon_v1_pool_update(): # ------------------------------------------------------------ carbon_v1_pool = CarbonV1Pool() - carbon_v1_pool.update_from_event(setup_data['carbon_v1_event_create_for_update'], {}) + carbon_v1_pool.update_from_event(Event.from_dict(setup_data['carbon_v1_event_create_for_update']), {}) strat_id = setup_data['carbon_v1_event_update']['args']['id'] assert (strat_id == carbon_v1_pool.state['strategy_id']) assert (carbon_v1_pool.state['y_0'] == 0) @@ -153,7 +154,7 @@ def test_test_carbon_v1_pool_update(): assert (carbon_v1_pool.state['z_1'] == 0) assert (carbon_v1_pool.state['A_1'] == 0) assert (carbon_v1_pool.state['B_1'] == 0) - carbon_v1_pool.update_from_event(setup_data['carbon_v1_event_update'], {}) + carbon_v1_pool.update_from_event(Event.from_dict(setup_data['carbon_v1_event_update']), {}) assert (carbon_v1_pool.state['y_0'] == setup_data['carbon_v1_event_update']['args']['order0'][0]) assert (carbon_v1_pool.state['z_0'] == setup_data['carbon_v1_event_update']['args']['order0'][1]) assert (carbon_v1_pool.state['A_0'] == setup_data['carbon_v1_event_update']['args']['order0'][2]) @@ -172,7 +173,7 @@ def test_test_carbon_v1_pool_delete(): # ------------------------------------------------------------ carbon_v1_pool = CarbonV1Pool() - carbon_v1_pool.update_from_event(setup_data['carbon_v1_event_create_for_delete'], {}) + carbon_v1_pool.update_from_event(Event.from_dict(setup_data['carbon_v1_event_create_for_delete']), {}) strat_id = setup_data['carbon_v1_event_update']['args']['id'] assert (strat_id == carbon_v1_pool.state['strategy_id']) assert (carbon_v1_pool.state['y_0'] == setup_data['carbon_v1_event_delete']['args']['order0'][0]) @@ -183,7 +184,7 @@ def test_test_carbon_v1_pool_delete(): assert (carbon_v1_pool.state['z_1'] == setup_data['carbon_v1_event_delete']['args']['order1'][1]) assert (carbon_v1_pool.state['A_1'] == setup_data['carbon_v1_event_delete']['args']['order1'][2]) assert (carbon_v1_pool.state['B_1'] == setup_data['carbon_v1_event_delete']['args']['order1'][3]) - carbon_v1_pool.update_from_event(setup_data['carbon_v1_event_delete'], {}) + carbon_v1_pool.update_from_event(Event.from_dict(setup_data['carbon_v1_event_delete']), {}) assert (carbon_v1_pool.state['y_0'] == 0) assert (carbon_v1_pool.state['z_0'] == 0) assert (carbon_v1_pool.state['A_0'] == 0) @@ -206,7 +207,7 @@ def test_test_bancor_pol_token_traded_event(): bancor_pol_pool = BancorPolPool() bancor_pol_pool.state['tkn0_address'] = setup_data['bancor_pol_token_traded_event']['args']['token'] bancor_pol_pool.state['tkn0_balance'] = 10 + setup_data['bancor_pol_token_traded_event']['args']['amount'] - bancor_pol_pool.update_from_event(setup_data['bancor_pol_token_traded_event'], + bancor_pol_pool.update_from_event(Event.from_dict(setup_data['bancor_pol_token_traded_event']), {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, @@ -230,7 +231,7 @@ def test_test_bancor_pol_trading_enabled_event(): bancor_pol_pool = BancorPolPool() bancor_pol_pool.state['tkn0_address'] = None - bancor_pol_pool.update_from_event(setup_data['bancor_pol_trading_enabled_event'], + bancor_pol_pool.update_from_event(Event.from_dict(setup_data['bancor_pol_trading_enabled_event']), {'cid': '0x', 'fee': '0.000', 'fee_float': 0.0, diff --git a/fastlane_bot/tests/test_035_Utils.py b/fastlane_bot/tests/test_035_Utils.py index 712c0e53b..6f7ab23a1 100644 --- a/fastlane_bot/tests/test_035_Utils.py +++ b/fastlane_bot/tests/test_035_Utils.py @@ -12,6 +12,7 @@ from web3.types import HexBytes from fastlane_bot import Bot +from fastlane_bot.events.interfaces.event import Event from fastlane_bot.events.pools import UniswapV2Pool, UniswapV3Pool, BancorV3Pool, CarbonV1Pool from fastlane_bot.events.utils import filter_latest_events, complex_handler from fastlane_bot.tools.cpc import ConstantProductCurve as CPC @@ -47,10 +48,10 @@ def exchange_name_from_event(self, event): return 'uniswap_v2' mocked_mgr = MockManager() -event1 = AttributeDict({'args': {'reserve0': 100, 'reserve1': 100}, 'event': 'Sync', 'address': '0xabc', 'blockNumber': 5, 'transactionIndex': 0, 'logIndex': 0}) -event2 = AttributeDict({'args': {'reserve0': 200, 'reserve1': 200}, 'event': 'Sync', 'address': '0xabc', 'blockNumber': 10, 'transactionIndex': 1, 'logIndex': 1}) -event3 = AttributeDict({'args': {'reserve0': 300, 'reserve1': 300}, 'event': 'Sync', 'address': '0xdef', 'blockNumber': 7, 'transactionIndex': 1, 'logIndex': 1}) -mock_events = [[event1, event2, event3]] +event1 = Event.from_dict({'args': {'reserve0': 100, 'reserve1': 100}, 'event': 'Sync', 'address': '0xabc', 'blockNumber': 5, 'transactionIndex': 0, 'logIndex': 0, 'transactionHash': '', 'blockHash': ''}) +event2 = Event.from_dict({'args': {'reserve0': 200, 'reserve1': 200}, 'event': 'Sync', 'address': '0xabc', 'blockNumber': 10, 'transactionIndex': 1, 'logIndex': 1, 'transactionHash': '', 'blockHash': ''}) +event3 = Event.from_dict({'args': {'reserve0': 300, 'reserve1': 300}, 'event': 'Sync', 'address': '0xdef', 'blockNumber': 7, 'transactionIndex': 1, 'logIndex': 1, 'transactionHash': '', 'blockHash': ''}) +mock_events = [event1, event2, event3] # ------------------------------------------------------------ @@ -62,9 +63,9 @@ def test_test_filter_latest_events(): # ------------------------------------------------------------ result = filter_latest_events(mocked_mgr, mock_events) - assert (len(result) == len({event['address'] for event in result})) - pool_address = result[0]['address'] - pool_events = [event for event in mock_events[0] if (event['address'] == pool_address)] + assert (len(result) == len({event.address for event in result})) + pool_address = result[0].address + pool_events = [event for event in mock_events if (event.address == pool_address)] # ------------------------------------------------------------ diff --git a/fastlane_bot/tests/test_036_Manager.py b/fastlane_bot/tests/test_036_Manager.py index 8d5ae2772..511c76ff9 100644 --- a/fastlane_bot/tests/test_036_Manager.py +++ b/fastlane_bot/tests/test_036_Manager.py @@ -14,6 +14,7 @@ from unittest.mock import MagicMock from fastlane_bot import Bot, Config +from fastlane_bot.events.interfaces.event import Event from fastlane_bot.events.exchanges import UniswapV2, UniswapV3, CarbonV1, BancorV3 from fastlane_bot.events.managers.manager import Manager from fastlane_bot.events.pools.utils import get_pool_cid @@ -85,7 +86,7 @@ def test_test_update_from_event_uniswap_v2(): assert event['args']['reserve0'] != [pool['tkn0_balance'] for pool in manager.pool_data if pool['address'] == event['address']][0] - manager.update_from_event(event) + manager.update_from_event(Event.from_dict(event)) assert event['address'] in [pool['address'] for pool in manager.pool_data] assert event['args']['reserve0'] == [pool['tkn0_balance'] for pool in manager.pool_data if pool['address'] == event['address']][0] @@ -105,7 +106,7 @@ def test_test_update_from_event_uniswap_v3(): assert event['args']['liquidity'] != [pool['liquidity'] for pool in manager.pool_data if pool['address'] == event['address']][0] - manager.update_from_event(event) + manager.update_from_event(Event.from_dict(event)) assert event['address'] in [pool['address'] for pool in manager.pool_data] assert event['args']['liquidity'] == [pool['liquidity'] for pool in manager.pool_data if pool['address'] == event['address']][0] @@ -126,8 +127,8 @@ def test_test_update_from_event_carbon_v1_update(): event_create_for_update = event_data['carbon_v1_event_create_for_update'] event = event_data['carbon_v1_event_update'] - manager.update_from_event(event_create_for_update) - pools_to_add_from_contracts = [event[2]['args']['id'] for event in manager.pools_to_add_from_contracts] + manager.update_from_event(Event.from_dict(event_create_for_update)) + pools_to_add_from_contracts = [event[2].args['id'] for event in manager.pools_to_add_from_contracts] assert event['args']['id'] in pools_to_add_from_contracts @@ -148,7 +149,7 @@ def test_test_update_from_event_carbon_v1_create(): manager.pool_data = [pool for pool in manager.pool_data if pool['cid'] != event['args']['id']] assert event['args']['id'] not in [pool['cid'] for pool in manager.pool_data] - manager.update_from_event(event) + manager.update_from_event(Event.from_dict(event)) assert event['args']['id'] not in [pool['cid'] for pool in manager.pool_data] # - @@ -171,7 +172,7 @@ def test_test_update_from_event_carbon_v1_delete(): manager.pool_data = [pool for pool in manager.pool_data if pool['cid'] != cid] assert cid not in [pool['cid'] for pool in manager.pool_data] - manager.update_from_event(event) + manager.update_from_event(Event.from_dict(event)) assert cid in [p[-1] for p in manager.pools_to_add_from_contracts] fake_pool_data = { "address": event['address'], @@ -183,5 +184,5 @@ def test_test_update_from_event_carbon_v1_delete(): manager.pool_data.append(fake_pool_data) event['event'] = 'StrategyDeleted' - manager.update_from_event(event) + manager.update_from_event(Event.from_dict(event)) assert cid not in [pool['cid'] for pool in manager.pool_data] \ No newline at end of file diff --git a/fastlane_bot/tests/test_037_Exchanges.py b/fastlane_bot/tests/test_037_Exchanges.py index 81a51b5ea..27f1ad82e 100644 --- a/fastlane_bot/tests/test_037_Exchanges.py +++ b/fastlane_bot/tests/test_037_Exchanges.py @@ -11,6 +11,7 @@ import json from fastlane_bot import Bot +from fastlane_bot.events.interfaces.event import Event from fastlane_bot.events.exchanges.balancer import Balancer from fastlane_bot.tools.cpc import ConstantProductCurve as CPC from fastlane_bot.events.exchanges import UniswapV2, UniswapV3, CarbonV1, BancorV3, BancorV2, BancorPol, SolidlyV2 @@ -261,7 +262,7 @@ def test_test_bancor_v2_exchange(): async def test_bancor_v2_exchange(): assert (bancor_v2_exchange.get_abi() == BANCOR_V2_CONVERTER_ABI) assert (await bancor_v2_exchange.get_fee('', mocked_contract) == (3000, 0.003)) - assert (await bancor_v2_exchange.get_tkn0('', mocked_contract, setup_data['bancor_v2_event']) == '0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE') + assert (await bancor_v2_exchange.get_tkn0('', mocked_contract, Event.from_dict(setup_data['bancor_v2_event'])) == '0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE') # Run the test in an event loop asyncio.run(test_bancor_v2_exchange()) @@ -283,7 +284,7 @@ def test_test_carbon_v1_exchange_update(): async def test_carbon_v1_exchange_update(): assert (carbon_v1_exchange.get_abi() == CARBON_CONTROLLER_ABI) assert (await carbon_v1_exchange.get_fee('', mocked_contract) == ('2000', 0.002)) - assert (await carbon_v1_exchange.get_tkn0('', mocked_contract, setup_data['carbon_v1_event_update']) == setup_data['carbon_v1_event_update']['args']['token0']) + assert (await carbon_v1_exchange.get_tkn0('', mocked_contract, Event.from_dict(setup_data['carbon_v1_event_update'])) == setup_data['carbon_v1_event_update']['args']['token0']) # Run the test in an event loop asyncio.run(test_carbon_v1_exchange_update()) @@ -305,7 +306,7 @@ def test_test_carbon_v1_exchange_create(): async def test_carbon_v1_exchange_create(): assert (carbon_v1_exchange.get_abi() == CARBON_CONTROLLER_ABI) assert (await carbon_v1_exchange.get_fee('', mocked_contract) == ('2000', 0.002)) - assert (await carbon_v1_exchange.get_tkn0('', mocked_contract, setup_data['carbon_v1_event_create']) == setup_data['carbon_v1_event_create']['args']['token0']) + assert (await carbon_v1_exchange.get_tkn0('', mocked_contract, Event.from_dict(setup_data['carbon_v1_event_create'])) == setup_data['carbon_v1_event_create']['args']['token0']) # Run the test in an event loop asyncio.run(test_carbon_v1_exchange_create()) @@ -333,7 +334,7 @@ def test_test_carbon_v1_exchange_delete(): async def test_bancor_pol_exchange(): assert (bancor_pol_exchange.get_abi() == BANCOR_POL_ABI) assert (await bancor_pol_exchange.get_fee('', mocked_contract) == ('0.000', 0.0)) - assert (await bancor_pol_exchange.get_tkn0('', mocked_contract, setup_data['bancor_pol_trading_enabled_event']) == "0x86772b1409b61c639EaAc9Ba0AcfBb6E238e5F83") + assert (await bancor_pol_exchange.get_tkn0('', mocked_contract, Event.from_dict(setup_data['bancor_pol_trading_enabled_event'])) == "0x86772b1409b61c639EaAc9Ba0AcfBb6E238e5F83") # Run the test in an event loop asyncio.run(test_bancor_pol_exchange()) \ No newline at end of file From 6b20b6a06e02afb85bfade3c65e817a0ba44a1a7 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Fri, 10 May 2024 14:33:03 +0300 Subject: [PATCH 11/58] fix: tests --- fastlane_bot/events/interfaces/event.py | 26 +++++++++---------- .../tests/test_049_CustomTradingFees.py | 5 ++-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/fastlane_bot/events/interfaces/event.py b/fastlane_bot/events/interfaces/event.py index 98bb5c2fc..5687f3079 100644 --- a/fastlane_bot/events/interfaces/event.py +++ b/fastlane_bot/events/interfaces/event.py @@ -1,27 +1,27 @@ from dataclasses import dataclass -from typing import Any, Dict +from typing import Any, Dict, Optional @dataclass class Event: args: Dict[str, Any] event: str - log_index: int - transaction_index: int - transaction_hash: str - address: str - block_hash: str - block_number: int + log_index: Optional[int] + transaction_index: Optional[int] + transaction_hash: Optional[str] + address: Optional[str] + block_hash: Optional[str] + block_number: Optional[int] @classmethod def from_dict(cls, data): return cls( args=data["args"], event=data["event"], - log_index=data["logIndex"], - transaction_index=data["transactionIndex"], - transaction_hash=data["transactionHash"], - address=data["address"], - block_hash=data["blockHash"], - block_number=data["blockNumber"], + log_index=data.get("logIndex", None), + transaction_index=data.get("transactionIndex", None), + transaction_hash=data.get("transactionHash", None), + address=data.get("address", None), + block_hash=data.get("blockHash", None), + block_number=data.get("blockNumber", None), ) diff --git a/fastlane_bot/tests/test_049_CustomTradingFees.py b/fastlane_bot/tests/test_049_CustomTradingFees.py index e6b2be248..c759db5df 100644 --- a/fastlane_bot/tests/test_049_CustomTradingFees.py +++ b/fastlane_bot/tests/test_049_CustomTradingFees.py @@ -14,6 +14,7 @@ from unittest.mock import MagicMock from fastlane_bot import Bot, Config +from fastlane_bot.events.interfaces.event import Event from fastlane_bot.events.exchanges import UniswapV2, UniswapV3, CarbonV1, BancorV3 from fastlane_bot.events.managers.manager import Manager Base = None @@ -77,7 +78,7 @@ def test_test_update_from_event_carbon_v1_pair_create(): assert (event['args']['token0'], event['args']['token1']) not in manager.fee_pairs[exchange_name] - manager.update_from_event(event) + manager.update_from_event(Event.from_dict(event)) assert (event['args']['token0'], event['args']['token1']) in manager.fee_pairs[exchange_name] @@ -112,7 +113,7 @@ async def test_update_from_event_carbon_v1_trading_fee_updated(): # find all pools with fee==prevFeePPM prev_default_pools = [idx for idx, pool in enumerate(manager.pool_data) if pool['fee'] == prevFeePPM] - manager.update_from_event(event) + manager.update_from_event(Event.from_dict(event)) for idx in prev_default_pools: assert manager.pool_data[idx]['fee'] == newFeePPM From 66654319bb83b3ba950cb068dc91fe8d061859d7 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Fri, 10 May 2024 14:57:22 +0300 Subject: [PATCH 12/58] fix: tests --- fastlane_bot/events/event_gatherer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane_bot/events/event_gatherer.py b/fastlane_bot/events/event_gatherer.py index 989793716..29eb78cc6 100644 --- a/fastlane_bot/events/event_gatherer.py +++ b/fastlane_bot/events/event_gatherer.py @@ -36,7 +36,7 @@ def __init__( for exchange_name, exchange in exchanges.items(): subscriptions = exchange.get_subscriptions(event_contracts[exchange_name]) for sub in subscriptions: - if sub.get_topic not in unique_topics: + if sub.topic not in unique_topics: unique_topics.add(sub.topic) self._subscriptions.append(sub) From 13aa6a1535c1a44e3f3bb7acef6fb7d92d9acdc0 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Fri, 10 May 2024 15:09:38 +0300 Subject: [PATCH 13/58] fix: tests --- fastlane_bot/events/pools/bancor_pol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane_bot/events/pools/bancor_pol.py b/fastlane_bot/events/pools/bancor_pol.py index d0f1a6065..15091e159 100644 --- a/fastlane_bot/events/pools/bancor_pol.py +++ b/fastlane_bot/events/pools/bancor_pol.py @@ -58,7 +58,7 @@ def event_matches_format( True if the event matches the format of a Bancor v3 event, False otherwise. """ - event_args = event["args"] + event_args = event.args return ("token" in event_args) and ("token0" not in event_args) and (event_args['token'] == "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE") def update_from_event( From 5364213a3d2d9502e0ec73cc8932e40250b838c2 Mon Sep 17 00:00:00 2001 From: Lesigh-3100 Date: Sun, 5 May 2024 20:16:20 +0300 Subject: [PATCH 14/58] Pairfinder Automatically searches for Carbon pairs or triangles (depending on mode) and finds pools to create viable routes for them. --- fastlane_bot/config/constants.py | 3 + fastlane_bot/data/abi.py | 67 +++++ fastlane_bot/events/exchanges/balancer.py | 8 +- fastlane_bot/events/exchanges/bancor_pol.py | 6 + fastlane_bot/events/exchanges/bancor_v2.py | 6 + fastlane_bot/events/exchanges/bancor_v3.py | 5 + fastlane_bot/events/exchanges/base.py | 9 + fastlane_bot/events/exchanges/carbon_v1.py | 5 + fastlane_bot/events/exchanges/solidly_v2.py | 79 +++++- fastlane_bot/events/exchanges/uniswap_v2.py | 14 +- fastlane_bot/events/exchanges/uniswap_v3.py | 15 +- fastlane_bot/events/managers/base.py | 19 +- fastlane_bot/events/pools/base.py | 2 +- fastlane_bot/helpers/poolandtokens.py | 38 +-- fastlane_bot/pool_finder.py | 233 ++++++++++++++++++ fastlane_bot/tests/test_073_TestPoolFinder.py | 96 ++++++++ main.py | 19 +- 17 files changed, 589 insertions(+), 35 deletions(-) create mode 100644 fastlane_bot/pool_finder.py create mode 100644 fastlane_bot/tests/test_073_TestPoolFinder.py diff --git a/fastlane_bot/config/constants.py b/fastlane_bot/config/constants.py index 0c6357ce0..ad6641ac0 100644 --- a/fastlane_bot/config/constants.py +++ b/fastlane_bot/config/constants.py @@ -19,6 +19,8 @@ } ETHEREUM = "ethereum" +UNISWAP_V2_NAME = "uniswap_v2" +UNISWAP_V3_NAME = "uniswap_v3" PANCAKESWAP_V2_NAME = "pancakeswap_v2" PANCAKESWAP_V3_NAME = "pancakeswap_v3" BUTTER_V3_NAME = "butter_v3" @@ -31,3 +33,4 @@ ECHODEX_V3_NAME = "echodex_v3" SECTA_V3_NAME = "secta_v3" METAVAULT_V3_NAME = "metavault_v3" +ZERO_ADDRESS = "0x0000000000000000000000000000000000000000" diff --git a/fastlane_bot/data/abi.py b/fastlane_bot/data/abi.py index ec6f102f4..daaa04a03 100644 --- a/fastlane_bot/data/abi.py +++ b/fastlane_bot/data/abi.py @@ -164,6 +164,15 @@ "name": "PairCreated", "anonymous": False, "inputs": [{"indexed": True, "internalType": "address", "name": "token0", "type": "address"}, {"indexed": True, "internalType": "address", "name": "token1", "type": "address"}, {"indexed": False, "internalType": "address", "name": "pair", "type": "address"}, {"indexed": False, "internalType": "uint256", "name": "", "type": "uint256"}] + }, + { + "type": "function", + "name": "getPair", + "stateMutability": "view", + "constant": True, + "inputs": [{"internalType": "address", "name": "", "type": "address"}, {"internalType": "address", "name": "", "type": "address"}], + "outputs": [{"internalType": "address", "name": "", "type": "address"}], + "payable": False, } ] @@ -173,6 +182,13 @@ "name": "PoolCreated", "anonymous": False, "inputs": [{"indexed": True, "internalType": "address", "name": "token0", "type": "address"}, {"indexed": True, "internalType": "address", "name": "token1", "type": "address"}, {"indexed": True, "internalType": "uint24", "name": "fee", "type": "uint24"}, {"indexed": False, "internalType": "int24", "name": "tickSpacing", "type": "int24"}, {"indexed": False, "internalType": "address", "name": "pool", "type": "address"}] + }, + { + "type": "function", + "name": "getPool", + "stateMutability": "view", + "inputs": [{"internalType": "address", "name": "", "type": "address"}, {"internalType": "address", "name": "", "type": "address"}, {"internalType": "uint24", "name": "", "type": "uint24"}], + "outputs": [{"internalType": "address", "name": "", "type": "address"}], } ] @@ -189,6 +205,13 @@ "stateMutability": "view", "inputs": [{"internalType": "address", "name": "pool", "type": "address"}, {"internalType": "bool", "name": "_stable", "type": "bool"}], "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}] + }, + { + "name": "getPool", + "stateMutability": "view", + "type": "function", + "inputs": [{"internalType": "address", "name": "tokenA", "type": "address"}, {"internalType": "address", "name": "tokenB", "type": "address"}, {"internalType": "bool", "name": "stable", "type": "bool"}], + "outputs": [{"internalType": "address", "name": "", "type": "address"}], } ] @@ -205,6 +228,13 @@ "stateMutability": "view", "inputs": [{"internalType": "address", "name": "_pair", "type": "address"}], "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}] + }, + { + "type": "function", + "name": "getPair", + "stateMutability": "view", + "inputs": [{"internalType": "address", "name": "", "type": "address"}, {"internalType": "address", "name": "", "type": "address"}, {"internalType": "bool", "name": "", "type": "bool"}], + "outputs": [{"internalType": "address", "name": "", "type": "address"}] } ] @@ -221,6 +251,15 @@ "stateMutability": "view", "inputs": [{"internalType": "address", "name": "_pair", "type": "address"}], "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}] + }, + { + "type": "function", + "name": "getPair", + "stateMutability": "view", + "inputs": [{"internalType": "address", "name": "", "type": "address"}, + {"internalType": "address", "name": "", "type": "address"}, + {"internalType": "bool", "name": "", "type": "bool"}], + "outputs": [{"internalType": "address", "name": "", "type": "address"}] } ] @@ -237,6 +276,13 @@ "stateMutability": "view", "inputs": [{"type": "address", "name": "_pair", "internalType": "address"}, {"type": "bool", "name": "_stable", "internalType": "bool"}], "outputs": [{"type": "uint256", "name": "", "internalType": "uint256"}] + }, + { + "type": "function", + "name": "getPair", + "stateMutability": "view", + "inputs": [{"internalType": "address", "name": "", "type": "address"}, {"internalType": "address", "name": "", "type": "address"}, {"internalType": "bool", "name": "", "type": "bool"}], + "outputs": [{"internalType": "address", "name": "", "type": "address"}] } ] @@ -253,6 +299,13 @@ "stateMutability": "view", "inputs": [{"internalType": "bool", "name": "_stable", "type": "bool"}], "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}] + }, + { + "type": "function", + "name": "getPair", + "stateMutability": "view", + "inputs": [{"internalType": "address", "name": "", "type": "address"}, {"internalType": "address", "name": "", "type": "address"}, {"internalType": "bool", "name": "", "type": "bool"}], + "outputs": [{"internalType": "address", "name": "", "type": "address"}] } ] @@ -269,6 +322,13 @@ "stateMutability": "view", "inputs": [{"internalType": "address", "name": "_pool", "type": "address"}], "outputs": [{"internalType": "uint256", "name": "fee", "type": "uint256"}] + }, + { + "type": "function", + "name": "getPair", + "stateMutability": "view", + "inputs": [{"internalType": "address", "name": "", "type": "address"}, {"internalType": "address", "name": "", "type": "address"}, {"internalType": "bool", "name": "", "type": "bool"}], + "outputs": [{"internalType": "address", "name": "", "type": "address"}] } ] @@ -285,6 +345,13 @@ "stateMutability": "view", "inputs": [], "outputs": [{"internalType": "address", "name": "", "type": "address"}] + }, + { + "type": "function", + "name": "getPool", + "stateMutability": "view", + "inputs": [{"internalType": "address", "name": "_token", "type": "address"}], + "outputs": [{"internalType": "address", "name": "pool", "type": "address"}], } ] diff --git a/fastlane_bot/events/exchanges/balancer.py b/fastlane_bot/events/exchanges/balancer.py index 791180f0c..3481d5fe1 100644 --- a/fastlane_bot/events/exchanges/balancer.py +++ b/fastlane_bot/events/exchanges/balancer.py @@ -87,4 +87,10 @@ async def get_tkn_n(self, address: str, contract: Contract, event: Any, index: i pool_balances = await contract.caller.getPoolTokens(address) tokens = pool_balances[0] token_balances = pool_balances[1] - return token_balances[index] \ No newline at end of file + return token_balances[index] + + def get_pool_function(self, factory_contract: Contract): + """ + This function is unused for Carbon. + """ + pass \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/bancor_pol.py b/fastlane_bot/events/exchanges/bancor_pol.py index 4e40c7397..7d0075f80 100644 --- a/fastlane_bot/events/exchanges/bancor_pol.py +++ b/fastlane_bot/events/exchanges/bancor_pol.py @@ -63,6 +63,12 @@ async def get_tkn0(self, address: str, contract: Contract, event: Event) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: return self.ETH_ADDRESS if event.args["token"] not in self.ETH_ADDRESS else self.BNT_ADDRESS + def get_pool_function(self, factory_contract: Contract): + """ + This function is unused for Bancor POL. + """ + pass + def save_strategy( self, token: str, diff --git a/fastlane_bot/events/exchanges/bancor_v2.py b/fastlane_bot/events/exchanges/bancor_v2.py index d1b4b149e..ab23a3b86 100644 --- a/fastlane_bot/events/exchanges/bancor_v2.py +++ b/fastlane_bot/events/exchanges/bancor_v2.py @@ -77,3 +77,9 @@ async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: async def get_anchor(self, contract: Contract) -> str: return await contract.caller.anchor() + + def get_pool_function(self, factory_contract: Contract): + """ + This function is unused for Bancor V2. + """ + pass \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/bancor_v3.py b/fastlane_bot/events/exchanges/bancor_v3.py index da2c76733..53c584ef3 100644 --- a/fastlane_bot/events/exchanges/bancor_v3.py +++ b/fastlane_bot/events/exchanges/bancor_v3.py @@ -64,3 +64,8 @@ async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: if event.args["pool"] != self.BNT_ADDRESS else event.args["tkn_address"] ) + def get_pool_function(self, factory_contract: Contract): + """ + This function is unused for Bancor V3. + """ + pass \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/base.py b/fastlane_bot/events/exchanges/base.py index ed641d974..fd13105d2 100644 --- a/fastlane_bot/events/exchanges/base.py +++ b/fastlane_bot/events/exchanges/base.py @@ -30,6 +30,7 @@ class Exchange(ABC): exchange_name: str base_exchange_name: str = '' pools: Dict[str, Pool] = field(default_factory=dict) + sync_factory_contract: Contract = None __VERSION__ = "0.0.3" __DATE__ = "2024-03-20" @@ -122,6 +123,14 @@ async def get_fee(address: str, contract: AsyncContract) -> float: """ pass + @staticmethod + @abstractmethod + def get_pool_function(contract: Contract, *args): + """ + Returns the Factory contract function used to fetch liquidity pools. + """ + pass + @staticmethod @abstractmethod async def get_tkn0(address: str, contract: AsyncContract, event: Any) -> str: diff --git a/fastlane_bot/events/exchanges/carbon_v1.py b/fastlane_bot/events/exchanges/carbon_v1.py index 2957e04cf..e95416c6e 100644 --- a/fastlane_bot/events/exchanges/carbon_v1.py +++ b/fastlane_bot/events/exchanges/carbon_v1.py @@ -245,3 +245,8 @@ def save_strategy( block_number=block_number, ) + def get_pool_function(self, factory_contract: Contract): + """ + This function is unused for Carbon. + """ + pass \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/solidly_v2.py b/fastlane_bot/events/exchanges/solidly_v2.py index d6f5aa02a..5d3951c11 100644 --- a/fastlane_bot/events/exchanges/solidly_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2.py @@ -59,17 +59,70 @@ async def _get_tkn0_B(contract: Contract) -> str: async def _get_tkn1_B(contract: Contract) -> str: return "0xe5D7C2a44FfDDf6b295A15c148167daaAf5Cf34f" # TODO Use the constant WRAPPED_GAS_TOKEN_ADDRESS for this network + +def _get_pool_function_1(factory_contract): + """ Function to get pools from Factory. + This function is intended to be used with a Multicall. It fetches pools from a Solidly fork Factory contract. + + Args: + factory_contract: The factory contract. + + Returns: + The function. + + """ + return factory_contract.functions.getPair +def _get_pool_function_2(factory_contract): + """ Function to get pools from Factory. + This function is intended to be used with a Multicall. It fetches pools from a Solidly fork Factory contract. + + Args: + factory_contract: The factory contract. + + Returns: + The function. + + """ + return factory_contract.functions.getPool + + +def _get_pool_args_1(tkn0, tkn1, stable): + """ Function to manage args input to get pools from Factory. + + Args: + tkn0: The first token address. + tkn1: The second token address. + stable: (bool) If True, indicates a stable pool. If False, indicates a Volatile pool. + Returns: + The function returns the arguments necessary to get pool addresses from the factory contract. + + """ + return tkn0, tkn1, stable + +def _get_pool_args_2(tkn0, tkn1, stable): + """ Function to manage args input to get pools from Factory. + + Args: + tkn0: The first token address. + tkn1: The second token address. + stable: (bool) If True, indicates a stable pool. If False, indicates a Volatile pool. + Returns: + The function returns the arguments necessary to get pool addresses from the factory contract. + + """ + return tkn0 + EXCHANGE_INFO = { - "velocimeter_v2": {"decimals": 4, "factory_abi": VELOCIMETER_V2_FACTORY_ABI, "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_1, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A}, - "equalizer_v2" : {"decimals": 4, "factory_abi": SCALE_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_2, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A}, - "aerodrome_v2" : {"decimals": 4, "factory_abi": SOLIDLY_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_3, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A}, - "velodrome_v2" : {"decimals": 4, "factory_abi": SOLIDLY_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_3, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A}, - "scale_v2" : {"decimals": 18, "factory_abi": SCALE_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_2, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A}, - "cleopatra_v2" : {"decimals": 4, "factory_abi": CLEOPATRA_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_4, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A}, - "stratum_v2" : {"decimals": 4, "factory_abi": VELOCIMETER_V2_FACTORY_ABI, "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_1, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A}, - "lynex_v2" : {"decimals": 4, "factory_abi": LYNEX_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_5, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A}, - "nile_v2" : {"decimals": 4, "factory_abi": NILE_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_6, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A}, - "xfai_v0" : {"decimals": 4, "factory_abi": XFAI_V0_FACTORY_ABI , "pool_abi": XFAI_V0_POOL_ABI , "get_fee": _get_fee_7, "get_tkn0": _get_tkn0_B, "get_tkn1": _get_tkn1_B}, + "velocimeter_v2": {"decimals": 4, "factory_abi": VELOCIMETER_V2_FACTORY_ABI, "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_1, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, + "equalizer_v2" : {"decimals": 4, "factory_abi": SCALE_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_2, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, + "aerodrome_v2" : {"decimals": 4, "factory_abi": SOLIDLY_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_3, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_2, "get_pool_args": _get_pool_args_1}, + "velodrome_v2" : {"decimals": 4, "factory_abi": SOLIDLY_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_3, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_2, "get_pool_args": _get_pool_args_1}, + "scale_v2" : {"decimals": 18, "factory_abi": SCALE_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_2, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, + "cleopatra_v2" : {"decimals": 4, "factory_abi": CLEOPATRA_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_4, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, + "stratum_v2" : {"decimals": 4, "factory_abi": VELOCIMETER_V2_FACTORY_ABI, "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_1, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, + "lynex_v2" : {"decimals": 4, "factory_abi": LYNEX_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_5, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, + "nile_v2" : {"decimals": 4, "factory_abi": NILE_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_6, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, + "xfai_v0" : {"decimals": 4, "factory_abi": XFAI_V0_FACTORY_ABI , "pool_abi": XFAI_V0_POOL_ABI , "get_fee": _get_fee_7, "get_tkn0": _get_tkn0_B, "get_tkn1": _get_tkn1_B, "get_pool_function": _get_pool_function_2, "get_pool_args": _get_pool_args_2}, } @dataclass @@ -120,3 +173,9 @@ async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return await EXCHANGE_INFO[self.exchange_name]["get_tkn1"](contract) + + def get_pool_function(self, factory_contract): + return EXCHANGE_INFO[self.exchange_name]["get_pool_function"](factory_contract) + + def get_pool_args(self, tkn0, tkn1, stable): + return EXCHANGE_INFO[self.exchange_name]["get_pool_args"](tkn0, tkn1, stable) diff --git a/fastlane_bot/events/exchanges/uniswap_v2.py b/fastlane_bot/events/exchanges/uniswap_v2.py index 870b938bf..b08e0ef27 100644 --- a/fastlane_bot/events/exchanges/uniswap_v2.py +++ b/fastlane_bot/events/exchanges/uniswap_v2.py @@ -32,7 +32,6 @@ class UniswapV2(Exchange): fee: str = None router_address: str = None exchange_initialized: bool = False - @property def fee_float(self): return float(self.fee) @@ -63,3 +62,16 @@ async def get_tkn0(address: str, contract: AsyncContract, event: Any) -> str: @staticmethod async def get_tkn1(address: str, contract: AsyncContract, event: Any) -> str: return await contract.caller.token1() + + def get_pool_function(self, factory_contract: Contract): + """ Function to get pools from Factory. + This function is intended to be used with a Multicall. It fetches pools from a Uniswap V2 fork Factory contract. + + Args: + factory_contract: The factory contract. + + Returns: + The function. + + """ + return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/uniswap_v3.py b/fastlane_bot/events/exchanges/uniswap_v3.py index 2136a32a2..0eb8fb6c4 100644 --- a/fastlane_bot/events/exchanges/uniswap_v3.py +++ b/fastlane_bot/events/exchanges/uniswap_v3.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from typing import List, Type, Tuple, Any -from web3.contract import Contract +from web3.contract import Contract, AsyncContract from fastlane_bot.config.constants import AGNI_V3_NAME, PANCAKESWAP_V3_NAME, FUSIONX_V3_NAME, ECHODEX_V3_NAME, SECTA_V3_NAME from fastlane_bot.data.abi import UNISWAP_V3_POOL_ABI, UNISWAP_V3_FACTORY_ABI, PANCAKESWAP_V3_POOL_ABI @@ -59,3 +59,16 @@ async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return await contract.caller.token1() + + def get_pool_function(self, factory_contract: Contract): + """ Function to get pools from Factory. + This function is intended to be used with a Multicall. It fetches pools from a Uniswap V3 fork Factory contract. + + Args: + factory_contract: The factory contract. + + Returns: + The function. + + """ + return factory_contract.functions.getPool diff --git a/fastlane_bot/events/managers/base.py b/fastlane_bot/events/managers/base.py index cf85d906a..6600c9304 100644 --- a/fastlane_bot/events/managers/base.py +++ b/fastlane_bot/events/managers/base.py @@ -17,7 +17,7 @@ from fastlane_bot import Config from fastlane_bot.config.constants import PANCAKESWAP_V2_NAME, PANCAKESWAP_V3_NAME, VELOCIMETER_V2_NAME, AGNI_V3_NAME, \ - SOLIDLY_V2_NAME, FUSIONX_V3_NAME + SOLIDLY_V2_NAME, FUSIONX_V3_NAME, UNISWAP_V2_NAME, UNISWAP_V3_NAME from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.events.exchanges import exchange_factory from fastlane_bot.events.exchanges.base import Exchange @@ -78,6 +78,7 @@ class BaseManager: token_contracts: Dict[str, Contract or Type[Contract]] = field(default_factory=dict) erc20_contracts: Dict[str, Contract or Type[Contract]] = field(default_factory=dict) exchanges: Dict[str, Exchange] = field(default_factory=dict) + sync_factory_contracts: Dict[str, Contract or Type[Contract]] = field(default_factory=dict) factory_contracts: Dict[str, Contract or Type[Contract]] = field(default_factory=dict) uniswap_v2_event_mappings: Dict[str, str] = field(default_factory=dict) uniswap_v3_event_mappings: Dict[str, str] = field(default_factory=dict) @@ -129,8 +130,10 @@ def __post_init__(self): self.SUPPORTED_BASE_EXCHANGES.append(base_exchange_name) self.exchanges[exchange_name] = exchange_factory.get_exchange(key=exchange_name, cfg=self.cfg, exchange_initialized=initialize_events) - if base_exchange_name in SOLIDLY_V2_NAME: + if base_exchange_name == SOLIDLY_V2_NAME: self.exchanges[exchange_name] = self.handle_solidly_exchanges(exchange=self.exchanges[exchange_name]) + if base_exchange_name in [SOLIDLY_V2_NAME, UNISWAP_V2_NAME, UNISWAP_V3_NAME]: + self.exchanges[exchange_name] = self.initialize_factory_contract(exchange=self.exchanges[exchange_name]) self.init_exchange_contracts() self.set_carbon_v1_fee_pairs() @@ -149,6 +152,18 @@ def handle_solidly_exchanges(self, exchange): return exchange + def initialize_factory_contract(self, exchange): + """ + Initialize factory contract for exchange. + """ + exchange_name = exchange.exchange_name + self.sync_factory_contracts[exchange_name] = self.web3.eth.contract( + address=self.cfg.FACTORY_MAPPING[exchange_name], + abi=exchange.get_factory_abi, + ) + exchange.sync_factory_contract = self.sync_factory_contracts[exchange.exchange_name] + + return exchange @property def fee_pairs(self) -> Dict: """ diff --git a/fastlane_bot/events/pools/base.py b/fastlane_bot/events/pools/base.py index 4aa92999a..a220ce3fd 100644 --- a/fastlane_bot/events/pools/base.py +++ b/fastlane_bot/events/pools/base.py @@ -36,7 +36,7 @@ class Pool(ABC): __DATE__ = "2023-07-03" state: Dict[str, Any] = field(default_factory=dict) - + factory_contract = None @classmethod @abstractmethod def event_matches_format( diff --git a/fastlane_bot/helpers/poolandtokens.py b/fastlane_bot/helpers/poolandtokens.py index 08f82506d..120eaa549 100644 --- a/fastlane_bot/helpers/poolandtokens.py +++ b/fastlane_bot/helpers/poolandtokens.py @@ -26,6 +26,23 @@ class SolidlyV2StablePoolsNotSupported(Exception): pass +FEE_LOOKUP = { + 0.000001: Univ3Calculator.FEE1, + 0.000008: Univ3Calculator.FEE8, + 0.00001: Univ3Calculator.FEE10, + 0.00004: Univ3Calculator.FEE40, + 0.00008: Univ3Calculator.FEE80, + 0.0001: Univ3Calculator.FEE100, + 0.00025: Univ3Calculator.FEE250, + 0.0003: Univ3Calculator.FEE300, + 0.00045: Univ3Calculator.FEE450, + 0.0005: Univ3Calculator.FEE500, + 0.0010: Univ3Calculator.FEE1000, + 0.0025: Univ3Calculator.FEE2500, + 0.0030: Univ3Calculator.FEE3000, + 0.01: Univ3Calculator.FEE10000, + } + @dataclass class PoolAndTokens: """ @@ -559,22 +576,7 @@ def decimal_converter(idx): return lst - FEE_LOOKUP = { - 0.000001: Univ3Calculator.FEE1, - 0.000008: Univ3Calculator.FEE8, - 0.00001: Univ3Calculator.FEE10, - 0.00004: Univ3Calculator.FEE40, - 0.00008: Univ3Calculator.FEE80, - 0.0001: Univ3Calculator.FEE100, - 0.00025: Univ3Calculator.FEE250, - 0.0003: Univ3Calculator.FEE300, - 0.00045: Univ3Calculator.FEE450, - 0.0005: Univ3Calculator.FEE500, - 0.0010: Univ3Calculator.FEE1000, - 0.0025: Univ3Calculator.FEE2500, - 0.0030: Univ3Calculator.FEE3000, - 0.01: Univ3Calculator.FEE10000, - } + def _univ3_to_cpc(self) -> List[Any]: """ @@ -599,10 +601,10 @@ def _univ3_to_cpc(self) -> List[Any]: "tick": self.tick, "liquidity": self.liquidity, } - feeconst = self.FEE_LOOKUP.get(float(self.fee_float)) + feeconst = FEE_LOOKUP.get(float(self.fee_float)) if feeconst is None: raise ValueError( - f"Illegal fee for Uniswap v3 pool: {self.fee_float} [{self.FEE_LOOKUP}]]" + f"Illegal fee for Uniswap v3 pool: {self.fee_float} [{FEE_LOOKUP}]]" ) uni3 = Univ3Calculator.from_dict(args, feeconst, addrdec=self.ADDRDEC) params = uni3.cpc_params() diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py new file mode 100644 index 000000000..ae7a901d5 --- /dev/null +++ b/fastlane_bot/pool_finder.py @@ -0,0 +1,233 @@ +""" +Finds liquidity pools. + +--- +(c) Copyright Bprotocol foundation 2023-24. +All rights reserved. +Licensed under MIT. +""" +from collections import defaultdict + +from typing import List, Tuple, Dict, Any + +from fastlane_bot.config.constants import ZERO_ADDRESS, UNISWAP_V2_NAME, UNISWAP_V3_NAME, SOLIDLY_V2_NAME +from fastlane_bot.config.multicaller import MultiCaller +from fastlane_bot.events.exchanges.base import Exchange + +class PoolFinder: + """A class that provides methods to find unsupported carbon pairs and triangles + within a given set of flashloan tokens and external pairs. + """ + + multicallers = [] + + def __init__(self, uni_v2_forks: List[str], uni_v3_forks: List[str], solidly_v2_forks: List[str], carbon_forks: List[str], flashloan_tokens: List[str]): + self.uni_v2_forks = uni_v2_forks + self.uni_v3_forks = uni_v3_forks + self.solidly_v2_forks = solidly_v2_forks + self.carbon_forks = carbon_forks + self.flashloan_tokens = flashloan_tokens + self.uni_v3_fee_tiers = defaultdict(set) + self.carbon_pairs_seen = set() + + def init_exchanges(self, exchanges: List[Exchange], web3: Any, multicall_address: str): + """ This function initializes multicallers that will be used for each exchange. + + The function is separated from the main __init__ function to enable easier testing. + + Args: + exchanges (List[Exchange]): List of exchange objects for which to make multicalls. + web3 (Web3): Web3 object + multicall_address (str): The address of the multicall contract. + + + """ + self.multicallers = {ex_name: {"multicaller": MultiCaller(contract=exchange.sync_factory_contract, web3=web3, multicall_address=multicall_address), "exchange": exchange} for ex_name, exchange in exchanges.items() if ex_name in self.uni_v2_forks + self.uni_v3_forks + self.solidly_v2_forks} + + def extract_univ3_fee_tiers(self, pools: List[Dict[str, Any]]): + """ + Extracts unique fee tiers for each exchange listed under Uniswap V3 forks from the provided pool data. + + Args: + pools (List[Dict[str, Any]]): List of pool dictionaries containing 'exchange_name' and 'fee'. + + This function updates the 'uni_v3_fee_tiers' dictionary where each exchange name is mapped to a set of unique fees. + """ + for pool in pools: + if pool["exchange_name"] in self.uni_v3_forks: + self.uni_v3_fee_tiers[pool["exchange_name"]].add(int(pool["fee"])) + + + def get_pools_for_unsupported_pairs(self, pools: List[Dict[str, Any]], arb_mode: str): + """ + Main flow for Poolfinder. + + Args: + pools (List[Dict[str, Any]]): A list of pool data where each pool is a dictionary. The expected keys in + each dictionary should align with the requirements of the _extract_pairs, + _find_unsupported_pairs, and _find_unsupported_triangles methods. + + Returns: + Dict: Returns a dictionary with pools sorted into different exchange types (Uni V2 forks, Uni V3 forks, + and Solidly V2 forks), each associated with their specific supporting pools based on the unsupported + configurations identified. + """ + carbon_pairs, other_pairs = self._extract_pairs(pools=pools, carbon_forks=self.carbon_forks) + if not carbon_pairs: + return [], [], [] + self.extract_univ3_fee_tiers(pools) + func = PoolFinder._find_unsupported_triangles if arb_mode in ["triangle", "multi_triangle"] else PoolFinder._find_unsupported_pairs + unsupported = func(self.flashloan_tokens, carbon_pairs=carbon_pairs, external_pairs=other_pairs) + supporting_pools = self._find_pools(unsupported) + return self._sort_exchange_pools(supporting_pools, self.uni_v2_forks, self.uni_v3_forks, self.solidly_v2_forks) + + + def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: + """ + Collects pool addresses for each exchange, based on a set of unsupported token pairs + and flashloan tokens. The function constructs pairs of tokens from unsupported_pairs with each + flashloan token and retrieves pool data via multicall. It filters out invalid addresses. + + Args: + unsupported_pairs (List[Tuple]): A list of tuples, where each tuple contains two token addresses. + flashloan_tokens (List[str]): A list of token addresses available for flashloans. + + Returns: + Dict[str, List[str]]: A list of dictionaries, where each dictionary maps an exchange's name + to a list of valid pool addresses (i.e., non-zero addresses) obtained + from the multicall across all generated pairs. + + Raises: + Exception: An exception could be raised from the multicall operation depending on the + implementation specifics of the multicall context manager or the exchange's + get_pool_function method if it encounters a problem. + """ + pairs = [(tkn, token) for pair in unsupported_pairs for tkn in pair for token in self.flashloan_tokens] + + result_list = {} + for ex_name, ex_data in self.multicallers.items(): + mc = ex_data["multicaller"] + ex = ex_data["exchange"] + with mc: + for pair in pairs: + if ex.base_exchange_name == UNISWAP_V2_NAME: + mc.add_call(ex.get_pool_function(ex.sync_factory_contract), pair[0], pair[1]) + elif ex.base_exchange_name == UNISWAP_V3_NAME: + for fee in self.uni_v3_fee_tiers[ex.exchange_name]: + mc.add_call(ex.get_pool_function(ex.sync_factory_contract), pair[0], pair[1], fee) + elif ex.base_exchange_name == SOLIDLY_V2_NAME: + mc.add_call(ex.get_pool_function(ex.sync_factory_contract), *ex.get_pool_args(pair[0], pair[1], False)) + results = mc.multicall() + result_list[ex.exchange_name] = [mc.web3.to_checksum_address(addr) for addr in results if addr != ZERO_ADDRESS] + return result_list + + @staticmethod + def _sort_exchange_pools(ex_pools: Dict, uni_v2_forks: List[str], uni_v3_forks: List[str], solidly_v2_forks: List[str]): + """ + Categorizes pools based on the type of exchange they belong to into separate dictionaries. + + Args: + ex_pools (List[Dict]): A list of dictionary where keys are exchange names and values are lists of pool addresses. + + Returns: + Tuple[Dict, Dict, Dict]: Three dictionaries categorizing pool addresses into Uniswap V2 forks, + Uniswap V3 forks, and Solidly V2 forks. + """ + # Initialize separate dictionaries for each type of exchange fork + uni_v2_pools = {} + uni_v3_pools = {} + solidly_v2_pools = {} + + # Assign pools to the appropriate category based on the exchange type + for ex_name, pools in ex_pools.items(): + if ex_name in uni_v2_forks: + target_pools = uni_v2_pools + elif ex_name in uni_v3_forks: + target_pools = uni_v3_pools + elif ex_name in solidly_v2_forks: + target_pools = solidly_v2_pools + else: + continue # Skip exchanges that do not match any known category + + for addr in pools: + target_pools[addr] = ex_name # Map pool address to exchange name + + return uni_v2_pools, uni_v3_pools, solidly_v2_pools + + + def _extract_pairs(self, pools: List[Dict[str, Any]], carbon_forks: List[str]) -> (List, set): + """ + Extracts unique, order-insensitive pairs of tokens from pools, categorizing them + into carbon pairs and other pairs based on the exchange's presence in carbon_forks. + + Args: + pools (List[Dict[str, Any]]): List of pool dictionaries containing token addresses and exchange names. + carbon_forks (List[str]): List of exchange names categorized under carbon forks. + + Returns: + tuple: Two sets of unique token pairs, one for carbon forks and one for other exchanges. + """ + carbon_pairs = set() + other_pairs = set() + + for pool in pools: + # Create a frozenset for each pair to ensure the pair is treated as order-insensitive + pair = (pool["tkn0_address"], pool["tkn1_address"]) + frozen_pair = frozenset(pair) + + if pool["exchange_name"] in carbon_forks and frozen_pair not in self.carbon_pairs_seen: + carbon_pairs.add(pair) + self.carbon_pairs_seen.add(frozen_pair) + else: + other_pairs.add(pair) + + return list(carbon_pairs), other_pairs + + @staticmethod + def _find_unsupported_triangles(flashloan_tokens: List[str], carbon_pairs: List[Tuple], external_pairs: set) -> List[Tuple]: + """ + Identifies carbon pairs that cannot form a valid triangle with any of the flashloan tokens, + even though each side of the pair is supported externally. + + Args: + flashloan_tokens (List[str]): Tokens available for forming triangles. + carbon_pairs (List[Tuple]): Carbon pairs to check for triangle support. + external_pairs (List[Tuple[str, str]]): Pairs that are supported externally. + + Returns: + List[Tuple]: List of carbon pairs that cannot form a valid triangle. + """ + unsupported_triangles = [] + + for pair in carbon_pairs: + tkn0, tkn1 = pair[0], pair[1] + if not any((frozenset((tkn0, tkn)) in external_pairs and frozenset((tkn1, tkn)) in external_pairs) for tkn in flashloan_tokens): + unsupported_triangles.append(pair) + return unsupported_triangles + + @staticmethod + def _find_unsupported_pairs(flashloan_tokens: List[str], carbon_pairs: set, external_pairs: set): + """ + Determines which carbon pairs are unsupported based on the lack of token support and non-existence in external pairs. + + Args: + flashloan_tokens (List[str]): List of tokens supported for flashloans. + carbon_pairs (List[Tuple]): Carbon pairs to evaluate for support. + external_pairs (List[Tuple]): Pairs externally supported. + + Returns: + List[Tuple]: List of unsupported carbon pairs. + """ + unsupported_pairs = [] + for pair in carbon_pairs: + tkn0, tkn1 = pair[0], pair[1] + if (tkn0 not in flashloan_tokens and tkn1 not in flashloan_tokens): + unsupported_pairs.append(pair) + continue + if pair not in external_pairs: + unsupported_pairs.append(pair) + + return unsupported_pairs + + + diff --git a/fastlane_bot/tests/test_073_TestPoolFinder.py b/fastlane_bot/tests/test_073_TestPoolFinder.py new file mode 100644 index 000000000..4b893b36b --- /dev/null +++ b/fastlane_bot/tests/test_073_TestPoolFinder.py @@ -0,0 +1,96 @@ +from fastlane_bot.pool_finder import PoolFinder + + +def test_find_unsupported_pairs(): + flashloan_tokens = ['TokenA', 'TokenB'] + carbon_pairs = [('TokenA', 'TokenC'), ('TokenC', 'TokenD'), ('TokenB', 'TokenE')] + external_pairs = [('TokenA', 'TokenB'), ('TokenC', 'TokenE')] + # Expected result + # ('TokenA', 'TokenC') is supported by flashloan_tokens but not in external_pairs + # ('TokenC', 'TokenD') is unsupported by flashloan_tokens and not in external_pairs + # ('TokenB', 'TokenE') is supported by flashloan_tokens but not in external_pairs + expected_result = [('TokenA', 'TokenC'), ('TokenC', 'TokenD'), ('TokenB', 'TokenE')] + + # Function under test + result = PoolFinder._find_unsupported_pairs(flashloan_tokens, carbon_pairs, external_pairs) + + # Check that the function returns the correct list of unsupported pairs + assert sorted(result) == sorted(expected_result) + +def test_find_unsupported_triangles(): + flashloan_tokens = ['TokenA', 'TokenB'] + carbon_pairs = [('TokenA', 'TokenC'), ('TokenC', 'TokenD'), ('TokenB', 'TokenE')] + external_pairs = {('TokenA', 'TokenC'), ('TokenA', 'TokenD')} + # Expected result + # ('TokenA', 'TokenC') is unsupported by triangles + # ('TokenC', 'TokenD') is supported by triangles + # ('TokenB', 'TokenE') is unsupported by triangles + expected_result = [('TokenA', 'TokenC'), ('TokenB', 'TokenE')] + + # Function under test + result = PoolFinder._find_unsupported_triangles(flashloan_tokens, carbon_pairs, external_pairs) + + # Check that the function returns the correct list of unsupported pairs + assert sorted(result) == sorted(expected_result) + + +def test_extract_pairs(): + # Sample data for testing + uni_v2_exchanges = ["bob_ex"] + uni_v3_exchanges = ["fred_ex"] + solidly_v2_exchanges = ["george_ex", "moose_ex"] + flashloan_tokens = ["BNT"] + pools = [ + {"exchange_name": "CarbonX", "tkn0_address": "WBTC", "tkn1_address": "BNT"}, + {"exchange_name": "CarbonX", "tkn0_address": "BNT", "tkn1_address": "WBTC"}, # Reverse order, should be treated as same + {"exchange_name": "NonCarbon", "tkn0_address": "WETH", "tkn1_address": "USDT"}, + {"exchange_name": "NonCarbon", "tkn0_address": "USDC", "tkn1_address": "WBTC"}, + {"exchange_name": "CarbonX", "tkn0_address": "WETH", "tkn1_address": "USDC"} + ] + carbon_forks = ["CarbonX"] + + # Expected results + expected_carbon_pairs = {frozenset(('WBTC', 'BNT')), frozenset(('WETH', 'USDC'))} + expected_other_pairs = {frozenset(('WETH', 'USDT')), frozenset(('USDC', 'WBTC'))} + #expected_carbon_pairs = [('WBTC', 'BNT'), ('WETH', 'USDC')] + pool_finder = PoolFinder(uni_v2_exchanges, uni_v3_exchanges, solidly_v2_exchanges, carbon_forks, flashloan_tokens) + + # Call the function with test data + carbon_pairs, other_pairs = pool_finder._extract_pairs(pools, carbon_forks) + + # Assert the results are as expected + assert frozenset(carbon_pairs[0]) in expected_carbon_pairs + assert frozenset(carbon_pairs[1]) in expected_carbon_pairs + for _pair in other_pairs: + assert _pair in expected_other_pairs + #assert other_pairs == expected_other_pairs + assert len(carbon_pairs) == 2 + assert len(other_pairs) == 2 + +def test_sort_exchanges(): + exchange_pools = { + "bob_ex": ["1", "2", "3"], + "fred_ex": ["4", "5"], + "george_ex": ["6", "7", "8"], + "moose_ex": ["9", "10", "11", "12"] + } + + uni_v2_exchanges = ["bob_ex"] + uni_v3_exchanges = ["fred_ex"] + solidly_v2_exchanges = ["george_ex", "moose_ex"] + uni_v2_pools, uni_v3_pools, solidly_v2_pools = PoolFinder._sort_exchange_pools(ex_pools=exchange_pools, uni_v2_forks=uni_v2_exchanges, uni_v3_forks=uni_v3_exchanges, solidly_v2_forks=solidly_v2_exchanges) + + + assert len(uni_v2_pools.keys()) == 3 + assert len(uni_v3_pools.keys()) == 2 + assert len(solidly_v2_pools.keys()) == 7 + + assert uni_v2_pools["1"] == "bob_ex" + assert uni_v3_pools["5"] == "fred_ex" + assert solidly_v2_pools["7"] == "george_ex" + assert solidly_v2_pools["9"] == "moose_ex" + + + + + diff --git a/main.py b/main.py index f2dce7d9c..3165d3059 100644 --- a/main.py +++ b/main.py @@ -8,6 +8,7 @@ from fastlane_bot.events.event_gatherer import EventGatherer from fastlane_bot.exceptions import ReadOnlyException, FlashloanUnavailableException from fastlane_bot.events.version_utils import check_version_requirements +from fastlane_bot.pool_finder import PoolFinder from fastlane_bot.tools.cpc import T check_version_requirements(required_version="6.11.0", package_name="web3") @@ -224,6 +225,7 @@ def main(args: argparse.Namespace) -> None: prefix_path: {args.prefix_path} self_fund: {args.self_fund} read_only: {args.read_only} + pool_finder: {args.pool_finder} +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -304,6 +306,10 @@ def run(mgr, args, tenderly_uri=None) -> None: event_gatherer = EventGatherer(w3=mgr.w3_async, exchanges=mgr.exchanges, event_contracts=mgr.event_contracts) + pool_finder = PoolFinder(uni_v2_forks=mgr.cfg.network.UNI_V2_FORKS, uni_v3_forks=mgr.cfg.network.UNI_V3_FORKS, solidly_v2_forks=mgr.cfg.network.SOLIDLY_V2_FORKS, carbon_forks=mgr.cfg.network.CARBON_V1_FORKS, flashloan_tokens=args.flashloan_tokens) if args.pool_finder != -1 else None + if pool_finder: + pool_finder.init_exchanges(exchanges=mgr.exchanges, web3=mgr.web3, multicall_address=mgr.cfg.network.MULTICALL_CONTRACT_ADDRESS) + while True: try: # ensure 'last_updated_block' is in pool_data for all pools @@ -513,6 +519,12 @@ def run(mgr, args, tenderly_uri=None) -> None: mgr.solidly_v2_event_mappings = dict( solidly_v2_event_mappings[["address", "exchange"]].values ) + if args.pool_finder != -1 and (loop_idx % args.pool_finder == 0 or loop_idx == 1): + uni_v2, uni_v3, solidly_v2 = pool_finder.get_pools_for_unsupported_pairs(mgr.pool_data, arb_mode=args.arb_mode) + mgr.uniswap_v2_event_mappings.update(uni_v2) + mgr.uniswap_v3_event_mappings.update(uni_v3) + mgr.solidly_v2_event_mappings.update(solidly_v2) + last_block_queried = current_block total_iteration_time += time.time() - iteration_start_time @@ -668,7 +680,7 @@ def run(mgr, args, tenderly_uri=None) -> None: ) parser.add_argument( "--blockchain", - default="ethereum", + default="coinbase_base", help="A blockchain from the list. Blockchains not in this list do not have a deployed Fast Lane contract and are not supported.", choices=["ethereum", "coinbase_base", "fantom", "mantle", "linea", "sei"], ) @@ -709,6 +721,11 @@ def run(mgr, args, tenderly_uri=None) -> None: default=None, help="Custom RPC URL. If not set, the bot will use the default Alchemy RPC URL for the blockchain (if available).", ) + parser.add_argument( + "--pool_finder", + default=100, + help="If not -1, searches for pools that can service Carbon strategies that do not have viable routes.", + ) # Process the arguments args = parser.parse_args() From c9afde9b6322ae2a73bd0c8fc90915b9cf3dd7cd Mon Sep 17 00:00:00 2001 From: Lesigh-3100 Date: Wed, 8 May 2024 11:08:34 +0300 Subject: [PATCH 15/58] Fix to test & code --- fastlane_bot/pool_finder.py | 9 +++++---- fastlane_bot/tests/test_073_TestPoolFinder.py | 11 ++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index ae7a901d5..f692cc341 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -175,9 +175,10 @@ def _extract_pairs(self, pools: List[Dict[str, Any]], carbon_forks: List[str]) - pair = (pool["tkn0_address"], pool["tkn1_address"]) frozen_pair = frozenset(pair) - if pool["exchange_name"] in carbon_forks and frozen_pair not in self.carbon_pairs_seen: - carbon_pairs.add(pair) - self.carbon_pairs_seen.add(frozen_pair) + if pool["exchange_name"] in carbon_forks: + if frozen_pair not in self.carbon_pairs_seen: + carbon_pairs.add(pair) + self.carbon_pairs_seen.add(frozen_pair) else: other_pairs.add(pair) @@ -206,7 +207,7 @@ def _find_unsupported_triangles(flashloan_tokens: List[str], carbon_pairs: List[ return unsupported_triangles @staticmethod - def _find_unsupported_pairs(flashloan_tokens: List[str], carbon_pairs: set, external_pairs: set): + def _find_unsupported_pairs(flashloan_tokens: List[str], carbon_pairs: List[Tuple], external_pairs: set): """ Determines which carbon pairs are unsupported based on the lack of token support and non-existence in external pairs. diff --git a/fastlane_bot/tests/test_073_TestPoolFinder.py b/fastlane_bot/tests/test_073_TestPoolFinder.py index 4b893b36b..e6bbe1262 100644 --- a/fastlane_bot/tests/test_073_TestPoolFinder.py +++ b/fastlane_bot/tests/test_073_TestPoolFinder.py @@ -4,7 +4,7 @@ def test_find_unsupported_pairs(): flashloan_tokens = ['TokenA', 'TokenB'] carbon_pairs = [('TokenA', 'TokenC'), ('TokenC', 'TokenD'), ('TokenB', 'TokenE')] - external_pairs = [('TokenA', 'TokenB'), ('TokenC', 'TokenE')] + external_pairs = {frozenset(('TokenA', 'TokenB')), frozenset(('TokenC', 'TokenE'))} # Expected result # ('TokenA', 'TokenC') is supported by flashloan_tokens but not in external_pairs # ('TokenC', 'TokenD') is unsupported by flashloan_tokens and not in external_pairs @@ -20,7 +20,7 @@ def test_find_unsupported_pairs(): def test_find_unsupported_triangles(): flashloan_tokens = ['TokenA', 'TokenB'] carbon_pairs = [('TokenA', 'TokenC'), ('TokenC', 'TokenD'), ('TokenB', 'TokenE')] - external_pairs = {('TokenA', 'TokenC'), ('TokenA', 'TokenD')} + external_pairs = {frozenset(('TokenA', 'TokenC')), frozenset(('TokenA', 'TokenD'))} # Expected result # ('TokenA', 'TokenC') is unsupported by triangles # ('TokenC', 'TokenD') is supported by triangles @@ -31,7 +31,8 @@ def test_find_unsupported_triangles(): result = PoolFinder._find_unsupported_triangles(flashloan_tokens, carbon_pairs, external_pairs) # Check that the function returns the correct list of unsupported pairs - assert sorted(result) == sorted(expected_result) + assert len(expected_result) == len(result) + assert sorted(expected_result) == sorted(result) def test_extract_pairs(): @@ -53,7 +54,7 @@ def test_extract_pairs(): expected_carbon_pairs = {frozenset(('WBTC', 'BNT')), frozenset(('WETH', 'USDC'))} expected_other_pairs = {frozenset(('WETH', 'USDT')), frozenset(('USDC', 'WBTC'))} #expected_carbon_pairs = [('WBTC', 'BNT'), ('WETH', 'USDC')] - pool_finder = PoolFinder(uni_v2_exchanges, uni_v3_exchanges, solidly_v2_exchanges, carbon_forks, flashloan_tokens) + pool_finder = PoolFinder(uni_v2_forks=uni_v2_exchanges, uni_v3_forks=uni_v3_exchanges, solidly_v2_forks=solidly_v2_exchanges, carbon_forks=carbon_forks, flashloan_tokens=flashloan_tokens) # Call the function with test data carbon_pairs, other_pairs = pool_finder._extract_pairs(pools, carbon_forks) @@ -62,7 +63,7 @@ def test_extract_pairs(): assert frozenset(carbon_pairs[0]) in expected_carbon_pairs assert frozenset(carbon_pairs[1]) in expected_carbon_pairs for _pair in other_pairs: - assert _pair in expected_other_pairs + assert frozenset(_pair) in expected_other_pairs #assert other_pairs == expected_other_pairs assert len(carbon_pairs) == 2 assert len(other_pairs) == 2 From 11fd3b731f27baeb1075883d104e5ecb9b58db10 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Wed, 8 May 2024 21:45:14 +0200 Subject: [PATCH 16/58] feat: dedicated Exchange types for distinct solidly forks --- fastlane_bot/events/exchanges/solidly_v2.py | 181 ------------------ .../events/exchanges/solidly_v2/__init__.py | 25 +++ .../events/exchanges/solidly_v2/base.py | 78 ++++++++ .../exchanges/solidly_v2/cleopatra_v2.py | 22 +++ .../exchanges/solidly_v2/equalizer_v2.py | 22 +++ .../events/exchanges/solidly_v2/lynex_v2.py | 22 +++ .../events/exchanges/solidly_v2/nile_v2.py | 22 +++ .../events/exchanges/solidly_v2/scale_v2.py | 22 +++ .../exchanges/solidly_v2/velocimeter_v2.py | 22 +++ .../exchanges/solidly_v2/velodrome_v2.py | 22 +++ .../events/exchanges/solidly_v2/xfai_v0.py | 36 ++++ run_blockchain_terraformer.py | 4 +- 12 files changed, 295 insertions(+), 183 deletions(-) delete mode 100644 fastlane_bot/events/exchanges/solidly_v2.py create mode 100644 fastlane_bot/events/exchanges/solidly_v2/__init__.py create mode 100644 fastlane_bot/events/exchanges/solidly_v2/base.py create mode 100644 fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py create mode 100644 fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py create mode 100644 fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py create mode 100644 fastlane_bot/events/exchanges/solidly_v2/nile_v2.py create mode 100644 fastlane_bot/events/exchanges/solidly_v2/scale_v2.py create mode 100644 fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py create mode 100644 fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py create mode 100644 fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py diff --git a/fastlane_bot/events/exchanges/solidly_v2.py b/fastlane_bot/events/exchanges/solidly_v2.py deleted file mode 100644 index 5d3951c11..000000000 --- a/fastlane_bot/events/exchanges/solidly_v2.py +++ /dev/null @@ -1,181 +0,0 @@ -""" -Contains the exchange class for SolidlyV2. - -This class is responsible for handling SolidlyV2 events and updating the state of the pools. - - -[DOC-TODO-OPTIONAL-longer description in rst format] - ---- -(c) Copyright Bprotocol foundation 2023-24. -All rights reserved. -Licensed under MIT. -""" -from dataclasses import dataclass -from typing import List, Type, Tuple, Any - -from web3.contract import Contract, AsyncContract - -from fastlane_bot.data.abi import SOLIDLY_V2_POOL_ABI, VELOCIMETER_V2_FACTORY_ABI, SOLIDLY_V2_FACTORY_ABI, \ - SCALE_V2_FACTORY_ABI, CLEOPATRA_V2_FACTORY_ABI, LYNEX_V2_FACTORY_ABI, NILE_V2_FACTORY_ABI, \ - XFAI_V0_FACTORY_ABI, XFAI_V0_CORE_ABI, XFAI_V0_POOL_ABI -from ..exchanges.base import Exchange -from ..pools.base import Pool -from ..interfaces.subscription import Subscription - - -async def _get_fee_1(address: str, contract: Contract, factory_contract: Contract) -> int: - return await factory_contract.caller.getFee(address) - -async def _get_fee_2(address: str, contract: Contract, factory_contract: Contract) -> int: - return await factory_contract.caller.getRealFee(address) - -async def _get_fee_3(address: str, contract: Contract, factory_contract: Contract) -> int: - return await factory_contract.caller.getFee(address, await contract.caller.stable()) - -async def _get_fee_4(address: str, contract: Contract, factory_contract: Contract) -> int: - return await factory_contract.caller.getPairFee(address, await contract.caller.stable()) - -async def _get_fee_5(address: str, contract: Contract, factory_contract: Contract) -> int: - return await factory_contract.caller.getFee(await contract.caller.stable()) - -async def _get_fee_6(address: str, contract: Contract, factory_contract: Contract) -> int: - return await factory_contract.caller.pairFee(address) - -async def _get_fee_7(address: str, contract: Contract, factory_contract: Contract) -> int: - core_address = factory_contract.w3.to_checksum_address(await factory_contract.caller.getXfaiCore()) - core_contract = factory_contract.w3.eth.contract(address=core_address, abi=XFAI_V0_CORE_ABI) - return await core_contract.caller.getTotalFee() - -async def _get_tkn0_A(contract: Contract) -> str: - return await contract.caller.token0() - -async def _get_tkn1_A(contract: Contract) -> str: - return await contract.caller.token1() - -async def _get_tkn0_B(contract: Contract) -> str: - return await contract.caller.poolToken() - -async def _get_tkn1_B(contract: Contract) -> str: - return "0xe5D7C2a44FfDDf6b295A15c148167daaAf5Cf34f" # TODO Use the constant WRAPPED_GAS_TOKEN_ADDRESS for this network - - -def _get_pool_function_1(factory_contract): - """ Function to get pools from Factory. - This function is intended to be used with a Multicall. It fetches pools from a Solidly fork Factory contract. - - Args: - factory_contract: The factory contract. - - Returns: - The function. - - """ - return factory_contract.functions.getPair -def _get_pool_function_2(factory_contract): - """ Function to get pools from Factory. - This function is intended to be used with a Multicall. It fetches pools from a Solidly fork Factory contract. - - Args: - factory_contract: The factory contract. - - Returns: - The function. - - """ - return factory_contract.functions.getPool - - -def _get_pool_args_1(tkn0, tkn1, stable): - """ Function to manage args input to get pools from Factory. - - Args: - tkn0: The first token address. - tkn1: The second token address. - stable: (bool) If True, indicates a stable pool. If False, indicates a Volatile pool. - Returns: - The function returns the arguments necessary to get pool addresses from the factory contract. - - """ - return tkn0, tkn1, stable - -def _get_pool_args_2(tkn0, tkn1, stable): - """ Function to manage args input to get pools from Factory. - - Args: - tkn0: The first token address. - tkn1: The second token address. - stable: (bool) If True, indicates a stable pool. If False, indicates a Volatile pool. - Returns: - The function returns the arguments necessary to get pool addresses from the factory contract. - - """ - return tkn0 - -EXCHANGE_INFO = { - "velocimeter_v2": {"decimals": 4, "factory_abi": VELOCIMETER_V2_FACTORY_ABI, "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_1, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, - "equalizer_v2" : {"decimals": 4, "factory_abi": SCALE_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_2, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, - "aerodrome_v2" : {"decimals": 4, "factory_abi": SOLIDLY_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_3, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_2, "get_pool_args": _get_pool_args_1}, - "velodrome_v2" : {"decimals": 4, "factory_abi": SOLIDLY_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_3, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_2, "get_pool_args": _get_pool_args_1}, - "scale_v2" : {"decimals": 18, "factory_abi": SCALE_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_2, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, - "cleopatra_v2" : {"decimals": 4, "factory_abi": CLEOPATRA_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_4, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, - "stratum_v2" : {"decimals": 4, "factory_abi": VELOCIMETER_V2_FACTORY_ABI, "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_1, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, - "lynex_v2" : {"decimals": 4, "factory_abi": LYNEX_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_5, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, - "nile_v2" : {"decimals": 4, "factory_abi": NILE_V2_FACTORY_ABI , "pool_abi": SOLIDLY_V2_POOL_ABI, "get_fee": _get_fee_6, "get_tkn0": _get_tkn0_A, "get_tkn1": _get_tkn1_A, "get_pool_function": _get_pool_function_1, "get_pool_args": _get_pool_args_1}, - "xfai_v0" : {"decimals": 4, "factory_abi": XFAI_V0_FACTORY_ABI , "pool_abi": XFAI_V0_POOL_ABI , "get_fee": _get_fee_7, "get_tkn0": _get_tkn0_B, "get_tkn1": _get_tkn1_B, "get_pool_function": _get_pool_function_2, "get_pool_args": _get_pool_args_2}, -} - -@dataclass -class SolidlyV2(Exchange): - """ - SolidlyV2 exchange class - """ - - base_exchange_name: str = "solidly_v2" - exchange_name: str = None - fee: str = None - router_address: str = None - exchange_initialized: bool = False - - stable_fee: float = None - volatile_fee: float = None - factory_address: str = None - factory_contract: AsyncContract = None - - @property - def fee_float(self): - return float(self.fee) - - def add_pool(self, pool: Pool): - self.pools[pool.state["address"]] = pool - - def get_abi(self): - return EXCHANGE_INFO[self.exchange_name]["pool_abi"] - - @property - def get_factory_abi(self): - return EXCHANGE_INFO[self.exchange_name]["factory_abi"] - - def get_events(self, contract: Contract) -> List[Type[Contract]]: - return [contract.events.Sync] if self.exchange_initialized else [] - - def get_subscriptions(self, contract: Contract) -> List[Subscription]: - return [Subscription(contract.events.Sync)] - - async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: - exchange_info = EXCHANGE_INFO[self.exchange_name] - fee = await exchange_info["get_fee"](address, contract, self.factory_contract) - fee_float = float(fee) / 10 ** exchange_info["decimals"] - return str(fee_float), fee_float - - async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: - return await EXCHANGE_INFO[self.exchange_name]["get_tkn0"](contract) - - async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: - return await EXCHANGE_INFO[self.exchange_name]["get_tkn1"](contract) - - def get_pool_function(self, factory_contract): - return EXCHANGE_INFO[self.exchange_name]["get_pool_function"](factory_contract) - - def get_pool_args(self, tkn0, tkn1, stable): - return EXCHANGE_INFO[self.exchange_name]["get_pool_args"](tkn0, tkn1, stable) diff --git a/fastlane_bot/events/exchanges/solidly_v2/__init__.py b/fastlane_bot/events/exchanges/solidly_v2/__init__.py new file mode 100644 index 000000000..c81f0bca4 --- /dev/null +++ b/fastlane_bot/events/exchanges/solidly_v2/__init__.py @@ -0,0 +1,25 @@ +from ..base import Exchange +from .velocimeter_v2 import VelocimeterV2 +from .equalizer_v2 import EqualizerV2 +from .velodrome_v2 import VelodromeV2 +from .scale_v2 import ScaleV2 +from .cleopatra_v2 import CleopatraV2 +from .lynex_v2 import LynexV2 +from .nile_v2 import NileV2 +from .xfai_v0 import XFaiV2 + + +class SolidlyV2(Exchange): + def __new__(cls, **kwargs): + return { + "velocimeter_v2": VelocimeterV2, + "equalizer_v2": EqualizerV2, + "aerodrome_v2": VelodromeV2, + "velodrome_v2": VelodromeV2, + "scale_v2": ScaleV2, + "cleopatra_v2": CleopatraV2, + "stratum_v2": VelocimeterV2, + "lynex_v2": LynexV2, + "nile_v2": NileV2, + "xfai_v0": XFaiV2, + }[kwargs["exchange_name"]](**kwargs) diff --git a/fastlane_bot/events/exchanges/solidly_v2/base.py b/fastlane_bot/events/exchanges/solidly_v2/base.py new file mode 100644 index 000000000..56287b650 --- /dev/null +++ b/fastlane_bot/events/exchanges/solidly_v2/base.py @@ -0,0 +1,78 @@ +""" +Contains the exchange class for SolidlyV2. + +This class is responsible for handling SolidlyV2 events and updating the state of the pools. + + +[DOC-TODO-OPTIONAL-longer description in rst format] + +--- +(c) Copyright Bprotocol foundation 2023-24. +All rights reserved. +Licensed under MIT. +""" +from abc import abstractmethod +from dataclasses import dataclass +from typing import List, Type, Any + +from web3.contract import Contract, AsyncContract + +from fastlane_bot.data.abi import SOLIDLY_V2_POOL_ABI +from fastlane_bot.events.exchanges.base import Exchange +from ...exchanges.base import Exchange +from ...pools.base import Pool +from ...interfaces.subscription import Subscription + + +@dataclass +class SolidlyV2(Exchange): + """ + SolidlyV2 exchange class + """ + base_exchange_name: str = "solidly_v2" + exchange_name: str = None + fee: str = None + router_address: str = None + exchange_initialized: bool = False + + stable_fee: float = None + volatile_fee: float = None + factory_address: str = None + factory_contract: AsyncContract = None + + @property + def fee_float(self): # TODO: why is this here? + return float(self.fee) + + def add_pool(self, pool: Pool): + self.pools[pool.state["address"]] = pool + + def get_events(self, contract: Contract) -> List[Type[Contract]]: + return [contract.events.Sync] if self.exchange_initialized else [] + + def get_subscriptions(self, contract: Contract) -> List[Subscription]: + return [Subscription(contract.events.Sync)] + + def get_abi(self): + return SOLIDLY_V2_POOL_ABI + + async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: + return await contract.caller.token0() + + async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: + return await contract.caller.token1() + + def get_pool_args(self, tkn0, tkn1, stable): + return tkn0, tkn1, stable + + @property + @abstractmethod + def get_factory_abi(self): + ... + + @abstractmethod + async def get_fee(self, address: str, contract: Contract, factory_contract: Contract): + ... + + def get_pool_function(self, factory_contract): + ... diff --git a/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py b/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py new file mode 100644 index 000000000..71da0b657 --- /dev/null +++ b/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import Tuple + +from web3.contract import AsyncContract + +from fastlane_bot.data.abi import CLEOPATRA_V2_FACTORY_ABI +from .base import SolidlyV2 + + +@dataclass +class CleopatraV2(SolidlyV2): + @property + def get_factory_abi(self): + return CLEOPATRA_V2_FACTORY_ABI + + async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: + fee = await self.factory_contract.caller.getPairFee(address, await contract.caller.stable()) + fee_float = float(fee) / 10 ** 4 + return str(fee_float), fee_float + + def get_pool_function(self, factory_contract): + return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py b/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py new file mode 100644 index 000000000..06992a1b2 --- /dev/null +++ b/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import Tuple + +from web3.contract import AsyncContract + +from fastlane_bot.data.abi import SCALE_V2_FACTORY_ABI +from .base import SolidlyV2 + + +@dataclass +class EqualizerV2(SolidlyV2): + @property + def get_factory_abi(self): + return SCALE_V2_FACTORY_ABI + + async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: + fee = await self.factory_contract.caller.getRealFee(address) + fee_float = float(fee) / 10 ** 4 + return str(fee_float), fee_float + + def get_pool_function(self, factory_contract): + return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py b/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py new file mode 100644 index 000000000..ede370136 --- /dev/null +++ b/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import Tuple + +from web3.contract import AsyncContract + +from fastlane_bot.data.abi import LYNEX_V2_FACTORY_ABI +from .base import SolidlyV2 + + +@dataclass +class LynexV2(SolidlyV2): + @property + def get_factory_abi(self): + return LYNEX_V2_FACTORY_ABI + + async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: + fee = await self.factory_contract.caller.getFee(await contract.caller.stable()) + fee_float = float(fee) / 10 ** 4 + return str(fee_float), fee_float + + def get_pool_function(self, factory_contract): + return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py b/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py new file mode 100644 index 000000000..9b90f0e26 --- /dev/null +++ b/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import Tuple + +from web3.contract import AsyncContract + +from fastlane_bot.data.abi import NILE_V2_FACTORY_ABI +from .base import SolidlyV2 + + +@dataclass +class NileV2(SolidlyV2): + @property + def get_factory_abi(self): + return NILE_V2_FACTORY_ABI + + async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: + fee = await self.factory_contract.caller.pairFee(address) + fee_float = float(fee) / 10 ** 4 + return str(fee_float), fee_float + + def get_pool_function(self, factory_contract): + return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py b/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py new file mode 100644 index 000000000..29a53e118 --- /dev/null +++ b/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import Tuple + +from web3.contract import AsyncContract + +from fastlane_bot.data.abi import SCALE_V2_FACTORY_ABI +from .base import SolidlyV2 + + +@dataclass +class ScaleV2(SolidlyV2): + @property + def get_factory_abi(self): + return SCALE_V2_FACTORY_ABI + + async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: + fee = await self.factory_contract.caller.getRealFee(address) + fee_float = float(fee) / 10 ** 18 + return str(fee_float), fee_float + + def get_pool_function(self, factory_contract): + return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py b/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py new file mode 100644 index 000000000..51b1d88ec --- /dev/null +++ b/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import Tuple + +from web3.contract import AsyncContract + +from fastlane_bot.data.abi import VELOCIMETER_V2_FACTORY_ABI +from .base import SolidlyV2 + + +@dataclass +class VelocimeterV2(SolidlyV2): + @property + def get_factory_abi(self): + return VELOCIMETER_V2_FACTORY_ABI + + async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: + fee = await self.factory_contract.caller.getFee(address) + fee_float = float(fee) / 10 ** 4 + return str(fee_float), fee_float + + def get_pool_function(self, factory_contract): + return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py b/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py new file mode 100644 index 000000000..9de5fdf1a --- /dev/null +++ b/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import Tuple + +from web3.contract import AsyncContract + +from fastlane_bot.data.abi import SOLIDLY_V2_FACTORY_ABI +from .base import SolidlyV2 + + +@dataclass +class VelodromeV2(SolidlyV2): + @property + def get_factory_abi(self): + return SOLIDLY_V2_FACTORY_ABI + + async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: + fee = await self.factory_contract.caller.getFee(address, await contract.caller.stable()) + fee_float = float(fee) / 10 ** 4 + return str(fee_float), fee_float + + def get_pool_function(self, factory_contract): + return factory_contract.functions.getPool diff --git a/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py b/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py new file mode 100644 index 000000000..3ef5e2006 --- /dev/null +++ b/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py @@ -0,0 +1,36 @@ +from dataclasses import dataclass +from typing import Any, Tuple + +from web3.contract import Contract, AsyncContract + +from fastlane_bot.data.abi import XFAI_V0_POOL_ABI, XFAI_V0_FACTORY_ABI +from .base import SolidlyV2 + + +@dataclass +class XFaiV2(SolidlyV2): + def get_abi(self): + return XFAI_V0_POOL_ABI + + async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: + return await contract.caller.poolToken() + + async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: + return "0xe5D7C2a44FfDDf6b295A15c148167daaAf5Cf34f" # TODO Use the constant WRAPPED_GAS_TOKEN_ADDRESS for this network + + def get_pool_args(self, tkn0, tkn1, stable): + return tkn0 + + @property + def get_factory_abi(self): + return XFAI_V0_FACTORY_ABI + + async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: + core_address = self.factory_contract.w3.to_checksum_address(await self.factory_contract.caller.getXfaiCore()) + core_contract = self.factory_contract.w3.eth.contract(address=core_address, abi=self.get_abi()) + fee = await core_contract.caller.getTotalFee() + fee_float = float(fee) / 10 ** 4 + return str(fee_float), fee_float + + def get_pool_function(self, factory_contract): + return factory_contract.functions.getPool diff --git a/run_blockchain_terraformer.py b/run_blockchain_terraformer.py index 96269107f..1fd29fb27 100644 --- a/run_blockchain_terraformer.py +++ b/run_blockchain_terraformer.py @@ -15,7 +15,6 @@ from fastlane_bot.utils import safe_int from fastlane_bot.events.exchanges.solidly_v2 import SolidlyV2 -from fastlane_bot.events.exchanges.solidly_v2 import EXCHANGE_INFO as SOLIDLY_EXCHANGE_INFO from fastlane_bot.data.abi import ERC20_ABI, UNISWAP_V2_FACTORY_ABI, UNISWAP_V3_FACTORY_ABI import asyncio @@ -1024,8 +1023,9 @@ def terraform_blockchain(network_name: str): univ3_mapdf = pd.concat([univ3_mapdf, m_df], ignore_index=True) elif "solidly" in fork: add_to_exchange_ids(exchange=exchange_name, fork=fork) + solidly_exchange = SolidlyV2(exchange_name=exchange_name) + factory_abi = solidly_exchange.get_factory_abi - factory_abi = SOLIDLY_EXCHANGE_INFO[exchange_name]["factory_abi"] factory_contract = web3.eth.contract( address=factory_address, abi=factory_abi ) From c3b2849e8ec06c310eb1dcb4b54b514eec9e0cf2 Mon Sep 17 00:00:00 2001 From: Lesigh-3100 Date: Thu, 9 May 2024 12:45:23 +0300 Subject: [PATCH 17/58] Add chunking to Poolfinder & results logging --- fastlane_bot/pool_finder.py | 29 +++++++++++++++++------------ main.py | 3 +++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index f692cc341..71ce763b6 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -103,22 +103,27 @@ def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: get_pool_function method if it encounters a problem. """ pairs = [(tkn, token) for pair in unsupported_pairs for tkn in pair for token in self.flashloan_tokens] - + chunk_size = 400 + # Create the list of chunks + chunked_pairs = [pairs[i:i + chunk_size] for i in range(0, len(pairs), chunk_size)] result_list = {} + for ex_name, ex_data in self.multicallers.items(): mc = ex_data["multicaller"] ex = ex_data["exchange"] - with mc: - for pair in pairs: - if ex.base_exchange_name == UNISWAP_V2_NAME: - mc.add_call(ex.get_pool_function(ex.sync_factory_contract), pair[0], pair[1]) - elif ex.base_exchange_name == UNISWAP_V3_NAME: - for fee in self.uni_v3_fee_tiers[ex.exchange_name]: - mc.add_call(ex.get_pool_function(ex.sync_factory_contract), pair[0], pair[1], fee) - elif ex.base_exchange_name == SOLIDLY_V2_NAME: - mc.add_call(ex.get_pool_function(ex.sync_factory_contract), *ex.get_pool_args(pair[0], pair[1], False)) - results = mc.multicall() - result_list[ex.exchange_name] = [mc.web3.to_checksum_address(addr) for addr in results if addr != ZERO_ADDRESS] + for pair_chunk in chunked_pairs: + with mc: + for pair in pair_chunk: + if ex.base_exchange_name == UNISWAP_V2_NAME: + mc.add_call(ex.get_pool_function(ex.sync_factory_contract), pair[0], pair[1]) + elif ex.base_exchange_name == UNISWAP_V3_NAME: + for fee in self.uni_v3_fee_tiers[ex.exchange_name]: + mc.add_call(ex.get_pool_function(ex.sync_factory_contract), pair[0], pair[1], fee) + elif ex.base_exchange_name == SOLIDLY_V2_NAME: + mc.add_call(ex.get_pool_function(ex.sync_factory_contract), *ex.get_pool_args(pair[0], pair[1], False)) + results = mc.multicall() + mc._contract_calls = [] + result_list[ex.exchange_name] = [mc.web3.to_checksum_address(addr) for addr in results if addr != ZERO_ADDRESS] return result_list @staticmethod diff --git a/main.py b/main.py index 3165d3059..3d51e1131 100644 --- a/main.py +++ b/main.py @@ -520,7 +520,10 @@ def run(mgr, args, tenderly_uri=None) -> None: solidly_v2_event_mappings[["address", "exchange"]].values ) if args.pool_finder != -1 and (loop_idx % args.pool_finder == 0 or loop_idx == 1): + mgr.cfg.logger.info(f"Searching for unsupported Carbon pairs.") uni_v2, uni_v3, solidly_v2 = pool_finder.get_pools_for_unsupported_pairs(mgr.pool_data, arb_mode=args.arb_mode) + result = f"Added {len(uni_v2) + len(uni_v3) + len(solidly_v2)} pools." if (uni_v2 or uni_v3 or solidly_v2) else f"No pools added." + mgr.cfg.logger.info(result) mgr.uniswap_v2_event_mappings.update(uni_v2) mgr.uniswap_v3_event_mappings.update(uni_v3) mgr.solidly_v2_event_mappings.update(solidly_v2) From 6619516263f92cdc074416c43589c38e0de130cf Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Thu, 9 May 2024 14:50:54 +0200 Subject: [PATCH 18/58] refactor: pool_finder --- fastlane_bot/events/exchanges/balancer.py | 4 +- fastlane_bot/events/exchanges/bancor_pol.py | 2 +- fastlane_bot/events/exchanges/bancor_v2.py | 2 +- fastlane_bot/events/exchanges/bancor_v3.py | 3 +- fastlane_bot/events/exchanges/carbon_v1.py | 2 +- fastlane_bot/pool_finder.py | 153 +++++++------------- main.py | 16 +- 7 files changed, 75 insertions(+), 107 deletions(-) diff --git a/fastlane_bot/events/exchanges/balancer.py b/fastlane_bot/events/exchanges/balancer.py index 3481d5fe1..c9bb4e453 100644 --- a/fastlane_bot/events/exchanges/balancer.py +++ b/fastlane_bot/events/exchanges/balancer.py @@ -91,6 +91,6 @@ async def get_tkn_n(self, address: str, contract: Contract, event: Any, index: i def get_pool_function(self, factory_contract: Contract): """ - This function is unused for Carbon. + This function is unused for Balancer. """ - pass \ No newline at end of file + raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/bancor_pol.py b/fastlane_bot/events/exchanges/bancor_pol.py index 7d0075f80..f86fd8a77 100644 --- a/fastlane_bot/events/exchanges/bancor_pol.py +++ b/fastlane_bot/events/exchanges/bancor_pol.py @@ -67,7 +67,7 @@ def get_pool_function(self, factory_contract: Contract): """ This function is unused for Bancor POL. """ - pass + raise NotImplementedError def save_strategy( self, diff --git a/fastlane_bot/events/exchanges/bancor_v2.py b/fastlane_bot/events/exchanges/bancor_v2.py index ab23a3b86..b445e4e47 100644 --- a/fastlane_bot/events/exchanges/bancor_v2.py +++ b/fastlane_bot/events/exchanges/bancor_v2.py @@ -82,4 +82,4 @@ def get_pool_function(self, factory_contract: Contract): """ This function is unused for Bancor V2. """ - pass \ No newline at end of file + raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/bancor_v3.py b/fastlane_bot/events/exchanges/bancor_v3.py index 53c584ef3..1d5328dba 100644 --- a/fastlane_bot/events/exchanges/bancor_v3.py +++ b/fastlane_bot/events/exchanges/bancor_v3.py @@ -64,8 +64,9 @@ async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: if event.args["pool"] != self.BNT_ADDRESS else event.args["tkn_address"] ) + def get_pool_function(self, factory_contract: Contract): """ This function is unused for Bancor V3. """ - pass \ No newline at end of file + raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/carbon_v1.py b/fastlane_bot/events/exchanges/carbon_v1.py index e95416c6e..2e117736b 100644 --- a/fastlane_bot/events/exchanges/carbon_v1.py +++ b/fastlane_bot/events/exchanges/carbon_v1.py @@ -249,4 +249,4 @@ def get_pool_function(self, factory_contract: Contract): """ This function is unused for Carbon. """ - pass \ No newline at end of file + raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index 71ce763b6..7133b8fe4 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -18,31 +18,24 @@ class PoolFinder: """A class that provides methods to find unsupported carbon pairs and triangles within a given set of flashloan tokens and external pairs. """ - - multicallers = [] - - def __init__(self, uni_v2_forks: List[str], uni_v3_forks: List[str], solidly_v2_forks: List[str], carbon_forks: List[str], flashloan_tokens: List[str]): - self.uni_v2_forks = uni_v2_forks - self.uni_v3_forks = uni_v3_forks - self.solidly_v2_forks = solidly_v2_forks - self.carbon_forks = carbon_forks - self.flashloan_tokens = flashloan_tokens - self.uni_v3_fee_tiers = defaultdict(set) - self.carbon_pairs_seen = set() - - def init_exchanges(self, exchanges: List[Exchange], web3: Any, multicall_address: str): - """ This function initializes multicallers that will be used for each exchange. - - The function is separated from the main __init__ function to enable easier testing. - - Args: - exchanges (List[Exchange]): List of exchange objects for which to make multicalls. - web3 (Web3): Web3 object - multicall_address (str): The address of the multicall contract. - - - """ - self.multicallers = {ex_name: {"multicaller": MultiCaller(contract=exchange.sync_factory_contract, web3=web3, multicall_address=multicall_address), "exchange": exchange} for ex_name, exchange in exchanges.items() if ex_name in self.uni_v2_forks + self.uni_v3_forks + self.solidly_v2_forks} + def __init__( + self, + carbon_forks: List[str], + uni_v3_forks: List[str], + flashloan_tokens: List[str], + exchanges: List[Exchange], + web3: Any, + multicall_address: str + ): + self._carbon_forks = carbon_forks + self._uni_v3_forks = uni_v3_forks + self._flashloan_tokens = flashloan_tokens + self._uni_v3_fee_tiers = defaultdict(set) + self._carbon_pairs_seen = set() + + self._exchanges = list(filter(lambda e: e.base_exchange_name in [UNISWAP_V2_NAME, UNISWAP_V3_NAME, SOLIDLY_V2_NAME], exchanges.values())) + self._web3 = web3 + self._multicall_address = multicall_address def extract_univ3_fee_tiers(self, pools: List[Dict[str, Any]]): """ @@ -54,8 +47,8 @@ def extract_univ3_fee_tiers(self, pools: List[Dict[str, Any]]): This function updates the 'uni_v3_fee_tiers' dictionary where each exchange name is mapped to a set of unique fees. """ for pool in pools: - if pool["exchange_name"] in self.uni_v3_forks: - self.uni_v3_fee_tiers[pool["exchange_name"]].add(int(pool["fee"])) + if pool["exchange_name"] in self._uni_v3_forks: + self._uni_v3_fee_tiers[pool["exchange_name"]].add(int(pool["fee"])) def get_pools_for_unsupported_pairs(self, pools: List[Dict[str, Any]], arb_mode: str): @@ -72,14 +65,16 @@ def get_pools_for_unsupported_pairs(self, pools: List[Dict[str, Any]], arb_mode: and Solidly V2 forks), each associated with their specific supporting pools based on the unsupported configurations identified. """ - carbon_pairs, other_pairs = self._extract_pairs(pools=pools, carbon_forks=self.carbon_forks) + carbon_pairs, other_pairs = self._extract_pairs(pools=pools) if not carbon_pairs: return [], [], [] - self.extract_univ3_fee_tiers(pools) - func = PoolFinder._find_unsupported_triangles if arb_mode in ["triangle", "multi_triangle"] else PoolFinder._find_unsupported_pairs - unsupported = func(self.flashloan_tokens, carbon_pairs=carbon_pairs, external_pairs=other_pairs) - supporting_pools = self._find_pools(unsupported) - return self._sort_exchange_pools(supporting_pools, self.uni_v2_forks, self.uni_v3_forks, self.solidly_v2_forks) + self.extract_univ3_fee_tiers(pools) # TODO: these should be configured per exchange + if arb_mode in ["triangle", "multi_triangle"]: + unsupported_pairs = PoolFinder._find_unsupported_triangles(self._flashloan_tokens, carbon_pairs=carbon_pairs, external_pairs=other_pairs) + else: + unsupported_pairs = PoolFinder._find_unsupported_pairs(self._flashloan_tokens, carbon_pairs=carbon_pairs, external_pairs=other_pairs) + missing_pools = self._find_pools(unsupported_pairs) + return missing_pools[UNISWAP_V2_NAME], missing_pools[UNISWAP_V3_NAME], missing_pools[SOLIDLY_V2_NAME] def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: @@ -102,72 +97,40 @@ def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: implementation specifics of the multicall context manager or the exchange's get_pool_function method if it encounters a problem. """ - pairs = [(tkn, token) for pair in unsupported_pairs for tkn in pair for token in self.flashloan_tokens] + pairs = [(tkn, token) for pair in unsupported_pairs for tkn in pair for token in self._flashloan_tokens] chunk_size = 400 # Create the list of chunks chunked_pairs = [pairs[i:i + chunk_size] for i in range(0, len(pairs), chunk_size)] - result_list = {} + result = defaultdict(dict) - for ex_name, ex_data in self.multicallers.items(): - mc = ex_data["multicaller"] - ex = ex_data["exchange"] + for exchange in self._exchanges: for pair_chunk in chunked_pairs: - with mc: - for pair in pair_chunk: - if ex.base_exchange_name == UNISWAP_V2_NAME: - mc.add_call(ex.get_pool_function(ex.sync_factory_contract), pair[0], pair[1]) - elif ex.base_exchange_name == UNISWAP_V3_NAME: - for fee in self.uni_v3_fee_tiers[ex.exchange_name]: - mc.add_call(ex.get_pool_function(ex.sync_factory_contract), pair[0], pair[1], fee) - elif ex.base_exchange_name == SOLIDLY_V2_NAME: - mc.add_call(ex.get_pool_function(ex.sync_factory_contract), *ex.get_pool_args(pair[0], pair[1], False)) - results = mc.multicall() - mc._contract_calls = [] - result_list[ex.exchange_name] = [mc.web3.to_checksum_address(addr) for addr in results if addr != ZERO_ADDRESS] - return result_list - - @staticmethod - def _sort_exchange_pools(ex_pools: Dict, uni_v2_forks: List[str], uni_v3_forks: List[str], solidly_v2_forks: List[str]): - """ - Categorizes pools based on the type of exchange they belong to into separate dictionaries. - - Args: - ex_pools (List[Dict]): A list of dictionary where keys are exchange names and values are lists of pool addresses. - - Returns: - Tuple[Dict, Dict, Dict]: Three dictionaries categorizing pool addresses into Uniswap V2 forks, - Uniswap V3 forks, and Solidly V2 forks. - """ - # Initialize separate dictionaries for each type of exchange fork - uni_v2_pools = {} - uni_v3_pools = {} - solidly_v2_pools = {} - - # Assign pools to the appropriate category based on the exchange type - for ex_name, pools in ex_pools.items(): - if ex_name in uni_v2_forks: - target_pools = uni_v2_pools - elif ex_name in uni_v3_forks: - target_pools = uni_v3_pools - elif ex_name in solidly_v2_forks: - target_pools = solidly_v2_pools - else: - continue # Skip exchanges that do not match any known category - - for addr in pools: - target_pools[addr] = ex_name # Map pool address to exchange name - - return uni_v2_pools, uni_v3_pools, solidly_v2_pools - - - def _extract_pairs(self, pools: List[Dict[str, Any]], carbon_forks: List[str]) -> (List, set): + mc = MultiCaller(contract=exchange.sync_factory_contract, web3=self._web3, multicall_address=self._multicall_address) + for pair in pair_chunk: + if exchange.base_exchange_name == UNISWAP_V2_NAME: + mc.add_call(exchange.get_pool_function(exchange.sync_factory_contract), pair[0], pair[1]) + elif exchange.base_exchange_name == UNISWAP_V3_NAME: + for fee in self._uni_v3_fee_tiers[exchange.exchange_name]: + mc.add_call(exchange.get_pool_function(exchange.sync_factory_contract), pair[0], pair[1], fee) + elif exchange.base_exchange_name == SOLIDLY_V2_NAME: + mc.add_call(exchange.get_pool_function(exchange.sync_factory_contract), *exchange.get_pool_args(pair[0], pair[1], False)) + response = mc.multicall() + result[exchange.base_exchange_name] = { + mc.web3.to_checksum_address(addr): exchange.exchange_name + for addr + in response + if addr != ZERO_ADDRESS + } + return result + + + def _extract_pairs(self, pools: List[Dict[str, Any]]) -> Tuple[List, set]: """ Extracts unique, order-insensitive pairs of tokens from pools, categorizing them into carbon pairs and other pairs based on the exchange's presence in carbon_forks. Args: pools (List[Dict[str, Any]]): List of pool dictionaries containing token addresses and exchange names. - carbon_forks (List[str]): List of exchange names categorized under carbon forks. Returns: tuple: Two sets of unique token pairs, one for carbon forks and one for other exchanges. @@ -178,12 +141,12 @@ def _extract_pairs(self, pools: List[Dict[str, Any]], carbon_forks: List[str]) - for pool in pools: # Create a frozenset for each pair to ensure the pair is treated as order-insensitive pair = (pool["tkn0_address"], pool["tkn1_address"]) - frozen_pair = frozenset(pair) - if pool["exchange_name"] in carbon_forks: - if frozen_pair not in self.carbon_pairs_seen: + if pool["exchange_name"] in self._carbon_forks: + frozen_pair = frozenset(pair) + if frozen_pair not in self._carbon_pairs_seen: carbon_pairs.add(pair) - self.carbon_pairs_seen.add(frozen_pair) + self._carbon_pairs_seen.add(frozen_pair) else: other_pairs.add(pair) @@ -229,11 +192,7 @@ def _find_unsupported_pairs(flashloan_tokens: List[str], carbon_pairs: List[Tupl tkn0, tkn1 = pair[0], pair[1] if (tkn0 not in flashloan_tokens and tkn1 not in flashloan_tokens): unsupported_pairs.append(pair) - continue - if pair not in external_pairs: + elif pair not in external_pairs: unsupported_pairs.append(pair) return unsupported_pairs - - - diff --git a/main.py b/main.py index 3d51e1131..a4df7f892 100644 --- a/main.py +++ b/main.py @@ -306,9 +306,17 @@ def run(mgr, args, tenderly_uri=None) -> None: event_gatherer = EventGatherer(w3=mgr.w3_async, exchanges=mgr.exchanges, event_contracts=mgr.event_contracts) - pool_finder = PoolFinder(uni_v2_forks=mgr.cfg.network.UNI_V2_FORKS, uni_v3_forks=mgr.cfg.network.UNI_V3_FORKS, solidly_v2_forks=mgr.cfg.network.SOLIDLY_V2_FORKS, carbon_forks=mgr.cfg.network.CARBON_V1_FORKS, flashloan_tokens=args.flashloan_tokens) if args.pool_finder != -1 else None - if pool_finder: - pool_finder.init_exchanges(exchanges=mgr.exchanges, web3=mgr.web3, multicall_address=mgr.cfg.network.MULTICALL_CONTRACT_ADDRESS) + if args.pool_finder != -1: + pool_finder = PoolFinder( + carbon_forks=mgr.cfg.network.CARBON_V1_FORKS, + uni_v3_forks=mgr.cfg.network.UNI_V3_FORKS, + flashloan_tokens=args.flashloan_tokens, + exchanges=mgr.exchanges, + web3=mgr.web3, + multicall_address=mgr.cfg.network.MULTICALL_CONTRACT_ADDRESS + ) + else: + pool_finder = None while True: try: @@ -519,7 +527,7 @@ def run(mgr, args, tenderly_uri=None) -> None: mgr.solidly_v2_event_mappings = dict( solidly_v2_event_mappings[["address", "exchange"]].values ) - if args.pool_finder != -1 and (loop_idx % args.pool_finder == 0 or loop_idx == 1): + if pool_finder is not None and (loop_idx % args.pool_finder == 0 or loop_idx == 1): mgr.cfg.logger.info(f"Searching for unsupported Carbon pairs.") uni_v2, uni_v3, solidly_v2 = pool_finder.get_pools_for_unsupported_pairs(mgr.pool_data, arb_mode=args.arb_mode) result = f"Added {len(uni_v2) + len(uni_v3) + len(solidly_v2)} pools." if (uni_v2 or uni_v3 or solidly_v2) else f"No pools added." From ab2e57342cd7415300848d0073f61e586c632910 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Thu, 9 May 2024 15:02:57 +0200 Subject: [PATCH 19/58] refactor: pool multical --- fastlane_bot/events/exchanges/balancer.py | 5 +++-- fastlane_bot/events/exchanges/bancor_pol.py | 5 +++-- fastlane_bot/events/exchanges/bancor_v2.py | 5 +++-- fastlane_bot/events/exchanges/bancor_v3.py | 5 +++-- fastlane_bot/events/exchanges/base.py | 6 +++--- fastlane_bot/events/exchanges/carbon_v1.py | 3 ++- fastlane_bot/events/exchanges/solidly_v2/base.py | 4 ++++ .../events/exchanges/solidly_v2/cleopatra_v2.py | 3 --- .../events/exchanges/solidly_v2/equalizer_v2.py | 3 --- .../events/exchanges/solidly_v2/lynex_v2.py | 3 --- .../events/exchanges/solidly_v2/nile_v2.py | 3 --- .../events/exchanges/solidly_v2/scale_v2.py | 3 --- .../events/exchanges/solidly_v2/velocimeter_v2.py | 3 --- .../events/exchanges/solidly_v2/velodrome_v2.py | 5 +++-- .../events/exchanges/solidly_v2/xfai_v0.py | 5 +++-- fastlane_bot/events/exchanges/uniswap_v2.py | 15 +++------------ fastlane_bot/events/exchanges/uniswap_v3.py | 15 +++------------ fastlane_bot/pool_finder.py | 8 +++----- 18 files changed, 36 insertions(+), 63 deletions(-) diff --git a/fastlane_bot/events/exchanges/balancer.py b/fastlane_bot/events/exchanges/balancer.py index c9bb4e453..ade0dc1e8 100644 --- a/fastlane_bot/events/exchanges/balancer.py +++ b/fastlane_bot/events/exchanges/balancer.py @@ -16,6 +16,7 @@ from web3.contract import Contract +from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import BALANCER_VAULT_ABI, BALANCER_POOL_ABI_V1 from ..exchanges.base import Exchange from ..pools.base import Pool @@ -89,8 +90,8 @@ async def get_tkn_n(self, address: str, contract: Contract, event: Any, index: i token_balances = pool_balances[1] return token_balances[index] - def get_pool_function(self, factory_contract: Contract): + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): """ - This function is unused for Balancer. + This function is unused for Carbon. """ raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/bancor_pol.py b/fastlane_bot/events/exchanges/bancor_pol.py index f86fd8a77..64beb7840 100644 --- a/fastlane_bot/events/exchanges/bancor_pol.py +++ b/fastlane_bot/events/exchanges/bancor_pol.py @@ -16,6 +16,7 @@ from web3.contract import Contract +from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import BANCOR_POL_ABI from fastlane_bot import Config from ..exchanges.base import Exchange @@ -63,9 +64,9 @@ async def get_tkn0(self, address: str, contract: Contract, event: Event) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: return self.ETH_ADDRESS if event.args["token"] not in self.ETH_ADDRESS else self.BNT_ADDRESS - def get_pool_function(self, factory_contract: Contract): + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): """ - This function is unused for Bancor POL. + This function is unused for Carbon. """ raise NotImplementedError diff --git a/fastlane_bot/events/exchanges/bancor_v2.py b/fastlane_bot/events/exchanges/bancor_v2.py index b445e4e47..9f783c30d 100644 --- a/fastlane_bot/events/exchanges/bancor_v2.py +++ b/fastlane_bot/events/exchanges/bancor_v2.py @@ -17,6 +17,7 @@ from web3 import AsyncWeb3 from web3.contract import Contract, AsyncContract +from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import BANCOR_V2_CONVERTER_ABI from ..exchanges.base import Exchange from ..pools.base import Pool @@ -78,8 +79,8 @@ async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: async def get_anchor(self, contract: Contract) -> str: return await contract.caller.anchor() - def get_pool_function(self, factory_contract: Contract): + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): """ - This function is unused for Bancor V2. + This function is unused for Carbon. """ raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/bancor_v3.py b/fastlane_bot/events/exchanges/bancor_v3.py index 1d5328dba..f1f5d9092 100644 --- a/fastlane_bot/events/exchanges/bancor_v3.py +++ b/fastlane_bot/events/exchanges/bancor_v3.py @@ -16,6 +16,7 @@ from web3.contract import Contract +from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import BANCOR_V3_POOL_COLLECTION_ABI from ..exchanges.base import Exchange from ..pools.base import Pool @@ -65,8 +66,8 @@ async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: else event.args["tkn_address"] ) - def get_pool_function(self, factory_contract: Contract): + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): """ - This function is unused for Bancor V3. + This function is unused for Carbon. """ raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/base.py b/fastlane_bot/events/exchanges/base.py index fd13105d2..2f6095efd 100644 --- a/fastlane_bot/events/exchanges/base.py +++ b/fastlane_bot/events/exchanges/base.py @@ -17,6 +17,7 @@ from web3.contract import Contract, AsyncContract from fastlane_bot.config.constants import CARBON_V1_NAME +from fastlane_bot.config.multicaller import MultiCaller from ..pools.base import Pool from ..interfaces.subscription import Subscription @@ -123,13 +124,12 @@ async def get_fee(address: str, contract: AsyncContract) -> float: """ pass - @staticmethod @abstractmethod - def get_pool_function(contract: Contract, *args): + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2, *args): """ Returns the Factory contract function used to fetch liquidity pools. """ - pass + ... @staticmethod @abstractmethod diff --git a/fastlane_bot/events/exchanges/carbon_v1.py b/fastlane_bot/events/exchanges/carbon_v1.py index 2e117736b..fa1ebe985 100644 --- a/fastlane_bot/events/exchanges/carbon_v1.py +++ b/fastlane_bot/events/exchanges/carbon_v1.py @@ -17,6 +17,7 @@ from fastlane_bot import Config from web3.contract import Contract +from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import CARBON_CONTROLLER_ABI from ..exchanges.base import Exchange from ..pools.base import Pool @@ -245,7 +246,7 @@ def save_strategy( block_number=block_number, ) - def get_pool_function(self, factory_contract: Contract): + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): """ This function is unused for Carbon. """ diff --git a/fastlane_bot/events/exchanges/solidly_v2/base.py b/fastlane_bot/events/exchanges/solidly_v2/base.py index 56287b650..58c1433f2 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/base.py +++ b/fastlane_bot/events/exchanges/solidly_v2/base.py @@ -17,6 +17,7 @@ from web3.contract import Contract, AsyncContract +from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import SOLIDLY_V2_POOL_ABI from fastlane_bot.events.exchanges.base import Exchange from ...exchanges.base import Exchange @@ -62,6 +63,9 @@ async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return await contract.caller.token1() + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): + mc.add_call(self.sync_factory_contract.functions.getPair, addr1, addr2, True) + def get_pool_args(self, tkn0, tkn1, stable): return tkn0, tkn1, stable diff --git a/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py b/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py index 71da0b657..94662910a 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py @@ -17,6 +17,3 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee = await self.factory_contract.caller.getPairFee(address, await contract.caller.stable()) fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - - def get_pool_function(self, factory_contract): - return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py b/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py index 06992a1b2..861e53777 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py @@ -17,6 +17,3 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee = await self.factory_contract.caller.getRealFee(address) fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - - def get_pool_function(self, factory_contract): - return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py b/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py index ede370136..b3564b306 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py @@ -17,6 +17,3 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee = await self.factory_contract.caller.getFee(await contract.caller.stable()) fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - - def get_pool_function(self, factory_contract): - return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py b/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py index 9b90f0e26..3e4102ecb 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py @@ -17,6 +17,3 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee = await self.factory_contract.caller.pairFee(address) fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - - def get_pool_function(self, factory_contract): - return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py b/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py index 29a53e118..f0d50dec1 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py @@ -17,6 +17,3 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee = await self.factory_contract.caller.getRealFee(address) fee_float = float(fee) / 10 ** 18 return str(fee_float), fee_float - - def get_pool_function(self, factory_contract): - return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py b/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py index 51b1d88ec..858d36ecd 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py @@ -17,6 +17,3 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee = await self.factory_contract.caller.getFee(address) fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - - def get_pool_function(self, factory_contract): - return factory_contract.functions.getPair diff --git a/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py b/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py index 9de5fdf1a..eb9392ac7 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py @@ -3,6 +3,7 @@ from web3.contract import AsyncContract +from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import SOLIDLY_V2_FACTORY_ABI from .base import SolidlyV2 @@ -18,5 +19,5 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - def get_pool_function(self, factory_contract): - return factory_contract.functions.getPool + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): + mc.add_call(self.sync_factory_contract.functions.getPool, addr1, addr2, True) diff --git a/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py b/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py index 3ef5e2006..6d27b210d 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py +++ b/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py @@ -3,6 +3,7 @@ from web3.contract import Contract, AsyncContract +from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import XFAI_V0_POOL_ABI, XFAI_V0_FACTORY_ABI from .base import SolidlyV2 @@ -32,5 +33,5 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - def get_pool_function(self, factory_contract): - return factory_contract.functions.getPool + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): + mc.add_call(self.sync_factory_contract.functions.getPool, addr1) diff --git a/fastlane_bot/events/exchanges/uniswap_v2.py b/fastlane_bot/events/exchanges/uniswap_v2.py index b08e0ef27..a21067fae 100644 --- a/fastlane_bot/events/exchanges/uniswap_v2.py +++ b/fastlane_bot/events/exchanges/uniswap_v2.py @@ -16,6 +16,7 @@ from web3.contract import Contract, AsyncContract +from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import UNISWAP_V2_POOL_ABI, UNISWAP_V2_FACTORY_ABI from ..exchanges.base import Exchange from ..pools.base import Pool @@ -63,15 +64,5 @@ async def get_tkn0(address: str, contract: AsyncContract, event: Any) -> str: async def get_tkn1(address: str, contract: AsyncContract, event: Any) -> str: return await contract.caller.token1() - def get_pool_function(self, factory_contract: Contract): - """ Function to get pools from Factory. - This function is intended to be used with a Multicall. It fetches pools from a Uniswap V2 fork Factory contract. - - Args: - factory_contract: The factory contract. - - Returns: - The function. - - """ - return factory_contract.functions.getPair + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): + mc.add_call(self.sync_factory_contract.functions.getPair, addr1, addr2) diff --git a/fastlane_bot/events/exchanges/uniswap_v3.py b/fastlane_bot/events/exchanges/uniswap_v3.py index 0eb8fb6c4..14f31855d 100644 --- a/fastlane_bot/events/exchanges/uniswap_v3.py +++ b/fastlane_bot/events/exchanges/uniswap_v3.py @@ -17,6 +17,7 @@ from web3.contract import Contract, AsyncContract from fastlane_bot.config.constants import AGNI_V3_NAME, PANCAKESWAP_V3_NAME, FUSIONX_V3_NAME, ECHODEX_V3_NAME, SECTA_V3_NAME +from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import UNISWAP_V3_POOL_ABI, UNISWAP_V3_FACTORY_ABI, PANCAKESWAP_V3_POOL_ABI from ..exchanges.base import Exchange from ..pools.base import Pool @@ -60,15 +61,5 @@ async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return await contract.caller.token1() - def get_pool_function(self, factory_contract: Contract): - """ Function to get pools from Factory. - This function is intended to be used with a Multicall. It fetches pools from a Uniswap V3 fork Factory contract. - - Args: - factory_contract: The factory contract. - - Returns: - The function. - - """ - return factory_contract.functions.getPool + def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2, fee): + mc.add_call(self.sync_factory_contract.functions.getPool, addr1, addr2, fee) diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index 7133b8fe4..3a6969dfe 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -107,13 +107,11 @@ def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: for pair_chunk in chunked_pairs: mc = MultiCaller(contract=exchange.sync_factory_contract, web3=self._web3, multicall_address=self._multicall_address) for pair in pair_chunk: - if exchange.base_exchange_name == UNISWAP_V2_NAME: - mc.add_call(exchange.get_pool_function(exchange.sync_factory_contract), pair[0], pair[1]) + if exchange.base_exchange_name in [UNISWAP_V2_NAME, SOLIDLY_V2_NAME]: + exchange.get_pool_with_multicall(mc, pair[0], pair[1]) elif exchange.base_exchange_name == UNISWAP_V3_NAME: for fee in self._uni_v3_fee_tiers[exchange.exchange_name]: - mc.add_call(exchange.get_pool_function(exchange.sync_factory_contract), pair[0], pair[1], fee) - elif exchange.base_exchange_name == SOLIDLY_V2_NAME: - mc.add_call(exchange.get_pool_function(exchange.sync_factory_contract), *exchange.get_pool_args(pair[0], pair[1], False)) + exchange.get_pool_with_multicall(mc, pair[0], pair[1], fee) response = mc.multicall() result[exchange.base_exchange_name] = { mc.web3.to_checksum_address(addr): exchange.exchange_name From 654a0eb4a1761b994bc0c0f4f2eed1564bef575d Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Thu, 9 May 2024 15:10:16 +0200 Subject: [PATCH 20/58] fix: PR review --- fastlane_bot/events/managers/base.py | 1 + main.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fastlane_bot/events/managers/base.py b/fastlane_bot/events/managers/base.py index 6600c9304..a410135f7 100644 --- a/fastlane_bot/events/managers/base.py +++ b/fastlane_bot/events/managers/base.py @@ -164,6 +164,7 @@ def initialize_factory_contract(self, exchange): exchange.sync_factory_contract = self.sync_factory_contracts[exchange.exchange_name] return exchange + @property def fee_pairs(self) -> Dict: """ diff --git a/main.py b/main.py index a4df7f892..d926d5614 100644 --- a/main.py +++ b/main.py @@ -691,7 +691,7 @@ def run(mgr, args, tenderly_uri=None) -> None: ) parser.add_argument( "--blockchain", - default="coinbase_base", + default="ethereum", help="A blockchain from the list. Blockchains not in this list do not have a deployed Fast Lane contract and are not supported.", choices=["ethereum", "coinbase_base", "fantom", "mantle", "linea", "sei"], ) From b035165afd627642a61231a45bbfbfc21148ed19 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Thu, 9 May 2024 19:39:21 +0200 Subject: [PATCH 21/58] refactor: factory contract and abi --- fastlane_bot/events/exchanges/balancer.py | 2 +- fastlane_bot/events/exchanges/bancor_pol.py | 2 +- fastlane_bot/events/exchanges/bancor_v2.py | 2 +- fastlane_bot/events/exchanges/bancor_v3.py | 2 +- fastlane_bot/events/exchanges/base.py | 4 +-- fastlane_bot/events/exchanges/carbon_v1.py | 2 +- fastlane_bot/events/exchanges/factory.py | 13 ++++++- .../events/exchanges/solidly_v2/base.py | 10 ++---- .../exchanges/solidly_v2/cleopatra_v2.py | 2 +- .../exchanges/solidly_v2/equalizer_v2.py | 2 +- .../events/exchanges/solidly_v2/lynex_v2.py | 2 +- .../events/exchanges/solidly_v2/nile_v2.py | 2 +- .../events/exchanges/solidly_v2/scale_v2.py | 2 +- .../exchanges/solidly_v2/velocimeter_v2.py | 2 +- .../exchanges/solidly_v2/velodrome_v2.py | 4 +-- .../events/exchanges/solidly_v2/xfai_v0.py | 7 ++-- fastlane_bot/events/exchanges/uniswap_v2.py | 4 +-- fastlane_bot/events/exchanges/uniswap_v3.py | 4 +-- fastlane_bot/events/managers/base.py | 34 +------------------ fastlane_bot/pool_finder.py | 4 +-- run_blockchain_terraformer.py | 2 +- 21 files changed, 39 insertions(+), 69 deletions(-) diff --git a/fastlane_bot/events/exchanges/balancer.py b/fastlane_bot/events/exchanges/balancer.py index ade0dc1e8..2f9f99fc9 100644 --- a/fastlane_bot/events/exchanges/balancer.py +++ b/fastlane_bot/events/exchanges/balancer.py @@ -39,7 +39,7 @@ def get_abi(self): return BALANCER_VAULT_ABI @property - def get_factory_abi(self): + def factory_abi(self): # Not used for Balancer return BALANCER_VAULT_ABI diff --git a/fastlane_bot/events/exchanges/bancor_pol.py b/fastlane_bot/events/exchanges/bancor_pol.py index 64beb7840..3fbb0fbb0 100644 --- a/fastlane_bot/events/exchanges/bancor_pol.py +++ b/fastlane_bot/events/exchanges/bancor_pol.py @@ -42,7 +42,7 @@ def get_abi(self): return BANCOR_POL_ABI @property - def get_factory_abi(self): + def factory_abi(self): # Not used for Bancor POL return BANCOR_POL_ABI diff --git a/fastlane_bot/events/exchanges/bancor_v2.py b/fastlane_bot/events/exchanges/bancor_v2.py index 9f783c30d..41ff63860 100644 --- a/fastlane_bot/events/exchanges/bancor_v2.py +++ b/fastlane_bot/events/exchanges/bancor_v2.py @@ -41,7 +41,7 @@ def get_abi(self): return BANCOR_V2_CONVERTER_ABI @property - def get_factory_abi(self): + def factory_abi(self): # Not used for Bancor V2 return BANCOR_V2_CONVERTER_ABI diff --git a/fastlane_bot/events/exchanges/bancor_v3.py b/fastlane_bot/events/exchanges/bancor_v3.py index f1f5d9092..b21c6f7bb 100644 --- a/fastlane_bot/events/exchanges/bancor_v3.py +++ b/fastlane_bot/events/exchanges/bancor_v3.py @@ -43,7 +43,7 @@ def get_abi(self): return BANCOR_V3_POOL_COLLECTION_ABI @property - def get_factory_abi(self): + def factory_abi(self): # Not used for Bancor V3 return BANCOR_V3_POOL_COLLECTION_ABI diff --git a/fastlane_bot/events/exchanges/base.py b/fastlane_bot/events/exchanges/base.py index 2f6095efd..93020c3b5 100644 --- a/fastlane_bot/events/exchanges/base.py +++ b/fastlane_bot/events/exchanges/base.py @@ -31,7 +31,7 @@ class Exchange(ABC): exchange_name: str base_exchange_name: str = '' pools: Dict[str, Pool] = field(default_factory=dict) - sync_factory_contract: Contract = None + factory_contract: Contract = None __VERSION__ = "0.0.3" __DATE__ = "2024-03-20" @@ -195,7 +195,7 @@ def get_pool(self, key: str) -> Pool: return self.pools[key] if key in self.pools else None @abstractmethod - def get_factory_abi(self): + def factory_abi(self): """ Get the ABI of the exchange's Factory contract diff --git a/fastlane_bot/events/exchanges/carbon_v1.py b/fastlane_bot/events/exchanges/carbon_v1.py index fa1ebe985..b3708c822 100644 --- a/fastlane_bot/events/exchanges/carbon_v1.py +++ b/fastlane_bot/events/exchanges/carbon_v1.py @@ -68,7 +68,7 @@ def get_abi(self): return CARBON_CONTROLLER_ABI @property - def get_factory_abi(self): + def factory_abi(self): return CARBON_CONTROLLER_ABI def get_events(self, contract: Contract) -> List[Type[Contract]]: diff --git a/fastlane_bot/events/exchanges/factory.py b/fastlane_bot/events/exchanges/factory.py index 670bdc5df..9fb50823d 100644 --- a/fastlane_bot/events/exchanges/factory.py +++ b/fastlane_bot/events/exchanges/factory.py @@ -12,6 +12,8 @@ """ from typing import Dict, Any +from fastlane_bot.config.constants import SOLIDLY_V2_NAME, UNISWAP_V2_NAME, UNISWAP_V3_NAME + class ExchangeFactory: """ @@ -60,7 +62,16 @@ def get_exchange(self, key, cfg: Any, exchange_initialized: bool = None): creator = self._creators.get(fork_name) args = self.get_fork_extras(exchange_name=key, cfg=cfg, exchange_initialized=exchange_initialized) - return creator(**args) + exchange = creator(**args) + + base_exchange_name = cfg.network.exchange_name_base_from_fork(exchange_name=key) + if base_exchange_name in [SOLIDLY_V2_NAME, UNISWAP_V2_NAME, UNISWAP_V3_NAME]: + exchange.factory_contract = self.w3_async.eth.contract( + address=cfg.FACTORY_MAPPING[key], + abi=exchange.factory_abi, + ) + + return exchange def get_fork_extras(self, exchange_name: str, cfg: Any, exchange_initialized: bool) -> Dict[str, str]: """ diff --git a/fastlane_bot/events/exchanges/solidly_v2/base.py b/fastlane_bot/events/exchanges/solidly_v2/base.py index 58c1433f2..52a971e7e 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/base.py +++ b/fastlane_bot/events/exchanges/solidly_v2/base.py @@ -64,19 +64,13 @@ async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return await contract.caller.token1() def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - mc.add_call(self.sync_factory_contract.functions.getPair, addr1, addr2, True) - - def get_pool_args(self, tkn0, tkn1, stable): - return tkn0, tkn1, stable + mc.add_call(self.factory_contract.functions.getPair, addr1, addr2, True) @property @abstractmethod - def get_factory_abi(self): + def factory_abi(self): ... @abstractmethod async def get_fee(self, address: str, contract: Contract, factory_contract: Contract): ... - - def get_pool_function(self, factory_contract): - ... diff --git a/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py b/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py index 94662910a..ecff33375 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/cleopatra_v2.py @@ -10,7 +10,7 @@ @dataclass class CleopatraV2(SolidlyV2): @property - def get_factory_abi(self): + def factory_abi(self): return CLEOPATRA_V2_FACTORY_ABI async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: diff --git a/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py b/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py index 861e53777..820f9a8ec 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/equalizer_v2.py @@ -10,7 +10,7 @@ @dataclass class EqualizerV2(SolidlyV2): @property - def get_factory_abi(self): + def factory_abi(self): return SCALE_V2_FACTORY_ABI async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: diff --git a/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py b/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py index b3564b306..79339519d 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/lynex_v2.py @@ -10,7 +10,7 @@ @dataclass class LynexV2(SolidlyV2): @property - def get_factory_abi(self): + def factory_abi(self): return LYNEX_V2_FACTORY_ABI async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: diff --git a/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py b/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py index 3e4102ecb..fe81c9921 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/nile_v2.py @@ -10,7 +10,7 @@ @dataclass class NileV2(SolidlyV2): @property - def get_factory_abi(self): + def factory_abi(self): return NILE_V2_FACTORY_ABI async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: diff --git a/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py b/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py index f0d50dec1..cb2744792 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/scale_v2.py @@ -10,7 +10,7 @@ @dataclass class ScaleV2(SolidlyV2): @property - def get_factory_abi(self): + def factory_abi(self): return SCALE_V2_FACTORY_ABI async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: diff --git a/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py b/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py index 858d36ecd..65634dc5a 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/velocimeter_v2.py @@ -10,7 +10,7 @@ @dataclass class VelocimeterV2(SolidlyV2): @property - def get_factory_abi(self): + def factory_abi(self): return VELOCIMETER_V2_FACTORY_ABI async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: diff --git a/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py b/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py index eb9392ac7..1224c6df6 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py @@ -11,7 +11,7 @@ @dataclass class VelodromeV2(SolidlyV2): @property - def get_factory_abi(self): + def factory_abi(self): return SOLIDLY_V2_FACTORY_ABI async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: @@ -20,4 +20,4 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo return str(fee_float), fee_float def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - mc.add_call(self.sync_factory_contract.functions.getPool, addr1, addr2, True) + mc.add_call(self.factory_contract.functions.getPool, addr1, addr2, True) diff --git a/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py b/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py index 6d27b210d..e8d7eb55e 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py +++ b/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py @@ -19,11 +19,8 @@ async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return "0xe5D7C2a44FfDDf6b295A15c148167daaAf5Cf34f" # TODO Use the constant WRAPPED_GAS_TOKEN_ADDRESS for this network - def get_pool_args(self, tkn0, tkn1, stable): - return tkn0 - @property - def get_factory_abi(self): + def factory_abi(self): # TODO: change to staticmethod return XFAI_V0_FACTORY_ABI async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, float]: @@ -34,4 +31,4 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo return str(fee_float), fee_float def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - mc.add_call(self.sync_factory_contract.functions.getPool, addr1) + mc.add_call(self.factory_contract.functions.getPool, addr1) diff --git a/fastlane_bot/events/exchanges/uniswap_v2.py b/fastlane_bot/events/exchanges/uniswap_v2.py index a21067fae..c7ed2615f 100644 --- a/fastlane_bot/events/exchanges/uniswap_v2.py +++ b/fastlane_bot/events/exchanges/uniswap_v2.py @@ -38,7 +38,7 @@ def fee_float(self): return float(self.fee) @property - def get_factory_abi(self): + def factory_abi(self): return UNISWAP_V2_FACTORY_ABI def add_pool(self, pool: Pool): @@ -65,4 +65,4 @@ async def get_tkn1(address: str, contract: AsyncContract, event: Any) -> str: return await contract.caller.token1() def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - mc.add_call(self.sync_factory_contract.functions.getPair, addr1, addr2) + mc.add_call(self.factory_contract.functions.getPair, addr1, addr2) diff --git a/fastlane_bot/events/exchanges/uniswap_v3.py b/fastlane_bot/events/exchanges/uniswap_v3.py index 14f31855d..604ba5af9 100644 --- a/fastlane_bot/events/exchanges/uniswap_v3.py +++ b/fastlane_bot/events/exchanges/uniswap_v3.py @@ -41,7 +41,7 @@ def get_abi(self): return UNISWAP_V3_POOL_ABI if self.exchange_name not in [PANCAKESWAP_V3_NAME, AGNI_V3_NAME, FUSIONX_V3_NAME, ECHODEX_V3_NAME, SECTA_V3_NAME] else PANCAKESWAP_V3_POOL_ABI @property - def get_factory_abi(self): + def factory_abi(self): return UNISWAP_V3_FACTORY_ABI def get_events(self, contract: Contract) -> List[Type[Contract]]: @@ -62,4 +62,4 @@ async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return await contract.caller.token1() def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2, fee): - mc.add_call(self.sync_factory_contract.functions.getPool, addr1, addr2, fee) + mc.add_call(self.factory_contract.functions.getPool, addr1, addr2, fee) diff --git a/fastlane_bot/events/managers/base.py b/fastlane_bot/events/managers/base.py index a410135f7..ca6361fbd 100644 --- a/fastlane_bot/events/managers/base.py +++ b/fastlane_bot/events/managers/base.py @@ -17,7 +17,7 @@ from fastlane_bot import Config from fastlane_bot.config.constants import PANCAKESWAP_V2_NAME, PANCAKESWAP_V3_NAME, VELOCIMETER_V2_NAME, AGNI_V3_NAME, \ - SOLIDLY_V2_NAME, FUSIONX_V3_NAME, UNISWAP_V2_NAME, UNISWAP_V3_NAME + FUSIONX_V3_NAME from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.events.exchanges import exchange_factory from fastlane_bot.events.exchanges.base import Exchange @@ -78,8 +78,6 @@ class BaseManager: token_contracts: Dict[str, Contract or Type[Contract]] = field(default_factory=dict) erc20_contracts: Dict[str, Contract or Type[Contract]] = field(default_factory=dict) exchanges: Dict[str, Exchange] = field(default_factory=dict) - sync_factory_contracts: Dict[str, Contract or Type[Contract]] = field(default_factory=dict) - factory_contracts: Dict[str, Contract or Type[Contract]] = field(default_factory=dict) uniswap_v2_event_mappings: Dict[str, str] = field(default_factory=dict) uniswap_v3_event_mappings: Dict[str, str] = field(default_factory=dict) solidly_v2_event_mappings: Dict[str, str] = field(default_factory=dict) @@ -130,41 +128,11 @@ def __post_init__(self): self.SUPPORTED_BASE_EXCHANGES.append(base_exchange_name) self.exchanges[exchange_name] = exchange_factory.get_exchange(key=exchange_name, cfg=self.cfg, exchange_initialized=initialize_events) - if base_exchange_name == SOLIDLY_V2_NAME: - self.exchanges[exchange_name] = self.handle_solidly_exchanges(exchange=self.exchanges[exchange_name]) - if base_exchange_name in [SOLIDLY_V2_NAME, UNISWAP_V2_NAME, UNISWAP_V3_NAME]: - self.exchanges[exchange_name] = self.initialize_factory_contract(exchange=self.exchanges[exchange_name]) self.init_exchange_contracts() self.set_carbon_v1_fee_pairs() self.init_tenderly_event_contracts() - def handle_solidly_exchanges(self, exchange): - """ - Handles getting stable & volatile fees for Solidly forks - """ - exchange_name = exchange.exchange_name - self.factory_contracts[exchange_name] = self.w3_async.eth.contract( - address=self.cfg.FACTORY_MAPPING[exchange_name], - abi=exchange.get_factory_abi, - ) - exchange.factory_contract = self.factory_contracts[exchange.exchange_name] - - return exchange - - def initialize_factory_contract(self, exchange): - """ - Initialize factory contract for exchange. - """ - exchange_name = exchange.exchange_name - self.sync_factory_contracts[exchange_name] = self.web3.eth.contract( - address=self.cfg.FACTORY_MAPPING[exchange_name], - abi=exchange.get_factory_abi, - ) - exchange.sync_factory_contract = self.sync_factory_contracts[exchange.exchange_name] - - return exchange - @property def fee_pairs(self) -> Dict: """ diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index 3a6969dfe..8f56436e4 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -95,7 +95,7 @@ def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: Raises: Exception: An exception could be raised from the multicall operation depending on the implementation specifics of the multicall context manager or the exchange's - get_pool_function method if it encounters a problem. + get_pool_with_multicall method if it encounters a problem. """ pairs = [(tkn, token) for pair in unsupported_pairs for tkn in pair for token in self._flashloan_tokens] chunk_size = 400 @@ -105,7 +105,7 @@ def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: for exchange in self._exchanges: for pair_chunk in chunked_pairs: - mc = MultiCaller(contract=exchange.sync_factory_contract, web3=self._web3, multicall_address=self._multicall_address) + mc = MultiCaller(contract=exchange.factory_contract, web3=self._web3, multicall_address=self._multicall_address) for pair in pair_chunk: if exchange.base_exchange_name in [UNISWAP_V2_NAME, SOLIDLY_V2_NAME]: exchange.get_pool_with_multicall(mc, pair[0], pair[1]) diff --git a/run_blockchain_terraformer.py b/run_blockchain_terraformer.py index 1fd29fb27..854119d40 100644 --- a/run_blockchain_terraformer.py +++ b/run_blockchain_terraformer.py @@ -1024,7 +1024,7 @@ def terraform_blockchain(network_name: str): elif "solidly" in fork: add_to_exchange_ids(exchange=exchange_name, fork=fork) solidly_exchange = SolidlyV2(exchange_name=exchange_name) - factory_abi = solidly_exchange.get_factory_abi + factory_abi = solidly_exchange.factory_abi factory_contract = web3.eth.contract( address=factory_address, abi=factory_abi From ba63ba953d73f25866e8d40c45760fcf083d2377 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Fri, 10 May 2024 01:14:51 +0300 Subject: [PATCH 22/58] fix: tests --- fastlane_bot/events/exchanges/factory.py | 2 +- fastlane_bot/tests/test_073_TestPoolFinder.py | 41 ++++--------------- 2 files changed, 10 insertions(+), 33 deletions(-) diff --git a/fastlane_bot/events/exchanges/factory.py b/fastlane_bot/events/exchanges/factory.py index 9fb50823d..fe98beea6 100644 --- a/fastlane_bot/events/exchanges/factory.py +++ b/fastlane_bot/events/exchanges/factory.py @@ -66,7 +66,7 @@ def get_exchange(self, key, cfg: Any, exchange_initialized: bool = None): base_exchange_name = cfg.network.exchange_name_base_from_fork(exchange_name=key) if base_exchange_name in [SOLIDLY_V2_NAME, UNISWAP_V2_NAME, UNISWAP_V3_NAME]: - exchange.factory_contract = self.w3_async.eth.contract( + exchange.factory_contract = cfg.w3_async.eth.contract( address=cfg.FACTORY_MAPPING[key], abi=exchange.factory_abi, ) diff --git a/fastlane_bot/tests/test_073_TestPoolFinder.py b/fastlane_bot/tests/test_073_TestPoolFinder.py index e6bbe1262..bfd1611c9 100644 --- a/fastlane_bot/tests/test_073_TestPoolFinder.py +++ b/fastlane_bot/tests/test_073_TestPoolFinder.py @@ -37,9 +37,7 @@ def test_find_unsupported_triangles(): def test_extract_pairs(): # Sample data for testing - uni_v2_exchanges = ["bob_ex"] uni_v3_exchanges = ["fred_ex"] - solidly_v2_exchanges = ["george_ex", "moose_ex"] flashloan_tokens = ["BNT"] pools = [ {"exchange_name": "CarbonX", "tkn0_address": "WBTC", "tkn1_address": "BNT"}, @@ -54,10 +52,17 @@ def test_extract_pairs(): expected_carbon_pairs = {frozenset(('WBTC', 'BNT')), frozenset(('WETH', 'USDC'))} expected_other_pairs = {frozenset(('WETH', 'USDT')), frozenset(('USDC', 'WBTC'))} #expected_carbon_pairs = [('WBTC', 'BNT'), ('WETH', 'USDC')] - pool_finder = PoolFinder(uni_v2_forks=uni_v2_exchanges, uni_v3_forks=uni_v3_exchanges, solidly_v2_forks=solidly_v2_exchanges, carbon_forks=carbon_forks, flashloan_tokens=flashloan_tokens) + pool_finder = PoolFinder( + uni_v3_forks=uni_v3_exchanges, + carbon_forks=carbon_forks, + flashloan_tokens=flashloan_tokens, + exchanges={}, + web3=None, + multicall_address="" + ) # Call the function with test data - carbon_pairs, other_pairs = pool_finder._extract_pairs(pools, carbon_forks) + carbon_pairs, other_pairs = pool_finder._extract_pairs(pools) # Assert the results are as expected assert frozenset(carbon_pairs[0]) in expected_carbon_pairs @@ -67,31 +72,3 @@ def test_extract_pairs(): #assert other_pairs == expected_other_pairs assert len(carbon_pairs) == 2 assert len(other_pairs) == 2 - -def test_sort_exchanges(): - exchange_pools = { - "bob_ex": ["1", "2", "3"], - "fred_ex": ["4", "5"], - "george_ex": ["6", "7", "8"], - "moose_ex": ["9", "10", "11", "12"] - } - - uni_v2_exchanges = ["bob_ex"] - uni_v3_exchanges = ["fred_ex"] - solidly_v2_exchanges = ["george_ex", "moose_ex"] - uni_v2_pools, uni_v3_pools, solidly_v2_pools = PoolFinder._sort_exchange_pools(ex_pools=exchange_pools, uni_v2_forks=uni_v2_exchanges, uni_v3_forks=uni_v3_exchanges, solidly_v2_forks=solidly_v2_exchanges) - - - assert len(uni_v2_pools.keys()) == 3 - assert len(uni_v3_pools.keys()) == 2 - assert len(solidly_v2_pools.keys()) == 7 - - assert uni_v2_pools["1"] == "bob_ex" - assert uni_v3_pools["5"] == "fred_ex" - assert solidly_v2_pools["7"] == "george_ex" - assert solidly_v2_pools["9"] == "moose_ex" - - - - - From 7c9df8f1eee0dd89a596f364e016ce42bdc6b279 Mon Sep 17 00:00:00 2001 From: NIXBNT <88088888+NIXBNT@users.noreply.github.com> Date: Thu, 2 May 2024 19:44:27 +1000 Subject: [PATCH 23/58] expand combo scope --- fastlane_bot/modes/base_pairwise.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastlane_bot/modes/base_pairwise.py b/fastlane_bot/modes/base_pairwise.py index d5df65ef9..7be12a719 100644 --- a/fastlane_bot/modes/base_pairwise.py +++ b/fastlane_bot/modes/base_pairwise.py @@ -49,10 +49,10 @@ def get_combos( """ all_tokens = CCm.tokens() - flashloan_tokens_intersect = all_tokens.intersection(set(flashloan_tokens)) + # flashloan_tokens_intersect = all_tokens.intersection(set(flashloan_tokens)) combos = [ (tkn0, tkn1) - for tkn0, tkn1 in itertools.product(all_tokens, flashloan_tokens_intersect) + for tkn0, tkn1 in itertools.product(all_tokens, flashloan_tokens) # tkn1 is always the token being flash loaned # note that the pair is tkn0/tkn1, ie tkn1 is the quote token if tkn0 != tkn1 From cc0dc5eea5854b3ed5f4f44ba68fb46a39159f45 Mon Sep 17 00:00:00 2001 From: NIXBNT <88088888+NIXBNT@users.noreply.github.com> Date: Thu, 2 May 2024 19:45:05 +1000 Subject: [PATCH 24/58] add carbon curves pairwise --- fastlane_bot/modes/pairwise_multi_all.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fastlane_bot/modes/pairwise_multi_all.py b/fastlane_bot/modes/pairwise_multi_all.py index 924bdb702..07fdd8237 100644 --- a/fastlane_bot/modes/pairwise_multi_all.py +++ b/fastlane_bot/modes/pairwise_multi_all.py @@ -67,6 +67,9 @@ def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_p if len(base_direction_two) > 0: curve_combos += [[curve] + base_direction_two for curve in not_carbon_curves] + if len(carbon_curves) >= 2: + curve_combos += [carbon_curves] + for curve_combo in curve_combos: src_token = tkn1 if len(curve_combo) < 2: From 5cbbf5da3390f42d5be7d9d438cda9cc950db5c6 Mon Sep 17 00:00:00 2001 From: NIXBNT <88088888+NIXBNT@users.noreply.github.com> Date: Mon, 6 May 2024 22:03:35 +1000 Subject: [PATCH 25/58] multi_triangle_complete mode added Increases the breadth and speed of carbon focused triangles --- fastlane_bot/bot.py | 2 + fastlane_bot/modes/base_triangle.py | 135 ++++++++++++++++++ fastlane_bot/modes/triangle_multi_complete.py | 114 +++++++++++++++ main.py | 1 + 4 files changed, 252 insertions(+) create mode 100644 fastlane_bot/modes/triangle_multi_complete.py diff --git a/fastlane_bot/bot.py b/fastlane_bot/bot.py index 5fe5800d5..31090b8c5 100644 --- a/fastlane_bot/bot.py +++ b/fastlane_bot/bot.py @@ -74,6 +74,7 @@ from .modes.pairwise_multi_pol import FindArbitrageMultiPairwisePol from .modes.pairwise_single import FindArbitrageSinglePairwise from .modes.triangle_multi import ArbitrageFinderTriangleMulti +from .modes.triangle_multi_complete import ArbitrageFinderTriangleMultiComplete from .modes.triangle_single import ArbitrageFinderTriangleSingle from .modes.triangle_bancor_v3_two_hop import ArbitrageFinderTriangleBancor3TwoHop from .utils import num_format @@ -109,6 +110,7 @@ class CarbonBot: "b3_two_hop": ArbitrageFinderTriangleBancor3TwoHop, "multi_pairwise_pol": FindArbitrageMultiPairwisePol, "multi_pairwise_all": FindArbitrageMultiPairwiseAll, + "multi_triangle_complete": ArbitrageFinderTriangleMultiComplete, } class NoArbAvailable(Exception): diff --git a/fastlane_bot/modes/base_triangle.py b/fastlane_bot/modes/base_triangle.py index bf5f0c6a4..d804ab24d 100644 --- a/fastlane_bot/modes/base_triangle.py +++ b/fastlane_bot/modes/base_triangle.py @@ -174,6 +174,141 @@ def get_combos( combos, ) return combos + + def sort_pairs(self, pairs): + # Clean up the pairs alphabetically + return ["/".join(sorted(pair.split('/'))) for pair in pairs] + + def flatten_nested_items_in_list(self, nested_list): + # unpack nested items + flattened_list = [] + for items in nested_list: + flat_list = [] + for item in items: + if isinstance(item, list): + flat_list.extend(item) + else: + flat_list.append(item) + flattened_list.append(flat_list) + return flattened_list + + def get_triangle_groups(self, flt, x_y_pairs): + # Get groups of triangles that conform to (flt/x , x/y, y/flt) where x!=y + triangle_groups = [] + for pair in x_y_pairs: + x,y = pair.split('/') + triangle_groups += [("/".join(sorted([flt,x])), pair, "/".join(sorted([flt,y])))] + return triangle_groups + + def get_all_relevant_pairs_info(self, CCm, all_relevant_pairs): + # Get pair info for the cohort to allow decision making at the triangle level + all_relevant_pairs_info = {} + for pair in all_relevant_pairs: + all_relevant_pairs_info[pair] = {} + pair_curves = CCm.bypair(pair) + carbon_curves = [] + non_carbon_curves = [] + for x in pair_curves: + if x.params.exchange in self.ConfigObj.CARBON_V1_FORKS: + carbon_curves += [x] + else: + non_carbon_curves += [x] + all_relevant_pairs_info[pair]['non_carbon_curves'] = non_carbon_curves + all_relevant_pairs_info[pair]['carbon_curves'] = carbon_curves + all_relevant_pairs_info[pair]['curves'] = non_carbon_curves + [carbon_curves] if len(carbon_curves) >0 else non_carbon_curves # condense carbon curves into a single list + all_relevant_pairs_info[pair]['all_counts'] = len(pair_curves) + all_relevant_pairs_info[pair]['carbon_counts'] = len(carbon_curves) + return all_relevant_pairs_info + + def get_triangle_groups_stats(self, triangle_groups, all_relevant_pairs_info, arb_mode=None): + # Get the stats on the triangle group cohort for decision making + triangle_group_stats = {} + for i,triangle in enumerate(triangle_groups): + triangle_group_stats[i] = {} + triangle_group_stats[i]['path'] = [] + triangle_group_stats[i]['has_path'] = False + triangle_group_stats[i]['has_carbon'] = False + for pair in triangle: + if all_relevant_pairs_info[pair]['all_counts'] > 0: + triangle_group_stats[i]['path'] += [1] + else: + triangle_group_stats[i]['path'] += [0] + if all_relevant_pairs_info[pair]['carbon_counts'] > 0: + triangle_group_stats[i]['has_carbon'] = True + if sum(triangle_group_stats[i]['path']) == 3: + triangle_group_stats[i]['has_path'] = True + + # identify valid paths for the given mode + valid_carbon_triangles = [triangle_groups[i] for i in triangle_group_stats.keys() if (triangle_group_stats[i]['has_path'] == True) and (triangle_group_stats[i]['has_carbon'] == True)] + return valid_carbon_triangles + + def get_analysis_set_per_flt(self, flt, valid_triangles, all_relevant_pairs_info): + flt_triangle_analysis_set = [] + for triangle in valid_triangles: + multiverse = [all_relevant_pairs_info[pair]['curves'] for pair in triangle] + product_of_triangle = list(itertools.product(multiverse[0], multiverse[1], multiverse[2])) + triangles_to_run = self.flatten_nested_items_in_list(product_of_triangle) + flt_triangle_analysis_set += list(zip([flt] * len(triangles_to_run), triangles_to_run)) + + self.ConfigObj.logger.debug(f"[base_triangle.get_analysis_set_per_flt] Length of flt_triangle_analysis_set: {flt, len(flt_triangle_analysis_set)}") + return flt_triangle_analysis_set + + def get_comprehensive_triangles( + self, flashloan_tokens: List[str], CCm: Any, arb_mode: str + ) -> Tuple[List[str], List[Any]]: + """ + Get comprehensive combos for triangular arbitrage + + Parameters + ---------- + flashloan_tokens : list + List of flashloan tokens + CCm : object + CCm object + arb_mode : str + Arbitrage mode + + Returns + ------- + combos : list + List of combos + + """ + combos = [] + for flt in flashloan_tokens: + + # Get the Carbon pairs + carbon_pairs = self.sort_pairs(set([x.pair for x in CCm.curves if x.params.exchange in self.ConfigObj.CARBON_V1_FORKS])) + + # Create a set of unique tokens, excluding 'flt' + x_tokens = {token for pair in carbon_pairs for token in pair.split('/') if token != flt} + + # Get relevant pairs containing the flashloan token + flt_x_pairs = self.sort_pairs([f"{x}/{flt}" for x in x_tokens]) + + # Generate all possible 2-item combinations from the unique tokens that arent the flashloan token + x_y_pairs = self.sort_pairs(["{}/{}".format(x, y) for x, y in itertools.combinations(x_tokens, 2)]) + + # Note the relevant pairs + all_relevant_pairs = flt_x_pairs + x_y_pairs + self.ConfigObj.logger.debug(f"len(all_relevant_pairs) {len(all_relevant_pairs)}") + + # Generate triangle groups + triangle_groups = self.get_triangle_groups(flt, x_y_pairs) + self.ConfigObj.logger.debug(f"len(triangle_groups) {len(triangle_groups)}") + + # Get pair info for the cohort + all_relevant_pairs_info = self.get_all_relevant_pairs_info(CCm, all_relevant_pairs) + + # Generate valid triangles for the groups base on arb_mode + valid_triangles = self.get_triangle_groups_stats(triangle_groups, all_relevant_pairs_info, arb_mode) + + # Get [(flt,curves)] analysis set for the flt + flt_triangle_analysis_set = self.get_analysis_set_per_flt(flt, valid_triangles, all_relevant_pairs_info) + + # The entire analysis set for all flts + combos.extend(flt_triangle_analysis_set) + return combos @staticmethod def get_mono_direction_carbon_curves( diff --git a/fastlane_bot/modes/triangle_multi_complete.py b/fastlane_bot/modes/triangle_multi_complete.py new file mode 100644 index 000000000..a03096d61 --- /dev/null +++ b/fastlane_bot/modes/triangle_multi_complete.py @@ -0,0 +1,114 @@ +""" +Defines the Triangular arbitrage finder class + +[DOC-TODO-OPTIONAL-longer description in rst format] + +--- +(c) Copyright Bprotocol foundation 2023-24. +All rights reserved. +Licensed under MIT. +""" +from typing import List, Any, Tuple, Union + +from fastlane_bot.modes.base_triangle import ArbitrageFinderTriangleBase +from fastlane_bot.tools.cpc import CPCContainer +from fastlane_bot.tools.optimizer import MargPOptimizer + + +class ArbitrageFinderTriangleMultiComplete(ArbitrageFinderTriangleBase): + """ + Triangular arbitrage finder mode + """ + + arb_mode = "multi_triangle_complete" + + def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_profit: float = 0, profit_src: float = 0) -> Union[List, Tuple]: + """ + see base.py + """ + + if candidates is None: + candidates = [] + + combos = self.get_comprehensive_triangles( + self.flashloan_tokens, self.CCm, arb_mode=self.arb_mode + ) + + for src_token, miniverse in combos: + # print(miniverse) + self.ConfigObj.logger.debug(f"{[x.cid for x in miniverse]}") + try: + r = None + CC_cc = CPCContainer(miniverse) + O = MargPOptimizer(CC_cc) + #try: + pstart = self.build_pstart(CC_cc, CC_cc.tokens(), src_token) + # print(pstart) + r = O.optimize(src_token, params=dict(pstart=pstart)) #debug=True, debug2=True, verbose=True + trade_instructions_dic = r.trade_instructions(O.TIF_DICTS) + # trade_instructions_df = r.trade_instructions(O.TIF_DFAGGR) + # print(trade_instructions_df) + if len(trade_instructions_dic) < 3: + # Failed to converge + continue + trade_instructions_df = r.trade_instructions(O.TIF_DFAGGR) + trade_instructions = r.trade_instructions() + + except Exception as e: + self.ConfigObj.logger.debug(f"[triangle multi] {str(e)}") + continue + if trade_instructions_dic is None: + continue + if len(trade_instructions_dic) < 2: + continue + profit_src = -r.result + + # Get the cids + cids = [ti["cid"] for ti in trade_instructions_dic] + + # Calculate the profit + profit = self.calculate_profit(src_token, profit_src, self.CCm, cids) + if str(profit) == "nan": + self.ConfigObj.logger.debug("profit is nan, skipping") + continue + + # Handle candidates based on conditions + candidates += self.handle_candidates( + best_profit, + profit, + trade_instructions_df, + trade_instructions_dic, + src_token, + trade_instructions, + ) + + # Find the best operations + best_profit, ops = self.find_best_operations( + best_profit, + ops, + profit, + trade_instructions_df, + trade_instructions_dic, + src_token, + trade_instructions, + ) + + return candidates if self.result == self.AO_CANDIDATES else ops + + def build_pstart(self, CCm, tkn0list, tkn1): + tkn0list = [x for x in tkn0list if x not in [tkn1]] + pstart = {} + for tkn0 in tkn0list: + try: + price = CCm.bytknx(tkn0).bytkny(tkn1)[0].p + except: + try: + price = 1/CCm.bytknx(tkn1).bytkny(tkn0)[0].p + except Exception as e: + print(str(e)) + self.ConfigObj.logger.debug(f"[pstart build] {tkn0} not supported. w {tkn1} {str(e)}") + pstart[tkn0]=price + pstart[tkn1] = 1 + return pstart + + diff --git a/main.py b/main.py index d926d5614..cdff59ead 100644 --- a/main.py +++ b/main.py @@ -588,6 +588,7 @@ def run(mgr, args, tenderly_uri=None) -> None: "b3_two_hop", "multi_pairwise_pol", "multi_pairwise_all", + "multi_triangle_complete" ], ) parser.add_argument( From 5766f970733f6fc532952b335f260097231e5d2c Mon Sep 17 00:00:00 2001 From: NIXBNT <88088888+NIXBNT@users.noreply.github.com> Date: Tue, 7 May 2024 14:05:37 +1000 Subject: [PATCH 26/58] updates following comments --- fastlane_bot/modes/base_triangle.py | 104 +++++++++++------------ fastlane_bot/modes/pairwise_multi_all.py | 4 +- 2 files changed, 52 insertions(+), 56 deletions(-) diff --git a/fastlane_bot/modes/base_triangle.py b/fastlane_bot/modes/base_triangle.py index d804ab24d..5da54b134 100644 --- a/fastlane_bot/modes/base_triangle.py +++ b/fastlane_bot/modes/base_triangle.py @@ -17,6 +17,49 @@ from fastlane_bot.modes.base import ArbitrageFinderBase from fastlane_bot.tools.cpc import T +@staticmethod +def sort_pairs(pairs): + # Clean up the pairs alphabetically + return ["/".join(sorted(pair.split('/'))) for pair in pairs] + +@staticmethod +def flatten_nested_items_in_list(nested_list): + # unpack nested items + flattened_list = [] + for items in nested_list: + flat_list = [] + for item in items: + if isinstance(item, list): + flat_list.extend(item) + else: + flat_list.append(item) + flattened_list.append(flat_list) + return flattened_list + +@staticmethod +def get_triangle_groups(flt, x_y_pairs): + # Get groups of triangles that conform to (flt/x , x/y, y/flt) where x!=y + triangle_groups = [] + for pair in x_y_pairs: + x,y = pair.split('/') + triangle_groups += [("/".join(sorted([flt,x])), pair, "/".join(sorted([flt,y])))] + return triangle_groups + +@staticmethod +def get_triangle_groups_stats(triangle_groups, all_relevant_pairs_info): + # Get the stats on the triangle group cohort for decision making + valid_carbon_triangles = [] + for triangle in triangle_groups: + path_len = 0 + has_carbon = False + for pair in triangle: + if all_relevant_pairs_info[pair]['all_counts'] > 0: + path_len += 1 + if all_relevant_pairs_info[pair]['carbon_counts'] > 0: + has_carbon = True + if path_len == 3 and has_carbon == True: + valid_carbon_triangles.append(triangle) + return valid_carbon_triangles class ArbitrageFinderTriangleBase(ArbitrageFinderBase): """ @@ -175,31 +218,6 @@ def get_combos( ) return combos - def sort_pairs(self, pairs): - # Clean up the pairs alphabetically - return ["/".join(sorted(pair.split('/'))) for pair in pairs] - - def flatten_nested_items_in_list(self, nested_list): - # unpack nested items - flattened_list = [] - for items in nested_list: - flat_list = [] - for item in items: - if isinstance(item, list): - flat_list.extend(item) - else: - flat_list.append(item) - flattened_list.append(flat_list) - return flattened_list - - def get_triangle_groups(self, flt, x_y_pairs): - # Get groups of triangles that conform to (flt/x , x/y, y/flt) where x!=y - triangle_groups = [] - for pair in x_y_pairs: - x,y = pair.split('/') - triangle_groups += [("/".join(sorted([flt,x])), pair, "/".join(sorted([flt,y])))] - return triangle_groups - def get_all_relevant_pairs_info(self, CCm, all_relevant_pairs): # Get pair info for the cohort to allow decision making at the triangle level all_relevant_pairs_info = {} @@ -215,39 +233,17 @@ def get_all_relevant_pairs_info(self, CCm, all_relevant_pairs): non_carbon_curves += [x] all_relevant_pairs_info[pair]['non_carbon_curves'] = non_carbon_curves all_relevant_pairs_info[pair]['carbon_curves'] = carbon_curves - all_relevant_pairs_info[pair]['curves'] = non_carbon_curves + [carbon_curves] if len(carbon_curves) >0 else non_carbon_curves # condense carbon curves into a single list + all_relevant_pairs_info[pair]['curves'] = non_carbon_curves + [carbon_curves] if len(carbon_curves) > 0 else non_carbon_curves # condense carbon curves into a single list all_relevant_pairs_info[pair]['all_counts'] = len(pair_curves) all_relevant_pairs_info[pair]['carbon_counts'] = len(carbon_curves) return all_relevant_pairs_info - def get_triangle_groups_stats(self, triangle_groups, all_relevant_pairs_info, arb_mode=None): - # Get the stats on the triangle group cohort for decision making - triangle_group_stats = {} - for i,triangle in enumerate(triangle_groups): - triangle_group_stats[i] = {} - triangle_group_stats[i]['path'] = [] - triangle_group_stats[i]['has_path'] = False - triangle_group_stats[i]['has_carbon'] = False - for pair in triangle: - if all_relevant_pairs_info[pair]['all_counts'] > 0: - triangle_group_stats[i]['path'] += [1] - else: - triangle_group_stats[i]['path'] += [0] - if all_relevant_pairs_info[pair]['carbon_counts'] > 0: - triangle_group_stats[i]['has_carbon'] = True - if sum(triangle_group_stats[i]['path']) == 3: - triangle_group_stats[i]['has_path'] = True - - # identify valid paths for the given mode - valid_carbon_triangles = [triangle_groups[i] for i in triangle_group_stats.keys() if (triangle_group_stats[i]['has_path'] == True) and (triangle_group_stats[i]['has_carbon'] == True)] - return valid_carbon_triangles - def get_analysis_set_per_flt(self, flt, valid_triangles, all_relevant_pairs_info): flt_triangle_analysis_set = [] for triangle in valid_triangles: multiverse = [all_relevant_pairs_info[pair]['curves'] for pair in triangle] product_of_triangle = list(itertools.product(multiverse[0], multiverse[1], multiverse[2])) - triangles_to_run = self.flatten_nested_items_in_list(product_of_triangle) + triangles_to_run = flatten_nested_items_in_list(product_of_triangle) flt_triangle_analysis_set += list(zip([flt] * len(triangles_to_run), triangles_to_run)) self.ConfigObj.logger.debug(f"[base_triangle.get_analysis_set_per_flt] Length of flt_triangle_analysis_set: {flt, len(flt_triangle_analysis_set)}") @@ -278,30 +274,30 @@ def get_comprehensive_triangles( for flt in flashloan_tokens: # Get the Carbon pairs - carbon_pairs = self.sort_pairs(set([x.pair for x in CCm.curves if x.params.exchange in self.ConfigObj.CARBON_V1_FORKS])) + carbon_pairs = sort_pairs(set([x.pair for x in CCm.curves if x.params.exchange in self.ConfigObj.CARBON_V1_FORKS])) # Create a set of unique tokens, excluding 'flt' x_tokens = {token for pair in carbon_pairs for token in pair.split('/') if token != flt} # Get relevant pairs containing the flashloan token - flt_x_pairs = self.sort_pairs([f"{x}/{flt}" for x in x_tokens]) + flt_x_pairs = sort_pairs([f"{x}/{flt}" for x in x_tokens]) # Generate all possible 2-item combinations from the unique tokens that arent the flashloan token - x_y_pairs = self.sort_pairs(["{}/{}".format(x, y) for x, y in itertools.combinations(x_tokens, 2)]) + x_y_pairs = sort_pairs(["{}/{}".format(x, y) for x, y in itertools.combinations(x_tokens, 2)]) # Note the relevant pairs all_relevant_pairs = flt_x_pairs + x_y_pairs self.ConfigObj.logger.debug(f"len(all_relevant_pairs) {len(all_relevant_pairs)}") # Generate triangle groups - triangle_groups = self.get_triangle_groups(flt, x_y_pairs) + triangle_groups = get_triangle_groups(flt, x_y_pairs) self.ConfigObj.logger.debug(f"len(triangle_groups) {len(triangle_groups)}") # Get pair info for the cohort all_relevant_pairs_info = self.get_all_relevant_pairs_info(CCm, all_relevant_pairs) # Generate valid triangles for the groups base on arb_mode - valid_triangles = self.get_triangle_groups_stats(triangle_groups, all_relevant_pairs_info, arb_mode) + valid_triangles = get_triangle_groups_stats(triangle_groups, all_relevant_pairs_info) # Get [(flt,curves)] analysis set for the flt flt_triangle_analysis_set = self.get_analysis_set_per_flt(flt, valid_triangles, all_relevant_pairs_info) diff --git a/fastlane_bot/modes/pairwise_multi_all.py b/fastlane_bot/modes/pairwise_multi_all.py index 07fdd8237..20cdbb46f 100644 --- a/fastlane_bot/modes/pairwise_multi_all.py +++ b/fastlane_bot/modes/pairwise_multi_all.py @@ -67,8 +67,8 @@ def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_p if len(base_direction_two) > 0: curve_combos += [[curve] + base_direction_two for curve in not_carbon_curves] - if len(carbon_curves) >= 2: - curve_combos += [carbon_curves] + if len(carbon_curves) >= 2: + curve_combos += [carbon_curves] for curve_combo in curve_combos: src_token = tkn1 From 1c9f5a02625aa1260f7221de33a6d9f36de6eeec Mon Sep 17 00:00:00 2001 From: NIXBNT <88088888+NIXBNT@users.noreply.github.com> Date: Tue, 7 May 2024 14:17:47 +1000 Subject: [PATCH 27/58] cleanup pstart and error messanges --- fastlane_bot/modes/triangle_multi_complete.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/fastlane_bot/modes/triangle_multi_complete.py b/fastlane_bot/modes/triangle_multi_complete.py index a03096d61..eb16e655c 100644 --- a/fastlane_bot/modes/triangle_multi_complete.py +++ b/fastlane_bot/modes/triangle_multi_complete.py @@ -35,19 +35,13 @@ def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_p ) for src_token, miniverse in combos: - # print(miniverse) - self.ConfigObj.logger.debug(f"{[x.cid for x in miniverse]}") try: r = None CC_cc = CPCContainer(miniverse) O = MargPOptimizer(CC_cc) - #try: pstart = self.build_pstart(CC_cc, CC_cc.tokens(), src_token) - # print(pstart) r = O.optimize(src_token, params=dict(pstart=pstart)) #debug=True, debug2=True, verbose=True trade_instructions_dic = r.trade_instructions(O.TIF_DICTS) - # trade_instructions_df = r.trade_instructions(O.TIF_DFAGGR) - # print(trade_instructions_df) if len(trade_instructions_dic) < 3: # Failed to converge continue @@ -100,14 +94,12 @@ def build_pstart(self, CCm, tkn0list, tkn1): pstart = {} for tkn0 in tkn0list: try: - price = CCm.bytknx(tkn0).bytkny(tkn1)[0].p + pstart[tkn0] = CCm.bytknx(tkn0).bytkny(tkn1)[0].p except: try: - price = 1/CCm.bytknx(tkn1).bytkny(tkn0)[0].p + pstart[tkn0] = 1/CCm.bytknx(tkn1).bytkny(tkn0)[0].p except Exception as e: - print(str(e)) - self.ConfigObj.logger.debug(f"[pstart build] {tkn0} not supported. w {tkn1} {str(e)}") - pstart[tkn0]=price + self.ConfigObj.logger.info(f"[pstart build] {tkn0} not supported. w {tkn1} {e}") pstart[tkn1] = 1 return pstart From 0dbb4dd75816ec230b093a112f76a57f0d773a64 Mon Sep 17 00:00:00 2001 From: Nicholas Welch Date: Wed, 8 May 2024 08:49:46 +1000 Subject: [PATCH 28/58] carbon<>carbon can support mixing directions --- fastlane_bot/tests/test_064_TestMultiAllMode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane_bot/tests/test_064_TestMultiAllMode.py b/fastlane_bot/tests/test_064_TestMultiAllMode.py index 8af2f4b8c..033818012 100644 --- a/fastlane_bot/tests/test_064_TestMultiAllMode.py +++ b/fastlane_bot/tests/test_064_TestMultiAllMode.py @@ -224,7 +224,7 @@ def test_test_expected_output(): assert len(r) >= 25, f"[NBTest64 TestMultiPairwiseAll Mode] Expected at least 25 arbs, found {len(r)}" assert multi_carbon_count > 0, f"[NBTest64 TestMultiPairwiseAll Mode] Not finding arbs with multiple Carbon curves." - assert carbon_wrong_direction_count == 0, f"[NBTest64 TestMultiPairwiseAll Mode] Expected all Carbon curves to have the same tkn in and tkn out. Mixing is currently not supported." + # assert carbon_wrong_direction_count == 0, f"[NBTest64 TestMultiPairwiseAll Mode] Expected all Carbon curves to have the same tkn in and tkn out. Mixing is currently not supported." # - \ No newline at end of file From 293ba785b40ee5aa0df4198da0e43ac64ae8206a Mon Sep 17 00:00:00 2001 From: Nicholas Welch Date: Thu, 9 May 2024 09:45:51 +1000 Subject: [PATCH 29/58] Revert "expand combo scope" This reverts commit 3dd85f279df065048b7a9eaf58c4b06beda59896. --- fastlane_bot/modes/base_pairwise.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastlane_bot/modes/base_pairwise.py b/fastlane_bot/modes/base_pairwise.py index 7be12a719..d5df65ef9 100644 --- a/fastlane_bot/modes/base_pairwise.py +++ b/fastlane_bot/modes/base_pairwise.py @@ -49,10 +49,10 @@ def get_combos( """ all_tokens = CCm.tokens() - # flashloan_tokens_intersect = all_tokens.intersection(set(flashloan_tokens)) + flashloan_tokens_intersect = all_tokens.intersection(set(flashloan_tokens)) combos = [ (tkn0, tkn1) - for tkn0, tkn1 in itertools.product(all_tokens, flashloan_tokens) + for tkn0, tkn1 in itertools.product(all_tokens, flashloan_tokens_intersect) # tkn1 is always the token being flash loaned # note that the pair is tkn0/tkn1, ie tkn1 is the quote token if tkn0 != tkn1 From 8afeacc0aa9aef77f29704425212e5254a0263f7 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Fri, 10 May 2024 20:48:09 +0300 Subject: [PATCH 30/58] fix: tests --- fastlane_bot/events/event_gatherer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fastlane_bot/events/event_gatherer.py b/fastlane_bot/events/event_gatherer.py index 29eb78cc6..5f70913b2 100644 --- a/fastlane_bot/events/event_gatherer.py +++ b/fastlane_bot/events/event_gatherer.py @@ -1,4 +1,5 @@ from itertools import chain +from typing import Dict import asyncio import nest_asyncio @@ -21,8 +22,8 @@ class EventGatherer: def __init__( self, w3: AsyncWeb3, - exchanges: dict[str, Exchange], - event_contracts: dict[str, Contract], + exchanges: Dict[str, Exchange], + event_contracts: Dict[str, Contract], ): """ Initializes the EventManager. Args: From fa25f931900d4407424133a22142b9ca925767fb Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Mon, 13 May 2024 10:32:44 +0300 Subject: [PATCH 31/58] Fix several issue in the triangle-mode implementation --- fastlane_bot/modes/base_triangle.py | 8 +------- fastlane_bot/modes/triangle_multi_complete.py | 13 +++---------- fastlane_bot/tests/test_064_TestMultiAllMode.py | 2 +- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/fastlane_bot/modes/base_triangle.py b/fastlane_bot/modes/base_triangle.py index 5da54b134..8ce78a7d1 100644 --- a/fastlane_bot/modes/base_triangle.py +++ b/fastlane_bot/modes/base_triangle.py @@ -17,12 +17,10 @@ from fastlane_bot.modes.base import ArbitrageFinderBase from fastlane_bot.tools.cpc import T -@staticmethod def sort_pairs(pairs): # Clean up the pairs alphabetically return ["/".join(sorted(pair.split('/'))) for pair in pairs] -@staticmethod def flatten_nested_items_in_list(nested_list): # unpack nested items flattened_list = [] @@ -36,7 +34,6 @@ def flatten_nested_items_in_list(nested_list): flattened_list.append(flat_list) return flattened_list -@staticmethod def get_triangle_groups(flt, x_y_pairs): # Get groups of triangles that conform to (flt/x , x/y, y/flt) where x!=y triangle_groups = [] @@ -45,7 +42,6 @@ def get_triangle_groups(flt, x_y_pairs): triangle_groups += [("/".join(sorted([flt,x])), pair, "/".join(sorted([flt,y])))] return triangle_groups -@staticmethod def get_triangle_groups_stats(triangle_groups, all_relevant_pairs_info): # Get the stats on the triangle group cohort for decision making valid_carbon_triangles = [] @@ -250,7 +246,7 @@ def get_analysis_set_per_flt(self, flt, valid_triangles, all_relevant_pairs_info return flt_triangle_analysis_set def get_comprehensive_triangles( - self, flashloan_tokens: List[str], CCm: Any, arb_mode: str + self, flashloan_tokens: List[str], CCm: Any ) -> Tuple[List[str], List[Any]]: """ Get comprehensive combos for triangular arbitrage @@ -261,8 +257,6 @@ def get_comprehensive_triangles( List of flashloan tokens CCm : object CCm object - arb_mode : str - Arbitrage mode Returns ------- diff --git a/fastlane_bot/modes/triangle_multi_complete.py b/fastlane_bot/modes/triangle_multi_complete.py index eb16e655c..e2ee8c947 100644 --- a/fastlane_bot/modes/triangle_multi_complete.py +++ b/fastlane_bot/modes/triangle_multi_complete.py @@ -30,30 +30,23 @@ def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_p if candidates is None: candidates = [] - combos = self.get_comprehensive_triangles( - self.flashloan_tokens, self.CCm, arb_mode=self.arb_mode - ) + combos = self.get_comprehensive_triangles(self.flashloan_tokens, self.CCm) for src_token, miniverse in combos: try: - r = None CC_cc = CPCContainer(miniverse) O = MargPOptimizer(CC_cc) pstart = self.build_pstart(CC_cc, CC_cc.tokens(), src_token) r = O.optimize(src_token, params=dict(pstart=pstart)) #debug=True, debug2=True, verbose=True trade_instructions_dic = r.trade_instructions(O.TIF_DICTS) - if len(trade_instructions_dic) < 3: + if trade_instructions_dic is None or len(trade_instructions_dic) < 3: # Failed to converge continue trade_instructions_df = r.trade_instructions(O.TIF_DFAGGR) trade_instructions = r.trade_instructions() except Exception as e: - self.ConfigObj.logger.debug(f"[triangle multi] {str(e)}") - continue - if trade_instructions_dic is None: - continue - if len(trade_instructions_dic) < 2: + self.ConfigObj.logger.info(f"[triangle multi] {e}") continue profit_src = -r.result diff --git a/fastlane_bot/tests/test_064_TestMultiAllMode.py b/fastlane_bot/tests/test_064_TestMultiAllMode.py index 033818012..6d7ff45be 100644 --- a/fastlane_bot/tests/test_064_TestMultiAllMode.py +++ b/fastlane_bot/tests/test_064_TestMultiAllMode.py @@ -224,7 +224,7 @@ def test_test_expected_output(): assert len(r) >= 25, f"[NBTest64 TestMultiPairwiseAll Mode] Expected at least 25 arbs, found {len(r)}" assert multi_carbon_count > 0, f"[NBTest64 TestMultiPairwiseAll Mode] Not finding arbs with multiple Carbon curves." - # assert carbon_wrong_direction_count == 0, f"[NBTest64 TestMultiPairwiseAll Mode] Expected all Carbon curves to have the same tkn in and tkn out. Mixing is currently not supported." + assert carbon_wrong_direction_count == 6, f"[NBTest64 TestMultiPairwiseAll Mode] Expected 6 Carbon curves to be in the opposite direction." # - \ No newline at end of file From 7e667e49ea0c30e66c3477023534eb5383c99b6d Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Mon, 13 May 2024 11:09:33 +0300 Subject: [PATCH 32/58] Fix function `build_pstart` in class `ArbitrageFinderTriangleBase`, and remove that function in each one of the classes which inherits that class --- fastlane_bot/modes/base_triangle.py | 11 +++++------ fastlane_bot/modes/triangle_multi.py | 18 ------------------ fastlane_bot/modes/triangle_multi_complete.py | 16 ---------------- 3 files changed, 5 insertions(+), 40 deletions(-) diff --git a/fastlane_bot/modes/base_triangle.py b/fastlane_bot/modes/base_triangle.py index 8ce78a7d1..ce07df8bc 100644 --- a/fastlane_bot/modes/base_triangle.py +++ b/fastlane_bot/modes/base_triangle.py @@ -345,18 +345,17 @@ def get_mono_direction_carbon_curves( wrong_direction_cids.append(idx) return [curve for curve in miniverse if curve.cid not in wrong_direction_cids] + def build_pstart(self, CCm, tkn0list, tkn1): tkn0list = [x for x in tkn0list if x not in [tkn1]] pstart = {} for tkn0 in tkn0list: try: - price = CCm.bytknx(tkn0).bytkny(tkn1)[0].p + pstart[tkn0] = CCm.bytknx(tkn0).bytkny(tkn1)[0].p except: try: - price = 1/CCm.bytknx(tkn1).bytkny(tkn0)[0].p + pstart[tkn0] = 1/CCm.bytknx(tkn1).bytkny(tkn0)[0].p except Exception as e: - print(str(e)) - self.ConfigObj.logger.debug(f"[pstart build] {tkn0} not supported. w {tkn1} {str(e)}") - pstart[tkn0]=price + self.ConfigObj.logger.info(f"[pstart build] {tkn0}/{tkn1} price error {e}") pstart[tkn1] = 1 - return pstart \ No newline at end of file + return pstart diff --git a/fastlane_bot/modes/triangle_multi.py b/fastlane_bot/modes/triangle_multi.py index 50e335880..6a0e2d926 100644 --- a/fastlane_bot/modes/triangle_multi.py +++ b/fastlane_bot/modes/triangle_multi.py @@ -92,21 +92,3 @@ def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_p ) return candidates if self.result == self.AO_CANDIDATES else ops - - def build_pstart(self, CCm, tkn0list, tkn1): - tkn0list = [x for x in tkn0list if x not in [tkn1]] - pstart = {} - for tkn0 in tkn0list: - try: - price = CCm.bytknx(tkn0).bytkny(tkn1)[0].p - except: - try: - price = 1/CCm.bytknx(tkn1).bytkny(tkn0)[0].p - except Exception as e: - print(str(e)) - self.ConfigObj.logger.debug(f"[pstart build] {tkn0} not supported. w {tkn1} {str(e)}") - pstart[tkn0]=price - pstart[tkn1] = 1 - return pstart - - diff --git a/fastlane_bot/modes/triangle_multi_complete.py b/fastlane_bot/modes/triangle_multi_complete.py index e2ee8c947..171fe032f 100644 --- a/fastlane_bot/modes/triangle_multi_complete.py +++ b/fastlane_bot/modes/triangle_multi_complete.py @@ -81,19 +81,3 @@ def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_p ) return candidates if self.result == self.AO_CANDIDATES else ops - - def build_pstart(self, CCm, tkn0list, tkn1): - tkn0list = [x for x in tkn0list if x not in [tkn1]] - pstart = {} - for tkn0 in tkn0list: - try: - pstart[tkn0] = CCm.bytknx(tkn0).bytkny(tkn1)[0].p - except: - try: - pstart[tkn0] = 1/CCm.bytknx(tkn1).bytkny(tkn0)[0].p - except Exception as e: - self.ConfigObj.logger.info(f"[pstart build] {tkn0} not supported. w {tkn1} {e}") - pstart[tkn1] = 1 - return pstart - - From 16efe4e7bc17e1b845777b72d5ccbee16b61905c Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Mon, 13 May 2024 11:57:21 +0300 Subject: [PATCH 33/58] Fix the handling of trade instructions in function ArbitrageFinderTriangleMulti.find_arbitrage --- fastlane_bot/modes/triangle_multi.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/fastlane_bot/modes/triangle_multi.py b/fastlane_bot/modes/triangle_multi.py index 6a0e2d926..f3a7c5289 100644 --- a/fastlane_bot/modes/triangle_multi.py +++ b/fastlane_bot/modes/triangle_multi.py @@ -42,22 +42,17 @@ def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_p r = None CC_cc = CPCContainer(miniverse) O = MargPOptimizer(CC_cc) - #try: pstart = self.build_pstart(CC_cc, CC_cc.tokens(), src_token) r = O.optimize(src_token, params=dict(pstart=pstart)) #debug=True, debug2=True trade_instructions_dic = r.trade_instructions(O.TIF_DICTS) - if len(trade_instructions_dic) < 3: + if trade_instructions_dic is None or len(trade_instructions_dic) < 3: # Failed to converge continue trade_instructions_df = r.trade_instructions(O.TIF_DFAGGR) trade_instructions = r.trade_instructions() except Exception as e: - self.ConfigObj.logger.debug(f"[triangle multi] {str(e)}") - continue - if trade_instructions_dic is None: - continue - if len(trade_instructions_dic) < 2: + self.ConfigObj.logger.info(f"[triangle multi] {e}") continue profit_src = -r.result From e9c19598d0ae3b4b53b5349bbd58e3b980b5171b Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Mon, 13 May 2024 12:00:52 +0300 Subject: [PATCH 34/58] Remove unused code --- fastlane_bot/modes/triangle_multi.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fastlane_bot/modes/triangle_multi.py b/fastlane_bot/modes/triangle_multi.py index f3a7c5289..917260f49 100644 --- a/fastlane_bot/modes/triangle_multi.py +++ b/fastlane_bot/modes/triangle_multi.py @@ -39,7 +39,6 @@ def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_p for src_token, miniverse in combos: try: - r = None CC_cc = CPCContainer(miniverse) O = MargPOptimizer(CC_cc) pstart = self.build_pstart(CC_cc, CC_cc.tokens(), src_token) From 4dfd33654a01c8ea460c3dbc66f562d675f33dbb Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Mon, 13 May 2024 13:10:26 +0300 Subject: [PATCH 35/58] Remove unused code --- fastlane_bot/modes/base_triangle.py | 46 ----------------------------- 1 file changed, 46 deletions(-) diff --git a/fastlane_bot/modes/base_triangle.py b/fastlane_bot/modes/base_triangle.py index ce07df8bc..522283b36 100644 --- a/fastlane_bot/modes/base_triangle.py +++ b/fastlane_bot/modes/base_triangle.py @@ -300,52 +300,6 @@ def get_comprehensive_triangles( combos.extend(flt_triangle_analysis_set) return combos - @staticmethod - def get_mono_direction_carbon_curves( - miniverse: List[Any], trade_instructions_df: pd.DataFrame, token_in: str=None - ) -> List[Any]: - """ - Get mono direction carbon curves for triangular arbitrage - - Parameters - ---------- - miniverse : list - List of miniverses - token_in : str - Token in - trade_instructions_df : DataFrame - Trade instructions dataframe - - Returns - ------- - mono_direction_carbon_curves : list - List of mono direction carbon curves - - """ - - if token_in is None: - columns = trade_instructions_df.columns - check_nan = trade_instructions_df.copy().fillna(0) - first_bancor_v3_pool = check_nan.iloc[0] - second_bancor_v3_pool = check_nan.iloc[1] - - for idx, token in enumerate(columns): - if token == T.BNT: - continue - if first_bancor_v3_pool[token] < 0: - token_in = token - break - if second_bancor_v3_pool[token] < 0: - token_in = token - break - - wrong_direction_cids = [] - for idx, row in trade_instructions_df.iterrows(): - if (row[token_in] < 0) and ("-0" in idx or "-1" in idx): - wrong_direction_cids.append(idx) - - return [curve for curve in miniverse if curve.cid not in wrong_direction_cids] - def build_pstart(self, CCm, tkn0list, tkn1): tkn0list = [x for x in tkn0list if x not in [tkn1]] pstart = {} From c2c4f02e20c4323a75dad272f8df48cc461df398 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Mon, 13 May 2024 13:12:14 +0300 Subject: [PATCH 36/58] Cosmetic --- fastlane_bot/modes/triangle_multi.py | 6 ++---- fastlane_bot/modes/triangle_multi_complete.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/fastlane_bot/modes/triangle_multi.py b/fastlane_bot/modes/triangle_multi.py index 917260f49..e6d1f6672 100644 --- a/fastlane_bot/modes/triangle_multi.py +++ b/fastlane_bot/modes/triangle_multi.py @@ -33,16 +33,14 @@ def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_p if candidates is None: candidates = [] - combos = self.get_combos( - self.flashloan_tokens, self.CCm, arb_mode=self.arb_mode - ) + combos = self.get_combos(self.flashloan_tokens, self.CCm, arb_mode=self.arb_mode) for src_token, miniverse in combos: try: CC_cc = CPCContainer(miniverse) O = MargPOptimizer(CC_cc) pstart = self.build_pstart(CC_cc, CC_cc.tokens(), src_token) - r = O.optimize(src_token, params=dict(pstart=pstart)) #debug=True, debug2=True + r = O.optimize(src_token, params=dict(pstart=pstart)) trade_instructions_dic = r.trade_instructions(O.TIF_DICTS) if trade_instructions_dic is None or len(trade_instructions_dic) < 3: # Failed to converge diff --git a/fastlane_bot/modes/triangle_multi_complete.py b/fastlane_bot/modes/triangle_multi_complete.py index 171fe032f..d053ae0fc 100644 --- a/fastlane_bot/modes/triangle_multi_complete.py +++ b/fastlane_bot/modes/triangle_multi_complete.py @@ -37,7 +37,7 @@ def find_arbitrage(self, candidates: List[Any] = None, ops: Tuple = None, best_p CC_cc = CPCContainer(miniverse) O = MargPOptimizer(CC_cc) pstart = self.build_pstart(CC_cc, CC_cc.tokens(), src_token) - r = O.optimize(src_token, params=dict(pstart=pstart)) #debug=True, debug2=True, verbose=True + r = O.optimize(src_token, params=dict(pstart=pstart)) trade_instructions_dic = r.trade_instructions(O.TIF_DICTS) if trade_instructions_dic is None or len(trade_instructions_dic) < 3: # Failed to converge From 03b1ef8fb70f40b84b958765eaa292202e6aa026 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Mon, 13 May 2024 11:26:43 +0300 Subject: [PATCH 37/58] fix: type annotation --- fastlane_bot/config/multicaller.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fastlane_bot/config/multicaller.py b/fastlane_bot/config/multicaller.py index d68760942..76b031bf6 100644 --- a/fastlane_bot/config/multicaller.py +++ b/fastlane_bot/config/multicaller.py @@ -6,9 +6,10 @@ All rights reserved. Licensed under MIT. """ -from typing import Callable, Any, List, Dict +from typing import Any, List, Dict from eth_abi import decode +from web3.contract.contract import ContractFunction from fastlane_bot.data.abi import MULTICALL_ABI @@ -45,10 +46,10 @@ class MultiCaller: def __init__(self, web3: Any, multicall_contract_address: str): self.multicall_contract = web3.eth.contract(abi=MULTICALL_ABI, address=multicall_contract_address) - self.contract_calls: List[Callable] = [] + self.contract_calls: List[ContractFunction] = [] self.output_types_list: List[List[str]] = [] - def add_call(self, call: Callable): + def add_call(self, call: ContractFunction): self.contract_calls.append({'target': call.address, 'callData': call._encode_transaction_data()}) self.output_types_list.append([collapse_if_tuple(item) for item in call.abi['outputs']]) From ec9ca60ae78f2fc7a48c0e60352088f8701e02ba Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Sun, 12 May 2024 12:58:56 +0300 Subject: [PATCH 38/58] Fix the pool-finder-period handling --- main.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/main.py b/main.py index cdff59ead..ae475d7db 100644 --- a/main.py +++ b/main.py @@ -94,6 +94,7 @@ def int_or_none(x): "self_fund": is_true, "read_only": is_true, "is_args_test": is_true, + "pool_finder_period": int, } # Apply the transformations @@ -225,7 +226,7 @@ def main(args: argparse.Namespace) -> None: prefix_path: {args.prefix_path} self_fund: {args.self_fund} read_only: {args.read_only} - pool_finder: {args.pool_finder} + pool_finder_period: {args.pool_finder_period} +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ @@ -306,17 +307,14 @@ def run(mgr, args, tenderly_uri=None) -> None: event_gatherer = EventGatherer(w3=mgr.w3_async, exchanges=mgr.exchanges, event_contracts=mgr.event_contracts) - if args.pool_finder != -1: - pool_finder = PoolFinder( - carbon_forks=mgr.cfg.network.CARBON_V1_FORKS, - uni_v3_forks=mgr.cfg.network.UNI_V3_FORKS, - flashloan_tokens=args.flashloan_tokens, - exchanges=mgr.exchanges, - web3=mgr.web3, - multicall_address=mgr.cfg.network.MULTICALL_CONTRACT_ADDRESS - ) - else: - pool_finder = None + pool_finder = PoolFinder( + carbon_forks=mgr.cfg.network.CARBON_V1_FORKS, + uni_v3_forks=mgr.cfg.network.UNI_V3_FORKS, + flashloan_tokens=args.flashloan_tokens, + exchanges=mgr.exchanges, + web3=mgr.web3, + multicall_address=mgr.cfg.network.MULTICALL_CONTRACT_ADDRESS + ) while True: try: @@ -527,7 +525,7 @@ def run(mgr, args, tenderly_uri=None) -> None: mgr.solidly_v2_event_mappings = dict( solidly_v2_event_mappings[["address", "exchange"]].values ) - if pool_finder is not None and (loop_idx % args.pool_finder == 0 or loop_idx == 1): + if args.pool_finder_period > 0 and loop_idx % args.pool_finder_period == 0: mgr.cfg.logger.info(f"Searching for unsupported Carbon pairs.") uni_v2, uni_v3, solidly_v2 = pool_finder.get_pools_for_unsupported_pairs(mgr.pool_data, arb_mode=args.arb_mode) result = f"Added {len(uni_v2) + len(uni_v3) + len(solidly_v2)} pools." if (uni_v2 or uni_v3 or solidly_v2) else f"No pools added." @@ -734,9 +732,9 @@ def run(mgr, args, tenderly_uri=None) -> None: help="Custom RPC URL. If not set, the bot will use the default Alchemy RPC URL for the blockchain (if available).", ) parser.add_argument( - "--pool_finder", + "--pool_finder_period", default=100, - help="If not -1, searches for pools that can service Carbon strategies that do not have viable routes.", + help="Searches for pools that can service Carbon strategies that do not have viable routes.", ) # Process the arguments From d13af71971e658741bee5702e2870fb712213034 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Sun, 12 May 2024 15:32:50 +0300 Subject: [PATCH 39/58] Remove the multicall dependency from all Exchange classes --- fastlane_bot/events/exchanges/balancer.py | 7 ------- fastlane_bot/events/exchanges/bancor_pol.py | 7 ------- fastlane_bot/events/exchanges/bancor_v2.py | 10 +--------- fastlane_bot/events/exchanges/bancor_v3.py | 9 +-------- fastlane_bot/events/exchanges/base.py | 8 -------- fastlane_bot/events/exchanges/carbon_v1.py | 7 ------- .../events/exchanges/solidly_v2/base.py | 5 ++--- .../events/exchanges/solidly_v2/velodrome_v2.py | 5 ++--- .../events/exchanges/solidly_v2/xfai_v0.py | 5 ++--- fastlane_bot/events/exchanges/uniswap_v2.py | 5 ++--- fastlane_bot/events/exchanges/uniswap_v3.py | 7 +++---- fastlane_bot/pool_finder.py | 17 +++++------------ main.py | 3 +-- 13 files changed, 19 insertions(+), 76 deletions(-) diff --git a/fastlane_bot/events/exchanges/balancer.py b/fastlane_bot/events/exchanges/balancer.py index 2f9f99fc9..2ec32119b 100644 --- a/fastlane_bot/events/exchanges/balancer.py +++ b/fastlane_bot/events/exchanges/balancer.py @@ -16,7 +16,6 @@ from web3.contract import Contract -from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import BALANCER_VAULT_ABI, BALANCER_POOL_ABI_V1 from ..exchanges.base import Exchange from ..pools.base import Pool @@ -89,9 +88,3 @@ async def get_tkn_n(self, address: str, contract: Contract, event: Any, index: i tokens = pool_balances[0] token_balances = pool_balances[1] return token_balances[index] - - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - """ - This function is unused for Carbon. - """ - raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/bancor_pol.py b/fastlane_bot/events/exchanges/bancor_pol.py index 3fbb0fbb0..bc66bb7c5 100644 --- a/fastlane_bot/events/exchanges/bancor_pol.py +++ b/fastlane_bot/events/exchanges/bancor_pol.py @@ -16,7 +16,6 @@ from web3.contract import Contract -from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import BANCOR_POL_ABI from fastlane_bot import Config from ..exchanges.base import Exchange @@ -64,12 +63,6 @@ async def get_tkn0(self, address: str, contract: Contract, event: Event) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: return self.ETH_ADDRESS if event.args["token"] not in self.ETH_ADDRESS else self.BNT_ADDRESS - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - """ - This function is unused for Carbon. - """ - raise NotImplementedError - def save_strategy( self, token: str, diff --git a/fastlane_bot/events/exchanges/bancor_v2.py b/fastlane_bot/events/exchanges/bancor_v2.py index 41ff63860..8da7d1745 100644 --- a/fastlane_bot/events/exchanges/bancor_v2.py +++ b/fastlane_bot/events/exchanges/bancor_v2.py @@ -12,12 +12,10 @@ Licensed under MIT. """ from dataclasses import dataclass -from typing import List, Type, Tuple, Any +from typing import List, Type, Tuple -from web3 import AsyncWeb3 from web3.contract import Contract, AsyncContract -from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import BANCOR_V2_CONVERTER_ABI from ..exchanges.base import Exchange from ..pools.base import Pool @@ -78,9 +76,3 @@ async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: async def get_anchor(self, contract: Contract) -> str: return await contract.caller.anchor() - - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - """ - This function is unused for Carbon. - """ - raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/bancor_v3.py b/fastlane_bot/events/exchanges/bancor_v3.py index b21c6f7bb..2af08a103 100644 --- a/fastlane_bot/events/exchanges/bancor_v3.py +++ b/fastlane_bot/events/exchanges/bancor_v3.py @@ -12,11 +12,10 @@ Licensed under MIT. """ from dataclasses import dataclass -from typing import List, Type, Tuple, Any +from typing import List, Type, Tuple from web3.contract import Contract -from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import BANCOR_V3_POOL_COLLECTION_ABI from ..exchanges.base import Exchange from ..pools.base import Pool @@ -65,9 +64,3 @@ async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: if event.args["pool"] != self.BNT_ADDRESS else event.args["tkn_address"] ) - - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - """ - This function is unused for Carbon. - """ - raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/base.py b/fastlane_bot/events/exchanges/base.py index 93020c3b5..bf450ff12 100644 --- a/fastlane_bot/events/exchanges/base.py +++ b/fastlane_bot/events/exchanges/base.py @@ -17,7 +17,6 @@ from web3.contract import Contract, AsyncContract from fastlane_bot.config.constants import CARBON_V1_NAME -from fastlane_bot.config.multicaller import MultiCaller from ..pools.base import Pool from ..interfaces.subscription import Subscription @@ -124,13 +123,6 @@ async def get_fee(address: str, contract: AsyncContract) -> float: """ pass - @abstractmethod - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2, *args): - """ - Returns the Factory contract function used to fetch liquidity pools. - """ - ... - @staticmethod @abstractmethod async def get_tkn0(address: str, contract: AsyncContract, event: Any) -> str: diff --git a/fastlane_bot/events/exchanges/carbon_v1.py b/fastlane_bot/events/exchanges/carbon_v1.py index b3708c822..df0540661 100644 --- a/fastlane_bot/events/exchanges/carbon_v1.py +++ b/fastlane_bot/events/exchanges/carbon_v1.py @@ -17,7 +17,6 @@ from fastlane_bot import Config from web3.contract import Contract -from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import CARBON_CONTROLLER_ABI from ..exchanges.base import Exchange from ..pools.base import Pool @@ -245,9 +244,3 @@ def save_strategy( ), block_number=block_number, ) - - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - """ - This function is unused for Carbon. - """ - raise NotImplementedError \ No newline at end of file diff --git a/fastlane_bot/events/exchanges/solidly_v2/base.py b/fastlane_bot/events/exchanges/solidly_v2/base.py index 52a971e7e..e58a474ff 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/base.py +++ b/fastlane_bot/events/exchanges/solidly_v2/base.py @@ -17,7 +17,6 @@ from web3.contract import Contract, AsyncContract -from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import SOLIDLY_V2_POOL_ABI from fastlane_bot.events.exchanges.base import Exchange from ...exchanges.base import Exchange @@ -63,8 +62,8 @@ async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return await contract.caller.token1() - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - mc.add_call(self.factory_contract.functions.getPair, addr1, addr2, True) + def get_pool(self, addr1, addr2): + return self.factory_contract.functions.getPair(addr1, addr2, False) @property @abstractmethod diff --git a/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py b/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py index 1224c6df6..e15d17bf2 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py @@ -3,7 +3,6 @@ from web3.contract import AsyncContract -from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import SOLIDLY_V2_FACTORY_ABI from .base import SolidlyV2 @@ -19,5 +18,5 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - mc.add_call(self.factory_contract.functions.getPool, addr1, addr2, True) + def get_pool(self, addr1, addr2): + return self.factory_contract.functions.getPool(addr1, addr2, False) diff --git a/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py b/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py index e8d7eb55e..75cc7cb00 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py +++ b/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py @@ -3,7 +3,6 @@ from web3.contract import Contract, AsyncContract -from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import XFAI_V0_POOL_ABI, XFAI_V0_FACTORY_ABI from .base import SolidlyV2 @@ -30,5 +29,5 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - mc.add_call(self.factory_contract.functions.getPool, addr1) + def get_pool(self, addr1, addr2): + return self.factory_contract.functions.getPool(addr1) diff --git a/fastlane_bot/events/exchanges/uniswap_v2.py b/fastlane_bot/events/exchanges/uniswap_v2.py index c7ed2615f..1043f7afe 100644 --- a/fastlane_bot/events/exchanges/uniswap_v2.py +++ b/fastlane_bot/events/exchanges/uniswap_v2.py @@ -16,7 +16,6 @@ from web3.contract import Contract, AsyncContract -from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import UNISWAP_V2_POOL_ABI, UNISWAP_V2_FACTORY_ABI from ..exchanges.base import Exchange from ..pools.base import Pool @@ -64,5 +63,5 @@ async def get_tkn0(address: str, contract: AsyncContract, event: Any) -> str: async def get_tkn1(address: str, contract: AsyncContract, event: Any) -> str: return await contract.caller.token1() - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2): - mc.add_call(self.factory_contract.functions.getPair, addr1, addr2) + def get_pool(self, addr1, addr2): + return self.factory_contract.functions.getPair(addr1, addr2) diff --git a/fastlane_bot/events/exchanges/uniswap_v3.py b/fastlane_bot/events/exchanges/uniswap_v3.py index 604ba5af9..569fd6bee 100644 --- a/fastlane_bot/events/exchanges/uniswap_v3.py +++ b/fastlane_bot/events/exchanges/uniswap_v3.py @@ -14,10 +14,9 @@ from dataclasses import dataclass from typing import List, Type, Tuple, Any -from web3.contract import Contract, AsyncContract +from web3.contract import Contract from fastlane_bot.config.constants import AGNI_V3_NAME, PANCAKESWAP_V3_NAME, FUSIONX_V3_NAME, ECHODEX_V3_NAME, SECTA_V3_NAME -from fastlane_bot.config.multicaller import MultiCaller from fastlane_bot.data.abi import UNISWAP_V3_POOL_ABI, UNISWAP_V3_FACTORY_ABI, PANCAKESWAP_V3_POOL_ABI from ..exchanges.base import Exchange from ..pools.base import Pool @@ -61,5 +60,5 @@ async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return await contract.caller.token1() - def get_pool_with_multicall(self, mc: MultiCaller, addr1, addr2, fee): - mc.add_call(self.factory_contract.functions.getPool, addr1, addr2, fee) + def get_pool(self, addr1, addr2, fee): + return self.factory_contract.functions.getPool(addr1, addr2, fee) diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index 8f56436e4..988e2e7da 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -91,11 +91,6 @@ def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: Dict[str, List[str]]: A list of dictionaries, where each dictionary maps an exchange's name to a list of valid pool addresses (i.e., non-zero addresses) obtained from the multicall across all generated pairs. - - Raises: - Exception: An exception could be raised from the multicall operation depending on the - implementation specifics of the multicall context manager or the exchange's - get_pool_with_multicall method if it encounters a problem. """ pairs = [(tkn, token) for pair in unsupported_pairs for tkn in pair for token in self._flashloan_tokens] chunk_size = 400 @@ -108,17 +103,15 @@ def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: mc = MultiCaller(contract=exchange.factory_contract, web3=self._web3, multicall_address=self._multicall_address) for pair in pair_chunk: if exchange.base_exchange_name in [UNISWAP_V2_NAME, SOLIDLY_V2_NAME]: - exchange.get_pool_with_multicall(mc, pair[0], pair[1]) + mc.add_call(exchange.get_pool(pair[0], pair[1])) elif exchange.base_exchange_name == UNISWAP_V3_NAME: for fee in self._uni_v3_fee_tiers[exchange.exchange_name]: - exchange.get_pool_with_multicall(mc, pair[0], pair[1], fee) + mc.add_call(exchange.get_pool(pair[0], pair[1], fee)) response = mc.multicall() - result[exchange.base_exchange_name] = { + result[exchange.base_exchange_name].update({ mc.web3.to_checksum_address(addr): exchange.exchange_name - for addr - in response - if addr != ZERO_ADDRESS - } + for addr in response if addr != ZERO_ADDRESS + }) return result diff --git a/main.py b/main.py index ae475d7db..c1d2adc90 100644 --- a/main.py +++ b/main.py @@ -528,8 +528,7 @@ def run(mgr, args, tenderly_uri=None) -> None: if args.pool_finder_period > 0 and loop_idx % args.pool_finder_period == 0: mgr.cfg.logger.info(f"Searching for unsupported Carbon pairs.") uni_v2, uni_v3, solidly_v2 = pool_finder.get_pools_for_unsupported_pairs(mgr.pool_data, arb_mode=args.arb_mode) - result = f"Added {len(uni_v2) + len(uni_v3) + len(solidly_v2)} pools." if (uni_v2 or uni_v3 or solidly_v2) else f"No pools added." - mgr.cfg.logger.info(result) + mgr.cfg.logger.info(f"Number of pools added: {len(uni_v2) + len(uni_v3) + len(solidly_v2)}") mgr.uniswap_v2_event_mappings.update(uni_v2) mgr.uniswap_v3_event_mappings.update(uni_v3) mgr.solidly_v2_event_mappings.update(solidly_v2) From 7be89e57519bff29faebe09f81834f9a8c5ce382 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Sun, 12 May 2024 15:39:08 +0300 Subject: [PATCH 40/58] Minor --- fastlane_bot/pool_finder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index 988e2e7da..fa71809ba 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -109,7 +109,7 @@ def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: mc.add_call(exchange.get_pool(pair[0], pair[1], fee)) response = mc.multicall() result[exchange.base_exchange_name].update({ - mc.web3.to_checksum_address(addr): exchange.exchange_name + self._web3.to_checksum_address(addr): exchange.exchange_name for addr in response if addr != ZERO_ADDRESS }) return result From c400d4a7a85df84da157a688a30c6a72dc780a21 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Sun, 12 May 2024 15:47:28 +0300 Subject: [PATCH 41/58] Simplify the code --- fastlane_bot/pool_finder.py | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index fa71809ba..606c12ef5 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -61,7 +61,7 @@ def get_pools_for_unsupported_pairs(self, pools: List[Dict[str, Any]], arb_mode: _find_unsupported_pairs, and _find_unsupported_triangles methods. Returns: - Dict: Returns a dictionary with pools sorted into different exchange types (Uni V2 forks, Uni V3 forks, + Dict: Returns a list of dictionaries with pools sorted into different exchange types (Uni V2 forks, Uni V3 forks, and Solidly V2 forks), each associated with their specific supporting pools based on the unsupported configurations identified. """ @@ -73,25 +73,7 @@ def get_pools_for_unsupported_pairs(self, pools: List[Dict[str, Any]], arb_mode: unsupported_pairs = PoolFinder._find_unsupported_triangles(self._flashloan_tokens, carbon_pairs=carbon_pairs, external_pairs=other_pairs) else: unsupported_pairs = PoolFinder._find_unsupported_pairs(self._flashloan_tokens, carbon_pairs=carbon_pairs, external_pairs=other_pairs) - missing_pools = self._find_pools(unsupported_pairs) - return missing_pools[UNISWAP_V2_NAME], missing_pools[UNISWAP_V3_NAME], missing_pools[SOLIDLY_V2_NAME] - - def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: - """ - Collects pool addresses for each exchange, based on a set of unsupported token pairs - and flashloan tokens. The function constructs pairs of tokens from unsupported_pairs with each - flashloan token and retrieves pool data via multicall. It filters out invalid addresses. - - Args: - unsupported_pairs (List[Tuple]): A list of tuples, where each tuple contains two token addresses. - flashloan_tokens (List[str]): A list of token addresses available for flashloans. - - Returns: - Dict[str, List[str]]: A list of dictionaries, where each dictionary maps an exchange's name - to a list of valid pool addresses (i.e., non-zero addresses) obtained - from the multicall across all generated pairs. - """ pairs = [(tkn, token) for pair in unsupported_pairs for tkn in pair for token in self._flashloan_tokens] chunk_size = 400 # Create the list of chunks @@ -112,7 +94,8 @@ def _find_pools(self, unsupported_pairs: List[Tuple]) -> Dict[str, List[str]]: self._web3.to_checksum_address(addr): exchange.exchange_name for addr in response if addr != ZERO_ADDRESS }) - return result + + return result[UNISWAP_V2_NAME], result[UNISWAP_V3_NAME], result[SOLIDLY_V2_NAME] def _extract_pairs(self, pools: List[Dict[str, Any]]) -> Tuple[List, set]: From f28f181572deee0f5ad2a307ac6fe4fce31952f7 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Sun, 12 May 2024 15:54:34 +0300 Subject: [PATCH 42/58] Rename `get_pool` to `get_pool_func_call` --- fastlane_bot/events/exchanges/solidly_v2/base.py | 2 +- fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py | 2 +- fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py | 2 +- fastlane_bot/events/exchanges/uniswap_v2.py | 2 +- fastlane_bot/events/exchanges/uniswap_v3.py | 2 +- fastlane_bot/pool_finder.py | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fastlane_bot/events/exchanges/solidly_v2/base.py b/fastlane_bot/events/exchanges/solidly_v2/base.py index e58a474ff..7bffef541 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/base.py +++ b/fastlane_bot/events/exchanges/solidly_v2/base.py @@ -62,7 +62,7 @@ async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return await contract.caller.token1() - def get_pool(self, addr1, addr2): + def get_pool_func_call(self, addr1, addr2): return self.factory_contract.functions.getPair(addr1, addr2, False) @property diff --git a/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py b/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py index e15d17bf2..27dc61bd4 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py +++ b/fastlane_bot/events/exchanges/solidly_v2/velodrome_v2.py @@ -18,5 +18,5 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - def get_pool(self, addr1, addr2): + def get_pool_func_call(self, addr1, addr2): return self.factory_contract.functions.getPool(addr1, addr2, False) diff --git a/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py b/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py index 75cc7cb00..2b40d98d3 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py +++ b/fastlane_bot/events/exchanges/solidly_v2/xfai_v0.py @@ -29,5 +29,5 @@ async def get_fee(self, address: str, contract: AsyncContract) -> Tuple[str, flo fee_float = float(fee) / 10 ** 4 return str(fee_float), fee_float - def get_pool(self, addr1, addr2): + def get_pool_func_call(self, addr1, addr2): return self.factory_contract.functions.getPool(addr1) diff --git a/fastlane_bot/events/exchanges/uniswap_v2.py b/fastlane_bot/events/exchanges/uniswap_v2.py index 1043f7afe..ce1e209cc 100644 --- a/fastlane_bot/events/exchanges/uniswap_v2.py +++ b/fastlane_bot/events/exchanges/uniswap_v2.py @@ -63,5 +63,5 @@ async def get_tkn0(address: str, contract: AsyncContract, event: Any) -> str: async def get_tkn1(address: str, contract: AsyncContract, event: Any) -> str: return await contract.caller.token1() - def get_pool(self, addr1, addr2): + def get_pool_func_call(self, addr1, addr2): return self.factory_contract.functions.getPair(addr1, addr2) diff --git a/fastlane_bot/events/exchanges/uniswap_v3.py b/fastlane_bot/events/exchanges/uniswap_v3.py index 569fd6bee..76d8a93d5 100644 --- a/fastlane_bot/events/exchanges/uniswap_v3.py +++ b/fastlane_bot/events/exchanges/uniswap_v3.py @@ -60,5 +60,5 @@ async def get_tkn0(self, address: str, contract: Contract, event: Any) -> str: async def get_tkn1(self, address: str, contract: Contract, event: Any) -> str: return await contract.caller.token1() - def get_pool(self, addr1, addr2, fee): + def get_pool_func_call(self, addr1, addr2, fee): return self.factory_contract.functions.getPool(addr1, addr2, fee) diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index 606c12ef5..2cdb8df7f 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -85,10 +85,10 @@ def get_pools_for_unsupported_pairs(self, pools: List[Dict[str, Any]], arb_mode: mc = MultiCaller(contract=exchange.factory_contract, web3=self._web3, multicall_address=self._multicall_address) for pair in pair_chunk: if exchange.base_exchange_name in [UNISWAP_V2_NAME, SOLIDLY_V2_NAME]: - mc.add_call(exchange.get_pool(pair[0], pair[1])) + mc.add_call(exchange.get_pool_func_call(pair[0], pair[1])) elif exchange.base_exchange_name == UNISWAP_V3_NAME: for fee in self._uni_v3_fee_tiers[exchange.exchange_name]: - mc.add_call(exchange.get_pool(pair[0], pair[1], fee)) + mc.add_call(exchange.get_pool_func_call(pair[0], pair[1], fee)) response = mc.multicall() result[exchange.base_exchange_name].update({ self._web3.to_checksum_address(addr): exchange.exchange_name From 600809c0debe7d36d3e2698504bedd8da3fe4807 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Sun, 12 May 2024 15:57:48 +0300 Subject: [PATCH 43/58] Semantic --- fastlane_bot/pool_finder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index 2cdb8df7f..5e047a026 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -89,10 +89,10 @@ def get_pools_for_unsupported_pairs(self, pools: List[Dict[str, Any]], arb_mode: elif exchange.base_exchange_name == UNISWAP_V3_NAME: for fee in self._uni_v3_fee_tiers[exchange.exchange_name]: mc.add_call(exchange.get_pool_func_call(pair[0], pair[1], fee)) - response = mc.multicall() + response = mc.multicall() # TODO: Handle failures gracefully once supported in class `MultiCaller` result[exchange.base_exchange_name].update({ - self._web3.to_checksum_address(addr): exchange.exchange_name - for addr in response if addr != ZERO_ADDRESS + self._web3.to_checksum_address(address): exchange.exchange_name + for address in response if address != ZERO_ADDRESS }) return result[UNISWAP_V2_NAME], result[UNISWAP_V3_NAME], result[SOLIDLY_V2_NAME] From 1ec14972d2acd7538567a97a9c8c10c6eff62d8a Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Sun, 12 May 2024 19:06:44 +0300 Subject: [PATCH 44/58] Cleanup --- fastlane_bot/data/abi.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/fastlane_bot/data/abi.py b/fastlane_bot/data/abi.py index daaa04a03..ffdcf65c5 100644 --- a/fastlane_bot/data/abi.py +++ b/fastlane_bot/data/abi.py @@ -169,10 +169,8 @@ "type": "function", "name": "getPair", "stateMutability": "view", - "constant": True, "inputs": [{"internalType": "address", "name": "", "type": "address"}, {"internalType": "address", "name": "", "type": "address"}], - "outputs": [{"internalType": "address", "name": "", "type": "address"}], - "payable": False, + "outputs": [{"internalType": "address", "name": "", "type": "address"}] } ] @@ -256,9 +254,7 @@ "type": "function", "name": "getPair", "stateMutability": "view", - "inputs": [{"internalType": "address", "name": "", "type": "address"}, - {"internalType": "address", "name": "", "type": "address"}, - {"internalType": "bool", "name": "", "type": "bool"}], + "inputs": [{"internalType": "address", "name": "", "type": "address"}, {"internalType": "address", "name": "", "type": "address"}, {"internalType": "bool", "name": "", "type": "bool"}], "outputs": [{"internalType": "address", "name": "", "type": "address"}] } ] From 9590ffb8b30c3ab7fc7ec024b134afd6d0eba6b4 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Sun, 12 May 2024 19:37:33 +0300 Subject: [PATCH 45/58] Minor --- fastlane_bot/data/abi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fastlane_bot/data/abi.py b/fastlane_bot/data/abi.py index ffdcf65c5..3c98f7000 100644 --- a/fastlane_bot/data/abi.py +++ b/fastlane_bot/data/abi.py @@ -205,9 +205,9 @@ "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}] }, { + "type": "function", "name": "getPool", "stateMutability": "view", - "type": "function", "inputs": [{"internalType": "address", "name": "tokenA", "type": "address"}, {"internalType": "address", "name": "tokenB", "type": "address"}, {"internalType": "bool", "name": "stable", "type": "bool"}], "outputs": [{"internalType": "address", "name": "", "type": "address"}], } From 6761c2ed42f20084dbf7a7ec3221c6775b2dde98 Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Sun, 12 May 2024 23:22:08 +0300 Subject: [PATCH 46/58] Fix usage of `MultiCall` in `PoolFinder` --- fastlane_bot/pool_finder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fastlane_bot/pool_finder.py b/fastlane_bot/pool_finder.py index 5e047a026..15ff0584f 100644 --- a/fastlane_bot/pool_finder.py +++ b/fastlane_bot/pool_finder.py @@ -82,17 +82,17 @@ def get_pools_for_unsupported_pairs(self, pools: List[Dict[str, Any]], arb_mode: for exchange in self._exchanges: for pair_chunk in chunked_pairs: - mc = MultiCaller(contract=exchange.factory_contract, web3=self._web3, multicall_address=self._multicall_address) + mc = MultiCaller(self._web3, self._multicall_address) for pair in pair_chunk: if exchange.base_exchange_name in [UNISWAP_V2_NAME, SOLIDLY_V2_NAME]: mc.add_call(exchange.get_pool_func_call(pair[0], pair[1])) elif exchange.base_exchange_name == UNISWAP_V3_NAME: for fee in self._uni_v3_fee_tiers[exchange.exchange_name]: mc.add_call(exchange.get_pool_func_call(pair[0], pair[1], fee)) - response = mc.multicall() # TODO: Handle failures gracefully once supported in class `MultiCaller` + addresses = mc.run_calls() result[exchange.base_exchange_name].update({ self._web3.to_checksum_address(address): exchange.exchange_name - for address in response if address != ZERO_ADDRESS + for address in addresses if address not in [None, ZERO_ADDRESS] }) return result[UNISWAP_V2_NAME], result[UNISWAP_V3_NAME], result[SOLIDLY_V2_NAME] From 705a78c180604d45ff650b733d15fc16a7cb8c1f Mon Sep 17 00:00:00 2001 From: barak manos <> Date: Mon, 13 May 2024 11:14:02 +0300 Subject: [PATCH 47/58] Ensure that the pool-finder runs on the first iteration when enabled (i.e., when its period is larger than 0) --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index c1d2adc90..71a5ca8d7 100644 --- a/main.py +++ b/main.py @@ -525,7 +525,7 @@ def run(mgr, args, tenderly_uri=None) -> None: mgr.solidly_v2_event_mappings = dict( solidly_v2_event_mappings[["address", "exchange"]].values ) - if args.pool_finder_period > 0 and loop_idx % args.pool_finder_period == 0: + if args.pool_finder_period > 0 and (loop_idx - 1) % args.pool_finder_period == 0: mgr.cfg.logger.info(f"Searching for unsupported Carbon pairs.") uni_v2, uni_v3, solidly_v2 = pool_finder.get_pools_for_unsupported_pairs(mgr.pool_data, arb_mode=args.arb_mode) mgr.cfg.logger.info(f"Number of pools added: {len(uni_v2) + len(uni_v3) + len(solidly_v2)}") From 3acf35485152b3b1e51f307efb32d19b5b5a8070 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Mon, 13 May 2024 14:45:38 +0300 Subject: [PATCH 48/58] chore: added get_pool_func_call to the interface --- fastlane_bot/events/exchanges/balancer.py | 3 +++ fastlane_bot/events/exchanges/bancor_pol.py | 3 +++ fastlane_bot/events/exchanges/bancor_v2.py | 3 +++ fastlane_bot/events/exchanges/bancor_v3.py | 3 +++ fastlane_bot/events/exchanges/base.py | 4 ++++ fastlane_bot/events/exchanges/carbon_v1.py | 3 +++ 6 files changed, 19 insertions(+) diff --git a/fastlane_bot/events/exchanges/balancer.py b/fastlane_bot/events/exchanges/balancer.py index 2ec32119b..384ee5fda 100644 --- a/fastlane_bot/events/exchanges/balancer.py +++ b/fastlane_bot/events/exchanges/balancer.py @@ -88,3 +88,6 @@ async def get_tkn_n(self, address: str, contract: Contract, event: Any, index: i tokens = pool_balances[0] token_balances = pool_balances[1] return token_balances[index] + + def get_pool_func_call(self, addr1, addr2): + raise NotImplementedError diff --git a/fastlane_bot/events/exchanges/bancor_pol.py b/fastlane_bot/events/exchanges/bancor_pol.py index bc66bb7c5..1dab4c8fe 100644 --- a/fastlane_bot/events/exchanges/bancor_pol.py +++ b/fastlane_bot/events/exchanges/bancor_pol.py @@ -105,3 +105,6 @@ def save_strategy( cid=cid, block_number=block_number, ) + + def get_pool_func_call(self, addr1, addr2): + raise NotImplementedError diff --git a/fastlane_bot/events/exchanges/bancor_v2.py b/fastlane_bot/events/exchanges/bancor_v2.py index 8da7d1745..a4ab5db4a 100644 --- a/fastlane_bot/events/exchanges/bancor_v2.py +++ b/fastlane_bot/events/exchanges/bancor_v2.py @@ -76,3 +76,6 @@ async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: async def get_anchor(self, contract: Contract) -> str: return await contract.caller.anchor() + + def get_pool_func_call(self, addr1, addr2): + raise NotImplementedError diff --git a/fastlane_bot/events/exchanges/bancor_v3.py b/fastlane_bot/events/exchanges/bancor_v3.py index 2af08a103..2b14afe0d 100644 --- a/fastlane_bot/events/exchanges/bancor_v3.py +++ b/fastlane_bot/events/exchanges/bancor_v3.py @@ -64,3 +64,6 @@ async def get_tkn1(self, address: str, contract: Contract, event: Event) -> str: if event.args["pool"] != self.BNT_ADDRESS else event.args["tkn_address"] ) + + def get_pool_func_call(self, addr1, addr2): + raise NotImplementedError diff --git a/fastlane_bot/events/exchanges/base.py b/fastlane_bot/events/exchanges/base.py index bf450ff12..8ff438a1b 100644 --- a/fastlane_bot/events/exchanges/base.py +++ b/fastlane_bot/events/exchanges/base.py @@ -123,6 +123,10 @@ async def get_fee(address: str, contract: AsyncContract) -> float: """ pass + @abstractmethod + def get_pool_func_call(self, addr1, addr2, *args, **kwargs): + ... + @staticmethod @abstractmethod async def get_tkn0(address: str, contract: AsyncContract, event: Any) -> str: diff --git a/fastlane_bot/events/exchanges/carbon_v1.py b/fastlane_bot/events/exchanges/carbon_v1.py index df0540661..fa4a40107 100644 --- a/fastlane_bot/events/exchanges/carbon_v1.py +++ b/fastlane_bot/events/exchanges/carbon_v1.py @@ -244,3 +244,6 @@ def save_strategy( ), block_number=block_number, ) + + def get_pool_func_call(self, addr1, addr2): + raise NotImplementedError From 99bdb50a3d2c5aaa88df38b0c42373df69112b17 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Tue, 14 May 2024 00:32:52 +0300 Subject: [PATCH 49/58] feat: add event topic test --- .../events/exchanges/solidly_v2/__init__.py | 4 +- fastlane_bot/tests/conftest.py | 36 +++++++++++++ fastlane_bot/tests/test_event_topics.py | 50 +++++++++++++++++++ 3 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 fastlane_bot/tests/conftest.py create mode 100644 fastlane_bot/tests/test_event_topics.py diff --git a/fastlane_bot/events/exchanges/solidly_v2/__init__.py b/fastlane_bot/events/exchanges/solidly_v2/__init__.py index c81f0bca4..7411376aa 100644 --- a/fastlane_bot/events/exchanges/solidly_v2/__init__.py +++ b/fastlane_bot/events/exchanges/solidly_v2/__init__.py @@ -1,4 +1,4 @@ -from ..base import Exchange +from .base import SolidlyV2 as SolidlyV2Base from .velocimeter_v2 import VelocimeterV2 from .equalizer_v2 import EqualizerV2 from .velodrome_v2 import VelodromeV2 @@ -9,7 +9,7 @@ from .xfai_v0 import XFaiV2 -class SolidlyV2(Exchange): +class SolidlyV2(SolidlyV2Base): def __new__(cls, **kwargs): return { "velocimeter_v2": VelocimeterV2, diff --git a/fastlane_bot/tests/conftest.py b/fastlane_bot/tests/conftest.py new file mode 100644 index 000000000..e1bd98a8a --- /dev/null +++ b/fastlane_bot/tests/conftest.py @@ -0,0 +1,36 @@ +import pytest + +import pandas as pd + +from fastlane_bot import Config +from fastlane_bot.events.managers.manager import Manager + + +@pytest.fixture +def config(): + return Config.new(config=Config.CONFIG_MAINNET) + + +@pytest.fixture +def manager(config): + static_pool_data_filename = "static_pool_data" + static_pool_data = pd.read_csv(f"fastlane_bot/data/{static_pool_data_filename}.csv", low_memory=False) + uniswap_v2_event_mappings = pd.read_csv("fastlane_bot/data/uniswap_v2_event_mappings.csv", low_memory=False) + + exchanges = "carbon_v1,bancor_v3,uniswap_v3,uniswap_v2,sushiswap_v2,bancor_pol,bancor_v2,balancer" + exchanges = exchanges.split(",") + + alchemy_max_block_fetch = 20 + + tokens = pd.read_csv("fastlane_bot/data/tokens.csv", low_memory=False) + + return Manager( + web3=config.w3, + w3_async=config.w3_async, + cfg=config, + pool_data=static_pool_data.to_dict(orient="records"), + SUPPORTED_EXCHANGES=exchanges, + alchemy_max_block_fetch=alchemy_max_block_fetch, + uniswap_v2_event_mappings=uniswap_v2_event_mappings, + tokens=tokens.to_dict(orient="records"), + ) diff --git a/fastlane_bot/tests/test_event_topics.py b/fastlane_bot/tests/test_event_topics.py new file mode 100644 index 000000000..bd7c9692c --- /dev/null +++ b/fastlane_bot/tests/test_event_topics.py @@ -0,0 +1,50 @@ +from fastlane_bot.events.exchanges.carbon_v1 import CarbonV1 +from fastlane_bot.events.exchanges.bancor_pol import BancorPol +from fastlane_bot.events.exchanges.bancor_v2 import BancorV2 +from fastlane_bot.events.exchanges.bancor_v3 import BancorV3 +from fastlane_bot.events.exchanges.uniswap_v2 import UniswapV2 +from fastlane_bot.events.exchanges.uniswap_v3 import UniswapV3 +from fastlane_bot.events.exchanges.solidly_v2 import SolidlyV2 + + +def test_event_topics(manager): + for exchange_name, exchange in manager.exchanges.items(): + contract = manager.event_contracts[exchange_name] + for subscription in exchange.get_subscriptions(contract): + if isinstance(exchange, CarbonV1): + if subscription._event == contract.events.StrategyCreated: + assert subscription.topic == "0xff24554f8ccfe540435cfc8854831f8dcf1cf2068708cfaf46e8b52a4ccc4c8d" + elif subscription._event == contract.events.StrategyUpdated: + assert subscription.topic == "0x720da23a5c920b1d8827ec83c4d3c4d90d9419eadb0036b88cb4c2ffa91aef7d" + elif subscription._event == contract.events.StrategyDeleted: + assert subscription.topic == "0x4d5b6e0627ea711d8e9312b6ba56f50e0b51d41816fd6fd38643495ac81d38b6" + elif subscription._event == contract.events.PairTradingFeePPMUpdated: + assert subscription.topic == "0x831434d05f3ad5f63be733ea463b2933c70d2162697fd200a22b5d56f5c454b6" + elif subscription._event == contract.events.TradingFeePPMUpdated: + assert subscription.topic == "0x66db0986e1156e2e747795714bf0301c7e1c695c149a738cb01bcf5cfead8465" + elif subscription._event == contract.events.PairCreated: + assert subscription.topic == "0x6365c594f5448f79c1cc1e6f661bdbf1d16f2e8f85747e13f8e80f1fd168b7c3" + elif isinstance(exchange, BancorPol): + if subscription._event == contract.events.TokenTraded: + assert subscription.topic == "0x16ddee9b3f1b2e6f797172fe2cd10a214e749294074e075e451f95aecd0b958c" + if subscription._event == contract.events.TradingEnabled: + assert subscription.topic == "0xe695080c3c54317994bff9c7069120ba78f950937caeb98bf02d395abf2a2867" + elif isinstance(exchange, BancorV2): + if subscription._event == contract.events.TokenRateUpdate: + assert subscription.topic == "0x77f29993cf2c084e726f7e802da0719d6a0ade3e204badc7a3ffd57ecb768c24" + elif isinstance(exchange, BancorV3): + if subscription._event == contract.events.TradingLiquidityUpdated: + assert subscription.topic == "0x6e96dc5343d067ec486a9920e0304c3610ed05c65e45cc029d9b9fe7ecfa7620" + elif isinstance(exchange, UniswapV2): + if subscription._event == contract.events.Sync: + assert subscription.topic == "0x1c411e9a96e071241c2f21f7726b17ae89e3cab4c78be50e062b03a9fffbbad1" + elif isinstance(exchange, UniswapV3): + if subscription._event == contract.events.Swap: + assert subscription.topic == "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67" + elif isinstance(exchange, SolidlyV2): + if subscription._event == contract.events.Sync: + assert subscription.topic == "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67" + else: + print(exchange_name) + print(subscription) + assert False From e9c994b0aefbbfbf05fbe1d6b30552456628cde7 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Tue, 14 May 2024 15:50:10 +0300 Subject: [PATCH 50/58] fix: bancor_pol event topic --- fastlane_bot/events/exchanges/bancor_pol.py | 2 +- fastlane_bot/events/exchanges/bancor_v3.py | 8 +- fastlane_bot/tests/conftest.py | 28 ------- fastlane_bot/tests/test_event_topics.py | 84 +++++++++++---------- 4 files changed, 50 insertions(+), 72 deletions(-) diff --git a/fastlane_bot/events/exchanges/bancor_pol.py b/fastlane_bot/events/exchanges/bancor_pol.py index 1dab4c8fe..23176a714 100644 --- a/fastlane_bot/events/exchanges/bancor_pol.py +++ b/fastlane_bot/events/exchanges/bancor_pol.py @@ -51,7 +51,7 @@ def get_events(self, contract: Contract) -> List[Type[Contract]]: def get_subscriptions(self, contract: Contract) -> List[Subscription]: return [ Subscription(contract.events.TokenTraded), - Subscription(contract.events.TradingEnabled), + Subscription(contract.events.TradingEnabled, "0xae3f48c001771f8e9868e24d47b9e4295b06b1d78072acf96f167074aa3fab64"), ] async def get_fee(self, address: str, contract: Contract) -> Tuple[str, float]: diff --git a/fastlane_bot/events/exchanges/bancor_v3.py b/fastlane_bot/events/exchanges/bancor_v3.py index 2b14afe0d..b9ea312b7 100644 --- a/fastlane_bot/events/exchanges/bancor_v3.py +++ b/fastlane_bot/events/exchanges/bancor_v3.py @@ -23,7 +23,8 @@ from ..interfaces.subscription import Subscription -LIQUIDITY_UPDATED_TOPIC = "0x6e96dc5343d067ec486a9920e0304c3610ed05c65e45cc029d9b9fe7ecfa7620" +TRADING_LIQUIDITY_UPDATED_TOPIC = "0x6e96dc5343d067ec486a9920e0304c3610ed05c65e45cc029d9b9fe7ecfa7620" +TOTAL_LIQUIDITY_UPDATED_TOPIC = "0x85a03952f50b8c00b32a521c32094023b64ef0b6d4511f423d44c480a62cb145" @dataclass @@ -50,7 +51,10 @@ def get_events(self, contract: Contract) -> List[Type[Contract]]: return [contract.events.TradingLiquidityUpdated] def get_subscriptions(self, contract: Contract) -> List[Subscription]: - return [Subscription(contract.events.TradingLiquidityUpdated, LIQUIDITY_UPDATED_TOPIC)] + return [ + Subscription(contract.events.TradingLiquidityUpdated, TRADING_LIQUIDITY_UPDATED_TOPIC), + # Subscription(contract.events.TotalLiquidityUpdated, TOTAL_LIQUIDITY_UPDATED_TOPIC), # Unused + ] async def get_fee(self, address: str, contract: Contract) -> Tuple[str, float]: return "0.000", 0.000 diff --git a/fastlane_bot/tests/conftest.py b/fastlane_bot/tests/conftest.py index e1bd98a8a..fa3fc27a7 100644 --- a/fastlane_bot/tests/conftest.py +++ b/fastlane_bot/tests/conftest.py @@ -1,36 +1,8 @@ import pytest -import pandas as pd - from fastlane_bot import Config -from fastlane_bot.events.managers.manager import Manager @pytest.fixture def config(): return Config.new(config=Config.CONFIG_MAINNET) - - -@pytest.fixture -def manager(config): - static_pool_data_filename = "static_pool_data" - static_pool_data = pd.read_csv(f"fastlane_bot/data/{static_pool_data_filename}.csv", low_memory=False) - uniswap_v2_event_mappings = pd.read_csv("fastlane_bot/data/uniswap_v2_event_mappings.csv", low_memory=False) - - exchanges = "carbon_v1,bancor_v3,uniswap_v3,uniswap_v2,sushiswap_v2,bancor_pol,bancor_v2,balancer" - exchanges = exchanges.split(",") - - alchemy_max_block_fetch = 20 - - tokens = pd.read_csv("fastlane_bot/data/tokens.csv", low_memory=False) - - return Manager( - web3=config.w3, - w3_async=config.w3_async, - cfg=config, - pool_data=static_pool_data.to_dict(orient="records"), - SUPPORTED_EXCHANGES=exchanges, - alchemy_max_block_fetch=alchemy_max_block_fetch, - uniswap_v2_event_mappings=uniswap_v2_event_mappings, - tokens=tokens.to_dict(orient="records"), - ) diff --git a/fastlane_bot/tests/test_event_topics.py b/fastlane_bot/tests/test_event_topics.py index bd7c9692c..67e456f71 100644 --- a/fastlane_bot/tests/test_event_topics.py +++ b/fastlane_bot/tests/test_event_topics.py @@ -1,50 +1,52 @@ +import pytest + from fastlane_bot.events.exchanges.carbon_v1 import CarbonV1 from fastlane_bot.events.exchanges.bancor_pol import BancorPol from fastlane_bot.events.exchanges.bancor_v2 import BancorV2 from fastlane_bot.events.exchanges.bancor_v3 import BancorV3 from fastlane_bot.events.exchanges.uniswap_v2 import UniswapV2 from fastlane_bot.events.exchanges.uniswap_v3 import UniswapV3 +from fastlane_bot.events.exchanges.balancer import Balancer from fastlane_bot.events.exchanges.solidly_v2 import SolidlyV2 -def test_event_topics(manager): - for exchange_name, exchange in manager.exchanges.items(): - contract = manager.event_contracts[exchange_name] - for subscription in exchange.get_subscriptions(contract): - if isinstance(exchange, CarbonV1): - if subscription._event == contract.events.StrategyCreated: - assert subscription.topic == "0xff24554f8ccfe540435cfc8854831f8dcf1cf2068708cfaf46e8b52a4ccc4c8d" - elif subscription._event == contract.events.StrategyUpdated: - assert subscription.topic == "0x720da23a5c920b1d8827ec83c4d3c4d90d9419eadb0036b88cb4c2ffa91aef7d" - elif subscription._event == contract.events.StrategyDeleted: - assert subscription.topic == "0x4d5b6e0627ea711d8e9312b6ba56f50e0b51d41816fd6fd38643495ac81d38b6" - elif subscription._event == contract.events.PairTradingFeePPMUpdated: - assert subscription.topic == "0x831434d05f3ad5f63be733ea463b2933c70d2162697fd200a22b5d56f5c454b6" - elif subscription._event == contract.events.TradingFeePPMUpdated: - assert subscription.topic == "0x66db0986e1156e2e747795714bf0301c7e1c695c149a738cb01bcf5cfead8465" - elif subscription._event == contract.events.PairCreated: - assert subscription.topic == "0x6365c594f5448f79c1cc1e6f661bdbf1d16f2e8f85747e13f8e80f1fd168b7c3" - elif isinstance(exchange, BancorPol): - if subscription._event == contract.events.TokenTraded: - assert subscription.topic == "0x16ddee9b3f1b2e6f797172fe2cd10a214e749294074e075e451f95aecd0b958c" - if subscription._event == contract.events.TradingEnabled: - assert subscription.topic == "0xe695080c3c54317994bff9c7069120ba78f950937caeb98bf02d395abf2a2867" - elif isinstance(exchange, BancorV2): - if subscription._event == contract.events.TokenRateUpdate: - assert subscription.topic == "0x77f29993cf2c084e726f7e802da0719d6a0ade3e204badc7a3ffd57ecb768c24" - elif isinstance(exchange, BancorV3): - if subscription._event == contract.events.TradingLiquidityUpdated: - assert subscription.topic == "0x6e96dc5343d067ec486a9920e0304c3610ed05c65e45cc029d9b9fe7ecfa7620" - elif isinstance(exchange, UniswapV2): - if subscription._event == contract.events.Sync: - assert subscription.topic == "0x1c411e9a96e071241c2f21f7726b17ae89e3cab4c78be50e062b03a9fffbbad1" - elif isinstance(exchange, UniswapV3): - if subscription._event == contract.events.Swap: - assert subscription.topic == "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67" - elif isinstance(exchange, SolidlyV2): - if subscription._event == contract.events.Sync: - assert subscription.topic == "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67" - else: - print(exchange_name) - print(subscription) - assert False +@pytest.mark.parametrize("cls,exchange_name,event_topics", [ + (CarbonV1, "carbon_v1", { + "StrategyCreated": "0xff24554f8ccfe540435cfc8854831f8dcf1cf2068708cfaf46e8b52a4ccc4c8d", + "StrategyUpdated": "0x720da23a5c920b1d8827ec83c4d3c4d90d9419eadb0036b88cb4c2ffa91aef7d", + "StrategyDeleted": "0x4d5b6e0627ea711d8e9312b6ba56f50e0b51d41816fd6fd38643495ac81d38b6", + "PairTradingFeePPMUpdated": "0x831434d05f3ad5f63be733ea463b2933c70d2162697fd200a22b5d56f5c454b6", + "TradingFeePPMUpdated": "0x66db0986e1156e2e747795714bf0301c7e1c695c149a738cb01bcf5cfead8465", + "PairCreated": "0x6365c594f5448f79c1cc1e6f661bdbf1d16f2e8f85747e13f8e80f1fd168b7c3", + }), + (BancorPol, "bancor_pol", { + "TokenTraded": "0x16ddee9b3f1b2e6f797172fe2cd10a214e749294074e075e451f95aecd0b958c", + "TradingEnabled": "0xae3f48c001771f8e9868e24d47b9e4295b06b1d78072acf96f167074aa3fab64", + }), + (BancorV2, "bancor_v2", { + "TokenRateUpdate": "0x77f29993cf2c084e726f7e802da0719d6a0ade3e204badc7a3ffd57ecb768c24", + }), + (BancorV3, "bancor_v3", { + "TradingLiquidityUpdated": "0x6e96dc5343d067ec486a9920e0304c3610ed05c65e45cc029d9b9fe7ecfa7620", + # "TotalLiquidityUpdated": "0x85a03952f50b8c00b32a521c32094023b64ef0b6d4511f423d44c480a62cb145", + }), + (UniswapV2, "uniswap_v2", { + "Sync": "0x1c411e9a96e071241c2f21f7726b17ae89e3cab4c78be50e062b03a9fffbbad1", + }), + (UniswapV3, "uniswap_v3", { + "Swap": "0xc42079f94a6350d7e6235f29174924f928cc2ac818eb64fed8004e115fbcca67", + }), + (UniswapV3, "pancakeswap_v3", { + "Swap": "0x19b47279256b2a23a1665c810c8d55a1758940ee09377d4f8d26497a3577dc83", + }), + (Balancer, "balancer", {}), + (SolidlyV2, "velocimeter_v2", { + "Sync": "0xcf2aa50876cdfbb541206f89af0ee78d44a2abf8d328e37fa4917f982149848a", + }), +]) +def test_event_topic(config, cls, exchange_name, event_topics): + exchange = cls(exchange_name=exchange_name) + contract = config.w3.eth.contract(abi=exchange.get_abi()) + for subscription in exchange.get_subscriptions(contract): + assert event_topics.pop(subscription._event.event_name) == subscription.topic + assert event_topics == {} From bf607176448f917853cc0b1d7addf845b653bd96 Mon Sep 17 00:00:00 2001 From: Nicholas Welch Date: Fri, 17 May 2024 15:36:38 +1000 Subject: [PATCH 51/58] update and save new pool_finder pools --- main.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 71a5ca8d7..f2a9c125e 100644 --- a/main.py +++ b/main.py @@ -529,9 +529,26 @@ def run(mgr, args, tenderly_uri=None) -> None: mgr.cfg.logger.info(f"Searching for unsupported Carbon pairs.") uni_v2, uni_v3, solidly_v2 = pool_finder.get_pools_for_unsupported_pairs(mgr.pool_data, arb_mode=args.arb_mode) mgr.cfg.logger.info(f"Number of pools added: {len(uni_v2) + len(uni_v3) + len(solidly_v2)}") - mgr.uniswap_v2_event_mappings.update(uni_v2) - mgr.uniswap_v3_event_mappings.update(uni_v3) - mgr.solidly_v2_event_mappings.update(solidly_v2) + + event_mappings = { + 'uniswap_v2': uni_v2, + 'uniswap_v3': uni_v3, + 'solidly_v2': solidly_v2, + } + for exchange_name, event_mapping_dict in event_mappings.items(): + # Update the manager's event mappings + getattr(mgr, f'{exchange_name}_event_mappings').update(event_mapping_dict) + + # Update the local event_mappings csvs + df = pd.DataFrame.from_dict(getattr(mgr, f'{exchange_name}_event_mappings'), orient='index').reset_index() + if len(df)>0: + df.columns = ['address', 'exchange'] + # if the csvs are always sorted then the diffs will be readable + df.sort_values(by=['exchange','address'], inplace=True) + df.to_csv(f"fastlane_bot/data/blockchain_data/{args.blockchain}/{exchange_name}_event_mappings.csv", index=False) + + # Update the static_pools data for later event filtering + handle_static_pools_update(mgr) last_block_queried = current_block From f7070f943ec2dcafda77d5799ceca5be3d9e4f88 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Fri, 17 May 2024 16:40:34 +0800 Subject: [PATCH 52/58] refactor --- fastlane_bot/events/utils.py | 13 +++++++++++++ main.py | 25 +++++-------------------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/fastlane_bot/events/utils.py b/fastlane_bot/events/utils.py index 2577ba460..87e378456 100644 --- a/fastlane_bot/events/utils.py +++ b/fastlane_bot/events/utils.py @@ -793,6 +793,19 @@ def save_events_to_json( mgr.cfg.logger.debug(f"[events.utils.save_events_to_json] Saved events to {path}") +def process_new_events(new_event_mappings, event_mappings, filename): + # Update the manager's event mappings + event_mappings.update(new_event_mappings) + + # Update the local event_mappings csvs + df = pd.DataFrame.from_dict(event_mappings, orient='index').reset_index() + if len(df)>0: + df.columns = ['address', 'exchange'] + # if the csvs are always sorted then the diffs will be readable + df.sort_values(by=['exchange','address'], inplace=True) + df.to_csv(filename, index=False) + + def update_pools_from_events(n_jobs: int, mgr: Any, latest_events: List[Event]): """ Updates the pools with the given events. diff --git a/main.py b/main.py index f2a9c125e..fcc722efe 100644 --- a/main.py +++ b/main.py @@ -39,6 +39,7 @@ get_config, get_loglevel, update_pools_from_events, + process_new_events, write_pool_data_to_disk, init_bot, get_cached_events, @@ -528,27 +529,11 @@ def run(mgr, args, tenderly_uri=None) -> None: if args.pool_finder_period > 0 and (loop_idx - 1) % args.pool_finder_period == 0: mgr.cfg.logger.info(f"Searching for unsupported Carbon pairs.") uni_v2, uni_v3, solidly_v2 = pool_finder.get_pools_for_unsupported_pairs(mgr.pool_data, arb_mode=args.arb_mode) - mgr.cfg.logger.info(f"Number of pools added: {len(uni_v2) + len(uni_v3) + len(solidly_v2)}") - - event_mappings = { - 'uniswap_v2': uni_v2, - 'uniswap_v3': uni_v3, - 'solidly_v2': solidly_v2, - } - for exchange_name, event_mapping_dict in event_mappings.items(): - # Update the manager's event mappings - getattr(mgr, f'{exchange_name}_event_mappings').update(event_mapping_dict) - - # Update the local event_mappings csvs - df = pd.DataFrame.from_dict(getattr(mgr, f'{exchange_name}_event_mappings'), orient='index').reset_index() - if len(df)>0: - df.columns = ['address', 'exchange'] - # if the csvs are always sorted then the diffs will be readable - df.sort_values(by=['exchange','address'], inplace=True) - df.to_csv(f"fastlane_bot/data/blockchain_data/{args.blockchain}/{exchange_name}_event_mappings.csv", index=False) - - # Update the static_pools data for later event filtering + process_new_events(uni_v2, mgr.uniswap_v2_event_mappings, f"fastlane_bot/data/blockchain_data/{args.blockchain}/uniswap_v2_event_mappings.csv") + process_new_events(uni_v3, mgr.uniswap_v3_event_mappings, f"fastlane_bot/data/blockchain_data/{args.blockchain}/uniswap_v3_event_mappings.csv") + process_new_events(solidly_v2, mgr.solidly_v2_event_mappings, f"fastlane_bot/data/blockchain_data/{args.blockchain}/solidly_v2_event_mappings.csv") handle_static_pools_update(mgr) + mgr.cfg.logger.info(f"Number of pools added: {len(uni_v2) + len(uni_v3) + len(solidly_v2)}") last_block_queried = current_block From b1538c14c6b431a059ff98b4d33ce61d701ade1c Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Fri, 17 May 2024 20:25:35 +0800 Subject: [PATCH 53/58] feat: collect bancor_pol events from block 0 --- fastlane_bot/events/event_gatherer.py | 9 ++++++++- fastlane_bot/events/exchanges/bancor_pol.py | 4 ++-- fastlane_bot/events/interfaces/subscription.py | 7 ++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/fastlane_bot/events/event_gatherer.py b/fastlane_bot/events/event_gatherer.py index 5f70913b2..1659f5807 100644 --- a/fastlane_bot/events/event_gatherer.py +++ b/fastlane_bot/events/event_gatherer.py @@ -42,7 +42,14 @@ def __init__( self._subscriptions.append(sub) def get_all_events(self, from_block: int, to_block: int): - results = asyncio.get_event_loop().run_until_complete(asyncio.gather(*[self._get_events_for_topic(from_block, to_block, sub) for sub in self._subscriptions])) + coroutines = [] + for sub in self._subscriptions: + if sub.collect_all: + from_block_ = 0 + else: + from_block_ = from_block + coroutines.append(self._get_events_for_topic(from_block_, to_block, sub)) + results = asyncio.get_event_loop().run_until_complete(asyncio.gather(*coroutines)) return list(chain.from_iterable(results)) async def _get_events_for_topic(self, from_block: int, to_block: int, subscription: Subscription): diff --git a/fastlane_bot/events/exchanges/bancor_pol.py b/fastlane_bot/events/exchanges/bancor_pol.py index 23176a714..25ba9be5f 100644 --- a/fastlane_bot/events/exchanges/bancor_pol.py +++ b/fastlane_bot/events/exchanges/bancor_pol.py @@ -50,8 +50,8 @@ def get_events(self, contract: Contract) -> List[Type[Contract]]: def get_subscriptions(self, contract: Contract) -> List[Subscription]: return [ - Subscription(contract.events.TokenTraded), - Subscription(contract.events.TradingEnabled, "0xae3f48c001771f8e9868e24d47b9e4295b06b1d78072acf96f167074aa3fab64"), + Subscription(contract.events.TokenTraded, collect_all=True), + Subscription(contract.events.TradingEnabled, "0xae3f48c001771f8e9868e24d47b9e4295b06b1d78072acf96f167074aa3fab64", collect_all=True), ] async def get_fee(self, address: str, contract: Contract) -> Tuple[str, float]: diff --git a/fastlane_bot/events/interfaces/subscription.py b/fastlane_bot/events/interfaces/subscription.py index b1165f757..7a178fe20 100644 --- a/fastlane_bot/events/interfaces/subscription.py +++ b/fastlane_bot/events/interfaces/subscription.py @@ -14,9 +14,10 @@ def _get_event_topic(event): class Subscription: - def __init__(self, event: ContractEvent, topic: Optional[str] = None): + def __init__(self, event: ContractEvent, topic: Optional[str] = None, collect_all: bool = False): self._event = event self._topic = _get_event_topic(event) if topic is None else topic + self._collect_all = collect_all self._subscription_id = None self._latest_event_index = (-1, -1) # (block_number, block_index) @@ -30,6 +31,10 @@ def subscription_id(self): @property def topic(self): return self._topic + + @property + def collect_all(self): + return self._collect_all def parse_log(self, log) -> Event: try: From a614db83443ebaebd770fed7ab6a923fc584b907 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Mon, 20 May 2024 19:23:11 +0800 Subject: [PATCH 54/58] feat: get logs batching --- fastlane_bot/config/constants.py | 13 +++++++ fastlane_bot/events/event_gatherer.py | 51 ++++++++++++++++++++++----- main.py | 7 +++- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/fastlane_bot/config/constants.py b/fastlane_bot/config/constants.py index ad6641ac0..ad855d546 100644 --- a/fastlane_bot/config/constants.py +++ b/fastlane_bot/config/constants.py @@ -34,3 +34,16 @@ SECTA_V3_NAME = "secta_v3" METAVAULT_V3_NAME = "metavault_v3" ZERO_ADDRESS = "0x0000000000000000000000000000000000000000" + +BLOCK_CHUNK_SIZE_MAP = { + "ethereum": 0, + "polygon": 0, + "polygon_zkevm": 0, + "arbitrum_one": 0, + "optimism": 0, + "coinbase_base": 0, + "fantom": 5000, + "mantle": 0, + "linea": 0, + "sei": 0, +} diff --git a/fastlane_bot/events/event_gatherer.py b/fastlane_bot/events/event_gatherer.py index 1659f5807..382343ee2 100644 --- a/fastlane_bot/events/event_gatherer.py +++ b/fastlane_bot/events/event_gatherer.py @@ -1,5 +1,5 @@ from itertools import chain -from typing import Dict +from typing import Dict, List import asyncio import nest_asyncio @@ -7,6 +7,7 @@ from web3 import AsyncWeb3 from web3.contract import Contract +from fastlane_bot.config.constants import BLOCK_CHUNK_SIZE_MAP from .interfaces.subscription import Subscription from .exchanges.base import Exchange @@ -21,6 +22,7 @@ class EventGatherer: def __init__( self, + blockchain: str, w3: AsyncWeb3, exchanges: Dict[str, Exchange], event_contracts: Dict[str, Contract], @@ -30,6 +32,7 @@ def __init__( manager: The Manager object w3: The connected AsyncWeb3 object. """ + self._blockchain = blockchain self._w3 = w3 self._subscriptions = [] unique_topics = set() @@ -48,16 +51,46 @@ def get_all_events(self, from_block: int, to_block: int): from_block_ = 0 else: from_block_ = from_block - coroutines.append(self._get_events_for_topic(from_block_, to_block, sub)) + coroutines.append(self._get_events_for_subscription(from_block_, to_block, sub)) results = asyncio.get_event_loop().run_until_complete(asyncio.gather(*coroutines)) return list(chain.from_iterable(results)) - async def _get_events_for_topic(self, from_block: int, to_block: int, subscription: Subscription): - events = await self._w3.eth.get_logs(filter_params={ - "fromBlock": from_block, - "toBlock": to_block, - "topics": [subscription.topic] - }) - return [subscription.parse_log(event) for event in events] + async def _get_events_for_subscription(self, from_block: int, to_block: int, subscription: Subscription): + return [subscription.parse_log(log) for log in await self._get_logs_for_topics(from_block, to_block, [subscription.topic])] + async def _get_logs_for_topics(self, from_block: int, to_block: int, topics: List[str]): + chunk_size = BLOCK_CHUNK_SIZE_MAP[self._blockchain] + if chunk_size > 0: + return await self._get_logs_iterative(from_block, to_block, topics, chunk_size) + else: + return await self._get_logs_recursive(from_block, to_block, topics) + async def _get_logs_iterative(self, from_block: int, to_block: int, topics: List[str], chunk_size: int) -> list: + block_numbers = list(range(from_block, to_block + 1, chunk_size)) + [to_block + 1] + log_lists = await asyncio.gather([ + self._w3.eth.get_logs(filter_params={ + "fromBlock": r[0], + "toBlock": r[1], + "topics": topics + }) + for r in zip(block_numbers, map(lambda n: n - 1, block_numbers[1:])) + ]) + return [log for log_list in log_lists for log in log_list] + + async def _get_logs_recursive(self, from_block: int, to_block: int, topics: List[str]) -> list: + if from_block <= to_block: + try: + return await self._w3.eth.get_logs(filter_params={ + "fromBlock": from_block, + "toBlock": to_block, + "topics": topics + }) + except Exception as e: + assert "eth_getLogs" in str(e), str(e) + mid_block = (from_block + to_block) // 2 + log_lists = await asyncio.gather( + self._get_logs_recursive(from_block, mid_block, topics), + self._get_logs_recursive(mid_block + 1, to_block, topics) + ) + return [log for log_list in log_lists for log in log_list] + return [] diff --git a/main.py b/main.py index fcc722efe..37a8d7490 100644 --- a/main.py +++ b/main.py @@ -306,7 +306,12 @@ def run(mgr, args, tenderly_uri=None) -> None: mainnet_uri = mgr.cfg.w3.provider.endpoint_uri handle_static_pools_update(mgr) - event_gatherer = EventGatherer(w3=mgr.w3_async, exchanges=mgr.exchanges, event_contracts=mgr.event_contracts) + event_gatherer = EventGatherer( + blockchain=args.blockchain, + w3=mgr.w3_async, + exchanges=mgr.exchanges, + event_contracts=mgr.event_contracts + ) pool_finder = PoolFinder( carbon_forks=mgr.cfg.network.CARBON_V1_FORKS, From 504ae10e5cb4529f9164619b8fa807a867766de6 Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Tue, 21 May 2024 16:33:54 +0800 Subject: [PATCH 55/58] fix: infinite recursion when fetching logs --- fastlane_bot/events/event_gatherer.py | 13 +++++++------ run_blockchain_terraformer.py | 9 +++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/fastlane_bot/events/event_gatherer.py b/fastlane_bot/events/event_gatherer.py index 382343ee2..e2016ffa0 100644 --- a/fastlane_bot/events/event_gatherer.py +++ b/fastlane_bot/events/event_gatherer.py @@ -87,10 +87,11 @@ async def _get_logs_recursive(self, from_block: int, to_block: int, topics: List }) except Exception as e: assert "eth_getLogs" in str(e), str(e) - mid_block = (from_block + to_block) // 2 - log_lists = await asyncio.gather( - self._get_logs_recursive(from_block, mid_block, topics), - self._get_logs_recursive(mid_block + 1, to_block, topics) - ) - return [log for log_list in log_lists for log in log_list] + if from_block < to_block: + mid_block = (from_block + to_block) // 2 + log_lists = await asyncio.gather( + self._get_logs_recursive(from_block, mid_block, topics), + self._get_logs_recursive(mid_block + 1, to_block, topics) + ) + return [log for log_list in log_lists for log in log_list] return [] diff --git a/run_blockchain_terraformer.py b/run_blockchain_terraformer.py index 854119d40..350affee0 100644 --- a/run_blockchain_terraformer.py +++ b/run_blockchain_terraformer.py @@ -680,10 +680,11 @@ def get_events_recursive(get_logs: any, start_block: int, end_block: int) -> lis return get_logs(fromBlock=start_block, toBlock=end_block) except Exception as e: assert "eth_getLogs" in str(e), str(e) - mid_block = (start_block + end_block) // 2 - event_list_1 = get_events_recursive(get_logs, start_block, mid_block) - event_list_2 = get_events_recursive(get_logs, mid_block + 1, end_block) - return event_list_1 + event_list_2 + if start_block < end_block: + mid_block = (start_block + end_block) // 2 + event_list_1 = get_events_recursive(get_logs, start_block, mid_block) + event_list_2 = get_events_recursive(get_logs, mid_block + 1, end_block) + return event_list_1 + event_list_2 return [] From bf3fca821670b26d64fe4de09eac43b2be298aab Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Tue, 21 May 2024 17:53:36 +0800 Subject: [PATCH 56/58] fix: print exception when failed to collect logs from blocks --- fastlane_bot/events/event_gatherer.py | 5 ++++- run_blockchain_terraformer.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/fastlane_bot/events/event_gatherer.py b/fastlane_bot/events/event_gatherer.py index e2016ffa0..9c7b67a33 100644 --- a/fastlane_bot/events/event_gatherer.py +++ b/fastlane_bot/events/event_gatherer.py @@ -1,7 +1,8 @@ +import asyncio +import traceback from itertools import chain from typing import Dict, List -import asyncio import nest_asyncio from web3 import AsyncWeb3 @@ -94,4 +95,6 @@ async def _get_logs_recursive(self, from_block: int, to_block: int, topics: List self._get_logs_recursive(mid_block + 1, to_block, topics) ) return [log for log_list in log_lists for log in log_list] + else: + traceback.print_exc(e) return [] diff --git a/run_blockchain_terraformer.py b/run_blockchain_terraformer.py index 350affee0..259c93532 100644 --- a/run_blockchain_terraformer.py +++ b/run_blockchain_terraformer.py @@ -1,4 +1,5 @@ import math +import traceback from typing import Tuple, List, Dict import pandas as pd @@ -685,6 +686,8 @@ def get_events_recursive(get_logs: any, start_block: int, end_block: int) -> lis event_list_1 = get_events_recursive(get_logs, start_block, mid_block) event_list_2 = get_events_recursive(get_logs, mid_block + 1, end_block) return event_list_1 + event_list_2 + else: + traceback.print_exc(e) return [] From 0a1d33e352dfe020f9ac4d07bd2c3b599db59e4d Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Tue, 21 May 2024 18:26:36 +0800 Subject: [PATCH 57/58] fix: error handling --- fastlane_bot/events/event_gatherer.py | 5 ++--- run_blockchain_terraformer.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/fastlane_bot/events/event_gatherer.py b/fastlane_bot/events/event_gatherer.py index 9c7b67a33..30b7bf7f3 100644 --- a/fastlane_bot/events/event_gatherer.py +++ b/fastlane_bot/events/event_gatherer.py @@ -1,5 +1,4 @@ import asyncio -import traceback from itertools import chain from typing import Dict, List @@ -96,5 +95,5 @@ async def _get_logs_recursive(self, from_block: int, to_block: int, topics: List ) return [log for log_list in log_lists for log in log_list] else: - traceback.print_exc(e) - return [] + raise e + raise Exception(f"Illegal log query range: {from_block} -> {to_block}") diff --git a/run_blockchain_terraformer.py b/run_blockchain_terraformer.py index 259c93532..df514dbee 100644 --- a/run_blockchain_terraformer.py +++ b/run_blockchain_terraformer.py @@ -1,5 +1,4 @@ import math -import traceback from typing import Tuple, List, Dict import pandas as pd @@ -687,8 +686,8 @@ def get_events_recursive(get_logs: any, start_block: int, end_block: int) -> lis event_list_2 = get_events_recursive(get_logs, mid_block + 1, end_block) return event_list_1 + event_list_2 else: - traceback.print_exc(e) - return [] + raise e + raise Exception(f"Illegal log query range: {start_block} -> {end_block}") def get_uni_v3_pools( From 7444e5e0df3b01372b2bee441f85d8718d77e59d Mon Sep 17 00:00:00 2001 From: Platon Floria Date: Tue, 21 May 2024 20:02:23 +0800 Subject: [PATCH 58/58] fix: take network name from config --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 37a8d7490..722539278 100644 --- a/main.py +++ b/main.py @@ -307,7 +307,7 @@ def run(mgr, args, tenderly_uri=None) -> None: handle_static_pools_update(mgr) event_gatherer = EventGatherer( - blockchain=args.blockchain, + blockchain=mgr.cfg.network.NETWORK, w3=mgr.w3_async, exchanges=mgr.exchanges, event_contracts=mgr.event_contracts