diff --git a/.ci/test.py b/.ci/test.py index a4f5ee742..7239d4594 100644 --- a/.ci/test.py +++ b/.ci/test.py @@ -241,13 +241,14 @@ def run_one(p: Plugin, workflow: str) -> bool: logging.info(f"Virtualenv at {vpath}") + num_workers = 1 if p.name == "sauron" else 5 cmd = [ str(pytest_path), "-vvv", "--timeout=600", "--timeout-method=thread", "--color=yes", - "-n=5", + f"-n={num_workers}", ] logging.info(f"Running `{' '.join(cmd)}` in directory {p.path.resolve()}") diff --git a/poncho b/poncho index e7795d763..a94da96ff 160000 --- a/poncho +++ b/poncho @@ -1 +1 @@ -Subproject commit e7795d763168d81435e7430105a7ef4c6985c45a +Subproject commit a94da96ff257cd10edda74ac897fc15a4344a08e diff --git a/sauron/.gitignore b/sauron/.gitignore new file mode 100644 index 000000000..21d0b898f --- /dev/null +++ b/sauron/.gitignore @@ -0,0 +1 @@ +.venv/ diff --git a/sauron/README.md b/sauron/README.md index d02dbe10e..ce023c808 100644 --- a/sauron/README.md +++ b/sauron/README.md @@ -24,6 +24,14 @@ Here is a fully reptilian example running against [blockstream.info](https://blo lightningd --mainnet --disable-plugin bcli --plugin $PWD/sauron.py --sauron-api-endpoint https://blockstream.info/api/ ``` + +Here is an example running against [mutinynet.com](https://mutinynet.com/): + +``` +lightningd --signet --disable-plugin bcli --plugin $PWD/sauron.py --sauron-api-endpoint https://mutinynet.com/api/ +``` + + You can use also proxy your requests through [Tor](https://www.torproject.org/) by specifying a SOCKS proxy to use with the `--sauron-tor-proxy` startup option, in the form `address:port`. diff --git a/sauron/requirements.txt b/sauron/requirements.txt index 1794cb833..7fd22f35e 100644 --- a/sauron/requirements.txt +++ b/sauron/requirements.txt @@ -1,2 +1,2 @@ -pyln-client>=23.2 +pyln-client>=23.2,<=24.5 requests[socks]>=2.23.0 diff --git a/sauron/sauron.py b/sauron/sauron.py index 3e2e34c9a..4165ef100 100755 --- a/sauron/sauron.py +++ b/sauron/sauron.py @@ -40,12 +40,29 @@ def fetch(url): @plugin.init() -def init(plugin, options, configuration, **kwargs): - plugin.api_endpoint = options["sauron-api-endpoint"] +def init(plugin, options, **kwargs): + plugin.api_endpoint = options.get("sauron-api-endpoint", None) if not plugin.api_endpoint: raise SauronError("You need to specify the sauron-api-endpoint option.") sys.exit(1) + # Test for Esplora or mempoolspace + try: + # Esplora API + feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) + feerate_req = fetch(feerate_url) + assert feerate_req.status_code == 200 and feerate_req.content != b'{}' + plugin.is_mempoolspace = False + except AssertionError: + try: + # MutinyNet API + feerate_url = "{}/v1/fees/recommended".format(plugin.api_endpoint) + feerate_req = fetch(feerate_url) + assert feerate_req.status_code == 200 + plugin.is_mempoolspace = True + except AssertionError as e1: + raise Exception("Sauron API cannot be reached") from e1 + if options["sauron-tor-proxy"]: address, port = options["sauron-tor-proxy"].split(":") socks5_proxy = "socks5h://{}:{}".format(address, port) @@ -55,7 +72,8 @@ def init(plugin, options, configuration, **kwargs): } plugin.log("Using proxy {} for requests".format(socks5_proxy)) - plugin.log("Sauron plugin initialized") + api = "mempool.space" if plugin.is_mempoolspace else "Esplora" + plugin.log(f"Sauron plugin initialized using {api} API") plugin.log(sauron_eye) @@ -188,12 +206,17 @@ def getutxout(plugin, txid, vout, **kwargs): @plugin.method("estimatefees") def estimatefees(plugin, **kwargs): - feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) + if plugin.is_mempoolspace: + # MutinyNet API + feerate_url = "{}/v1/fees/recommended".format(plugin.api_endpoint) + else: + # Blockstream API + feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) feerate_req = fetch(feerate_url) assert feerate_req.status_code == 200 feerates = feerate_req.json() - if plugin.sauron_network == "test" or plugin.sauron_network == "signet": + if plugin.sauron_network in ["test", "signet"]: # FIXME: remove the hack if the test API is "fixed" feerate = feerates.get("144", 1) slow = normal = urgent = very_urgent = int(feerate * 10**3) @@ -204,7 +227,7 @@ def estimatefees(plugin, **kwargs): urgent = int(feerates["6"] * 10**3) very_urgent = int(feerates["2"] * 10**3) - feerate_floor = int(feerates["1008"] * 10**3) + feerate_floor = int(feerates.get("1008", slow) * 10**3) feerates = [ {"blocks": 2, "feerate": very_urgent}, {"blocks": 6, "feerate": urgent}, @@ -229,7 +252,7 @@ def estimatefees(plugin, **kwargs): plugin.add_option( "sauron-api-endpoint", "", - "The URL of the esplora instance to hit (including '/api').", + "The URL of the esplora or mempool.space instance to hit (including '/api').", ) plugin.add_option( @@ -237,7 +260,7 @@ def estimatefees(plugin, **kwargs): "", "Tor's SocksPort address in the form address:port, don't specify the" " protocol. If you didn't modify your torrc you want to put" - "'localhost:9050' here.", + " 'localhost:9050' here.", ) diff --git a/sauron/tests/test_sauron_esplora.py b/sauron/tests/test_sauron_esplora.py new file mode 100644 index 000000000..952fb7feb --- /dev/null +++ b/sauron/tests/test_sauron_esplora.py @@ -0,0 +1,153 @@ +#!/usr/bin/python + +import os + +import pyln +import pytest +from pyln.testing import utils +from util import * # noqa: F403 + +pyln.testing.fixtures.network_daemons["bitcoin"] = utils.BitcoinD + + +class LightningNode(utils.LightningNode): + def __init__(self, *args, **kwargs): + pyln.testing.utils.TEST_NETWORK = "bitcoin" + utils.LightningNode.__init__(self, *args, **kwargs) + lightning_dir = args[1] + + self.daemon = LightningD(lightning_dir, None) # noqa: F405 + options = { + "disable-plugin": "bcli", + "network": "bitcoin", + "plugin": os.path.join(os.path.dirname(__file__), "../sauron.py"), + "sauron-api-endpoint": "https://blockstream.info/api", + } + self.daemon.opts.update(options) + + # Monkey patch + def set_feerates(self, feerates, wait_for_effect=True): + return None + + +@pytest.fixture +def node_cls(monkeypatch): + monkeypatch.setenv("TEST_NETWORK", "bitcoin") + yield LightningNode + + +def test_rpc_getchaininfo(ln_node): + """ + Test getchaininfo + """ + + response = ln_node.rpc.call("getchaininfo") + + assert ln_node.daemon.is_in_log("Sauron plugin initialized using Esplora API") + + expected_response_keys = ["chain", "blockcount", "headercount", "ibd"] + assert list(response.keys()) == expected_response_keys + assert response["chain"] == "main" + assert not response["ibd"] + + +def test_rpc_getrawblockbyheight(ln_node): + """ + Test getrawblockbyheight + """ + + response = ln_node.rpc.call("getrawblockbyheight", {"height": 0}) + + expected_response = { + "block": "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000", + "blockhash": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f", + } + assert response == expected_response + + +def test_rpc_sendrawtransaction_invalid(ln_node): + """ + Test sendrawtransaction + """ + + expected_response = { + "errmsg": 'sendrawtransaction RPC error: {"code":-22,"message":"TX decode failed. Make sure the tx has at least one input."}', + "success": False, + } + response = ln_node.rpc.call( + "sendrawtransaction", + {"tx": "invalid-raw-tx"}, + ) + + assert response == expected_response + + +def test_rpc_getutxout(ln_node): + """ + Test getutxout + """ + + expected_response = { + "amount": 1000000000, + "script": "4104b5abd412d4341b45056d3e376cd446eca43fa871b51961330deebd84423e740daa520690e1d9e074654c59ff87b408db903649623e86f1ca5412786f61ade2bfac", + } + response = ln_node.rpc.call( + "getutxout", + { + # block 181 + "txid": "a16f3ce4dd5deb92d98ef5cf8afeaf0775ebca408f708b2146c4fb42b41e14be", + "vout": 0, + }, + ) + assert response == expected_response + + +def test_rpc_estimatefees(ln_node): + """ + Test estimatefees + """ + + # Sample response + # { + # "opening": 4477, + # "mutual_close": 4477, + # "unilateral_close": 11929, + # "delayed_to_us": 4477, + # "htlc_resolution": 5652, + # "penalty": 5652, + # "min_acceptable": 1060, + # "max_acceptable": 119290, + # "feerate_floor": 1520, + # "feerates": [ + # {"blocks": 2, "feerate": 11929}, + # {"blocks": 6, "feerate": 5652}, + # {"blocks": 12, "feerate": 4477}, + # {"blocks": 144, "feerate": 2120} + # ] + # } + response = ln_node.rpc.call("estimatefees") + + expected_response_keys = [ + "opening", + "mutual_close", + "unilateral_close", + "delayed_to_us", + "htlc_resolution", + "penalty", + "min_acceptable", + "max_acceptable", + "feerate_floor", + "feerates", + ] + assert list(response.keys()) == expected_response_keys + + expected_feerates_keys = ("blocks", "feerate") + assert ( + list(set([tuple(entry.keys()) for entry in response["feerates"]]))[0] + == expected_feerates_keys + ) + + expected_feerates_blocks = [2, 6, 12, 144] + assert [ + entry["blocks"] for entry in response["feerates"] + ] == expected_feerates_blocks diff --git a/sauron/tests/test_sauron_esplora_tor_proxy.py b/sauron/tests/test_sauron_esplora_tor_proxy.py new file mode 100644 index 000000000..1ffd41d2d --- /dev/null +++ b/sauron/tests/test_sauron_esplora_tor_proxy.py @@ -0,0 +1,46 @@ +#!/usr/bin/python + +import os + +import pyln +import pytest +from pyln.testing import utils +from util import * # noqa: F403 + +pyln.testing.fixtures.network_daemons["bitcoin"] = utils.BitcoinD + + +class LightningNode(utils.LightningNode): + def __init__(self, *args, **kwargs): + pyln.testing.utils.TEST_NETWORK = "bitcoin" + utils.LightningNode.__init__(self, *args, **kwargs) + lightning_dir = args[1] + + self.daemon = LightningD(lightning_dir, None) # noqa: F405 + options = { + "disable-plugin": "bcli", + "network": "bitcoin", + "plugin": os.path.join(os.path.dirname(__file__), "../sauron.py"), + "sauron-api-endpoint": "https://blockstream.info/api", + "sauron-tor-proxy": "localhost:9050", + } + self.daemon.opts.update(options) + + # Monkey patch + def set_feerates(self, feerates, wait_for_effect=True): + return None + + +@pytest.fixture +def node_cls(monkeypatch): + monkeypatch.setenv("TEST_NETWORK", "bitcoin") + yield LightningNode + +@pytest.mark.skip(reason="TODO: Mock tor") +def test_tor_proxy(ln_node): + """ + Test for tor proxy + """ + + assert ln_node.daemon.opts["sauron-tor-proxy"] == "localhost:9050" + assert ln_node.daemon.is_in_log("Using proxy socks5h://localhost:9050 for requests") diff --git a/sauron/tests/test_sauron_mempoolspace.py b/sauron/tests/test_sauron_mempoolspace.py new file mode 100644 index 000000000..c3e93b287 --- /dev/null +++ b/sauron/tests/test_sauron_mempoolspace.py @@ -0,0 +1,154 @@ +#!/usr/bin/python + +import os + +import pyln +import pytest +from pyln.testing import utils +from util import * # noqa: F403 + +pyln.testing.fixtures.network_daemons["signet"] = utils.BitcoinD + + +class LightningNode(utils.LightningNode): + def __init__(self, *args, **kwargs): + pyln.testing.utils.TEST_NETWORK = "signet" + utils.LightningNode.__init__(self, *args, **kwargs) + lightning_dir = args[1] + + self.daemon = LightningD(lightning_dir, None) # noqa: F405 + options = { + "disable-plugin": "bcli", + "network": "signet", + "plugin": os.path.join(os.path.dirname(__file__), "../sauron.py"), + "sauron-api-endpoint": "https://mutinynet.com/api", + } + self.daemon.opts.update(options) + + # Monkey patch + def set_feerates(self, feerates, wait_for_effect=True): + return None + + +@pytest.fixture +def node_cls(monkeypatch): + monkeypatch.setenv("TEST_NETWORK", "signet") + yield LightningNode + + +def test_rpc_getchaininfo(ln_node): + """ + Test getchaininfo + """ + + response = ln_node.rpc.call("getchaininfo") + + assert ln_node.daemon.is_in_log("Sauron plugin initialized using mempool.space API") + + expected_response_keys = ["chain", "blockcount", "headercount", "ibd"] + assert list(response.keys()) == expected_response_keys + assert response["chain"] == "signet" + assert not response["ibd"] + + +def test_rpc_getrawblockbyheight(ln_node): + """ + Test getrawblockbyheight + """ + + response = ln_node.rpc.call("getrawblockbyheight", {"height": 0}) + + expected_response = { + "block": "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a008f4d5fae77031e8ad222030101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000", + "blockhash": "00000008819873e925422c1ff0f99f7cc9bbb232af63a077a480a3633bee1ef6", + } + assert response == expected_response + + +def test_rpc_sendrawtransaction_invalid(ln_node): + """ + Test sendrawtransaction + """ + + expected_response = { + "errmsg": 'sendrawtransaction RPC error: {"code":-22,"message":"TX decode failed. Make sure the tx has at least one input."}', + "success": False, + } + response = ln_node.rpc.call( + "sendrawtransaction", + {"tx": "invalid-raw-tx"}, + ) + + assert response == expected_response + + +def test_rpc_getutxout(ln_node): + """ + Test getutxout + """ + + expected_response = { + "amount": 5000000000, + "script": "4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac", + } + response = ln_node.rpc.call( + "getutxout", + { + # coinbase tx block 0 + "txid": "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + "vout": 0, + }, + ) + assert response == expected_response + + +def test_rpc_estimatefees(ln_node): + """ + Test estimatefees + """ + + # Sample response + # { + # "opening": 1000, + # "mutual_close": 1000, + # "unilateral_close": 1000, + # "delayed_to_us": 1000, + # "htlc_resolution": 1000, + # "penalty": 1000, + # "min_acceptable": 500, + # "max_acceptable": 10000, + # "feerate_floor": 1000000, + # "feerates": [ + # {"blocks": 2, "feerate": 1000}, + # {"blocks": 6, "feerate": 1000}, + # {"blocks": 12, "feerate": 1000}, + # {"blocks": 144, "feerate": 1000} + # ] + # } + + response = ln_node.rpc.call("estimatefees") + + expected_response_keys = [ + "opening", + "mutual_close", + "unilateral_close", + "delayed_to_us", + "htlc_resolution", + "penalty", + "min_acceptable", + "max_acceptable", + "feerate_floor", + "feerates", + ] + assert list(response.keys()) == expected_response_keys + + expected_feerates_keys = ("blocks", "feerate") + assert ( + list(set([tuple(entry.keys()) for entry in response["feerates"]]))[0] + == expected_feerates_keys + ) + + expected_feerates_blocks = [2, 6, 12, 144] + assert [ + entry["blocks"] for entry in response["feerates"] + ] == expected_feerates_blocks diff --git a/sauron/tests/util.py b/sauron/tests/util.py new file mode 100644 index 000000000..c0e01d2d1 --- /dev/null +++ b/sauron/tests/util.py @@ -0,0 +1,34 @@ +import logging + +import pytest +from pyln.testing import utils +from pyln.testing.fixtures import * # noqa: F403 + + +class LightningD(utils.LightningD): + def __init__(self, lightning_dir, *args, **kwargs): + super().__init__(lightning_dir, *args, **kwargs) + + opts_to_disable = [ + "bitcoin-datadir", + "bitcoin-rpcpassword", + "bitcoin-rpcuser", + "dev-bitcoind-poll", + ] + for opt in opts_to_disable: + self.opts.pop(opt) + + # Monkey patch + def start(self, stdin=None, wait_for_initialized=True, stderr_redir=False): + utils.TailableProc.start( + self, stdin, stdout_redir=False, stderr_redir=stderr_redir + ) + + if wait_for_initialized: + self.wait_for_log("Server started with public key") + logging.info("LightningD started") + + +@pytest.fixture +def ln_node(node_factory): # noqa: F811 + yield node_factory.get_node()