From cf2517b9791399c7166d6aef3a9b43a13b52e43c Mon Sep 17 00:00:00 2001 From: Fred De Backer Date: Fri, 7 Nov 2025 15:22:16 +0000 Subject: [PATCH 1/3] Stabilize Paramiko executions --- .../integration/ceos_post_integration.py | 24 ++++++++++++++----- .../integration/sros_post_integration.py | 12 ++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/clab_connector/services/integration/ceos_post_integration.py b/clab_connector/services/integration/ceos_post_integration.py index 6425be2..85a2f37 100644 --- a/clab_connector/services/integration/ceos_post_integration.py +++ b/clab_connector/services/integration/ceos_post_integration.py @@ -11,6 +11,7 @@ from pathlib import Path import paramiko +from rich.markup import escape logger = logging.getLogger(__name__) @@ -145,6 +146,10 @@ def execute_ssh_commands( try: commands = script_path.read_text().splitlines() + # This will send 5 empty commands at the end, making sure everything gets executed till the last bit + for _i in range(5): + commands.append("") + client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect( @@ -161,9 +166,15 @@ def execute_ssh_commands( time.sleep(2) for cmd in commands: - chan.send(cmd + "\n") + if cmd.strip() == "write": + time.sleep(2) # Wait 2 seconds before sending write - time.sleep(0.5) + if cmd.strip() == "": + time.sleep( + 0.5 + ) # Wait 0.5 seconds before sending on empty command + + chan.send(cmd + "\n") while not chan.recv_ready(): pass @@ -186,7 +197,8 @@ def execute_ssh_commands( node_name, sum(map(len, output)), ) - logger.debug("Output: %s", output) + textoutput = escape("".join(output)) + logger.debug("Output: %s", textoutput) return True except Exception as e: logger.error("SSH exec error on %s: %s", node_name, e) @@ -287,12 +299,12 @@ def _copy_files_and_config( _build_post_script(postscript_p, root) post_success = transfer_file( - postscript_p, root + "copy_certs.sh", username, mgmt_ip, working_pw, quiet + postscript_p, root + "copy-certs.sh", username, mgmt_ip, working_pw, quiet ) if post_success: - logger.info(f"Post script copied successfully to {root}copy_certs.sh") + logger.info(f"Post script copied successfully to {root}copy-certs.sh") else: - logger.warning(f"Failed to copy post script to {root}copy_certs.sh") + logger.warning(f"Failed to copy post script to {root}copy-certs.sh") continue cert_success = transfer_file( diff --git a/clab_connector/services/integration/sros_post_integration.py b/clab_connector/services/integration/sros_post_integration.py index d38fe82..2592ca1 100644 --- a/clab_connector/services/integration/sros_post_integration.py +++ b/clab_connector/services/integration/sros_post_integration.py @@ -12,6 +12,7 @@ from pathlib import Path import paramiko +from rich.markup import escape logger = logging.getLogger(__name__) @@ -146,6 +147,10 @@ def execute_ssh_commands( try: commands = script_path.read_text().splitlines() + # This will send 5 empty commands at the end, making sure everything gets executed till the last bit + for _i in range(5): + commands.append("") + client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect( @@ -163,6 +168,11 @@ def execute_ssh_commands( if cmd.strip() == "commit": time.sleep(2) # Wait 2 seconds before sending commit + if cmd.strip() == "": + time.sleep( + 0.5 + ) # Wait 0.5 seconds before sending on empty command + chan.send(cmd + "\n") while not chan.recv_ready(): pass @@ -185,6 +195,8 @@ def execute_ssh_commands( node_name, sum(map(len, output)), ) + textoutput = escape("".join(output)) + logger.debug("Output: %s", textoutput) return True except Exception as e: logger.error("SSH exec error on %s: %s", node_name, e) From 025697745b8f6f92e4eaa4ca40cab17bded14a40 Mon Sep 17 00:00:00 2001 From: Fred De Backer Date: Fri, 7 Nov 2025 15:25:54 +0000 Subject: [PATCH 2/3] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e00e5a2..0e08c7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "clab-connector" -version = "0.8.2" +version = "0.8.3" description = "EDA Containerlab Connector" readme = "README.md" requires-python = ">=3.11" From dbc75e783261fc12b0b22f84c92c18d63ed715ad Mon Sep 17 00:00:00 2001 From: Fred De Backer Date: Mon, 10 Nov 2025 11:12:31 +0000 Subject: [PATCH 3/3] SCP retry; cEOS execute_ssh_commands wait for prompt --- .../integration/ceos_post_integration.py | 66 ++++++++++++++++--- .../integration/sros_post_integration.py | 21 +++++- 2 files changed, 77 insertions(+), 10 deletions(-) diff --git a/clab_connector/services/integration/ceos_post_integration.py b/clab_connector/services/integration/ceos_post_integration.py index 85a2f37..d4b04ee 100644 --- a/clab_connector/services/integration/ceos_post_integration.py +++ b/clab_connector/services/integration/ceos_post_integration.py @@ -109,6 +109,7 @@ def transfer_file( mgmt_ip: str, password: str, quiet: bool = False, + tries: int = 2, ) -> bool: """ SCP file to the target node using Paramiko SFTP. @@ -128,7 +129,25 @@ def transfer_file( except Exception as e: if not quiet: logger.debug("SCP failed: %s", e) - return False + tries -= 1 + if tries > 0: + logger.info( + "SCP failed! Waiting 20 seconds before retrying. Retrying %s more time%s", + tries, + "s" if tries > 1 else "", + ) + time.sleep(20) + return transfer_file( + src_path=src_path, + dest_path=dest_path, + username=username, + mgmt_ip=mgmt_ip, + password=password, + quiet=quiet, + tries=tries, + ) + else: + return False def execute_ssh_commands( @@ -138,17 +157,25 @@ def execute_ssh_commands( node_name: str, password: str, quiet: bool = False, + prompt_terminator_chars: list[str] | None = None, + prompt_termination_offset: int = -1, + timeout: float = 30.0, ) -> bool: """ Push the command file line-by-line over an interactive shell. - No timeouts version that will wait as long as needed for each command. + This waits for prompt after each command and raises exception if timeout is reached. """ + if prompt_terminator_chars is None: + prompt_terminator_chars = [">", "#", "$"] + try: commands = script_path.read_text().splitlines() - # This will send 5 empty commands at the end, making sure everything gets executed till the last bit - for _i in range(5): - commands.append("") + # This will add 1 empty commands at the end, making sure everything gets executed till the last bit + # 1 is enough because we check for prompt + commands.append("") + # This will insert 1 empty command at the beginning to wait for first prompt + commands.insert(0, "") client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -179,10 +206,31 @@ def execute_ssh_commands( while not chan.recv_ready(): pass - buffer = "" - while chan.recv_ready(): - buffer += chan.recv(4096).decode() - output.append(buffer) + command_timeout = timeout + while command_timeout > 0: + buffer = "" + while chan.recv_ready(): + buffer += chan.recv(4096).decode() + output.append(buffer) + + last_line = "".join(output).splitlines()[-1] + prompt_search_offset = ( + prompt_termination_offset if prompt_termination_offset < 0 else -1 + ) + if ( + len(last_line) >= -prompt_search_offset + and last_line[prompt_search_offset] in prompt_terminator_chars + ): + break + command_timeout -= 0.5 + if command_timeout < 0: + raise RuntimeError("Timeout reached while waiting for prompt!") + time.sleep(0.5) + logger.debug( + "Waited %s seconds for prompt after command '%s'", + timeout - command_timeout, + cmd, + ) # Get any remaining output while chan.recv_ready(): diff --git a/clab_connector/services/integration/sros_post_integration.py b/clab_connector/services/integration/sros_post_integration.py index 2592ca1..291e0a0 100644 --- a/clab_connector/services/integration/sros_post_integration.py +++ b/clab_connector/services/integration/sros_post_integration.py @@ -110,6 +110,7 @@ def transfer_file( mgmt_ip: str, password: str, quiet: bool = False, + tries: int = 2, ) -> bool: """ SCP file to the target node using Paramiko SFTP. @@ -129,7 +130,25 @@ def transfer_file( except Exception as e: if not quiet: logger.debug("SCP failed: %s", e) - return False + tries -= 1 + if tries > 0: + logger.info( + "SCP failed! Waiting 20 seconds before retrying. Retrying %s more time%s", + tries, + "s" if tries > 1 else "", + ) + time.sleep(20) + return transfer_file( + src_path=src_path, + dest_path=dest_path, + username=username, + mgmt_ip=mgmt_ip, + password=password, + quiet=quiet, + tries=tries, + ) + else: + return False def execute_ssh_commands(