fix(deploy): wait for stack to become idle before update and deploy

scripts/deploy.py

@@ -2,8 +2,8 @@
 """
 Deploy script for OpenHands Agent Canvas on Komodo Core.
 Reads docker-compose.yml, ensures the Komodo Stack exists on server-tsj08,
-updates the Compose configuration, triggers deployment via Komodo API,
-and waits for deployment completion.
+waits for any in-flight deployment to complete, updates the Compose configuration,
+triggers deployment via Komodo API, and monitors until completion.
 """
 
 import os
@@ -66,9 +66,8 @@ def api_call(host, key, secret, endpoint, payload, retries=5, retry_delay=5):
                 return json.loads(data) if data else {}
         except urllib.error.HTTPError as e:
             err = e.read().decode("utf-8", errors="replace")
-            # If stack is busy with another operation, wait and retry
             if ("Stack busy" in err or "busy" in err) and attempt < retries - 1:
-                print(f"[INFO] Stack is busy with a previous operation. Waiting {retry_delay}s (attempt {attempt + 1}/{retries})...")
+                print(f"[INFO] Stack is busy. Waiting {retry_delay}s before retry ({attempt + 1}/{retries})...")
                 time.sleep(retry_delay)
                 continue
             raise RuntimeError(f"Komodo API {endpoint} error {e.code}: {err}") from e
@@ -80,17 +79,37 @@ def api_call(host, key, secret, endpoint, payload, retries=5, retry_delay=5):
             raise
 
 
-def wait_for_update(host, key, secret, update_id, max_wait_sec=300):
+def wait_for_stack_idle(host, key, secret, stack_id, max_wait_sec=600):
+    """Waits for any active DeployStack operations targeting stack_id to finish."""
+    start_time = time.time()
+    while time.time() - start_time < max_wait_sec:
+        try:
+            updates = api_call(host, key, secret, "/read/ListUpdates", {}).get("updates", [])
+            in_progress = [
+                u for u in updates
+                if (u.get("target") or {}).get("id") == stack_id and u.get("status") == "InProgress"
+            ]
+            if not in_progress:
+                return True
+            print(f"[INFO] Waiting for in-progress operation {in_progress[0].get('operation')} ({in_progress[0].get('id')}) to finish...")
+        except Exception as e:
+            print(f"[WARN] Failed checking updates: {e}")
+        time.sleep(10)
+    print(f"[WARN] Stack did not become idle after {max_wait_sec}s.")
+    return False
+
+
+def wait_for_update(host, key, secret, update_id, max_wait_sec=600):
     print(f"[INFO] Monitoring deployment update {update_id}...")
     start_time = time.time()
     last_status = None
 
     while time.time() - start_time < max_wait_sec:
-        time.sleep(5)
+        time.sleep(10)
         try:
             updates_data = api_call(host, key, secret, "/read/ListUpdates", {})
             for u in updates_data.get("updates", []):
-                uid = (u.get("_id") or {}).get("$oid") or u.get("id")
+                uid = u.get("id")
                 if uid == update_id:
                     status = u.get("status")
                     if status != last_status:
@@ -165,13 +184,18 @@ def main():
             host, key, secret, "/write/CreateStack",
             {"name": STACK_NAME, "config": stack_config}
         )
-        print(f"[INFO] Created Stack successfully: {res.get('name', STACK_NAME)}")
+        stack_id = res.get("id") or res.get("_id", {}).get("$oid")
+        print(f"[INFO] Created Stack successfully: {STACK_NAME}")
     else:
+        stack_id = existing_stack["id"]
+        # Ensure stack is not currently executing another deployment
+        wait_for_stack_idle(host, key, secret, stack_id, max_wait_sec=600)
+
         print(f"[INFO] Updating Komodo Stack '{STACK_NAME}' compose definition...")
         api_call(
             host, key, secret, "/write/UpdateStack",
-            {"id": existing_stack["id"], "config": stack_config},
-            retries=12, retry_delay=5
+            {"id": stack_id, "config": stack_config},
+            retries=10, retry_delay=10
         )
         print(f"[INFO] Updated Stack configuration.")
 
@@ -180,11 +204,11 @@ def main():
     deploy_res = api_call(
         host, key, secret, "/execute/DeployStack",
         {"stack": STACK_NAME, "services": [], "stop_time": None},
-        retries=12, retry_delay=5
+        retries=10, retry_delay=10
     )
-    update_id = (deploy_res.get("_id") or {}).get("$oid") or deploy_res.get("id")
+    update_id = deploy_res.get("id") or (deploy_res.get("_id") or {}).get("$oid")
     if update_id:
-        ok = wait_for_update(host, key, secret, update_id, max_wait_sec=300)
+        ok = wait_for_update(host, key, secret, update_id, max_wait_sec=600)
         if not ok:
             sys.exit(1)
     else: