fix: handle stack busy with retry and match update id properly

scripts/deploy.py

@@ -47,25 +47,37 @@ def get_komodo_config():
     return host.rstrip("/"), key, secret
 
 
-def api_call(host, key, secret, endpoint, payload):
+def api_call(host, key, secret, endpoint, payload, retries=5, retry_delay=5):
     url = f"{host}{endpoint}"
-    req = urllib.request.Request(
-        url,
-        data=json.dumps(payload).encode("utf-8"),
-        headers={
-            "Content-Type": "application/json",
-            "x-api-key": key,
-            "x-api-secret": secret,
-        },
-        method="POST",
-    )
-    try:
-        with urllib.request.urlopen(req, timeout=120) as resp:
-            data = resp.read().decode("utf-8")
-            return json.loads(data) if data else {}
-    except urllib.error.HTTPError as e:
-        err = e.read().decode("utf-8", errors="replace")
-        raise RuntimeError(f"Komodo API {endpoint} error {e.code}: {err}") from e
+    for attempt in range(retries):
+        req = urllib.request.Request(
+            url,
+            data=json.dumps(payload).encode("utf-8"),
+            headers={
+                "Content-Type": "application/json",
+                "x-api-key": key,
+                "x-api-secret": secret,
+            },
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=120) as resp:
+                data = resp.read().decode("utf-8")
+                return json.loads(data) if data else {}
+        except urllib.error.HTTPError as e:
+            err = e.read().decode("utf-8", errors="replace")
+            # If stack is busy with another operation, wait and retry
+            if ("Stack busy" in err or "busy" in err) and attempt < retries - 1:
+                print(f"[INFO] Stack is busy with a previous operation. Waiting {retry_delay}s (attempt {attempt + 1}/{retries})...")
+                time.sleep(retry_delay)
+                continue
+            raise RuntimeError(f"Komodo API {endpoint} error {e.code}: {err}") from e
+        except Exception as e:
+            if attempt < retries - 1:
+                print(f"[WARN] Request to {endpoint} failed ({e}). Retrying in {retry_delay}s...")
+                time.sleep(retry_delay)
+                continue
+            raise
 
 
 def wait_for_update(host, key, secret, update_id, max_wait_sec=300):
@@ -97,7 +109,7 @@ def wait_for_update(host, key, secret, update_id, max_wait_sec=300):
         except Exception as e:
             print(f"[WARN] Error polling update status: {e}")
 
-    print(f"[WARN] Deployment is taking longer than {max_wait_sec}s (likely pulling large image). Continuing.")
+    print(f"[WARN] Deployment update still pending after {max_wait_sec}s. Continuing.")
     return True
 
 
@@ -158,7 +170,8 @@ def main():
         print(f"[INFO] Updating Komodo Stack '{STACK_NAME}' compose definition...")
         api_call(
             host, key, secret, "/write/UpdateStack",
-            {"id": existing_stack["id"], "config": stack_config}
+            {"id": existing_stack["id"], "config": stack_config},
+            retries=12, retry_delay=5
         )
         print(f"[INFO] Updated Stack configuration.")
 
@@ -166,7 +179,8 @@ def main():
     print(f"[INFO] Executing DeployStack for '{STACK_NAME}'...")
     deploy_res = api_call(
         host, key, secret, "/execute/DeployStack",
-        {"stack": STACK_NAME, "services": [], "stop_time": None}
+        {"stack": STACK_NAME, "services": [], "stop_time": None},
+        retries=12, retry_delay=5
     )
     update_id = (deploy_res.get("_id") or {}).get("$oid") or deploy_res.get("id")
     if update_id: