diff --git a/scripts/deploy.py b/scripts/deploy.py index 582b0f9..12c5aa0 100755 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -2,8 +2,8 @@ """ Deploy script for OpenHands Agent Canvas on Komodo Core. Reads docker-compose.yml, ensures the Komodo Stack exists on server-tsj08, -updates the Compose configuration, triggers deployment via Komodo API, -and waits for deployment completion. +waits for any in-flight deployment to complete, updates the Compose configuration, +triggers deployment via Komodo API, and monitors until completion. """ import os @@ -66,9 +66,8 @@ def api_call(host, key, secret, endpoint, payload, retries=5, retry_delay=5): return json.loads(data) if data else {} except urllib.error.HTTPError as e: err = e.read().decode("utf-8", errors="replace") - # If stack is busy with another operation, wait and retry if ("Stack busy" in err or "busy" in err) and attempt < retries - 1: - print(f"[INFO] Stack is busy with a previous operation. Waiting {retry_delay}s (attempt {attempt + 1}/{retries})...") + print(f"[INFO] Stack is busy. Waiting {retry_delay}s before retry ({attempt + 1}/{retries})...") time.sleep(retry_delay) continue raise RuntimeError(f"Komodo API {endpoint} error {e.code}: {err}") from e @@ -80,17 +79,37 @@ def api_call(host, key, secret, endpoint, payload, retries=5, retry_delay=5): raise -def wait_for_update(host, key, secret, update_id, max_wait_sec=300): +def wait_for_stack_idle(host, key, secret, stack_id, max_wait_sec=600): + """Waits for any active DeployStack operations targeting stack_id to finish.""" + start_time = time.time() + while time.time() - start_time < max_wait_sec: + try: + updates = api_call(host, key, secret, "/read/ListUpdates", {}).get("updates", []) + in_progress = [ + u for u in updates + if (u.get("target") or {}).get("id") == stack_id and u.get("status") == "InProgress" + ] + if not in_progress: + return True + print(f"[INFO] Waiting for in-progress operation {in_progress[0].get('operation')} ({in_progress[0].get('id')}) to finish...") + except Exception as e: + print(f"[WARN] Failed checking updates: {e}") + time.sleep(10) + print(f"[WARN] Stack did not become idle after {max_wait_sec}s.") + return False + + +def wait_for_update(host, key, secret, update_id, max_wait_sec=600): print(f"[INFO] Monitoring deployment update {update_id}...") start_time = time.time() last_status = None while time.time() - start_time < max_wait_sec: - time.sleep(5) + time.sleep(10) try: updates_data = api_call(host, key, secret, "/read/ListUpdates", {}) for u in updates_data.get("updates", []): - uid = (u.get("_id") or {}).get("$oid") or u.get("id") + uid = u.get("id") if uid == update_id: status = u.get("status") if status != last_status: @@ -165,13 +184,18 @@ def main(): host, key, secret, "/write/CreateStack", {"name": STACK_NAME, "config": stack_config} ) - print(f"[INFO] Created Stack successfully: {res.get('name', STACK_NAME)}") + stack_id = res.get("id") or res.get("_id", {}).get("$oid") + print(f"[INFO] Created Stack successfully: {STACK_NAME}") else: + stack_id = existing_stack["id"] + # Ensure stack is not currently executing another deployment + wait_for_stack_idle(host, key, secret, stack_id, max_wait_sec=600) + print(f"[INFO] Updating Komodo Stack '{STACK_NAME}' compose definition...") api_call( host, key, secret, "/write/UpdateStack", - {"id": existing_stack["id"], "config": stack_config}, - retries=12, retry_delay=5 + {"id": stack_id, "config": stack_config}, + retries=10, retry_delay=10 ) print(f"[INFO] Updated Stack configuration.") @@ -180,11 +204,11 @@ def main(): deploy_res = api_call( host, key, secret, "/execute/DeployStack", {"stack": STACK_NAME, "services": [], "stop_time": None}, - retries=12, retry_delay=5 + retries=10, retry_delay=10 ) - update_id = (deploy_res.get("_id") or {}).get("$oid") or deploy_res.get("id") + update_id = deploy_res.get("id") or (deploy_res.get("_id") or {}).get("$oid") if update_id: - ok = wait_for_update(host, key, secret, update_id, max_wait_sec=300) + ok = wait_for_update(host, key, secret, update_id, max_wait_sec=600) if not ok: sys.exit(1) else: