diff --git a/scripts/deploy.py b/scripts/deploy.py index 8fbb608..582b0f9 100755 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -47,25 +47,37 @@ def get_komodo_config(): return host.rstrip("/"), key, secret -def api_call(host, key, secret, endpoint, payload): +def api_call(host, key, secret, endpoint, payload, retries=5, retry_delay=5): url = f"{host}{endpoint}" - req = urllib.request.Request( - url, - data=json.dumps(payload).encode("utf-8"), - headers={ - "Content-Type": "application/json", - "x-api-key": key, - "x-api-secret": secret, - }, - method="POST", - ) - try: - with urllib.request.urlopen(req, timeout=120) as resp: - data = resp.read().decode("utf-8") - return json.loads(data) if data else {} - except urllib.error.HTTPError as e: - err = e.read().decode("utf-8", errors="replace") - raise RuntimeError(f"Komodo API {endpoint} error {e.code}: {err}") from e + for attempt in range(retries): + req = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "x-api-key": key, + "x-api-secret": secret, + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=120) as resp: + data = resp.read().decode("utf-8") + return json.loads(data) if data else {} + except urllib.error.HTTPError as e: + err = e.read().decode("utf-8", errors="replace") + # If stack is busy with another operation, wait and retry + if ("Stack busy" in err or "busy" in err) and attempt < retries - 1: + print(f"[INFO] Stack is busy with a previous operation. Waiting {retry_delay}s (attempt {attempt + 1}/{retries})...") + time.sleep(retry_delay) + continue + raise RuntimeError(f"Komodo API {endpoint} error {e.code}: {err}") from e + except Exception as e: + if attempt < retries - 1: + print(f"[WARN] Request to {endpoint} failed ({e}). Retrying in {retry_delay}s...") + time.sleep(retry_delay) + continue + raise def wait_for_update(host, key, secret, update_id, max_wait_sec=300): @@ -97,7 +109,7 @@ def wait_for_update(host, key, secret, update_id, max_wait_sec=300): except Exception as e: print(f"[WARN] Error polling update status: {e}") - print(f"[WARN] Deployment is taking longer than {max_wait_sec}s (likely pulling large image). Continuing.") + print(f"[WARN] Deployment update still pending after {max_wait_sec}s. Continuing.") return True @@ -158,7 +170,8 @@ def main(): print(f"[INFO] Updating Komodo Stack '{STACK_NAME}' compose definition...") api_call( host, key, secret, "/write/UpdateStack", - {"id": existing_stack["id"], "config": stack_config} + {"id": existing_stack["id"], "config": stack_config}, + retries=12, retry_delay=5 ) print(f"[INFO] Updated Stack configuration.") @@ -166,7 +179,8 @@ def main(): print(f"[INFO] Executing DeployStack for '{STACK_NAME}'...") deploy_res = api_call( host, key, secret, "/execute/DeployStack", - {"stack": STACK_NAME, "services": [], "stop_time": None} + {"stack": STACK_NAME, "services": [], "stop_time": None}, + retries=12, retry_delay=5 ) update_id = (deploy_res.get("_id") or {}).get("$oid") or deploy_res.get("id") if update_id: