merge: feat/65-retry-wrapper into freeze/0.22.0
# Conflicts: # CHANGELOG.md # README.md
This commit is contained in:
commit
2f5391554c
6 changed files with 220 additions and 0 deletions
11
CHANGELOG.md
11
CHANGELOG.md
|
|
@ -34,6 +34,17 @@ Answers "can this hardware run model X at quantization level Y?" — the missing
|
||||||
- Raises `ModelVramLookupError` on HF Hub API failures or missing safetensors metadata; raises `ValueError` for unrecognized quant levels.
|
- Raises `ModelVramLookupError` on HF Hub API failures or missing safetensors metadata; raises `ValueError` for unrecognized quant levels.
|
||||||
- Application points noted in the ticket: Avocet preflight (verify a checkpoint fits before benchmarking), cf-orch worker assignment (match model to GPU by VRAM fit), Peregrine/Kiwi onboarding wizard ("your GPU has X GB — here are models that will run well").
|
- Application points noted in the ticket: Avocet preflight (verify a checkpoint fits before benchmarking), cf-orch worker assignment (match model to GPU by VRAM fit), Peregrine/Kiwi onboarding wizard ("your GPU has X GB — here are models that will run well").
|
||||||
|
|
||||||
|
**`circuitforge_core.retry`** — standard retry/backoff wrapper over `backon` (closes #65)
|
||||||
|
|
||||||
|
Standardizes retry/backoff behavior for cf-core modules and products that make external calls, replacing ad-hoc per-product retry loops. Wraps [backon](https://github.com/Llucs/backon) (MIT, zero stdlib dependencies) rather than reimplementing retry logic — chosen over `tenacity`/`backoff` per the eval in #65 for its native async support, built-in circuit breaker/hedging primitives, and a process-wide enable/disable toggle that's ideal for tests.
|
||||||
|
|
||||||
|
- `on_exception(exception, *, max_tries=3, max_time=30.0, **backon_kwargs)` — decorator with CF's standard defaults (full-jitter exponential backoff); extra kwargs forward to `backon.on_exception` (e.g. `sleep=` for tests, `on_backoff=` for logging hooks).
|
||||||
|
- `retry(target, *args, exception=Exception, max_tries=3, max_time=30.0, **kwargs)` — call an already-defined callable with the same retry behavior, for use without decorator syntax.
|
||||||
|
- `disable_retries()` / `enable_retries()` — re-exported `backon` context managers for scoping retry-disable to a test or block.
|
||||||
|
- `disable_retries_globally()` / `enable_retries_globally()` — re-exported `backon.disable()`/`enable()` process-wide toggles, for a test-suite-level fixture.
|
||||||
|
- New core dependency: `backon>=4.0`.
|
||||||
|
- Scope note: only the standalone wrapper module ships in this PR. Wiring it into `llm`, `affiliates`, `reranker`, and `activitypub` (the call sites identified in #65) is left as follow-up work per module, rather than retrofitting `LLMRouter`'s existing fallback-chain error handling in the same change.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## [0.20.0] — 2026-05-05
|
## [0.20.0] — 2026-05-05
|
||||||
|
|
|
||||||
|
|
@ -84,6 +84,7 @@ pip install circuitforge-core[dev] # All dev dependencies
|
||||||
| `input` | Implemented | Input handling — MediaPipe gesture recognition |
|
| `input` | Implemented | Input handling — MediaPipe gesture recognition |
|
||||||
| `job_quality` | Implemented | Job listing quality scoring and signal extraction |
|
| `job_quality` | Implemented | Job listing quality scoring and signal extraction |
|
||||||
| `signal_bus` | Implemented | Generic SSE event publisher for real-time signal streams |
|
| `signal_bus` | Implemented | Generic SSE event publisher for real-time signal streams |
|
||||||
|
| `retry` | Implemented | Standard retry/backoff wrapper over `backon` for external-call modules |
|
||||||
| `vision` | Stub | Vision router (moondream2 / SigLIP dispatch — planned) |
|
| `vision` | Stub | Vision router (moondream2 / SigLIP dispatch — planned) |
|
||||||
| `wizard` | Stub | First-run wizard base class — products subclass `BaseWizard` |
|
| `wizard` | Stub | First-run wizard base class — products subclass `BaseWizard` |
|
||||||
| `pipeline` | Stub | Staging queue base — products provide concrete schema |
|
| `pipeline` | Stub | Staging queue base — products provide concrete schema |
|
||||||
|
|
|
||||||
93
circuitforge_core/retry.py
Normal file
93
circuitforge_core/retry.py
Normal file
|
|
@ -0,0 +1,93 @@
|
||||||
|
# circuitforge_core/retry.py
|
||||||
|
"""
|
||||||
|
circuitforge_core.retry — thin standard-retry wrapper over backon (cf-core #65).
|
||||||
|
|
||||||
|
Standardizes retry/backoff behavior across cf-core modules and products that
|
||||||
|
make external calls (LLM endpoints, affiliate links, reranker APIs, federated
|
||||||
|
ActivityPub delivery). Wraps `backon` (MIT, zero stdlib dependencies) rather
|
||||||
|
than reimplementing retry logic, and rather than each module hand-rolling its
|
||||||
|
own ad-hoc retry loop.
|
||||||
|
|
||||||
|
Typical usage::
|
||||||
|
|
||||||
|
from circuitforge_core.retry import on_exception
|
||||||
|
import requests
|
||||||
|
|
||||||
|
@on_exception(requests.RequestException, max_tries=3)
|
||||||
|
def fetch(url):
|
||||||
|
return requests.get(url, timeout=5)
|
||||||
|
|
||||||
|
Tests should disable retries globally rather than waiting out real backoff
|
||||||
|
delays::
|
||||||
|
|
||||||
|
from circuitforge_core.retry import disable_retries
|
||||||
|
|
||||||
|
def test_fetch_raises_immediately_on_failure():
|
||||||
|
with disable_retries():
|
||||||
|
with pytest.raises(requests.RequestException):
|
||||||
|
fetch("http://unreachable")
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Callable
|
||||||
|
from typing import TypeVar
|
||||||
|
|
||||||
|
import backon
|
||||||
|
from backon import disable as disable_retries_globally
|
||||||
|
from backon import disable_retries, enable_retries
|
||||||
|
from backon import enable as enable_retries_globally
|
||||||
|
|
||||||
|
T = TypeVar("T")
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"on_exception",
|
||||||
|
"retry",
|
||||||
|
"disable_retries",
|
||||||
|
"enable_retries",
|
||||||
|
"disable_retries_globally",
|
||||||
|
"enable_retries_globally",
|
||||||
|
]
|
||||||
|
|
||||||
|
_DEFAULT_MAX_TRIES = 3
|
||||||
|
_DEFAULT_MAX_TIME = 30.0
|
||||||
|
|
||||||
|
|
||||||
|
def on_exception(
|
||||||
|
exception: type[Exception] | tuple[type[Exception], ...] = Exception,
|
||||||
|
*,
|
||||||
|
max_tries: int = _DEFAULT_MAX_TRIES,
|
||||||
|
max_time: float | None = _DEFAULT_MAX_TIME,
|
||||||
|
**backon_kwargs,
|
||||||
|
) -> Callable[[Callable[..., T]], Callable[..., T]]:
|
||||||
|
"""
|
||||||
|
Decorator: retry a function on `exception` using full-jitter exponential
|
||||||
|
backoff — CF's standard defaults for external API calls.
|
||||||
|
|
||||||
|
Extra keyword arguments are forwarded to `backon.on_exception` (e.g.
|
||||||
|
`on_backoff=` for a logging callback, `sleep=` to override the sleep
|
||||||
|
function in tests).
|
||||||
|
"""
|
||||||
|
return backon.on_exception(
|
||||||
|
backon.expo(),
|
||||||
|
exception,
|
||||||
|
max_tries=max_tries,
|
||||||
|
max_time=max_time,
|
||||||
|
logger="circuitforge_core.retry",
|
||||||
|
**backon_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def retry(
|
||||||
|
target: Callable[..., T],
|
||||||
|
*args,
|
||||||
|
exception: type[Exception] | tuple[type[Exception], ...] = Exception,
|
||||||
|
max_tries: int = _DEFAULT_MAX_TRIES,
|
||||||
|
max_time: float | None = _DEFAULT_MAX_TIME,
|
||||||
|
**kwargs,
|
||||||
|
) -> T:
|
||||||
|
"""
|
||||||
|
Call `target(*args, **kwargs)` with CF's standard retry/backoff behavior,
|
||||||
|
for wrapping an already-defined callable without decorator syntax.
|
||||||
|
"""
|
||||||
|
wrapped = on_exception(exception, max_tries=max_tries, max_time=max_time)(target)
|
||||||
|
return wrapped(*args, **kwargs)
|
||||||
|
|
@ -11,6 +11,7 @@ dependencies = [
|
||||||
"pyyaml>=6.0",
|
"pyyaml>=6.0",
|
||||||
"requests>=2.31",
|
"requests>=2.31",
|
||||||
"openai>=1.0",
|
"openai>=1.0",
|
||||||
|
"backon>=4.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
|
|
|
||||||
0
tests/test_retry/__init__.py
Normal file
0
tests/test_retry/__init__.py
Normal file
114
tests/test_retry/test_retry.py
Normal file
114
tests/test_retry/test_retry.py
Normal file
|
|
@ -0,0 +1,114 @@
|
||||||
|
"""Tests for circuitforge_core.retry (cf-core #65)."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from circuitforge_core.retry import (
|
||||||
|
disable_retries,
|
||||||
|
enable_retries_globally,
|
||||||
|
on_exception,
|
||||||
|
retry,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FlakyError(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _flaky_after(n_failures):
|
||||||
|
"""Return a function that raises FlakyError n_failures times, then succeeds."""
|
||||||
|
state = {"calls": 0}
|
||||||
|
|
||||||
|
def fn():
|
||||||
|
state["calls"] += 1
|
||||||
|
if state["calls"] <= n_failures:
|
||||||
|
raise FlakyError(f"attempt {state['calls']}")
|
||||||
|
return state["calls"]
|
||||||
|
|
||||||
|
fn.state = state
|
||||||
|
return fn
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _ensure_retries_enabled():
|
||||||
|
"""Some tests disable retries; always restore the global state after."""
|
||||||
|
yield
|
||||||
|
enable_retries_globally()
|
||||||
|
|
||||||
|
|
||||||
|
class TestOnException:
|
||||||
|
def test_succeeds_without_retry_when_no_exception(self):
|
||||||
|
calls = {"n": 0}
|
||||||
|
|
||||||
|
@on_exception(FlakyError, max_tries=3)
|
||||||
|
def fn():
|
||||||
|
calls["n"] += 1
|
||||||
|
return "ok"
|
||||||
|
|
||||||
|
assert fn() == "ok"
|
||||||
|
assert calls["n"] == 1
|
||||||
|
|
||||||
|
def test_retries_until_success(self):
|
||||||
|
fn = _flaky_after(2)
|
||||||
|
wrapped = on_exception(FlakyError, max_tries=5, sleep=lambda s: None)(fn)
|
||||||
|
|
||||||
|
assert wrapped() == 3
|
||||||
|
assert fn.state["calls"] == 3
|
||||||
|
|
||||||
|
def test_gives_up_after_max_tries(self):
|
||||||
|
fn = _flaky_after(10)
|
||||||
|
wrapped = on_exception(FlakyError, max_tries=3, sleep=lambda s: None)(fn)
|
||||||
|
|
||||||
|
with pytest.raises(FlakyError):
|
||||||
|
wrapped()
|
||||||
|
assert fn.state["calls"] == 3
|
||||||
|
|
||||||
|
def test_only_retries_specified_exception_type(self):
|
||||||
|
@on_exception(FlakyError, max_tries=3, sleep=lambda s: None)
|
||||||
|
def fn():
|
||||||
|
raise ValueError("not a FlakyError")
|
||||||
|
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
fn()
|
||||||
|
|
||||||
|
def test_disable_retries_context_manager_calls_once(self):
|
||||||
|
fn = _flaky_after(10)
|
||||||
|
wrapped = on_exception(FlakyError, max_tries=5, sleep=lambda s: None)(fn)
|
||||||
|
|
||||||
|
with disable_retries():
|
||||||
|
with pytest.raises(FlakyError):
|
||||||
|
wrapped()
|
||||||
|
|
||||||
|
assert fn.state["calls"] == 1
|
||||||
|
|
||||||
|
def test_retries_resume_after_disable_retries_context_exits(self):
|
||||||
|
fn = _flaky_after(2)
|
||||||
|
wrapped = on_exception(FlakyError, max_tries=5, sleep=lambda s: None)(fn)
|
||||||
|
|
||||||
|
with disable_retries():
|
||||||
|
pass # just exercise enter/exit without calling wrapped()
|
||||||
|
|
||||||
|
assert wrapped() == 3
|
||||||
|
|
||||||
|
|
||||||
|
class TestRetry:
|
||||||
|
def test_retry_calls_target_with_args_and_kwargs(self):
|
||||||
|
def fn(a, b, *, c):
|
||||||
|
return a + b + c
|
||||||
|
|
||||||
|
assert retry(fn, 1, 2, c=3, max_tries=1) == 6
|
||||||
|
|
||||||
|
def test_retry_retries_on_failure(self):
|
||||||
|
fn = _flaky_after(2)
|
||||||
|
|
||||||
|
with patch("time.sleep"):
|
||||||
|
assert retry(fn, max_tries=5) == 3
|
||||||
|
assert fn.state["calls"] == 3
|
||||||
|
|
||||||
|
def test_retry_raises_after_exhausting_max_tries(self):
|
||||||
|
fn = _flaky_after(10)
|
||||||
|
|
||||||
|
with patch("time.sleep"), pytest.raises(FlakyError):
|
||||||
|
retry(fn, exception=FlakyError, max_tries=2)
|
||||||
Loading…
Reference in a new issue