Skip to content

reach.uncertainty

Statistical uncertainty estimation, Wilson score confidence intervals, and power analysis.

Calculate statistical confidence intervals and power for rate metrics.

DEFAULT_CONFIDENCE module-attribute 

DEFAULT_CONFIDENCE = 0.95

DEFAULT_POWER module-attribute 

DEFAULT_POWER = 0.8

Interval

Bases: BaseModel

Represent a statistical confidence interval with lower and upper bounds.

Source code in src/reach/uncertainty.py 
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
class Interval(BaseModel):
    """Represent a statistical confidence interval with lower and upper bounds."""

    model_config = ConfigDict(frozen=True)

    low: float = Field(ge=0.0, le=1.0)
    high: float = Field(ge=0.0, le=1.0)
    confidence: float = Field(default=DEFAULT_CONFIDENCE, gt=0.0, lt=1.0)

    @model_validator(mode="after")
    def _bounds_are_ordered(self) -> Self:
        """Validate that lower bound does not exceed upper bound."""
        if self.low > self.high:
            msg = f"interval bounds are inverted: [{self.low}, {self.high}]"
            raise ValueError(msg)
        return self

    @property
    def width(self) -> float:
        """Return the span (high - low) of the confidence interval."""
        return self.high - self.low

    def excludes(self, rate: float) -> bool:
        """Return True if rate falls strictly outside the interval bounds."""
        return not self.low <= rate <= self.high

    def overlaps(self, other: Interval) -> bool:
        """Return True if this interval intersects with another interval."""
        return self.low <= other.high and other.low <= self.high

width property 

width: float

Return the span (high - low) of the confidence interval.

excludes 

excludes(rate: float) -> bool

Return True if rate falls strictly outside the interval bounds.

Source code in src/reach/uncertainty.py 
84
85
86
def excludes(self, rate: float) -> bool:
    """Return True if rate falls strictly outside the interval bounds."""
    return not self.low <= rate <= self.high

overlaps 

overlaps(other: Interval) -> bool

Return True if this interval intersects with another interval.

Source code in src/reach/uncertainty.py 
88
89
90
def overlaps(self, other: Interval) -> bool:
    """Return True if this interval intersects with another interval."""
    return self.low <= other.high and other.low <= self.high

critical_value 

critical_value(
    confidence: float = DEFAULT_CONFIDENCE,
) -> float

Calculate two-sided normal critical value for a given confidence level.

Source code in src/reach/uncertainty.py 
57
58
59
def critical_value(confidence: float = DEFAULT_CONFIDENCE) -> float:
    """Calculate two-sided normal critical value for a given confidence level."""
    return _z((1.0 - _checked_confidence(confidence)) / 2.0)

detectable_delta 

detectable_delta(
    probes: int,
    confidence: float = DEFAULT_CONFIDENCE,
    power: float = DEFAULT_POWER,
) -> float | None

Calculate the minimum detectable rate delta for a given sample size per arm.

Source code in src/reach/uncertainty.py 
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
def detectable_delta(
    probes: int,
    confidence: float = DEFAULT_CONFIDENCE,
    power: float = DEFAULT_POWER,
) -> float | None:
    """Calculate the minimum detectable rate delta for a given sample size per arm."""
    if probes < 0:
        msg = f"probes cannot be negative, got {probes}"
        raise ValueError(msg)
    if not probes:
        return None
    alpha = 1.0 - _checked_confidence(confidence)
    if not 0.0 < power < 1.0:
        msg = f"power must lie strictly between 0 and 1, got {power}"
        raise ValueError(msg)
    return min(1.0, math.sqrt(_two_proportion_constant(alpha, power) / probes))

required_probes 

required_probes(
    delta: float,
    confidence: float = DEFAULT_CONFIDENCE,
    power: float = DEFAULT_POWER,
) -> int

Calculate sample size per arm required to detect a rate delta with power.

Source code in src/reach/uncertainty.py 
125
126
127
128
129
130
131
132
133
134
135
136
137
138
def required_probes(
    delta: float,
    confidence: float = DEFAULT_CONFIDENCE,
    power: float = DEFAULT_POWER,
) -> int:
    """Calculate sample size per arm required to detect a rate delta with power."""
    if not 0.0 < delta <= 1.0:
        msg = f"delta must lie in (0, 1], got {delta}"
        raise ValueError(msg)
    alpha = 1.0 - _checked_confidence(confidence)
    if not 0.0 < power < 1.0:
        msg = f"power must lie strictly between 0 and 1, got {power}"
        raise ValueError(msg)
    return math.ceil(_two_proportion_constant(alpha, power) / (delta * delta))

wilson_interval 

wilson_interval(
    hits: int,
    probes: int,
    confidence: float = DEFAULT_CONFIDENCE,
) -> Interval | None

Calculate the Wilson score confidence interval for a binomial proportion.

Source code in src/reach/uncertainty.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
def wilson_interval(
    hits: int,
    probes: int,
    confidence: float = DEFAULT_CONFIDENCE,
) -> Interval | None:
    """Calculate the Wilson score confidence interval for a binomial proportion."""
    if probes < 0:
        msg = f"probes cannot be negative, got {probes}"
        raise ValueError(msg)
    if not 0 <= hits <= probes:
        msg = f"{hits} hits is not a possible count out of {probes} probes"
        raise ValueError(msg)
    if not probes:
        return None

    z = critical_value(confidence)
    z2 = z * z
    denominator = probes + z2
    center = (hits + z2 / 2.0) / denominator
    half = z / denominator * math.sqrt(hits * (probes - hits) / probes + z2 / 4.0)
    return Interval(
        low=0.0 if hits == 0 else max(0.0, center - half),
        high=1.0 if hits == probes else min(1.0, center + half),
        confidence=confidence,
    )