At Netflix, when a downstream service is unhealthy, we need to fail fast instead of waiting for timeouts. Implement a Circuit Breaker that prevents cascading failures in distributed systems.
┌─────────────────────────────────────────────────┐
│ │
▼ │
┌───────┐ failures > threshold ┌────────┐ │
│CLOSED │ ───────────────────────► │ OPEN │ │
│(Normal)│ │(Failing)│ │
└───┬───┘ └────┬───┘ │
│ │ │
│ success timeout expires │
│ │ │
│ ┌───────────┐ │ │
└─────────│ HALF-OPEN │◄────────────┘ │
│ (Testing) │ │
└─────┬─────┘ │
│ │
│ success: reset │
│ failure: back to open ─────────┘
▼package circuitbreaker
import (
"errors"
"sync"
"time"
)
type State int
const (
StateClosed State = iota
StateOpen
StateHalfOpen
)
var ErrCircuitOpen = errors.New("circuit breaker is open")
type CircuitBreaker struct {
mu sync.RWMutex
state State
failureCount int
successCount int
lastFailure time.Time
maxFailures int // Failures before opening
timeout time.Duration // Time to wait before half-open
halfOpenMax int // Successes needed to close
}
func New(maxFailures int, timeout time.Duration) *CircuitBreaker {
return &CircuitBreaker{
state: StateClosed,
maxFailures: maxFailures,
timeout: timeout,
halfOpenMax: 3,
}
}
func (cb *CircuitBreaker) Execute(fn func() error) error {
if !cb.allowRequest() {
return ErrCircuitOpen
}
err := fn()
cb.recordResult(err)
return err
}
func (cb *CircuitBreaker) allowRequest() bool {
cb.mu.Lock()
defer cb.mu.Unlock()
switch cb.state {
case StateClosed:
return true
case StateOpen:
if time.Since(cb.lastFailure) > cb.timeout {
cb.state = StateHalfOpen
cb.successCount = 0
return true
}
return false
case StateHalfOpen:
return true
}
return false
}
func (cb *CircuitBreaker) recordResult(err error) {
cb.mu.Lock()
defer cb.mu.Unlock()
if err != nil {
cb.onFailure()
} else {
cb.onSuccess()
}
}
func (cb *CircuitBreaker) onSuccess() {
switch cb.state {
case StateClosed:
cb.failureCount = 0
case StateHalfOpen:
cb.successCount++
if cb.successCount >= cb.halfOpenMax {
cb.state = StateClosed
cb.failureCount = 0
}
}
}
func (cb *CircuitBreaker) onFailure() {
cb.lastFailure = time.Now()
switch cb.state {
case StateClosed:
cb.failureCount++
if cb.failureCount >= cb.maxFailures {
cb.state = StateOpen
}
case StateHalfOpen:
cb.state = StateOpen
}
}
func (cb *CircuitBreaker) State() State {
cb.mu.RLock()
defer cb.mu.RUnlock()
return cb.state
}func main() {
cb := circuitbreaker.New(5, 30*time.Second)
client := &http.Client{Timeout: 5 * time.Second}
for i := 0; i < 100; i++ {
err := cb.Execute(func() error {
resp, err := client.Get("https://unreliable-service.com/api")
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode >= 500 {
return fmt.Errorf("server error: %d", resp.StatusCode)
}
return nil
})
if errors.Is(err, circuitbreaker.ErrCircuitOpen) {
log.Println("Circuit is open, using fallback")
useFallback()
} else if err != nil {
log.Println("Request failed:", err)
}
}
}// Use github.com/sony/gobreaker in production
import "github.com/sony/gobreaker"
cb := gobreaker.NewCircuitBreaker(gobreaker.Settings{
Name: "payment-service",
MaxRequests: 3,
Interval: 10 * time.Second,
Timeout: 30 * time.Second,
ReadyToTrip: func(counts gobreaker.Counts) bool {
return counts.ConsecutiveFailures > 5
},
})