الـ Timeouts والـ Retries والـ Circuit Breakers: إزاي تتعامل مع dependency بطيئة أو واقعة

// load.js: خمس مراحل: ok، و flaky، و slow، و down، و ok تاني. كل مرحلة: الـ statuses، والـ latency، والـ upstream اتضرب كام مرة
const B = "http://localhost:3000", UP = "http://localhost:4000";
const hits = async () => (await (await fetch(`${UP}/hits`)).json()).hits;
const hit = async (i) => { const s = performance.now(); const r = await fetch(`${B}/product?sku=${i}`); const b = await r.json(); return { status: r.status, ms: performance.now() - s, err: b.error ?? (b.stale ? "stale" : undefined) }; };

async function phase(mode, n, parallel, waitMs = 0) {
  await new Promise((r) => setTimeout(r, waitMs));
  await fetch(`${UP}/mode?m=${mode}`);
  const h0 = await hits(), s = performance.now();
  const rs = parallel ? await Promise.all(Array.from({ length: n }, (_, i) => hit(i))) : [];
  if (!parallel) for (let i = 0; i < n; i++) rs.push(await hit(i));
  const by = {};
  for (const r of rs) { const k = r.err ? `${r.status} ${r.err}` : r.status; by[k] = (by[k] ?? 0) + 1; }
  const ms = rs.map((r) => r.ms).sort((a, b) => a - b);
  console.log(`${mode.padEnd(5)} | ${JSON.stringify(by)} | p50 ${ms[Math.floor(n / 2)].toFixed(0)}ms، max ${ms.at(-1).toFixed(0)}ms | المرحلة ${(performance.now() - s).toFixed(0)}ms | upstream اتضرب ${(await hits()) - h0} مرة`);
}

await phase("ok", 10, false);
await phase("flaky", 10, false);
await phase("slow", 5, true);
await phase("down", 20, false, 1100); // بعد أكتر من تانية: الـ breaker هيجرّب تاني
await phase("ok", 10, false, 1100); // الـ upstream رجع

$ node server.js & S=$!; sleep 1; node load.js; kill $S
on 3000
ok    | {"200":10} | p50 4ms، max 62ms | المرحلة 97ms | upstream اتضرب 10 مرة
flaky | {"200":10} | p50 2ms، max 4ms | المرحلة 25ms | upstream اتضرب 19 مرة
slow  | {"200":5} | p50 3005ms، max 3006ms | المرحلة 3008ms | upstream اتضرب 5 مرة
down  | {"502 UPSTREAM":20} | p50 4ms، max 8ms | المرحلة 92ms | upstream اتضرب 80 مرة
ok    | {"200":10} | p50 2ms، max 3ms | المرحلة 19ms | upstream اتضرب 10 مرة

// resilience.js: timeout بـ AbortController، و retry بـ exponential backoff و full jitter، و circuit breaker
export class ResilienceError extends Error {
  constructor(code) { super(code); this.code = code; }
}

export function withTimeout(fn, ms) {
  const ac = new AbortController();
  let timer;
  const timeout = new Promise((_, reject) => {
    timer = setTimeout(() => { const e = new ResilienceError("TIMEOUT"); ac.abort(e); reject(e); }, ms);
  });
  return Promise.race([Promise.resolve().then(() => fn(ac.signal)), timeout]).finally(() => clearTimeout(timer));
}

// timeout، و 429، و 5xx، و network error (fetch بيرمي TypeError) ← نعيد. 4xx ← لأ، الغلط عندنا
export const isRetryable = (e) => e?.code === "TIMEOUT" || e?.status === 429 || e?.status >= 500 || e instanceof TypeError;

// full jitter: رقم عشوائي بين صفر والسقف، والسقف بيتضاعف مع كل محاولة لحد maxMs
export const backoff = (attempt, { baseMs = 100, maxMs = 1000, rand = Math.random } = {}) =>
  Math.round(rand() * Math.min(maxMs, baseMs * 2 ** attempt));

export async function retry(fn, { retries = 2, baseMs = 100, maxMs = 1000, retryIf = isRetryable, rand = Math.random, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {
  for (let attempt = 0; ; attempt++) {
    try {
      return await fn(attempt);
    } catch (e) {
      if (attempt >= retries || !retryIf(e)) throw e;
      await sleep(backoff(attempt, { baseMs, maxMs, rand }));
    }
  }
}

export function createBreaker({ failureThreshold = 5, resetMs = 1000, isFailure = isRetryable, now = Date.now } = {}) {
  let state = "CLOSED", failures = 0, openedAt = 0, trialInFlight = false;
  const open = () => { state = "OPEN"; openedAt = now(); };

  async function run(fn) {
    if (state === "OPEN") {
      if (now() - openedAt < resetMs) throw new ResilienceError("CIRCUIT_OPEN"); // fail fast من غير ما نكلّمه
      state = "HALF_OPEN";
    }
    const isTrial = state === "HALF_OPEN";
    if (isTrial) {
      if (trialInFlight) throw new ResilienceError("CIRCUIT_OPEN"); // محاولة تجربة واحدة بس
      trialInFlight = true;
    }
    try {
      const v = await fn();
      if (isTrial || state === "CLOSED") { state = "CLOSED"; failures = 0; }
      return v;
    } catch (e) {
      if (!isFailure(e)) { if (isTrial) { state = "CLOSED"; failures = 0; } }
      else if (isTrial) open();
      else if (state === "CLOSED" && ++failures >= failureThreshold) open();
      throw e;
    } finally {
      if (isTrial) trialInFlight = false;
    }
  }
  return { run, stats: () => ({ state, failures }) };
}

$ node server.js & S=$!; sleep 1; node load.js; kill $S
on 3000
ok    | {"200":10} | p50 4ms، max 63ms | المرحلة 95ms | upstream اتضرب 10 مرة
flaky | {"200":10} | p50 23ms، max 50ms | المرحلة 250ms | upstream اتضرب 19 مرة
slow  | {"200 stale":5} | p50 964ms، max 1009ms | المرحلة 1010ms | upstream اتضرب 15 مرة
down  | {"200 stale":10,"503 CIRCUIT_OPEN":10} | p50 1ms، max 34ms | المرحلة 50ms | upstream اتضرب 3 مرة
ok    | {"200":10} | p50 1ms، max 3ms | المرحلة 15ms | upstream اتضرب 10 مرة

// server.js: كل call للـ upstream: breaker ← retry ← timeout. ولو فشل: آخر سعر نعرفه، أو 503 بسرعة
import express from "express";
import "./upstream.js";
import { withTimeout, retry, createBreaker } from "./resilience.js";

const UP = "http://localhost:4000";
const breaker = createBreaker({ failureThreshold: 5, resetMs: 1000 });
const lastGood = new Map(); // fallback: آخر سعر نجح لكل sku

const getPrice = (sku) => (signal) =>
  fetch(`${UP}/price?sku=${sku}`, { signal }).then(async (r) => {
    if (!r.ok) throw Object.assign(new Error(`upstream ${r.status}`), { status: r.status });
    return r.json();
  });

const app = express();
app.get("/product", async (req, res) => {
  const sku = req.query.sku;
  try {
    const p = await breaker.run(() => retry(() => withTimeout(getPrice(sku), 300), { retries: 2, baseMs: 50, maxMs: 400 }));
    lastGood.set(sku, p);
    res.json(p);
  } catch (e) {
    if (lastGood.has(sku)) return res.json({ ...lastGood.get(sku), stale: true });
    res.set("Retry-After", "1").status(503).json({ error: e.code ?? "UPSTREAM" });
  }
});
app.get("/stats", (req, res) => res.json(breaker.stats()));
app.listen(3000, () => console.log("on 3000"));

// upstream.js: service الأسعار اللي إحنا معتمدين عليها، على port أربعة آلاف. الـ mode بيتغير من بره
import express from "express";

let mode = "ok", hits = 0;
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const up = express();
up.get("/price", async (req, res) => {
  hits++;
  if (mode === "slow") await sleep(3000); // الـ service مش واقعة، بس بطيئة أوي
  if (mode === "down") return res.status(500).json({ error: "boom" });
  if (mode === "flaky" && hits % 2 === 0) return res.status(503).json({ error: "busy" }); // واحدة آه وواحدة لأ
  res.json({ sku: req.query.sku, price: 100 });
});
up.get("/mode", (req, res) => { mode = req.query.m; res.json({ mode }); });
up.get("/hits", (req, res) => res.json({ hits }));
up.listen(4000);

import { ResilienceError, withTimeout, isRetryable, backoff, retry, createBreaker } from "./resilience.js";

let ok = 0, all = 0;
const check = (name, cond, extra = "") => { all++; if (cond) ok++; console.log(cond ? "✅" : "❌", name, extra); };
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const err = (p) => p.then(() => null, (e) => e);
const B = "http://localhost:3000", UP = "http://localhost:4000";
const httpErr = (status) => Object.assign(new Error(`status ${status}`), { status });

// withTimeout
check("withTimeout: سريع ← القيمة", (await withTimeout(async () => "تمام", 100)) === "تمام");
let sig;
const t0 = performance.now();
const te = await err(withTimeout((signal) => { sig = signal; return sleep(500); }, 50));
const tookMs = performance.now() - t0;
check("withTimeout: بطيء ← TIMEOUT بعد حوالي خمسين ms، والـ signal اتعمله abort", te instanceof ResilienceError && te.code === "TIMEOUT" && tookMs < 150 && sig.aborted, `${tookMs.toFixed(0)}ms`);

// isRetryable و backoff
check("isRetryable: TIMEOUT و 503 و 429 و TypeError آه، و 400 و 404 لأ",
  isRetryable(new ResilienceError("TIMEOUT")) && isRetryable(httpErr(503)) && isRetryable(httpErr(429)) && isRetryable(new TypeError("fetch failed")) && !isRetryable(httpErr(400)) && !isRetryable(httpErr(404)));
const one = () => 1, half = () => 0.5;
check("backoff: السقف بيتضاعف لحد maxMs", [0, 1, 2, 3, 4].map((a) => backoff(a, { baseMs: 100, maxMs: 1000, rand: one })).join() === "100,200,400,800,1000");
check("backoff: full jitter ← rand في السقف", backoff(2, { baseMs: 100, maxMs: 1000, rand: half }) === 200 && backoff(3, { baseMs: 100, maxMs: 1000, rand: () => 0 }) === 0);

// retry
{
  const slept = [];
  let calls = 0;
  const v = await retry(async (attempt) => { calls++; if (attempt < 2) throw httpErr(503); return "نجح"; }, { retries: 3, baseMs: 100, rand: one, sleep: async (ms) => slept.push(ms) });
  check("retry: فشل مرتين ونجح في التالتة، و sleep مِيّة ومِيّتين", v === "نجح" && calls === 3 && slept.join() === "100,200", slept.join());
  calls = 0;
  const e = await err(retry(async () => { calls++; throw httpErr(400); }, { retries: 3, sleep: async () => {} }));
  check("retry: 400 ← مفيش retry، call واحدة", e?.status === 400 && calls === 1);
  calls = 0;
  const e2 = await err(retry(async () => { calls++; throw httpErr(500 + calls); }, { retries: 2, sleep: async () => {} }));
  check("retry: بيستسلم بعد retries + واحد، وبيرمي آخر error", calls === 3 && e2?.status === 503);
}

// breaker بساعة وهمية
{
  let now = 0;
  const b = createBreaker({ failureThreshold: 3, resetMs: 1000, now: () => now });
  const fail = () => Promise.reject(httpErr(500));
  for (const f of [fail, fail, () => "تمام", fail, fail]) await err(b.run(f));
  check("breaker: نجاح في النص بيصفّر العد ← لسه CLOSED", b.stats().state === "CLOSED" && b.stats().failures === 2, JSON.stringify(b.stats()));
  await err(b.run(fail));
  let called = false;
  const oe = await err(b.run(() => { called = true; }));
  check("brea
…