<mxfile host="Electron" modified="2026-04-14T00:00:00Z" version="26.0.0">
  <diagram id="billing-incident-seq" name="Billing Webhook Incident">
    <mxGraphModel dx="1422" dy="900" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1400" pageHeight="1000" math="0" shadow="0">
      <root>
        <mxCell id="0" />
        <mxCell id="1" parent="0" />
        
        <!-- Title -->
        <mxCell id="title" value="Billing Webhook Incident - Failed Rollout &amp; Recovery" style="text;html=1;strokeColor=none;fillColor=none;align=center;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontSize=16;fontStyle=1;" vertex="1" parent="1">
          <mxGeometry x="400" y="10" width="600" height="30" as="geometry" />
        </mxCell>
        
        <!-- Actors/Services -->
        <mxCell id="billing-svc" value="Billing Service" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1">
          <mxGeometry x="60" y="60" width="120" height="40" as="geometry" />
        </mxCell>
        <mxCell id="webhook-v2" value="Webhook Handler&lt;br&gt;(v2 - NEW)" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;fontStyle=1;" vertex="1" parent="1">
          <mxGeometry x="240" y="60" width="140" height="40" as="geometry" />
        </mxCell>
        <mxCell id="database" value="Database" style="shape=mxgraph.flowchart.database;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;" vertex="1" parent="1">
          <mxGeometry x="450" y="60" width="100" height="40" as="geometry" />
        </mxCell>
        <mxCell id="retry-queue" value="Retry Queue" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;" vertex="1" parent="1">
          <mxGeometry x="620" y="60" width="120" height="40" as="geometry" />
        </mxCell>
        <mxCell id="dlq" value="DLQ" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="1">
          <mxGeometry x="810" y="60" width="100" height="40" as="geometry" />
        </mxCell>
        <mxCell id="pagerduty" value="PagerDuty" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;fontStyle=1;" vertex="1" parent="1">
          <mxGeometry x="980" y="60" width="120" height="40" as="geometry" />
        </mxCell>
        <mxCell id="loadbalancer" value="Load Balancer" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;" vertex="1" parent="1">
          <mxGeometry x="1170" y="60" width="120" height="40" as="geometry" />
        </mxCell>
        
        <!-- Lifelines -->
        <mxCell id="life-1" value="" style="shape=line;strokeColor=#999999;dashed=1;html=1;" vertex="1" parent="1">
          <mxGeometry x="120" y="110" width="1" height="820" as="geometry" />
        </mxCell>
        <mxCell id="life-2" value="" style="shape=line;strokeColor=#b85450;dashed=1;html=1;strokeWidth=2;" vertex="1" parent="1">
          <mxGeometry x="310" y="110" width="1" height="820" as="geometry" />
        </mxCell>
        <mxCell id="life-3" value="" style="shape=line;strokeColor=#999999;dashed=1;html=1;" vertex="1" parent="1">
          <mxGeometry x="500" y="110" width="1" height="820" as="geometry" />
        </mxCell>
        <mxCell id="life-4" value="" style="shape=line;strokeColor=#999999;dashed=1;html=1;" vertex="1" parent="1">
          <mxGeometry x="680" y="110" width="1" height="820" as="geometry" />
        </mxCell>
        <mxCell id="life-5" value="" style="shape=line;strokeColor=#999999;dashed=1;html=1;" vertex="1" parent="1">
          <mxGeometry x="860" y="110" width="1" height="820" as="geometry" />
        </mxCell>
        <mxCell id="life-6" value="" style="shape=line;strokeColor=#999999;dashed=1;html=1;" vertex="1" parent="1">
          <mxGeometry x="1040" y="110" width="1" height="820" as="geometry" />
        </mxCell>
        <mxCell id="life-7" value="" style="shape=line;strokeColor=#999999;dashed=1;html=1;" vertex="1" parent="1">
          <mxGeometry x="1230" y="110" width="1" height="820" as="geometry" />
        </mxCell>
        
        <!-- Sequence Flow -->
        
        <!-- Initial webhook event -->
        <mxCell id="e1" value="POST /webhook/billing" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="120" y="140" as="sourcePoint" />
            <mxPoint x="310" y="140" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <!-- Activation box for webhook handler -->
        <mxCell id="act-1" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;opacity=30;" vertex="1" parent="1">
          <mxGeometry x="305" y="140" width="10" height="180" as="geometry" />
        </mxCell>
        
        <!-- DB query attempt -->
        <mxCell id="e2" value="Query customer data" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="315" y="170" as="sourcePoint" />
            <mxPoint x="500" y="170" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <!-- Timeout -->
        <mxCell id="note-1" value="⏱️ TIMEOUT (30s)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;fontStyle=1;dashed=1;" vertex="1" parent="1">
          <mxGeometry x="420" y="200" width="140" height="30" as="geometry" />
        </mxCell>
        
        <mxCell id="e3" value="❌ Timeout Exception" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;dashed=1;strokeColor=#d79b00;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="500" y="250" as="sourcePoint" />
            <mxPoint x="315" y="250" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <!-- Error handling - send to retry queue -->
        <mxCell id="e4" value="Enqueue for retry (attempt 1)" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;strokeColor=#d79b00;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="315" y="290" as="sourcePoint" />
            <mxPoint x="680" y="290" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <!-- Return 500 -->
        <mxCell id="e5" value="500 Internal Error" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;dashed=1;strokeColor=#b85450;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="310" y="310" as="sourcePoint" />
            <mxPoint x="120" y="310" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <!-- Retry 1 -->
        <mxCell id="note-2" value="⏰ Retry #1 (after 5min)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;fontStyle=1;" vertex="1" parent="1">
          <mxGeometry x="600" y="340" width="160" height="30" as="geometry" />
        </mxCell>
        
        <mxCell id="e6" value="Dequeue &amp; retry" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="680" y="380" as="sourcePoint" />
            <mxPoint x="315" y="380" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <mxCell id="act-2" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;opacity=30;" vertex="1" parent="1">
          <mxGeometry x="305" y="380" width="10" height="100" as="geometry" />
        </mxCell>
        
        <mxCell id="e7" value="Query customer data" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="315" y="410" as="sourcePoint" />
            <mxPoint x="500" y="410" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <mxCell id="e8" value="❌ Timeout Exception" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;dashed=1;strokeColor=#d79b00;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="500" y="450" as="sourcePoint" />
            <mxPoint x="315" y="450" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <mxCell id="e9" value="Re-enqueue (attempt 2)" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;strokeColor=#d79b00;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="315" y="470" as="sourcePoint" />
            <mxPoint x="680" y="470" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <!-- Retry 2 -->
        <mxCell id="note-3" value="⏰ Retry #2 (after 15min)" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;fontStyle=1;" vertex="1" parent="1">
          <mxGeometry x="600" y="500" width="160" height="30" as="geometry" />
        </mxCell>
        
        <mxCell id="e10" value="Dequeue &amp; retry" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="680" y="540" as="sourcePoint" />
            <mxPoint x="315" y="540" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <mxCell id="act-3" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;opacity=30;" vertex="1" parent="1">
          <mxGeometry x="305" y="540" width="10" height="100" as="geometry" />
        </mxCell>
        
        <mxCell id="e11" value="Query customer data" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="315" y="570" as="sourcePoint" />
            <mxPoint x="500" y="570" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <mxCell id="e12" value="❌ Timeout Exception" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;dashed=1;strokeColor=#d79b00;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="500" y="610" as="sourcePoint" />
            <mxPoint x="315" y="610" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <!-- Max retries exceeded - send to DLQ -->
        <mxCell id="note-4" value="🚨 Max retries exceeded" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f8cecc;strokeColor=#b85450;fontStyle=1;" vertex="1" parent="1">
          <mxGeometry x="230" y="640" width="160" height="30" as="geometry" />
        </mxCell>
        
        <mxCell id="e13" value="Send to DLQ" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;strokeColor=#b85450;strokeWidth=2;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="315" y="680" as="sourcePoint" />
            <mxPoint x="860" y="680" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <!-- DLQ triggers alert -->
        <mxCell id="e14" value="Trigger alert (threshold exceeded)" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;strokeColor=#b85450;strokeWidth=2;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="860" y="720" as="sourcePoint" />
            <mxPoint x="1040" y="720" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <!-- PagerDuty incident -->
        <mxCell id="note-5" value="📟 INCIDENT CREATED&lt;br&gt;P1: Billing webhooks failing" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;fontStyle=1;" vertex="1" parent="1">
          <mxGeometry x="960" y="750" width="180" height="50" as="geometry" />
        </mxCell>
        
        <!-- Rollback action -->
        <mxCell id="note-6" value="🔧 ROLLBACK INITIATED" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;fontStyle=1;" vertex="1" parent="1">
          <mxGeometry x="1140" y="820" width="180" height="30" as="geometry" />
        </mxCell>
        
        <mxCell id="e15" value="Scale down v2 to 0%" style="edgeStyle=orthogonalEdgeStyle;html=1;endArrow=block;endFill=1;strokeColor=#82b366;strokeWidth=2;" edge="1" parent="1">
          <mxGeometry relative="1" as="geometry">
            <mxPoint x="1230" y="860" as="sourcePoint" />
            <mxPoint x="315" y="860" as="targetPoint" />
          </mxGeometry>
        </mxCell>
        
        <mxCell id="note-7" value="❌ v2 TERMINATED" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;strokeColor=#666666;fontStyle=1;" vertex="1" parent="1">
          <mxGeometry x="230" y="880" width="160" height="30" as="geometry" />
        </mxCell>
        
        <!-- Root cause annotation -->
        <mxCell id="root-cause" value="ROOT CAUSE:&lt;br&gt;v2 webhook handler had inefficient DB query&lt;br&gt;causing connection pool exhaustion &amp;amp; timeouts" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;align=left;verticalAlign=top;fontStyle=1;" vertex="1" parent="1">
          <mxGeometry x="40" y="820" width="300" height="70" as="geometry" />
        </mxCell>
        
        <!-- Timeline annotations -->
        <mxCell id="time-1" value="T+0min" style="text;html=1;strokeColor=none;fillColor=none;align=right;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontStyle=2;" vertex="1" parent="1">
          <mxGeometry x="10" y="130" width="40" height="20" as="geometry" />
        </mxCell>
        <mxCell id="time-2" value="T+5min" style="text;html=1;strokeColor=none;fillColor=none;align=right;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontStyle=2;" vertex="1" parent="1">
          <mxGeometry x="10" y="370" width="40" height="20" as="geometry" />
        </mxCell>
        <mxCell id="time-3" value="T+20min" style="text;html=1;strokeColor=none;fillColor=none;align=right;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontStyle=2;" vertex="1" parent="1">
          <mxGeometry x="10" y="530" width="40" height="20" as="geometry" />
        </mxCell>
        <mxCell id="time-4" value="T+25min" style="text;html=1;strokeColor=none;fillColor=none;align=right;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontStyle=2;" vertex="1" parent="1">
          <mxGeometry x="10" y="710" width="40" height="20" as="geometry" />
        </mxCell>
        <mxCell id="time-5" value="T+30min" style="text;html=1;strokeColor=none;fillColor=none;align=right;verticalAlign=middle;whiteSpace=wrap;rounded=0;fontStyle=2;" vertex="1" parent="1">
          <mxGeometry x="10" y="850" width="40" height="20" as="geometry" />
        </mxCell>
        
      </root>
    </mxGraphModel>
  </diagram>
</mxfile>
