AIRBRAKE → PR AUTOMATION PIPELINE

#!/bin/bash
# Scan for fixable Airbrake issues and launch Claude Code to fix them.
# Runs every 60 seconds via systemd timer.

set -euo pipefail

LOCKFILE="/tmp/fix-airbrakes.lock"

# Prevent concurrent runs with flock
exec 200>"$LOCKFILE"
if ! flock -n 200; then
  echo "$(date): Already running, skipping"
  exit 0
fi

cd /home/deploy/projects/crowdcow

export HOME=/home/deploy
export PATH="$HOME/.local/bin:$PATH"
export AIRBRAKE_REPO="Crowd-Cow/crowdcow"

# Load secrets from Doppler
set +u
eval "$(doppler secrets download --project devbox --config prd --no-file --format env 2>/dev/null)"
unset DOPPLER_PROJECT DOPPLER_CONFIG DOPPLER_ENVIRONMENT
eval "$(doppler secrets download --project crowdcow --config dev --no-file --format env 2>/dev/null)"
unset DOPPLER_PROJECT DOPPLER_CONFIG DOPPLER_ENVIRONMENT
set -u

# Check for fixable issues
issues=$("$HOME/.local/bin/airbrake-issues" fixable 2>&1)

if [ "$issues" = "[]" ] || [ -z "$issues" ]; then
  echo "$(date): No fixable issues found"
  exit 0
fi

# Launch Claude to fix the issue
LOGFILE="$HOME/logs/fix-airbrake/$(date +%Y%m%d-%H%M%S).log"
echo "$(date): Found fixable issues, launching claude (log: $LOGFILE)"
claude --dangerously-skip-permissions -p "/fix-airbrakes" --model opus >> "$LOGFILE" 2>&1

#!/usr/bin/env bash
set -euo pipefail

# Airbrake issue manager for automated fix workflows.
# Tracks which issues are being worked on to prevent duplicate PRs.
#
# Prerequisites: gh (authenticated), jq
#
# Usage:
#   airbrake-issues list                     # List open airbrake issues (JSON)
#   airbrake-issues fixable                  # List issues not yet addressed
#   airbrake-issues addressed                # Issue numbers with open PRs
#   airbrake-issues claim ISSUE [BRANCH]     # Mark issue as in-progress
#   airbrake-issues unclaim ISSUE            # Remove in-progress mark
#   airbrake-issues status                   # Show tracking file
#   airbrake-issues comment ISSUE 'msg'      # Post a comment on issue
#   airbrake-issues resolve ISSUE            # Resolve Airbrake error group

# Auto-detect repo from git remote
REPO="${AIRBRAKE_REPO:-$(gh repo view --json nameWithOwner -q .nameWithOwner 2>/dev/null || echo "")}"
if [[ -z "$REPO" ]]; then
  echo "Error: Could not detect GitHub repo. Set AIRBRAKE_REPO or run from a git repo." >&2
  exit 1
fi

TRACKING_FILE="$HOME/.claude/airbrake-in-progress.json"

# Load env vars from Claude settings if not already set
if [[ -z "${AIRBRAKE_PROJECT_ID:-}" || -z "${AIRBRAKE_API_KEY:-}" ]] && [[ -f "$HOME/.claude/settings.local.json" ]]; then
  AIRBRAKE_PROJECT_ID="${AIRBRAKE_PROJECT_ID:-$(jq -r '.env.AIRBRAKE_PROJECT_ID // empty' "$HOME/.claude/settings.local.json" 2>/dev/null)}"
  AIRBRAKE_API_KEY="${AIRBRAKE_API_KEY:-$(jq -r '.env.AIRBRAKE_API_KEY // empty' "$HOME/.claude/settings.local.json" 2>/dev/null)}"
fi

AIRBRAKE_PROJECT_ID="${AIRBRAKE_PROJECT_ID:?Set AIRBRAKE_PROJECT_ID env var or in ~/.claude/settings.local.json}"

ensure_tracking_file() {
  mkdir -p "$(dirname "$TRACKING_FILE")"
  if [[ ! -f "$TRACKING_FILE" ]]; then
    echo '{"issues":{}}' > "$TRACKING_FILE"
  fi
}

# Remove entries older than 2 hours (prevents stale locks)
prune_stale() {
  ensure_tracking_file
  local cutoff
  cutoff=$(date -u -v-2H +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -d '2 hours ago' +%Y-%m-%dT%H:%M:%SZ)

  local pruned
  pruned=$(jq --arg cutoff "$cutoff" '
    .issues |= with_entries(select(.value.started_at > $cutoff))
  ' "$TRACKING_FILE")
  echo "$pruned" > "$TRACKING_FILE"
}

cmd_list() {
  gh issue list --repo "$REPO" --state open --limit 200 \
    --json number,title,body,labels,createdAt,comments
}

cmd_addressed() {
  # Find issue numbers referenced by open PRs (Fixes/Closes/Resolves #NNN)
  local prs
  prs=$(gh pr list --repo "$REPO" --state open --limit 100 --json number,title,body,headRefName)

  echo "$prs" | jq -r '
    [.[] | (
      # Check PR body for Fixes/Closes/Resolves #NNN
      (.body // "" | [scan("(?i)(?:fixes|closes|resolves)\\s+#(\\d+)")] | .[][] // empty),
      # Check branch name for airbrake-NNN pattern
      (.headRefName | capture("airbrake[/-](?<n>\\d+)")? | .n // empty)
    )] | map(select(. != null and . != "")) | unique | sort | .[]
  '
}

cmd_fixable() {
  prune_stale

  local issues addressed in_progress

  issues=$(cmd_list)
  addressed=$(cmd_addressed)
  in_progress=$(jq -r '.issues | keys[]' "$TRACKING_FILE")

  # Combine addressed + in_progress into a skip set
  local skip_set
  skip_set=$(printf '%s\n%s' "$addressed" "$in_progress" | sort -u | grep -v '^$' || true)

  # Filter out already-triaged issues
  local filtered
  filtered=$(echo "$issues" | jq '
    [.[] | select(
      (.labels // []) | map(.name) | any(. == "wont-fix" or . == "unsure") | not
    )]
  ')

  if [[ -z "$skip_set" ]]; then
    echo "$filtered"
  else
    echo "$filtered" | jq --argjson skip "$(echo "$skip_set" | jq -R 'tonumber' | jq -s '.')" '
      [.[] | select(.number as $n | ($skip | index($n)) | not)]
    '
  fi
}

cmd_claim() {
  local issue="${1:-}"
  local branch="${2:-}"

  if [[ -z "$issue" ]]; then
    echo "Usage: airbrake-issues claim ISSUE [BRANCH]" >&2
    exit 1
  fi

  ensure_tracking_file
  prune_stale

  local now
  now=$(date -u +%Y-%m-%dT%H:%M:%SZ)

  jq --arg issue "$issue" --arg branch "$branch" --arg now "$now" '
    .issues[$issue] = {started_at: $now, branch: $branch}
  ' "$TRACKING_FILE" > "${TRACKING_FILE}.tmp" && mv "${TRACKING_FILE}.tmp" "$TRACKING_FILE"

  echo "Claimed issue #$issue"
}

cmd_unclaim() {
  local issue="${1:-}"
  if [[ -z "$issue" ]]; then
    echo "Usage: airbrake-issues unclaim ISSUE" >&2; exit 1
  fi
  ensure_tracking_file
  jq --arg issue "$issue" 'del(.issues[$issue])' "$TRACKING_FILE" \
    > "${TRACKING_FILE}.tmp" && mv "${TRACKING_FILE}.tmp" "$TRACKING_FILE"
  echo "Unclaimed issue #$issue"
}

cmd_status() {
  ensure_tracking_file
  prune_stale
  jq '.' "$TRACKING_FILE"
}

cmd_comment() {
  local issue="${1:-}"
  local message="${2:-}"
  if [[ -z "$issue" || -z "$message" ]]; then
    echo "Usage: airbrake-issues comment ISSUE 'message'" >&2; exit 1
  fi
  gh issue comment "$issue" --repo "$REPO" --body "$message"
  echo "Commented on issue #$issue"
}

cmd_extract_group_id() {
  local issue="${1:-}"
  if [[ -z "$issue" ]]; then
    echo "Usage: airbrake-issues extract-group-id ISSUE" >&2; exit 1
  fi
  local body
  body=$(gh issue view "$issue" --repo "$REPO" --json body -q .body)
  echo "$body" | grep -oE 'airbrake\.io/projects/[0-9]+/groups/[0-9]+' | head -1 | grep -oE '[0-9]+$'
}

cmd_resolve() {
  local issue="${1:-}"
  if [[ -z "$issue" ]]; then
    echo "Usage: airbrake-issues resolve ISSUE_NUMBER" >&2; exit 1
  fi

  : "${AIRBRAKE_API_KEY:?Set AIRBRAKE_API_KEY}"

  local group_id
  group_id=$(cmd_extract_group_id "$issue")
  if [[ -z "$group_id" ]]; then
    echo "Error: could not extract Airbrake group ID from issue #$issue" >&2; exit 1
  fi

  echo "Resolving Airbrake group $group_id (from issue #$issue)..."

  local response
  response=$(curl -s -w "\n%{http_code}" \
    -X PUT \
    "https://api.airbrake.io/api/v4/projects/${AIRBRAKE_PROJECT_ID}/groups/${group_id}/resolved?key=${AIRBRAKE_API_KEY}")

  local http_code body
  http_code=$(echo "$response" | tail -1)
  body=$(echo "$response" | sed '$d')

  if [[ "$http_code" -ge 200 && "$http_code" -lt 300 ]]; then
    echo "Resolved Airbrake group $group_id (HTTP $http_code)"
  else
    echo "Error resolving group $group_id (HTTP $http_code):" >&2
    echo "$body" >&2; exit 1
  fi
}

case "${1:-}" in
  list)             cmd_list ;;
  fixable)          cmd_fixable ;;
  addressed)        cmd_addressed ;;
  claim)            shift; cmd_claim "$@" ;;
  unclaim)          shift; cmd_unclaim "$@" ;;
  status)           cmd_status ;;
  comment)          shift; cmd_comment "$@" ;;
  extract-group-id) shift; cmd_extract_group_id "$@" ;;
  resolve)          shift; cmd_resolve "$@" ;;
  *)                echo "Usage: airbrake-issues <command> [args]" >&2; exit 1 ;;
esac

---
name: fix-airbrakes
description: Scan open Airbrake GitHub issues, group duplicates, and create
  PRs to fix them. Uses worktrees and /ship-it for each fix. Skips issues
  already addressed by open PRs.
---

# Fix Airbrakes

Automatically scan open Airbrake-labeled GitHub issues, identify fixable
bugs, and create PRs.

## CLI Helper

Use `airbrake-issues` for dedup, tracking, and Airbrake API:
- `airbrake-issues fixable` — returns open issues NOT already addressed
  by a PR or in-progress (full JSON with bodies)
- `airbrake-issues claim ISSUE BRANCH` — mark issue as in-progress
- `airbrake-issues unclaim ISSUE` — remove in-progress mark
- `airbrake-issues status` — show what's currently being tracked
- `airbrake-issues comment ISSUE 'message'` — post a comment on issue
- `airbrake-issues resolve ISSUE` — resolve the Airbrake error group

## Workflow (strict order)

### 1. Get and read all fixable issues

```
airbrake-issues fixable
```

Read ALL the issue bodies. Each contains error type, message, backtrace,
URL, occurrences, and severity. Use this to:
- Group issues that are the same underlying bug (same error type +
  same code path)
- Identify which are fixable vs transient/external
- Pick the best one to fix

If no fixable issues, report "No fixable Airbrake issues found" and stop.

### 2. Triage unfixable and uncertain issues

**Unfixable issues** — external API errors, gem-only backtraces, timeouts,
transient issues, `[safely]`-wrapped errors, 503/502 errors from external
services — close with `wont-fix`:

```bash
gh issue close ISSUE_NUMBER --repo Crowd-Cow/crowdcow \
  --comment "[Airbrake Bot] Closing: <reason>"
gh issue edit ISSUE_NUMBER --repo Crowd-Cow/crowdcow --add-label wont-fix
```

**Uncertain issues** — you can't tell from the backtrace alone whether it's
fixable — label with `unsure` and leave open:

```bash
gh issue edit ISSUE_NUMBER --repo Crowd-Cow/crowdcow --add-label unsure
gh issue comment ISSUE_NUMBER --repo Crowd-Cow/crowdcow \
  --body "[Airbrake Bot] Unsure: <what you'd need to know>"
```

Do this for ALL unfixable/uncertain issues in the batch.

### 3. Select an issue group to fix

Pick at most 1 issue group per run. Prioritize by:
1. Groups with higher issue counts (more impact)
2. Groups with clearer, more localized fixes
3. Older groups first

### 4. Claim the issue

```
airbrake-issues claim ISSUE_NUMBER BRANCH_NAME
```

### 5. Fix the bug in a worktree

Use `EnterWorktree` to create a worktree named `airbrake-NNNN`.

After entering the worktree:
- Copy `.env` if needed
- Run `bundle install -j4` and `npm ci`

Then:
1. Read and understand the relevant source files from the backtrace
2. Identify the root cause
3. Implement the fix — keep it minimal
4. If there are relevant specs, run them to verify

### 6. Ship it

Run the `ship-it` workflow. This will:
- Run rubocop + specs
- Commit, push, and create a PR
- Wait for CI
- Request and address Copilot reviews

**IMPORTANT:** PR title should start with `[Airbrake]` and body
must include `Fixes #NNNN` for each issue in the group.

### 7. Clean up tracking and resolve Airbrake

```
airbrake-issues unclaim ISSUE_NUMBER
airbrake-issues resolve ISSUE_NUMBER
```

### 8. Report results

Report:
- Which issues were processed
- PR URL if created
- Whether Airbrake errors were resolved
- Any issues that were skipped and why

## Important rules

- One fix per run. Do not attempt multiple unrelated fixes.
- Stay in the worktree for the entire workflow including ship-it.
- Do not fix issues in gem code. Only fix code under PROJECT_ROOT.
- If you can't figure out the fix, close the issue. Add wont-fix.
- Never mutate production data.

---
name: ship-it
description: Full end-to-end workflow that lands work, requests a GitHub
  Copilot review, waits for feedback, and addresses it in a loop.
---

# Ship It

Full end-to-end workflow that lands work, requests a GitHub Copilot review,
waits for feedback, and addresses it in a loop.

## CLI tools

These CLIs wrap complex APIs and handle pagination, polling, and edge cases.
Always use them instead of raw `gh api` calls.

- `buildkite` — CI status, wait-for-completion, failure logs
- `gh-reviews` — Review threads, replies, resolution, Copilot review

## Workflow (strict order)

1. Run the `land-the-plane` workflow.
   - This handles rubocop, specs, commit, sync, push, and PR creation.

2. Wait for Buildkite CI to pass.
   - Run the `check-ci` workflow (uses `buildkite wait` to poll).
   - If CI fails and the failure is fixable (rubocop, specs), fix it,
     commit, push, and re-run `check-ci`.
   - If CI fails with infrastructure issue, report and continue.

3. Request a Copilot review.
   - `gh-reviews copilot-review`

4. Wait for review to complete.
   - `gh-reviews copilot-wait` (polls automatically, 10min max)
   - If timeout, continue anyway.

5. Address feedback loop.
   - If there are unresolved threads, run `check-for-reviews`.
   - Always use `gh-reviews` for thread operations.
   - After addressing, re-request review with `gh-reviews copilot-review`,
     then repeat wait/poll until no unresolved threads remain.
   - After pushing review fixes, re-run `check-ci`.

6. Confirm and report PR URL.
   - `gh pr view --json url -q .url`

## Completion criteria

You are not done until ALL of the following are true:
- CI is green.
- Every review thread is either resolved (fix applied) or has a
  reply explaining why it was not fixed.
- PR URL is confirmed.

## Notes

- This is an iterative workflow:
  land → CI → review → fix → CI → review → fix → done
- Never use `gh api` for review operations — `gh-reviews` handles it.
- Never use `gh pr checks` for CI — `buildkite` has wait/poll support.

# Role: airbrake_fixer
# Automated Airbrake issue fixer — runs Claude Code every 60 seconds
# to scan for fixable Airbrake GitHub issues and create PRs.
#
# Requires: claude_code role, projects role
---
- name: Deploy airbrake-issues CLI
  ansible.builtin.copy:
    src: airbrake-issues
    dest: "/home/{{ deploy_user }}/.local/bin/airbrake-issues"
    mode: "0755"

- name: Deploy fix-airbrakes cron script
  ansible.builtin.template:
    src: fix-airbrakes.sh.j2
    dest: "/home/{{ deploy_user }}/.local/bin/fix-airbrakes.sh"
    mode: "0755"

- name: Create log directory
  ansible.builtin.file:
    path: "/home/{{ deploy_user }}/logs/fix-airbrake"
    state: directory
    mode: "0755"

- name: Populate settings.local.json with Airbrake keys from Doppler
  ansible.builtin.shell: |
    AIRBRAKE_PROJECT_ID=$(doppler secrets get AIRBRAKE_PROJECT_ID \
      --project crowdcow --config dev --plain 2>/dev/null)
    AIRBRAKE_API_KEY=$(doppler secrets get AIRBRAKE_API_KEY \
      --project crowdcow --config dev --plain 2>/dev/null)
    if [ -z "$AIRBRAKE_PROJECT_ID" ] || [ -z "$AIRBRAKE_API_KEY" ]; then
      echo "Failed to fetch Airbrake keys from Doppler" >&2
      exit 1
    fi
    SETTINGS="$HOME/.claude/settings.local.json"
    [ -f "$SETTINGS" ] || echo '{}' > "$SETTINGS"
    jq --arg pid "$AIRBRAKE_PROJECT_ID" --arg key "$AIRBRAKE_API_KEY" \
      '.env = (.env // {}) |
       .env.AIRBRAKE_PROJECT_ID = $pid |
       .env.AIRBRAKE_API_KEY = $key' \
      "$SETTINGS" > "${SETTINGS}.tmp" && mv "${SETTINGS}.tmp" "$SETTINGS"
    chmod 600 "$SETTINGS"
  become_user: "{{ deploy_user }}"

- name: Create Claude commands directory
  ansible.builtin.file:
    path: "/home/{{ deploy_user }}/.claude/commands"
    state: directory
    mode: "0755"

- name: Deploy fix-airbrakes command
  ansible.builtin.copy:
    src: fix-airbrakes.md
    dest: "/home/{{ deploy_user }}/.claude/commands/fix-airbrakes.md"
    mode: "0644"

- name: Deploy ship-it command
  ansible.builtin.copy:
    src: ship-it.md
    dest: "/home/{{ deploy_user }}/.claude/commands/ship-it.md"
    mode: "0644"

- name: Deploy airbrake-fixer systemd user service
  ansible.builtin.template:
    src: airbrake-fixer.service.j2
    dest: "~/.config/systemd/user/airbrake-fixer.service"

- name: Deploy airbrake-fixer systemd user timer
  ansible.builtin.template:
    src: airbrake-fixer.timer.j2
    dest: "~/.config/systemd/user/airbrake-fixer.timer"

- name: Enable and start airbrake-fixer timer
  ansible.builtin.systemd:
    name: airbrake-fixer.timer
    enabled: true
    state: started
    daemon_reload: true
    scope: user