github-scan-secrets

#!/usr/bin/env bash
# Usage:
#   github-scan-secrets -u <username>                       # scan non-fork repos of a user
#   github-scan-secrets -u <username> -v public             # only public
#   github-scan-secrets -u <username> -v private            # only private
#   github-scan-secrets -u <username> -F                    # include forks
#   github-scan-secrets -f repos.txt                        # repos from a file
#   echo "https://github.com/o/r" | github-scan-secrets     # repos from stdin
#   github-scan-secrets owner/repo another-owner/repo       # positional slugs
#   github-scan-secrets -u <username> -g 'my_secret|token'  # custom grep regex
#   github-scan-secrets -u <username> -d /path/to/reports   # custom reports directory
#   github-scan-secrets --version                           # show version
#   github-scan-secrets --help                              # show help
#
# For each repo:
#   1. clone history into a temporary directory (authenticated with the gh
#      token when available, so private repos work; never prompts)
#   2. run gitleaks on the full history (`gitleaks git`, or `gitleaks detect`
#      on older gitleaks versions)
#   3. search `git log -p --all` with a secret grep pattern
#   4. save reports to <results dir>/<owner>__<repo>/ (default: $PWD/scan-results)
#   5. clean up the clone
#
# A results directory created by this script is chmod 700; an existing one
# keeps its permissions and only gets a warning when it is not owner-only.
# Reports are partially redacted: gitleaks runs with --redact, and in the grep
# matches every token-like string of 20+ characters is masked with [REDACTED].
# SHORT secrets (e.g. `password=hunter2`) are NOT masked - treat the reports as
# sensitive. Use the reported file/commit context to inspect the repo itself.
#
# Env:
#   CONCURRENCY    how many repos to scan in parallel (default: 4)
#
# Exit codes: 0 = clean, 1 = at least one scan failed (clone error, gitleaks
# or git log failure), 2 = scans succeeded but findings were reported.
#
# Input: positional slugs, -f files and -u listings are combined and
# deduplicated; stdin is read only when none of them yielded a repo.
#
# Requires: git, gitleaks (brew install gitleaks), jq; gh CLI for -u and for
# cloning private repos

set -u

script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source-path=SCRIPTDIR
# shellcheck source=../shared/__shared.sh
source "$script_dir/../shared/__shared.sh"

RESULTS_DIR="$PWD/scan-results"
CLONE_DIR="$(mktemp -d -t scan-secrets.XXXXXX)"
RESULTS_LOG="$(mktemp -t scan-secrets-log.XXXXXX)"
GREP_PATTERN="secret|password|passwd|token|api[_-]?key|private[_-]?key|aws[_-]?access|bearer"
CONCURRENCY="${CONCURRENCY:-4}"

# Replaces the shared library's EXIT trap, so it also cleans up GH_LIST_FAILED.
trap 'rm -rf "$CLONE_DIR"; rm -f "$RESULTS_LOG" "$GH_LIST_FAILED"' EXIT INT TERM

if [[ $# -eq 0 && -t 0 ]]; then
    show_usage
    exit 0
fi

while [[ $# -gt 0 ]]; do
    case "$1" in
        -u|--user)
            require_value --user "${2:-}" "a username"
            GH_USER="$2"; shift 2 ;;
        -v|--visibility)
            require_value --visibility "${2:-}" "a value"
            VISIBILITY="$2"; shift 2 ;;
        -F|--include-forks)
            INCLUDE_FORKS=1; shift ;;
        -f|--file)
            require_value --file "${2:-}" "a path"
            INPUT_FILE="$2"; shift 2 ;;
        -g|--grep-pattern)
            require_value --grep-pattern "${2:-}" "a regex"
            GREP_PATTERN="$2"; shift 2 ;;
        -d|--results-dir)
            require_value --results-dir "${2:-}" "a path"
            RESULTS_DIR="$2"; shift 2 ;;
        -V|--version)
            show_version ;;
        -h|--help)
            show_usage; exit 0 ;;
        -*)
            echo "Unknown flag: $1 (use --help)" >&2; exit 1 ;;
        *)
            POSITIONAL+=("$1"); shift ;;
    esac
done

require_cmd git "brew install git"
require_cmd gitleaks "brew install gitleaks"
require_cmd jq "brew install jq"
validate_visibility "$VISIBILITY"

# Clones go through git_authed: authenticated with the gh token when available
# (private repos), anonymous otherwise, and never prompting for credentials
# (a prompt would hang the parallel workers).
optional_gh_token

# gitleaks 8.19+ deprecates `detect` in favour of the `git` subcommand - pick
# the mode once and pass it to the workers.
GITLEAKS_MODE="detect"
gitleaks git --help > /dev/null 2>&1 && GITLEAKS_MODE="git"

# shellcheck disable=SC2329  # invoked indirectly by the xargs workers
print_scan_row() {
    local status="$1"
    local repo="$2"
    local leaks="${3:--}"
    local grep_n="${4:--}"
    local details="${5:-}"
    printf "%s  %-45s  %8s  %10s  %s\n" \
        "$(paint_status "$status")" "$repo" "$leaks" "$grep_n" "$details"
}

print_scan_header() {
    printf "%s%-6s  %-45s  %8s  %10s  %s%s\n" \
        "$C_BOLD" "STATUS" "REPO" "GITLEAKS" "GREP" "DETAILS" "$C_RESET"
    printf "%s\n" "$(printf '%.0s-' {1..100})"
}

# --- collect slugs ---
collect_slugs() {
    local slugs=()

    # 1. positional args
    local arg
    for arg in "${POSITIONAL[@]+"${POSITIONAL[@]}"}"; do
        local s
        s=$(url_to_slug "$arg")
        [[ -n "$s" ]] && slugs+=("$s")
    done

    # 2. file
    if [[ -n "$INPUT_FILE" ]]; then
        if [[ ! -f "$INPUT_FILE" ]]; then
            echo "ERROR: file does not exist: $INPUT_FILE" >&2
            exit 1
        fi
        echo "${C_BOLD}Source:${C_RESET} file $INPUT_FILE" >&2
        while IFS= read -r line; do
            [[ "$line" =~ ^[[:space:]]*(#|$) ]] && continue
            local s
            s=$(url_to_slug "$line")
            [[ -n "$s" ]] && slugs+=("$s")
        done < "$INPUT_FILE"
    fi

    # 3. user (gh)
    if [[ -n "$GH_USER" ]]; then
        if ! command -v gh > /dev/null 2>&1; then
            echo "ERROR: gh CLI not found. Install with: brew install gh" >&2
            exit 1
        fi
        local vis_args=()
        [[ "$VISIBILITY" != "all" ]] && vis_args=(--visibility "$VISIBILITY")
        local jq_filter='.[]'
        [[ "$INCLUDE_FORKS" == "0" ]] && jq_filter='.[] | select(.isFork==false)'
        jq_filter="$jq_filter | .nameWithOwner"

        local note="(forks excluded)"
        [[ "$INCLUDE_FORKS" == "1" ]] && note="(forks included)"
        echo "${C_BOLD}Source:${C_RESET} gh repo list $GH_USER --visibility $VISIBILITY $note" >&2

        local listed=0
        while IFS= read -r slug; do
            [[ -n "$slug" ]] && { slugs+=("$slug"); listed=$((listed + 1)); }
        done < <(gh repo list "$GH_USER" "${vis_args[@]}" --limit "$REPO_LIMIT" \
            --json nameWithOwner,isFork --jq "$jq_filter")
        warn_if_truncated "$listed"
    fi

    # 4. stdin (only if nothing else provided yet)
    if [[ ${#slugs[@]} -eq 0 && ! -t 0 ]]; then
        echo "${C_BOLD}Source:${C_RESET} stdin" >&2
        while IFS= read -r line; do
            [[ "$line" =~ ^[[:space:]]*(#|$) ]] && continue
            local s
            s=$(url_to_slug "$line")
            [[ -n "$s" ]] && slugs+=("$s")
        done
    fi

    # dedupe
    printf "%s\n" "${slugs[@]+"${slugs[@]}"}" | awk '!seen[$0]++'
}

# --- scan a single repo (safe for parallel execution) ---
# Writes a pipe-separated result tuple to $RESULTS_LOG (O_APPEND, atomic for
# small writes) and prints the formatted row to stdout with a single printf
# (also atomic for short lines).
# shellcheck disable=SC2329  # invoked indirectly via xargs bash -c
scan_repo() {
    local slug="$1"

    if [[ -z "$slug" || "$slug" != */* ]]; then
        print_scan_row "SKIP" "$slug" "-" "-" "invalid slug"
        printf "SKIP|%s|invalid slug|-|-\n" "$slug" >> "$RESULTS_LOG"
        return
    fi

    local safe_name="${slug//\//__}"
    local out_dir="$RESULTS_DIR/$safe_name"
    local repo_dir="$CLONE_DIR/$safe_name"
    mkdir -p "$out_dir"

    if ! git_authed clone --quiet "https://github.com/$slug.git" "$repo_dir" \
            2>"$out_dir/clone.err"; then
        print_scan_row "FAIL" "$slug" "-" "-" "clone failed → $out_dir/clone.err"
        printf "FAIL|%s|clone failed → %s|-|-\n" "$slug" "$out_dir/clone.err" >> "$RESULTS_LOG"
        return
    fi

    local leaks_count=0
    local grep_count=0
    local log_rc=0

    local gitleaks_args=(
        --report-format json
        --report-path "$out_dir/gitleaks.json"
        --redact
        --no-banner
        --no-color
    )
    # gitleaks exits 0 (no leaks) or 1 (leaks found); anything else - and any
    # run that leaves no report behind - means the scan itself broke and must
    # not be reported as a clean repo.
    local gitleaks_rc=0
    if [[ "$GITLEAKS_MODE" == "git" ]]; then
        NO_COLOR=1 gitleaks git "$repo_dir" "${gitleaks_args[@]}" \
            > "$out_dir/gitleaks.stdout" 2>&1 || gitleaks_rc=$?
    else
        NO_COLOR=1 gitleaks detect --source "$repo_dir" "${gitleaks_args[@]}" \
            > "$out_dir/gitleaks.stdout" 2>&1 || gitleaks_rc=$?
    fi

    if (( gitleaks_rc > 1 )) || [[ ! -f "$out_dir/gitleaks.json" ]]; then
        rm -rf "$repo_dir"
        print_scan_row "FAIL" "$slug" "-" "-" "gitleaks failed (rc=$gitleaks_rc) → $out_dir/gitleaks.stdout"
        printf "FAIL|%s|gitleaks failed (rc=%s) → %s|-|-\n" \
            "$slug" "$gitleaks_rc" "$out_dir/gitleaks.stdout" >> "$RESULTS_LOG"
        return
    fi

    if [[ -s "$out_dir/gitleaks.json" ]]; then
        leaks_count=$(jq 'length' "$out_dir/gitleaks.json" 2>/dev/null || echo 0)
    fi

    # Long token-like strings are masked so the report does not store obvious
    # tokens in plaintext (the gitleaks report is redacted by gitleaks).
    # A failing `git log` must not be mistaken for "no matches".
    # PIPESTATUS must be read by the very next command - even a `local`
    # declaration or a trailing `|| true` would clobber it, so log_rc is
    # declared above. grep exits 1 when there are no matches, which is not an
    # error here; only git's status matters.
    git -C "$repo_dir" log -p --all 2>"$out_dir/git-log.err" \
        | grep -iE "$GREP_PATTERN" \
        | sed -E 's|[A-Za-z0-9+/=_-]{20,}|[REDACTED]|g' \
        > "$out_dir/grep-matches.txt" 2>/dev/null
    log_rc=${PIPESTATUS[0]}

    if (( log_rc != 0 )); then
        rm -rf "$repo_dir"
        print_scan_row "FAIL" "$slug" "$leaks_count" "-" "git log failed → $out_dir/git-log.err"
        printf "FAIL|%s|git log failed → %s|%s|-\n" \
            "$slug" "$out_dir/git-log.err" "$leaks_count" >> "$RESULTS_LOG"
        return
    fi

    if [[ -s "$out_dir/grep-matches.txt" ]]; then
        grep_count=$(wc -l < "$out_dir/grep-matches.txt" | tr -d ' ')
    fi

    rm -rf "$repo_dir"

    if (( leaks_count > 0 || grep_count > 0 )); then
        print_scan_row "WARN" "$slug" "$leaks_count" "$grep_count" "$out_dir"
        printf "WARN|%s|%s|%s|%s\n" "$slug" "$out_dir" "$leaks_count" "$grep_count" >> "$RESULTS_LOG"
    else
        print_scan_row "OK" "$slug" "$leaks_count" "$grep_count" ""
        printf "OK|%s||%s|%s\n" "$slug" "$leaks_count" "$grep_count" >> "$RESULTS_LOG"
    fi
}

# Export everything the worker subshell needs
export -f print_scan_row scan_repo
export CLONE_DIR RESULTS_DIR RESULTS_LOG GREP_PATTERN GITLEAKS_MODE

# --- summary (reads $RESULTS_LOG populated by parallel workers) ---
print_summary() {
    local ok=0 warn=0 fail=0 skip=0
    local line status
    while IFS= read -r line; do
        [[ -z "$line" ]] && continue
        status="${line%%|*}"
        case "$status" in
            OK)   ok=$((ok + 1)) ;;
            WARN) warn=$((warn + 1)) ;;
            FAIL) fail=$((fail + 1)) ;;
            SKIP) skip=$((skip + 1)) ;;
        esac
    done < "$RESULTS_LOG"
    local total=$((ok + warn + fail + skip))

    echo
    echo "${C_BOLD}Summary:${C_RESET}"
    printf "  %s%-5s%s  %d\n" "$C_GREEN"  "OK"    "$C_RESET" "$ok"
    printf "  %s%-5s%s  %d\n" "$C_YELLOW" "WARN"  "$C_RESET" "$warn"
    printf "  %s%-5s%s  %d\n" "$C_RED"    "FAIL"  "$C_RESET" "$fail"
    printf "  %s%-5s%s  %d\n" "$C_DIM"    "SKIP"  "$C_RESET" "$skip"
    printf "  %sTOTAL%s  %d\n" "$C_BOLD"  "$C_RESET" "$total"

    if (( warn + fail > 0 )); then
        echo
        echo "${C_BOLD}Repositories that need attention:${C_RESET}"
        # FAIL first, then WARN, both sorted by slug
        (
            awk -F'|' '$1=="FAIL"' "$RESULTS_LOG" | sort -t'|' -k2
            awk -F'|' '$1=="WARN"' "$RESULTS_LOG" | sort -t'|' -k2
        ) | while IFS='|' read -r status slug details leaks grep_n; do
            printf "  %s  %-45s  %s\n" "$(paint_status "$status")" "$slug" "${details:--}"
        done
    fi
}

# --- main ---
mapfile -t SLUGS < <(collect_slugs)

if [[ ${#SLUGS[@]} -eq 0 ]]; then
    echo "No repos to scan." >&2
    show_usage
    exit 1
fi

# Reports may reference sensitive findings, so a directory created by this
# script is owner-only. A directory that already exists keeps its permissions
# (it may be shared on purpose) - warn instead of silently narrowing it.
if [[ -d "$RESULTS_DIR" ]]; then
    # BSD stat (macOS) uses -f, GNU stat uses -c; fall back to an empty string.
    perms=$(stat -f '%OLp' "$RESULTS_DIR" 2>/dev/null || stat -c '%a' "$RESULTS_DIR" 2>/dev/null || echo "")
    if [[ -n "$perms" && "$perms" != "700" ]]; then
        echo "${C_YELLOW}WARN:${C_RESET} $RESULTS_DIR already exists and is not owner-only (mode $perms)." >&2
        echo "      Reports may reference sensitive findings - run: chmod 700 $RESULTS_DIR" >&2
    fi
else
    mkdir -p "$RESULTS_DIR" || { echo "ERROR: cannot create results directory: $RESULTS_DIR" >&2; exit 1; }
    chmod 700 "$RESULTS_DIR" || { echo "ERROR: cannot lock down results directory: $RESULTS_DIR" >&2; exit 1; }
fi
: > "$RESULTS_LOG"

print_kv "Results" "$RESULTS_DIR"
print_kv "Tmp" "$CLONE_DIR"
print_kv "Repos" "${#SLUGS[@]}"
print_kv "Concurrency" "$CONCURRENCY"
echo
print_scan_header

printf "%s\n" "${SLUGS[@]}" \
    | xargs -n 1 -P "$CONCURRENCY" -I {} bash -c 'scan_repo "$@"' _ {}

print_summary
echo
echo "Done. Reports in: $RESULTS_DIR"

# Exit codes: 1 when any scan failed (clone error), 2 when scans succeeded but
# findings were reported (gitleaks-style), 0 when everything is clean.
fails=$(awk -F'|' '$1=="FAIL"' "$RESULTS_LOG" | grep -c . || true)
warns=$(awk -F'|' '$1=="WARN"' "$RESULTS_LOG" | grep -c . || true)
[[ "$fails" -gt 0 ]] && exit 1
[[ "$warns" -gt 0 ]] && exit 2
exit 0