Grafana 장애 알림에서 Stack Trace 기반 소스 코드 수정 권고안까지 연결하기

Grafana Alert Webhook
        │
        ├─ Loki 로그 조회
        ├─ Prometheus 메트릭 조회 (optional)
        │
        ├─ LLM 분석 요청
        │    alert context
        │    + log section
        │    + metric section
        │
        └─ AnalyzeFiringRecord 저장 및 WebSocket push

val logFuture = CompletableFuture.supplyAsync({ executeFindLog(request) }, executor)
val metricFuture = CompletableFuture.supplyAsync({ executeFindMetric(request) }, executor)
val checkoutFuture = CompletableFuture.supplyAsync({ getSourcePath(targetApplication) }, executor)

val logResults: LokiQueryResult = logFuture.get()
val metricResults: PrometheusQueryResult? = metricFuture.get()
val sourcePath: Path? = checkoutFuture.get()

private fun resolveAccessToken(gitUrl: String): String? {
    val lower = gitUrl.lowercase()
    return when {
        lower.contains("github") -> githubService.getToken()
        lower.contains("gitlab") -> gitlabService.getToken()
        else -> null
    }
}

fun rawLogText(): String {
    return data?.result
        ?.asSequence()
        ?.flatMap { stream -> stream.values.asSequence() }
        ?.mapNotNull { entry -> entry.getOrNull(1) }
        ?.joinToString(System.lineSeparator())
        ?: ""
}

[
  ["1710000000000000000", "actual log line"]
]

at com.example.service.FooService.doWork(FooService.kt:42)
at com.example.controller.FooController.handle(FooController.java:19)

private val frameRegex =
    Regex("""\s*at\s+([\w.$]+)\.([\w$<>]+)\(([^():]+)(?::(\d+))?\)""")

data class StackTraceFrame(
    val className: String,
    val methodName: String,
    val fileName: String?,
    val lineNumber: Int?,
) {
    val packageName: String
        get() = className.substringBeforeLast('.', "")

    val simpleClassName: String
        get() = className.substringAfterLast('.').substringBefore('$')
}

fun MatchResult.toStackTraceFrame(): StackTraceFrame {
    return StackTraceFrame(
        className = groupValues[1],
        methodName = groupValues[2],
        fileName = groupValues[3].takeIf { it != "Unknown Source" && it != "Native Method" },
        lineNumber = groupValues.getOrNull(4)?.takeIf { it.isNotBlank() }?.toIntOrNull(),
    )
}

@Component
class StackTraceParser {
    fun parse(logText: String, maxFrameCount: Int = 5): List<StackTraceFrame> {
        if (logText.isBlank() || maxFrameCount <= 0) {
            return emptyList()
        }

        return frameRegex.findAll(logText)
            .map { matchResult -> matchResult.toStackTraceFrame() }
            .filterNot { frame -> isExternalLibraryFrame(frame) }
            .distinctBy { frame -> "${frame.className}.${frame.methodName}:${frame.fileName}:${frame.lineNumber}" }
            .take(maxFrameCount)
            .toList()
    }
}

com.example.service.FooService.doWork(FooService.kt:42)

src/main/kotlin/com/example/service/FooService.kt
src/main/java/com/example/service/FooService.java
src/test/kotlin/com/example/service/FooService.kt
src/test/java/com/example/service/FooService.java

service-a/src/main/kotlin/...
domain/src/main/java/...

fun Path.resolveSourceFile(frame: StackTraceFrame): Path? {
    val directMatch = directSourceCandidates(frame).firstOrNull {
        it.exists() && it.isRegularFile()
    }
    if (directMatch != null) {
        return directMatch
    }

    val fallbackFileNames = frame.candidateFileNames()
    if (fallbackFileNames.isEmpty()) {
        return null
    }

    val matchedFiles = Files.walk(this).use { stream ->
        stream
            .filter { path -> path.isRegularFile() }
            .filter { path -> path.extension.lowercase() in sourceExtensions }
            .filter { path -> path.name in fallbackFileNames }
            .sorted()
            .toList()
    }

    if (matchedFiles.isEmpty()) {
        return null
    }

    return matchedFiles.firstOrNull { path -> path.hasPackageDeclaration(frame.packageName) }
        ?: matchedFiles.first()
}

private fun StackTraceFrame.candidateFileNames(): List<String> {
    return buildList {
        fileName?.takeIf { it.endsWith(".kt") || it.endsWith(".java") }?.let { add(it) }
        add("${simpleClassName}.kt")
        add("${simpleClassName}.java")
    }.distinct()
}

fun Path.extractSourceSnippet(
    repositoryRoot: Path,
    frame: StackTraceFrame,
    radius: Int = 40,
): SourceSnippet {
    val lines = readLines()
    val focusLine = frame.lineNumber?.takeIf { it in 1..lines.size }
    val startLine = if (focusLine != null) {
        (focusLine - radius).coerceAtLeast(1)
    } else {
        1
    }
    val endLine = if (focusLine != null) {
        (focusLine + radius).coerceAtMost(lines.size)
    } else {
        80.coerceAtMost(lines.size)
    }

    val content = (startLine..endLine).joinToString(System.lineSeparator()) { lineNumber ->
        val marker = if (lineNumber == focusLine) ">>" else "  "
        "$marker ${lineNumber.toString().padStart(4)} | ${lines[lineNumber - 1]}"
    }

    return SourceSnippet(
        filePath = repositoryRoot.relativize(this).invariantSeparatorsPathString,
        startLine = startLine,
        endLine = endLine,
        focusLine = focusLine,
        content = content,
    )
}

File: src/main/kotlin/com/example/service/FooService.kt
Lines: 2-82
Focus line: 42

   40 | val response = connector.call(...)
   41 | ...
>> 42 | return response.data.result.flatMap { ... }
   43 | ...

LokiQueryResult.rawLogText()
        │
        ▼
StackTraceParser.parse()
        │
        ▼
repositoryRoot.resolveSourceFile(frame)
        │
        ▼
sourceFile.extractSourceSnippet(repositoryRoot, frame)
        │
        ▼
IncidentSourceContext

@Service
class IncidentSourceContextService(
    private val stackTraceParser: StackTraceParser,
) {
    fun createContext(logResults: LokiQueryResult, repositoryRoot: Path?): IncidentSourceContext {
        val frames = stackTraceParser.parse(logResults.rawLogText())
        if (repositoryRoot == null || frames.isEmpty()) {
            return IncidentSourceContext(
                frames = frames,
                snippets = emptyList(),
                unresolvedFrames = frames,
            )
        }

        val snippets = mutableListOf<SourceSnippet>()
        val unresolvedFrames = mutableListOf<StackTraceFrame>()
        val resolvedSnippetKeys = mutableSetOf<String>()

        frames.forEach { frame ->
            val sourceFile = repositoryRoot.resolveSourceFile(frame)
            if (sourceFile == null) {
                unresolvedFrames.add(frame)
                return@forEach
            }

            val snippet = sourceFile.extractSourceSnippet(repositoryRoot, frame)
            val snippetKey = "${snippet.filePath}:${snippet.startLine}:${snippet.endLine}:${snippet.focusLine}"
            if (resolvedSnippetKeys.add(snippetKey)) {
                snippets.add(snippet)
            }
        }

        return IncidentSourceContext(
            frames = frames,
            snippets = snippets,
            unresolvedFrames = unresolvedFrames,
        )
    }
}

data class IncidentSourceContext(
    val frames: List<StackTraceFrame>,
    val snippets: List<SourceSnippet>,
    val unresolvedFrames: List<StackTraceFrame>,
) {
    fun createSourceSectionPrompt(): String {
        if (snippets.isEmpty()) {
            return ""
        }

        return buildString {
            appendLine("## Related source snippets")
            appendLine("The following snippets were selected from JVM stack trace frames. Treat them as focused source context, not as the full repository.")
            appendLine()
            snippets.forEach { snippet ->
                appendLine(snippet.createSourceSnippetPrompt())
            }
        }
    }
}

fun executeAnalyzeFiring(
    alertSection: String,
    logSection: String,
    metricSection: String = "",
    sourceSection: String = "",
): String

if (sourceSection.isNotBlank()) {
    append(sourceSection)
    appendLine()
}

---SOURCE_CODE_SUGGESTIONS_JSON_START---
[
  {
    "filePath": "relative/path/to/File.kt",
    "originalCode": "exact original code lines",
    "suggestionCode": "replacement code lines",
    "description": "Why this change helps",
    "lineNumber": 42
  }
]
---SOURCE_CODE_SUGGESTIONS_JSON_END---

public record SourceCodeSuggestion(
    String filePath,
    String originalCode,
    String suggestionCode,
    String description,
    Integer lineNumber
) { }

List<SourceCodeSuggestion> sourceCodeSuggestions

private fun parseAnalyzeFiringResponse(raw: String): Pair<String, List<SourceCodeSuggestion>> {
    val startIdx = raw.indexOf(SOURCE_CODE_SUGGESTIONS_START)
    if (startIdx == -1) {
        return Pair(raw.trim(), emptyList())
    }

    val markdown = raw.substring(0, startIdx).trim()
    val afterStart = raw.substring(startIdx + SOURCE_CODE_SUGGESTIONS_START.length)
    val endIdx = afterStart.indexOf(SOURCE_CODE_SUGGESTIONS_END)
    val jsonText = (if (endIdx == -1) afterStart else afterStart.substring(0, endIdx)).trim()
    val sanitized = codeAnalysisResultHandler.sanitizeControlChars(jsonText)

    val suggestions = runCatching {
        codeAnalysisResultHandler.parseJsonArray(sanitized, SourceCodeSuggestion::class.java)
    }.getOrElse {
        log.warn("Failed to parse source code suggestions JSON — attempting recovery: {}", it.message)
        codeAnalysisResultHandler.recoverIssuesFromJson(sanitized, SourceCodeSuggestion::class.java)
    }
    return Pair(markdown, suggestions)
}

function renderAnalysisLayers(appName, record) {
    ...
    return `
        ...
        <div class="analysis-layer">
            <div class="layer-header">AI Analysis<span class="layer-header-disclaimer">* AI-generated results may not always be accurate.</span></div>
            <div class="analysis-text markdown-body">${renderMarkdown(record.analyzeResults)}</div>
            ${renderSourceCodeSuggestions(record.sourceCodeSuggestions)}
        </div>`;
}

function renderSourceCodeSuggestions(suggestions) {
    const items = Array.isArray(suggestions) ? suggestions.filter(Boolean) : [];
    if (items.length === 0) return '';

    return `<div class="source-suggestion-list">
        <div class="source-suggestion-list-title">Source Code Suggestions</div>
        ${items.map((suggestion, idx) => {
            const filePath = suggestion.filePath || 'Unknown file';
            const line = suggestion.lineNumber == null ? '' : `:${suggestion.lineNumber}`;
            const description = suggestion.description ? `<div class="source-suggestion-summary">${escHtml(suggestion.description)}</div>` : '';
            return `<div class="source-suggestion-item">
                <button class="source-suggestion-link" data-suggestion-idx="${idx}" title="${escHtml(filePath)}">
                    ${escHtml(filePath)}${escHtml(line)}
                </button>
                ${description}
            </div>`;
        }).join('')}
    </div>`;
}

async function copySourceSuggestionCode() {
    const code = activeSourceSuggestion?.suggestionCode || '';
    if (!code) return;

    const copyBtn = document.getElementById('source-suggestion-copy-btn');
    try {
        await writeClipboardText(code);
        copyBtn.textContent = 'Copied';
    } catch (e) {
        copyBtn.textContent = 'Copy failed';
    }
}

node --check src/main/resources/static/js/aiops.js

./gradlew test --tests 'com.walter.spring.ai.ops.facade.ObservabilityFacadeTest'
./gradlew test --tests 'com.walter.spring.ai.ops.service.IncidentSourceContextServiceTest'
./gradlew test --tests 'com.walter.spring.ai.ops.util.PathExtensionsTest'

java.
javax.
jakarta.
kotlin.
org.springframework.
reactor.
io.netty.
...

Grafana Alert
  → Loki Logs
  → Stack Trace
  → Source File
  → Source Snippet
  → LLM Root Cause
  → Source Code Suggestion
  → UI Popup Compare