AI应用的可观测性：LLM调用的全链路追踪与异常诊断平台

AI应用的可观测性LLM调用的全链路追踪与异常诊断平台一、引言传统微服务的可观测性三支柱Logs、Metrics、Traces在AI应用中面临新的挑战一次LLM调用涉及Prompt构造、模型推理、Token计数和成本核算等多个维度且调用链路可能跨越多个模型提供商OpenAI、Claude、本地部署模型。当用户投诉AI回复质量差或响应太慢时运维团队需要的不仅是API调用成功的二进制判断而是能从Prompt质量、模型延迟、Token消耗等多个维度定位根因。本文将设计一套基于OpenTelemetry的LLM全链路追踪方案覆盖从Prompt构造到最终响应的完整观测。flowchart TB subgraph 应用层 APP[AI应用] SDK[观测SDKbr/OpenTelemetry集成] end subgraph 采集层 COL[OTel Collector] PROC[Span处理器br/Prompt脱敏/Token计算] end subgraph 存储与分析 TRACE[(Trace存储br/Jaeger/Tempo)] METRICS[(指标存储br/Prometheus/VictoriaMetrics)] LOGS[(日志存储br/Loki/ES)] end subgraph 诊断层 DASH[多维分析面板] ALERT[异常告警引擎] COST[成本归因分析] end APP -- SDK SDK --|Spans| COL COL -- PROC PROC -- TRACE PROC -- METRICS SDK --|结构化日志| LOGS TRACE -- DASH METRICS -- DASH LOGS -- DASH DASH -- ALERT DASH -- COST ALERT --|延迟异常| ONCALL[On-Call] ALERT --|成本异常| FINOPS[FinOps] COST --|按用户/场景归因| FINOPS style SDK fill:#4a90d9,color:#fff style PROC fill:#e67e22,color:#fff style DASH fill:#27ae60,color:#fff二、LLM调用的分布式追踪设计2.1 OpenTelemetry Span语义定义为LLM调用定义专门的Span属性捕获完整的调用上下文/** * LLM调用的OpenTelemetry Span构建器 * 为每个LLM调用创建结构化Trace Span */ Component public class LLMSpanBuilder { private final Tracer tracer; private final Meter meter; private final PromptSanitizer sanitizer; // LLM Span属性常量便于下游系统统一查询 public static final class Attributes { public static final String LLM_PROVIDER llm.provider; // openai/anthropic/local public static final String LLM_MODEL llm.model; // gpt-4o-mini/claude-3.5 public static final String LLM_PROMPT_HASH llm.prompt.hash; // Prompt SHA256脱敏 public static final String LLM_PROMPT_LENGTH llm.prompt.length; public static final String LLM_REQUEST_TOKENS llm.request_tokens; public static final String LLM_RESPONSE_TOKENS llm.response_tokens; public static final String LLM_TOTAL_TOKENS llm.total_tokens; public static final String LLM_TEMPERATURE llm.temperature; public static final String LLM_FINISH_REASON llm.finish_reason; // stop/length/content_filter public static final String LLM_COST llm.cost_usd; public static final String LLM_FIRST_TOKEN_LATENCY llm.first_token_latency_ms; public static final String LLM_ERROR_TYPE llm.error.type; // rate_limit/context_length/timeout public static final String SCENE_NAME scene.name; // 业务场景标识 public static final String USER_ID_HASH user.id_hash; // 用户标识脱敏 } public LLMSpanBuilder(OpenTelemetry openTelemetry, PromptSanitizer sanitizer) { this.tracer openTelemetry.getTracer(llm-instrumentation, 1.0.0); this.meter openTelemetry.getMeter(llm-instrumentation, 1.0.0); this.sanitizer sanitizer; } /** * 创建LLM调用Span自动注入所有标准属性 */ public Span startLLMSpan(LLMCallContext context) { Span span tracer.spanBuilder(llm.chat.completion) .setSpanKind(SpanKind.CLIENT) .setAttribute(Attributes.LLM_PROVIDER, context.getProvider()) .setAttribute(Attributes.LLM_MODEL, context.getModel()) .setAttribute(Attributes.LLM_PROMPT_HASH, hashPrompt(context.getFullPrompt())) .setAttribute(Attributes.LLM_PROMPT_LENGTH, context.getFullPrompt().length()) .setAttribute(Attributes.LLM_TEMPERATURE, context.getTemperature()) .setAttribute(Attributes.SCENE_NAME, context.getSceneName()) .setAttribute(Attributes.USER_ID_HASH, hashUserId(context.getUserId())) .startSpan(); // 将上下文注入Baggage跨服务传递 Baggage.current().toBuilder() .put(scene.name, context.getSceneName()) .put(llm.provider, context.getProvider()) .build() .storeInContext(Context.current().with(span)); return span; } /** * 填充LLM响应指标 */ public void recordResponse(Span span, LLMResponse response, Duration totalDuration, Duration firstTokenLatency) { span.setAttribute(Attributes.LLM_REQUEST_TOKENS, response.getPromptTokens()); span.setAttribute(Attributes.LLM_RESPONSE_TOKENS, response.getCompletionTokens()); span.setAttribute(Attributes.LLM_TOTAL_TOKENS, response.getTotalTokens()); span.setAttribute(Attributes.LLM_FINISH_REASON, response.getFinishReason()); span.setAttribute(Attributes.LLM_FIRST_TOKEN_LATENCY, firstTokenLatency.toMillis()); // 计算成本 double cost calculateCost( span.getAttribute(Attributes.LLM_MODEL), response.getPromptTokens(), response.getCompletionTokens() ); span.setAttribute(Attributes.LLM_COST, cost); // 记录指标 recordMetrics(span, response, totalDuration, firstTokenLatency, cost); } /** * 记录异常调用 */ public void recordError(Span span, LLMException exception) { span.setStatus(StatusCode.ERROR, exception.getMessage()); span.setAttribute(Attributes.LLM_ERROR_TYPE, exception.getErrorType()); span.recordException(exception); // 错误计数指标 meter.counterBuilder(llm.errors.total) .setDescription(Total LLM call errors) .build() .add(1, Attributes.of( io.opentelemetry.api.common.AttributeKey.stringKey(error.type), exception.getErrorType(), io.opentelemetry.api.common.AttributeKey.stringKey(provider), span.getAttribute(Attributes.LLM_PROVIDER) )); } }2.2 全链路追踪可视化一次完整的LLM调用链路包含以下Span层级Trace: ai-chat-completion-abc123 ├── Span: http.request [POST /api/chat] (200ms) │ └── Span: auth.verify_token (2ms) │ └── Span: llm.prompt.build [模版渲染变量注入] (5ms) │ │ └── Attribute: scene.name customer-service │ │ └── Attribute: llm.prompt.length 1245 │ └── Span: llm.chat.completion [调用OpenAI API] (180ms) │ │ └── Attribute: llm.provider openai │ │ └── Attribute: llm.model gpt-4o-mini │ │ └── Attribute: llm.request_tokens 312 │ │ └── Attribute: llm.response_tokens 89 │ │ └── Attribute: llm.first_token_latency_ms 95 │ │ └── Attribute: llm.cost_usd 0.00012 │ │ └── Span: http.client [POST api.openai.com] (180ms) │ │ └── Event: first_token_received 95ms │ │ └── Event: stream_completed 180ms │ └── Span: llm.response.postprocess [输出过滤/格式化] (3ms)三、多维监控与告警体系3.1 核心监控指标/** * LLM调用监控指标注册 * 覆盖延迟、Token、成本、质量四个维度 */ Component public class LLMMetricsRegistry { private final Meter meter; PostConstruct public void registerMetrics() { // 延迟维度 meter.histogramBuilder(llm.request.duration_ms) .setDescription(LLM请求总耗时ms) .setUnit(ms) .ofLongs() .buildWithCallback(measurement - { // 聚合Prometheus采集 }); meter.histogramBuilder(llm.first_token.latency_ms) .setDescription(首个Token返回延迟ms) .setUnit(ms) .build(); // Token维度 meter.counterBuilder(llm.tokens.input.total) .setDescription(输入Token总数) .setUnit(tokens) .build(); meter.counterBuilder(llm.tokens.output.total) .setDescription(输出Token总数) .setUnit(tokens) .build(); // 成本维度 meter.counterBuilder(llm.cost.total_usd) .setDescription(LLM调用总成本USD) .setUnit(USD) .build(); // 质量维度 meter.counterBuilder(llm.response.finish_reason) .setDescription(按finish_reason分类统计) .build(); } }3.2 关键告警规则# Prometheus AlertManager规则 groups: - name: llm_observability_alerts rules: # P99延迟超过3秒告警 - alert: LLMHighLatency expr: | histogram_quantile(0.99, rate(llm_request_duration_ms_bucket{scene_name!~batch.*}[5m]) ) 3000 for: 5m labels: severity: warning annotations: summary: LLM调用P99延迟超过3秒 description: 场景{{ $labels.scene_name }}的LLM调用P99延迟为{{ $value }}ms # Token消耗异常增长环比超过50% - alert: LLMTokenSpike expr: | rate(llm_tokens_output_total[15m]) / rate(llm_tokens_output_total[15m] offset 1h) 1.5 for: 10m labels: severity: warning annotations: summary: LLM Token消耗环比增长超过50% # 单日成本超过预算阈值 - alert: LLMCostOverBudget expr: | increase(llm_cost_total_usd[24h]) 500 labels: severity: critical annotations: summary: LLM单日成本超过500美元 # 错误率超过1% - alert: LLMHighErrorRate expr: | rate(llm_errors_total[5m]) / rate(llm_request_duration_ms_count[5m]) 0.01 for: 5m labels: severity: critical annotations: summary: LLM调用错误率超过1% description: 当前错误率: {{ $value | humanizePercentage }}四、异常调用自动诊断 LLM异常调用诊断引擎 - 自动归类异常模式并给出诊断建议 from dataclasses import dataclass, field from enum import Enum from typing import List, Dict, Optional class AnomalyType(Enum): LATENCY_SPIKE latency_spike # 延迟突增 TOKEN_EXPLOSION token_explosion # Token爆炸 EMPTY_RESPONSE empty_response # 空响应 TRUNCATED truncated # 输出截断 RATE_LIMITED rate_limited # 限流 CONTENT_FILTERED content_filtered # 内容过滤 PROMPT_INJECTION prompt_injection # 疑似Prompt注入 dataclass class AnomalyDiagnosis: 异常诊断结果 trace_id: str anomaly_type: AnomalyType severity: str evidence: List[str] field(default_factorylist) suggested_action: str class LLMAnomalyDiagnoser: LLM异常调用自动诊断器 def diagnose(self, span_data: Dict) - Optional[AnomalyDiagnosis]: 基于Span数据进行多维度异常诊断 anomalies [] # 检查延迟异常 latency_ms span_data.get(duration_ms, 0) first_token_ms span_data.get(first_token_latency_ms, 0) if latency_ms 5000: if first_token_ms 4000: anomalies.append(self._build_diagnosis( span_data, AnomalyType.LATENCY_SPIKE, critical, evidence[ f总延迟: {latency_ms}ms基线P99: 2000ms, f首Token延迟: {first_token_ms}ms基线P99: 800ms ], action检查模型提供方状态页考虑降级到备用模型 )) # 检查Token异常 output_tokens span_data.get(response_tokens, 0) expected_max span_data.get(max_tokens, 4096) if output_tokens expected_max * 0.95: anomalies.append(self._build_diagnosis( span_data, AnomalyType.TRUNCATED, high, evidence[ f输出Token: {output_tokens}达到max_tokens的{output_tokens/expected_max*100:.0f}% ], action提高max_tokens限制或优化Prompt以减少输出长度 )) # 检查内容过滤 finish_reason span_data.get(finish_reason, ) if finish_reason content_filter: anomalies.append(self._build_diagnosis( span_data, AnomalyType.CONTENT_FILTERED, medium, evidence[模型触发内容安全过滤], action检查Prompt和上下文是否包含敏感内容 )) # 检查疑似Prompt注入异常长的Prompt或特殊模式 prompt_length span_data.get(prompt_length, 0) avg_prompt_len span_data.get(avg_prompt_len_scene, 500) if prompt_length avg_prompt_len * 5: anomalies.append(self._build_diagnosis( span_data, AnomalyType.PROMPT_INJECTION, high, evidence[ fPrompt长度: {prompt_length}场景均值: {avg_prompt_len} ], action排查是否存在用户输入直接拼入System Prompt的情况 )) # 返回严重度最高的诊断 if anomalies: anomalies.sort(keylambda a: [low, medium, high, critical].index(a.severity), reverseTrue) return anomalies[0] return None def _build_diagnosis( self, span: Dict, anomaly_type: AnomalyType, severity: str, evidence: List[str], action: str ) - AnomalyDiagnosis: return AnomalyDiagnosis( trace_idspan.get(trace_id, ), anomaly_typeanomaly_type, severityseverity, evidenceevidence, suggested_actionaction )五、总结LLM应用的可观测性建设应遵循先打通链路再做多维分析最后引入自动诊断的渐进式路径。第一步确保每个LLM调用的Span包含provider、model、tokens、cost、latency和finish_reason六个核心属性——这是后续所有分析的数据基础。第二步搭建面向不同角色的面板——SRE关注延迟和错误率FinOps关注成本和Token分布产品团队关注finish_reason分布和场景级质量指标。第三步在积累2-4周基线数据后引入异常自动诊断引擎通过统计模型识别偏离基线的异常模式。在实践中需要特别注意Prompt数据的安全处理全链路追踪时必须对Prompt内容进行脱敏建议仅保留SHA256哈希避免将用户敏感信息写入可被广泛访问的Trace系统。成本统计建议精确到场景级别scene_name这样可以在月底清晰地看到每个业务场景的AI调用开销为后续的模型选型和Prompt优化提供数据支撑。