{"version":"0.40.0","built_at":"2026-10-31T14:19:31.362Z","license":"CC BY 4.0 © Mike Reams","canonical":"https://catalog.mikereams.com","category":{"id":"TR-110.60","name":"AI Observability & Evaluation","description":"Tracing what models and agents were asked, what they did and what it cost, and scoring their output against test sets and live traffic so quality changes show up in measurements rather than in user complaints.","aliases":["LLM Observability","LLM Evaluation","Evals","Agent Tracing"],"in_scope":["Traces of prompts, tool calls and agent steps","Offline evaluation against curated test sets","Online scoring, user feedback and human review queues","Token, cost and latency tracking per model call"],"out_of_scope":["General application performance monitoring (see TR-90.30)","Training, experiment tracking and model registry (see TR-110.20)","Serving and scaling the model endpoint itself (see TR-110.50)"],"capability_ids":["BC-770.30.40","BC-770.20.40","BC-770.10.30","BC-770.50.30","BC-730.60.30"],"examples":["Langfuse","LangSmith","Arize Phoenix","Braintrust","DeepEval","Ragas"],"domain_id":"TR-110","domain":"AI & Machine Learning","page_url":"https://catalog.mikereams.com/technology/TR-110.60","business_capabilities":[{"id":"BC-770.30.40","name":"Output Evaluation & Guardrails"},{"id":"BC-770.20.40","name":"Model Monitoring & Retraining"},{"id":"BC-770.10.30","name":"AI Risk & Model Review"},{"id":"BC-770.50.30","name":"AI Cost Management"},{"id":"BC-730.60.30","name":"Observability Platform"}],"technology_ids":["TE-amazon-bedrock-agentcore","TE-langfuse","TE-deepeval","TE-litellm"]},"domain":{"id":"TR-110","name":"AI & Machine Learning","description":"The models, platforms and plumbing behind machine learning and generative AI, from training to agents in production.","category_ids":["TR-110.10","TR-110.20","TR-110.30","TR-110.40","TR-110.50","TR-110.60"],"page_url":"https://catalog.mikereams.com/technology#TR-110"},"technologies":[{"id":"TE-amazon-bedrock-agentcore","name":"Amazon Bedrock AgentCore","kind":"cloud-service","page_url":"https://catalog.mikereams.com/technology/TE-amazon-bedrock-agentcore"},{"id":"TE-langfuse","name":"Langfuse","kind":"platform","page_url":"https://catalog.mikereams.com/technology/TE-langfuse"},{"id":"TE-deepeval","name":"DeepEval","kind":"framework","page_url":"https://catalog.mikereams.com/technology/TE-deepeval"},{"id":"TE-litellm","name":"LiteLLM","kind":"platform","page_url":"https://catalog.mikereams.com/technology/TE-litellm"}]}