diff --git a/bpf/errors.h b/bpf/errors.h new file mode 100644 index 000000000..74a31852d --- /dev/null +++ b/bpf/errors.h @@ -0,0 +1,27 @@ +#ifndef __ERRORS_H_ +#define __ERRORS_H_ + +#ifndef TASK_COMM_LEN +#define TASK_COMM_LEN 16 +#endif + +#ifndef ERR_MSG_LEN +#define ERR_MSG_LEN 128 +#endif + +#ifndef MAX_STACK_DEPTH +#define MAX_STACK_DEPTH 32 +#endif + +typedef __u64 stack_trace_t[MAX_STACK_DEPTH]; + +typedef struct error_event { + __u32 pid; + __u32 cpu_id; + char comm[TASK_COMM_LEN]; + __s32 ustack_sz; + stack_trace_t ustack; + u8 err_msg[ERR_MSG_LEN]; +} error_event; + +#endif /* __ERRORS_H_ */ diff --git a/bpf/go_nethttp.h b/bpf/go_nethttp.h index 3ad437db2..357e2fdd4 100644 --- a/bpf/go_nethttp.h +++ b/bpf/go_nethttp.h @@ -24,6 +24,7 @@ #include "tracing.h" #include "hpack.h" #include "ringbuf.h" +#include "errors.h" typedef struct http_client_data { u8 method[METHOD_MAX_LEN]; @@ -57,6 +58,13 @@ struct { __uint(max_entries, MAX_CONCURRENT_REQUESTS); } ongoing_http_server_requests SEC(".maps"); +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __type(key, go_addr_key_t); // key: pointer to the request goroutine + __type(value, struct error_event); + __uint(max_entries, MAX_CONCURRENT_REQUESTS); +} last_error SEC(".maps"); + /* HTTP Server */ // This instrumentation attaches uprobe to the following function: @@ -246,6 +254,9 @@ int uprobe_ServeHTTPReturns(struct pt_regs *ctx) { make_tp_string(tp_buf, &invocation->tp); bpf_dbg_printk("tp: %s", tp_buf); + struct error_event *error = bpf_map_lookup_elem(&last_error, &g_key); + bpf_map_delete_elem(&last_error, &g_key); + http_request_trace *trace = bpf_ringbuf_reserve(&events, sizeof(http_request_trace), 0); if (!trace) { bpf_dbg_printk("can't reserve space in the ringbuffer"); @@ -256,7 +267,9 @@ int uprobe_ServeHTTPReturns(struct pt_regs *ctx) { trace->type = EVENT_HTTP_REQUEST; trace->start_monotime_ns = invocation->start_monotime_ns; trace->end_monotime_ns = bpf_ktime_get_ns(); - + if (error) { + trace->error = *error; + } goroutine_metadata *g_metadata = bpf_map_lookup_elem(&ongoing_goroutines, &g_key); if (g_metadata) { trace->go_start_monotime_ns = g_metadata->timestamp; @@ -473,6 +486,79 @@ int uprobe_roundTripReturn(struct pt_regs *ctx) { return 0; } +SEC("uprobe/error") +int uprobe_error(struct pt_regs *ctx) { + bpf_dbg_printk("=== uprobe/proc error === "); + + void *goroutine_addr = GOROUTINE_PTR(ctx); + go_addr_key_t g_key = {}; + go_addr_key_from_id(&g_key, goroutine_addr); + bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); + + int pid = bpf_get_current_pid_tgid() >> 32; + int cpu_id = bpf_get_smp_processor_id(); + int BPF_F_USER_STACK = (1ULL << 8); + struct error_event event = { + .pid = pid, + .cpu_id = cpu_id, + }; + + if (bpf_get_current_comm(event.comm, sizeof(event.comm))) { + event.comm[0] = 0; + } + + // Read the stack trace + event.ustack_sz = bpf_get_stack(ctx, event.ustack, sizeof(event.ustack), BPF_F_USER_STACK); + + // Get the caller of the error function and store it in the first slot of the stack + void *sp_caller = STACK_PTR(ctx); + u64 caller = 0; + bpf_probe_read(&caller, sizeof(u64), sp_caller); + bpf_dbg_printk("sp_caller %lx caller %lx", sp_caller, caller); + event.ustack[0] = caller; + + // Write event + if (bpf_map_update_elem(&last_error, &g_key, &event, BPF_ANY)) { + bpf_dbg_printk("can't update event error map element"); + } + return 0; +} + +SEC("uprobe/error_return") +int uprobe_errorReturn(struct pt_regs *ctx) { + bpf_dbg_printk("=== uprobe/proc error return === "); + + void *goroutine_addr = GOROUTINE_PTR(ctx); + go_addr_key_t g_key = {}; + go_addr_key_from_id(&g_key, goroutine_addr); + bpf_dbg_printk("goroutine_addr %lx", goroutine_addr); + + error_event *event = bpf_map_lookup_elem(&last_error, &g_key); + if (event == NULL) { + bpf_dbg_printk("can't read error event"); + return 0; + } + + // Read the error message + // GO_PARAM1(ctx) is the pointer to the error message + // GO_PARAM2(ctx) is the length of the error message + void *msg_ptr = GO_PARAM1(ctx); + u64 len = (u64)GO_PARAM2(ctx); + u64 max_size = sizeof(event->err_msg); + u64 size = max_size < len ? max_size : len; + bpf_probe_read(&event->err_msg, size, msg_ptr); + if (size < max_size) { + ((char *)event->err_msg)[size] = 0; + } + bpf_dbg_printk("error msg %llx, %s", msg_ptr, event->err_msg); + + // Write event + if (bpf_map_update_elem(&last_error, &g_key, event, BPF_ANY)) { + bpf_dbg_printk("can't update event error map element"); + } + return 0; +} + #ifndef NO_HEADER_PROPAGATION // Context propagation through HTTP headers SEC("uprobe/header_writeSubset") diff --git a/bpf/headers/utils.h b/bpf/headers/utils.h index 37413daef..7ea2c6b27 100644 --- a/bpf/headers/utils.h +++ b/bpf/headers/utils.h @@ -21,35 +21,36 @@ #if defined(__TARGET_ARCH_x86) -#define GO_PARAM1(x) ((void*)(x)->ax) -#define GO_PARAM2(x) ((void*)(x)->bx) -#define GO_PARAM3(x) ((void*)(x)->cx) -#define GO_PARAM4(x) ((void*)(x)->di) -#define GO_PARAM5(x) ((void*)(x)->si) -#define GO_PARAM6(x) ((void*)(x)->r8) -#define GO_PARAM7(x) ((void*)(x)->r9) -#define GO_PARAM8(x) ((void*)(x)->r10) -#define GO_PARAM9(x) ((void*)(x)->r11) +#define GO_PARAM1(x) ((void *)(x)->ax) +#define GO_PARAM2(x) ((void *)(x)->bx) +#define GO_PARAM3(x) ((void *)(x)->cx) +#define GO_PARAM4(x) ((void *)(x)->di) +#define GO_PARAM5(x) ((void *)(x)->si) +#define GO_PARAM6(x) ((void *)(x)->r8) +#define GO_PARAM7(x) ((void *)(x)->r9) +#define GO_PARAM8(x) ((void *)(x)->r10) +#define GO_PARAM9(x) ((void *)(x)->r11) // In x86, current goroutine is pointed by r14, according to // https://go.googlesource.com/go/+/refs/heads/dev.regabi/src/cmd/compile/internal-abi.md#amd64-architecture -#define GOROUTINE_PTR(x) ((void*)(x)->r14) - +#define GOROUTINE_PTR(x) ((void *)(x)->r14) +#define STACK_PTR(x) ((void *)(x)->sp) #elif defined(__TARGET_ARCH_arm64) -#define GO_PARAM1(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[0]) -#define GO_PARAM2(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[1]) -#define GO_PARAM3(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[2]) -#define GO_PARAM4(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[3]) -#define GO_PARAM5(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[4]) -#define GO_PARAM6(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[5]) -#define GO_PARAM7(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[6]) -#define GO_PARAM8(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[7]) -#define GO_PARAM9(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[8]) +#define GO_PARAM1(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[0]) +#define GO_PARAM2(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[1]) +#define GO_PARAM3(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[2]) +#define GO_PARAM4(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[3]) +#define GO_PARAM5(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[4]) +#define GO_PARAM6(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[5]) +#define GO_PARAM7(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[6]) +#define GO_PARAM8(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[7]) +#define GO_PARAM9(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[8]) // In arm64, current goroutine is pointed by R28 according to // https://github.com/golang/go/blob/master/src/cmd/compile/abi-internal.md#arm64-architecture -#define GOROUTINE_PTR(x) ((void*)((PT_REGS_ARM64 *)(x))->regs[28]) +#define GOROUTINE_PTR(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[28]) +#define STACK_PTR(x) ((void *)((PT_REGS_ARM64 *)(x))->regs[13]) #endif /*defined(__TARGET_ARCH_arm64)*/ @@ -58,5 +59,5 @@ "%0 = %[max]\n" \ : "+r"(VAR) \ : [max] "i"(UMAX)) - + #endif /* __UTILS_H__ */ diff --git a/bpf/tracer_common.h b/bpf/tracer_common.h index 2fcb6b2c7..a21cc6ebe 100644 --- a/bpf/tracer_common.h +++ b/bpf/tracer_common.h @@ -15,6 +15,7 @@ #include "pid_types.h" #include "utils.h" +#include "errors.h" #include "http_types.h" #define PATH_MAX_LEN 100 @@ -40,6 +41,7 @@ typedef struct http_request_trace_t { u16 status; connection_info_t conn __attribute__((aligned(8))); s64 content_length; + error_event error; tp_info_t tp; pid_info pid; diff --git a/pkg/beyla/config.go b/pkg/beyla/config.go index cb55fba08..a6c2c8fe3 100644 --- a/pkg/beyla/config.go +++ b/pkg/beyla/config.go @@ -75,11 +75,12 @@ var DefaultConfig = Config{ TTL: defaultMetricsTTL, }, Traces: otel.TracesConfig{ - Protocol: otel.ProtocolUnset, - TracesProtocol: otel.ProtocolUnset, - MaxQueueSize: 4096, - MaxExportBatchSize: 4096, - ReportersCacheLen: ReporterLRUSize, + Protocol: otel.ProtocolUnset, + TracesProtocol: otel.ProtocolUnset, + MaxQueueSize: 4096, + MaxExportBatchSize: 4096, + ReportersCacheLen: ReporterLRUSize, + ReportExceptionEvents: false, Instrumentations: []string{ instrumentations.InstrumentationALL, }, diff --git a/pkg/export/alloy/traces.go b/pkg/export/alloy/traces.go index 55ff700d0..e9748ccfc 100644 --- a/pkg/export/alloy/traces.go +++ b/pkg/export/alloy/traces.go @@ -17,7 +17,7 @@ import ( func TracesReceiver( ctx context.Context, ctxInfo *global.ContextInfo, - cfg *beyla.TracesReceiverConfig, + cfg *beyla.Config, userAttribSelection attributes.Selection, ) pipe.FinalProvider[[]request.Span] { return (&tracesReceiver{ctx: ctx, cfg: cfg, attributes: userAttribSelection, hostID: ctxInfo.HostID}).provideLoop @@ -25,7 +25,7 @@ func TracesReceiver( type tracesReceiver struct { ctx context.Context - cfg *beyla.TracesReceiverConfig + cfg *beyla.Config attributes attributes.Selection hostID string } @@ -35,7 +35,7 @@ func (tr *tracesReceiver) spanDiscarded(span *request.Span) bool { } func (tr *tracesReceiver) provideLoop() (pipe.FinalFunc[[]request.Span], error) { - if !tr.cfg.Enabled() { + if !tr.cfg.TracesReceiver.Enabled() { return pipe.IgnoreFinal[[]request.Span](), nil } return func(in <-chan []request.Span) { @@ -53,8 +53,8 @@ func (tr *tracesReceiver) provideLoop() (pipe.FinalFunc[[]request.Span], error) } envResourceAttrs := otel.ResourceAttrsFromEnv(&span.ServiceID) - for _, tc := range tr.cfg.Traces { - traces := otel.GenerateTraces(span, tr.hostID, traceAttrs, envResourceAttrs) + for _, tc := range tr.cfg.TracesReceiver.Traces { + traces := otel.GenerateTraces(tr.cfg.Traces, span, tr.hostID, traceAttrs, envResourceAttrs) err := tc.ConsumeTraces(tr.ctx, traces) if err != nil { slog.Error("error sending trace to consumer", "error", err) diff --git a/pkg/export/alloy/traces_test.go b/pkg/export/alloy/traces_test.go index 35bec48b7..ef3a1800f 100644 --- a/pkg/export/alloy/traces_test.go +++ b/pkg/export/alloy/traces_test.go @@ -59,7 +59,7 @@ func TestTracesSkipsInstrumented(t *testing.T) { func makeTracesTestReceiver() *tracesReceiver { return &tracesReceiver{ ctx: context.Background(), - cfg: &beyla.TracesReceiverConfig{}, + cfg: &beyla.Config{}, attributes: attributes.Selection{}, hostID: "Alloy", } @@ -69,12 +69,13 @@ func generateTracesForSpans(t *testing.T, tr *tracesReceiver, spans []request.Sp res := []ptrace.Traces{} traceAttrs, err := otel.GetUserSelectedAttributes(tr.attributes) assert.NoError(t, err) + cfg := otel.TracesConfig{} for i := range spans { span := &spans[i] if tr.spanDiscarded(span) { continue } - res = append(res, otel.GenerateTraces(span, tr.hostID, traceAttrs, []attribute.KeyValue{})) + res = append(res, otel.GenerateTraces(cfg, span, tr.hostID, traceAttrs, []attribute.KeyValue{})) } return res diff --git a/pkg/export/debug/debug.go b/pkg/export/debug/debug.go index 8adf724bc..8a380f34f 100644 --- a/pkg/export/debug/debug.go +++ b/pkg/export/debug/debug.go @@ -115,6 +115,9 @@ func textPrinter(input <-chan []request.Span) { spans[i].ServiceID.SDKLanguage.String(), traceparent(&spans[i]), ) + if spans[i].ErrorMessage != "" { + fmt.Printf("error_message=%s stacktrace=\n%s\n", spans[i].ErrorMessage, spans[i].ErrorStacktrace) + } } } } diff --git a/pkg/export/otel/traces.go b/pkg/export/otel/traces.go index 958b76fe1..86a8a84e0 100644 --- a/pkg/export/otel/traces.go +++ b/pkg/export/otel/traces.go @@ -85,6 +85,9 @@ type TracesConfig struct { // BackOffMaxElapsedTime is the maximum amount of time (including retries) spent trying to send a request/batch. BackOffMaxElapsedTime time.Duration `yaml:"backoff_max_elapsed_time" env:"BEYLA_BACKOFF_MAX_ELAPSED_TIME"` + // ReportExceptionEvents enables the reporting of exception events. + ReportExceptionEvents bool `yaml:"report_exception_events" env:"BEYLA_TRACES_REPORT_EXCEPTION_EVENTS"` + ReportersCacheLen int `yaml:"reporters_cache_len" env:"BEYLA_TRACES_REPORT_CACHE_LEN"` // SDKLogLevel works independently from the global LogLevel because it prints GBs of logs in Debug mode @@ -195,7 +198,7 @@ func (tr *tracesOTELReceiver) processSpans(exp exporter.Traces, spans []request. } envResourceAttrs := ResourceAttrsFromEnv(&span.ServiceID) - traces := GenerateTracesWithAttributes(span, tr.ctxInfo.HostID, finalAttrs, envResourceAttrs) + traces := GenerateTracesWithAttributes(tr.cfg, span, tr.ctxInfo.HostID, finalAttrs, envResourceAttrs) err := exp.ConsumeTraces(tr.ctx, traces) if err != nil { slog.Error("error sending trace to consumer", "error", err) @@ -415,7 +418,7 @@ func traceAppResourceAttrs(hostID string, service *svc.ID) []attribute.KeyValue return attrs } -func GenerateTracesWithAttributes(span *request.Span, hostID string, attrs []attribute.KeyValue, envResourceAttrs []attribute.KeyValue) ptrace.Traces { +func GenerateTracesWithAttributes(cfg TracesConfig, span *request.Span, hostID string, attrs []attribute.KeyValue, envResourceAttrs []attribute.KeyValue) ptrace.Traces { t := span.Timings() start := spanStartTime(t) hasSubSpans := t.Start.After(start) @@ -457,6 +460,15 @@ func GenerateTracesWithAttributes(span *request.Span, hostID string, attrs []att m := attrsToMap(attrs) m.CopyTo(s.Attributes()) + // Set error message and stacktrace + if cfg.ReportExceptionEvents && span.ErrorMessage != "" { + e := s.Events().AppendEmpty() + e.SetName(semconv.ExceptionEventName) + e.Attributes().PutStr(string(semconv.ExceptionMessageKey), span.ErrorMessage) + e.Attributes().PutStr(string(semconv.ExceptionTypeKey), "error") + e.Attributes().PutStr(string(semconv.ExceptionStacktraceKey), span.ErrorStacktrace) + } + // Set status code statusCode := codeToStatusCode(request.SpanStatusCode(span)) s.Status().SetCode(statusCode) @@ -465,8 +477,8 @@ func GenerateTracesWithAttributes(span *request.Span, hostID string, attrs []att } // GenerateTraces creates a ptrace.Traces from a request.Span -func GenerateTraces(span *request.Span, hostID string, userAttrs map[attr.Name]struct{}, envResourceAttrs []attribute.KeyValue) ptrace.Traces { - return GenerateTracesWithAttributes(span, hostID, traceAttributes(span, userAttrs), envResourceAttrs) +func GenerateTraces(cfg TracesConfig, span *request.Span, hostID string, userAttrs map[attr.Name]struct{}, envResourceAttrs []attribute.KeyValue) ptrace.Traces { + return GenerateTracesWithAttributes(cfg, span, hostID, traceAttributes(span, userAttrs), envResourceAttrs) } // createSubSpans creates the internal spans for a request.Span diff --git a/pkg/export/otel/traces_test.go b/pkg/export/otel/traces_test.go index bef07b72d..bbea53920 100644 --- a/pkg/export/otel/traces_test.go +++ b/pkg/export/otel/traces_test.go @@ -322,18 +322,23 @@ func TestGenerateTraces(t *testing.T) { spanID, _ := trace.SpanIDFromHex("89cbc1f60aab3b01") traceID, _ := trace.TraceIDFromHex("eae56fbbec9505c102e8aabfc6b5c481") span := &request.Span{ - Type: request.EventTypeHTTP, - RequestStart: start.UnixNano(), - Start: start.Add(time.Second).UnixNano(), - End: start.Add(3 * time.Second).UnixNano(), - Method: "GET", - Route: "/test", - Status: 200, - ParentSpanID: parentSpanID, - TraceID: traceID, - SpanID: spanID, + Type: request.EventTypeHTTP, + RequestStart: start.UnixNano(), + Start: start.Add(time.Second).UnixNano(), + End: start.Add(3 * time.Second).UnixNano(), + Method: "GET", + Route: "/test", + Status: 200, + ParentSpanID: parentSpanID, + TraceID: traceID, + SpanID: spanID, + ErrorMessage: "crash", + ErrorStacktrace: "function\nline", } - traces := GenerateTraces(span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) + cfg := TracesConfig{ + ReportExceptionEvents: true, + } + traces := GenerateTraces(cfg, span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -361,6 +366,13 @@ func TestGenerateTraces(t *testing.T) { assert.NotEqual(t, spans.At(0).SpanID().String(), spans.At(1).SpanID().String()) assert.NotEqual(t, spans.At(1).SpanID().String(), spans.At(2).SpanID().String()) + + e := spans.At(2).Events().At(0) + val, _ := e.Attributes().Get(string(semconv.ExceptionMessageKey)) + assert.Equal(t, "crash", val.AsString()) + val, _ = e.Attributes().Get(string(semconv.ExceptionStacktraceKey)) + assert.Equal(t, "function\nline", val.AsString()) + }) t.Run("test with subtraces - ids set bpf layer", func(t *testing.T) { @@ -378,7 +390,8 @@ func TestGenerateTraces(t *testing.T) { SpanID: spanID, TraceID: traceID, } - traces := GenerateTraces(span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -414,7 +427,8 @@ func TestGenerateTraces(t *testing.T) { Route: "/test", Status: 200, } - traces := GenerateTraces(span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -451,7 +465,8 @@ func TestGenerateTraces(t *testing.T) { SpanID: spanID, TraceID: traceID, } - traces := GenerateTraces(span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -476,7 +491,8 @@ func TestGenerateTraces(t *testing.T) { ParentSpanID: parentSpanID, TraceID: traceID, } - traces := GenerateTraces(span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -496,7 +512,8 @@ func TestGenerateTraces(t *testing.T) { Method: "GET", Route: "/test", } - traces := GenerateTraces(span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -512,7 +529,8 @@ func TestGenerateTraces(t *testing.T) { func TestGenerateTracesAttributes(t *testing.T) { t.Run("test SQL trace generation, no statement", func(t *testing.T) { span := makeSQLRequestSpan("SELECT password FROM credentials WHERE username=\"bill\"") - traces := GenerateTraces(&span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, &span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -533,7 +551,8 @@ func TestGenerateTracesAttributes(t *testing.T) { t.Run("test SQL trace generation, unknown attribute", func(t *testing.T) { span := makeSQLRequestSpan("SELECT password FROM credentials WHERE username=\"bill\"") - traces := GenerateTraces(&span, "host-id", map[attr.Name]struct{}{"db.operation.name": {}}, []attribute.KeyValue{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, &span, "host-id", map[attr.Name]struct{}{"db.operation.name": {}}, []attribute.KeyValue{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -554,7 +573,8 @@ func TestGenerateTracesAttributes(t *testing.T) { t.Run("test SQL trace generation, unknown attribute", func(t *testing.T) { span := makeSQLRequestSpan("SELECT password FROM credentials WHERE username=\"bill\"") - traces := GenerateTraces(&span, "host-id", map[attr.Name]struct{}{attr.DBQueryText: {}}, []attribute.KeyValue{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, &span, "host-id", map[attr.Name]struct{}{attr.DBQueryText: {}}, []attribute.KeyValue{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -574,7 +594,8 @@ func TestGenerateTracesAttributes(t *testing.T) { }) t.Run("test Kafka trace generation", func(t *testing.T) { span := request.Span{Type: request.EventTypeKafkaClient, Method: "process", Path: "important-topic", Statement: "test"} - traces := GenerateTraces(&span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, &span, "host-id", map[attr.Name]struct{}{}, []attribute.KeyValue{}) assert.Equal(t, 1, traces.ResourceSpans().Len()) assert.Equal(t, 1, traces.ResourceSpans().At(0).ScopeSpans().Len()) @@ -594,7 +615,8 @@ func TestGenerateTracesAttributes(t *testing.T) { defer restoreEnvAfterExecution()() require.NoError(t, os.Setenv(envResourceAttrs, "deployment.environment=productions,source.upstream=beyla")) span := request.Span{Type: request.EventTypeHTTP, Method: "GET", Route: "/test", Status: 200} - traces := GenerateTraces(&span, "host-id", map[attr.Name]struct{}{}, ResourceAttrsFromEnv(&span.ServiceID)) + cfg := TracesConfig{} + traces := GenerateTraces(cfg, &span, "host-id", map[attr.Name]struct{}{}, ResourceAttrsFromEnv(&span.ServiceID)) assert.Equal(t, 1, traces.ResourceSpans().Len()) rs := traces.ResourceSpans().At(0) @@ -1428,7 +1450,8 @@ func generateTracesForSpans(t *testing.T, tr *tracesOTELReceiver, spans []reques if tr.spanDiscarded(span) { continue } - res = append(res, GenerateTraces(span, "host-id", traceAttrs, []attribute.KeyValue{})) + cfg := TracesConfig{} + res = append(res, GenerateTraces(cfg, span, "host-id", traceAttrs, []attribute.KeyValue{})) } return res diff --git a/pkg/internal/discover/attacher.go b/pkg/internal/discover/attacher.go index 6508aa1e9..6f533be58 100644 --- a/pkg/internal/discover/attacher.go +++ b/pkg/internal/discover/attacher.go @@ -2,6 +2,7 @@ package discover import ( "context" + "debug/gosym" "log/slog" "os" @@ -285,9 +286,13 @@ func (ta *TraceAttacher) monitorPIDs(tracer *ebpf.ProcessTracer, ie *ebpf.Instru ie.FileInfo.Service.SDKLanguage = ie.Type // allowing the tracer to forward traces from the discovered PID and its children processes - tracer.AllowPID(uint32(ie.FileInfo.Pid), ie.FileInfo.Ns, &ie.FileInfo.Service) + var symTab *gosym.Table + if ie.Offsets != nil && ie.Offsets.SymTab != nil { + symTab = ie.Offsets.SymTab + } + tracer.AllowPID(uint32(ie.FileInfo.Pid), ie.FileInfo.Ns, &ie.FileInfo.Service, symTab) for _, pid := range ie.ChildPids { - tracer.AllowPID(pid, ie.FileInfo.Ns, &ie.FileInfo.Service) + tracer.AllowPID(pid, ie.FileInfo.Ns, &ie.FileInfo.Service, symTab) } if ta.SpanSignalsShortcut != nil { spans := make([]request.Span, 0, len(ie.ChildPids)+1) diff --git a/pkg/internal/discover/typer.go b/pkg/internal/discover/typer.go index f392ab186..3d0f66156 100644 --- a/pkg/internal/discover/typer.go +++ b/pkg/internal/discover/typer.go @@ -158,7 +158,7 @@ func (t *typer) inspectOffsets(execElf *exec.FileInfo) (*goexec.Offsets, bool, e t.log.Debug("skipping inspection for Go functions", "pid", execElf.Pid, "comm", execElf.CmdExePath) } else { t.log.Debug("inspecting", "pid", execElf.Pid, "comm", execElf.CmdExePath) - offsets, err := goexec.InspectOffsets(execElf, t.allGoFunctions) + offsets, err := goexec.InspectOffsets(&t.cfg.Traces, execElf, t.allGoFunctions) if err != nil { t.log.Debug("couldn't find go specific tracers", "error", err) return nil, false, err diff --git a/pkg/internal/ebpf/common/bpf_arm64_bpfel.go b/pkg/internal/ebpf/common/bpf_arm64_bpfel.go index 17f827402..8fcd07fab 100644 --- a/pkg/internal/ebpf/common/bpf_arm64_bpfel.go +++ b/pkg/internal/ebpf/common/bpf_arm64_bpfel.go @@ -90,7 +90,16 @@ type bpfHttpRequestTrace struct { _ [2]byte Conn bpfConnectionInfoT ContentLength int64 - Tp struct { + Error struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 + } + Tp struct { TraceId [16]uint8 SpanId [8]uint8 ParentId [8]uint8 diff --git a/pkg/internal/ebpf/common/bpf_arm64_bpfel.o b/pkg/internal/ebpf/common/bpf_arm64_bpfel.o index 5ad5e9c95..fb9b70ae5 100644 --- a/pkg/internal/ebpf/common/bpf_arm64_bpfel.o +++ b/pkg/internal/ebpf/common/bpf_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:34d534081f766ad8d8e1ad3aa023b0160c094ad8169e0749992b08521734b822 -size 4432 +oid sha256:0d9473f1cdef0cce1da8be157aaac32d1e8dda385a4d667149b381ace9c1d612 +size 4728 diff --git a/pkg/internal/ebpf/common/bpf_x86_bpfel.go b/pkg/internal/ebpf/common/bpf_x86_bpfel.go index 12b31bd99..5ee25a7e3 100644 --- a/pkg/internal/ebpf/common/bpf_x86_bpfel.go +++ b/pkg/internal/ebpf/common/bpf_x86_bpfel.go @@ -90,7 +90,16 @@ type bpfHttpRequestTrace struct { _ [2]byte Conn bpfConnectionInfoT ContentLength int64 - Tp struct { + Error struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 + } + Tp struct { TraceId [16]uint8 SpanId [8]uint8 ParentId [8]uint8 diff --git a/pkg/internal/ebpf/common/bpf_x86_bpfel.o b/pkg/internal/ebpf/common/bpf_x86_bpfel.o index 5ad5e9c95..fb9b70ae5 100644 --- a/pkg/internal/ebpf/common/bpf_x86_bpfel.o +++ b/pkg/internal/ebpf/common/bpf_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:34d534081f766ad8d8e1ad3aa023b0160c094ad8169e0749992b08521734b822 -size 4432 +oid sha256:0d9473f1cdef0cce1da8be157aaac32d1e8dda385a4d667149b381ace9c1d612 +size 4728 diff --git a/pkg/internal/ebpf/common/common.go b/pkg/internal/ebpf/common/common.go index 3060a24cd..d823e99cf 100644 --- a/pkg/internal/ebpf/common/common.go +++ b/pkg/internal/ebpf/common/common.go @@ -103,8 +103,7 @@ func ReadBPFTraceAsSpan(record *ringbuf.Record, filter ServiceFilter) (request.S if err != nil { return request.Span{}, true, err } - - return HTTPRequestTraceToSpan(&event), false, nil + return HTTPRequestTraceToSpan(&event, filter), false, nil } func ReadSQLRequestTraceAsSpan(record *ringbuf.Record) (request.Span, bool, error) { diff --git a/pkg/internal/ebpf/common/pids.go b/pkg/internal/ebpf/common/pids.go index 3faa8601f..3b603e447 100644 --- a/pkg/internal/ebpf/common/pids.go +++ b/pkg/internal/ebpf/common/pids.go @@ -1,6 +1,7 @@ package ebpfcommon import ( + "debug/gosym" "log/slog" "sync" @@ -32,11 +33,12 @@ type PIDInfo struct { } type ServiceFilter interface { - AllowPID(uint32, uint32, *svc.ID, PIDType) + AllowPID(uint32, uint32, *svc.ID, PIDType, *gosym.Table) BlockPID(uint32, uint32) ValidPID(uint32, uint32, PIDType) bool Filter(inputSpans []request.Span) []request.Span CurrentPIDs(PIDType) map[uint32]map[uint32]svc.ID + GetSymTab(uint32) *gosym.Table } // PIDsFilter keeps a thread-safe copy of the PIDs whose traces are allowed to @@ -47,6 +49,7 @@ type PIDsFilter struct { current map[uint32]map[uint32]PIDInfo mux *sync.RWMutex detectOtel bool + symTabs map[uint32]*gosym.Table } var commonPIDsFilter *PIDsFilter @@ -58,6 +61,7 @@ func newPIDsFilter(c *services.DiscoveryConfig, log *slog.Logger) *PIDsFilter { current: map[uint32]map[uint32]PIDInfo{}, mux: &sync.RWMutex{}, detectOtel: c.ExcludeOTelInstrumentedServices, + symTabs: make(map[uint32]*gosym.Table), } } @@ -78,16 +82,17 @@ func CommonPIDsFilter(c *services.DiscoveryConfig) ServiceFilter { return commonPIDsFilter } -func (pf *PIDsFilter) AllowPID(pid, ns uint32, svc *svc.ID, pidType PIDType) { +func (pf *PIDsFilter) AllowPID(pid, ns uint32, svc *svc.ID, pidType PIDType, symTab *gosym.Table) { pf.mux.Lock() defer pf.mux.Unlock() - pf.addPID(pid, ns, svc, pidType) + pf.addPID(pid, ns, svc, pidType, symTab) } func (pf *PIDsFilter) BlockPID(pid, ns uint32) { pf.mux.Lock() defer pf.mux.Unlock() pf.removePID(pid, ns) + } func (pf *PIDsFilter) ValidPID(userPID, ns uint32, pidType PIDType) bool { @@ -158,13 +163,24 @@ func (pf *PIDsFilter) Filter(inputSpans []request.Span) []request.Span { return outputSpans } -func (pf *PIDsFilter) addPID(pid, nsid uint32, s *svc.ID, t PIDType) { +func (pf *PIDsFilter) GetSymTab(pid uint32) *gosym.Table { + pf.mux.RLock() + defer pf.mux.RUnlock() + return pf.symTabs[pid] +} + +func (pf *PIDsFilter) addPID(pid, nsid uint32, s *svc.ID, t PIDType, symTab *gosym.Table) { ns, nsExists := pf.current[nsid] if !nsExists { ns = make(map[uint32]PIDInfo) pf.current[nsid] = ns } + _, stExists := pf.symTabs[pid] + if !stExists { + pf.symTabs[pid] = symTab + } + allPids, err := readNamespacePIDs(int32(pid)) if err != nil { @@ -187,6 +203,7 @@ func (pf *PIDsFilter) removePID(pid, nsid uint32) { if len(ns) == 0 { delete(pf.current, nsid) } + delete(pf.symTabs, pid) } // IdentityPidsFilter is a PIDsFilter that does not filter anything. It is feasible @@ -195,7 +212,7 @@ type IdentityPidsFilter struct { detectOTel bool } -func (pf *IdentityPidsFilter) AllowPID(_ uint32, _ uint32, _ *svc.ID, _ PIDType) {} +func (pf *IdentityPidsFilter) AllowPID(_ uint32, _ uint32, _ *svc.ID, _ PIDType, _ *gosym.Table) {} func (pf *IdentityPidsFilter) BlockPID(_ uint32, _ uint32) {} @@ -219,6 +236,10 @@ func (pf *IdentityPidsFilter) Filter(inputSpans []request.Span) []request.Span { return inputSpans } +func (pf *IdentityPidsFilter) GetSymTab(_ uint32) *gosym.Table { + return nil +} + func serviceInfo(pid uint32) *svc.ID { cached, ok := activePids.Get(pid) if ok { diff --git a/pkg/internal/ebpf/common/pids_test.go b/pkg/internal/ebpf/common/pids_test.go index 0cddc5779..95386ec26 100644 --- a/pkg/internal/ebpf/common/pids_test.go +++ b/pkg/internal/ebpf/common/pids_test.go @@ -26,9 +26,9 @@ func TestFilter_SameNS(t *testing.T) { return []uint32{uint32(pid)}, nil } pf := newPIDsFilter(&services.DiscoveryConfig{}, slog.With("env", "testing")) - pf.AllowPID(123, 33, &svc.ID{}, PIDTypeGo) - pf.AllowPID(456, 33, &svc.ID{}, PIDTypeGo) - pf.AllowPID(789, 33, &svc.ID{}, PIDTypeGo) + pf.AllowPID(123, 33, &svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(456, 33, &svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(789, 33, &svc.ID{}, PIDTypeGo, nil) // with the same namespace, it filters by user PID, as it is the PID // that is seen by Beyla's process discovery @@ -44,9 +44,9 @@ func TestFilter_DifferentNS(t *testing.T) { return []uint32{uint32(pid)}, nil } pf := newPIDsFilter(&services.DiscoveryConfig{}, slog.With("env", "testing")) - pf.AllowPID(123, 22, &svc.ID{}, PIDTypeGo) - pf.AllowPID(456, 22, &svc.ID{}, PIDTypeGo) - pf.AllowPID(666, 22, &svc.ID{}, PIDTypeGo) + pf.AllowPID(123, 22, &svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(456, 22, &svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(666, 22, &svc.ID{}, PIDTypeGo, nil) // with the same namespace, it filters by user PID, as it is the PID // that is seen by Beyla's process discovery @@ -58,8 +58,8 @@ func TestFilter_Block(t *testing.T) { return []uint32{uint32(pid)}, nil } pf := newPIDsFilter(&services.DiscoveryConfig{}, slog.With("env", "testing")) - pf.AllowPID(123, 33, &svc.ID{}, PIDTypeGo) - pf.AllowPID(456, 33, &svc.ID{}, PIDTypeGo) + pf.AllowPID(123, 33, &svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(456, 33, &svc.ID{}, PIDTypeGo, nil) pf.BlockPID(123, 33) // with the same namespace, it filters by user PID, as it is the PID @@ -76,9 +76,9 @@ func TestFilter_NewNSLater(t *testing.T) { return []uint32{uint32(pid)}, nil } pf := newPIDsFilter(&services.DiscoveryConfig{}, slog.With("env", "testing")) - pf.AllowPID(123, 33, &svc.ID{}, PIDTypeGo) - pf.AllowPID(456, 33, &svc.ID{}, PIDTypeGo) - pf.AllowPID(789, 33, &svc.ID{}, PIDTypeGo) + pf.AllowPID(123, 33, &svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(456, 33, &svc.ID{}, PIDTypeGo, nil) + pf.AllowPID(789, 33, &svc.ID{}, PIDTypeGo, nil) // with the same namespace, it filters by user PID, as it is the PID // that is seen by Beyla's process discovery @@ -88,7 +88,7 @@ func TestFilter_NewNSLater(t *testing.T) { {Pid: request.PidInfo{UserPID: 789, HostPID: 234, Namespace: 33}}, }, pf.Filter(spanSet)) - pf.AllowPID(1000, 44, &svc.ID{}, PIDTypeGo) + pf.AllowPID(1000, 44, &svc.ID{}, PIDTypeGo, nil) assert.Equal(t, []request.Span{ {Pid: request.PidInfo{UserPID: 123, HostPID: 333, Namespace: 33}}, diff --git a/pkg/internal/ebpf/common/ringbuf_test.go b/pkg/internal/ebpf/common/ringbuf_test.go index 3f1440ac3..0b8de8411 100644 --- a/pkg/internal/ebpf/common/ringbuf_test.go +++ b/pkg/internal/ebpf/common/ringbuf_test.go @@ -3,6 +3,7 @@ package ebpfcommon import ( "bytes" "context" + "debug/gosym" "encoding/binary" "log/slog" "sync" @@ -32,7 +33,7 @@ func TestForwardRingbuf_CapacityFull(t *testing.T) { metrics := &metricsReporter{} forwardedMessages := make(chan []request.Span, 100) fltr := TestPidsFilter{services: map[uint32]svc.ID{}} - fltr.AllowPID(1, 1, &svc.ID{Name: "myService"}, PIDTypeGo) + fltr.AllowPID(1, 1, &svc.ID{Name: "myService"}, PIDTypeGo, nil) go ForwardRingbuf( &config.EPPFTracer{BatchLength: 10}, nil, // the source ring buffer can be null @@ -84,7 +85,7 @@ func TestForwardRingbuf_Deadline(t *testing.T) { metrics := &metricsReporter{} forwardedMessages := make(chan []request.Span, 100) fltr := TestPidsFilter{services: map[uint32]svc.ID{}} - fltr.AllowPID(1, 1, &svc.ID{Name: "myService"}, PIDTypeGo) + fltr.AllowPID(1, 1, &svc.ID{Name: "myService"}, PIDTypeGo, nil) go ForwardRingbuf( &config.EPPFTracer{BatchLength: 10, BatchTimeout: 20 * time.Millisecond}, nil, // the source ring buffer can be null @@ -220,7 +221,7 @@ type TestPidsFilter struct { services map[uint32]svc.ID } -func (pf *TestPidsFilter) AllowPID(p uint32, _ uint32, s *svc.ID, _ PIDType) { +func (pf *TestPidsFilter) AllowPID(p uint32, _ uint32, s *svc.ID, _ PIDType, _ *gosym.Table) { pf.services[p] = *s } @@ -236,6 +237,10 @@ func (pf *TestPidsFilter) CurrentPIDs(_ PIDType) map[uint32]map[uint32]svc.ID { return nil } +func (pf *TestPidsFilter) GetSymTab(_ uint32) *gosym.Table { + return nil +} + func (pf *TestPidsFilter) Filter(inputSpans []request.Span) []request.Span { for i := range inputSpans { s := &inputSpans[i] diff --git a/pkg/internal/ebpf/common/spanner.go b/pkg/internal/ebpf/common/spanner.go index 43fad0cd6..610397e2c 100644 --- a/pkg/internal/ebpf/common/spanner.go +++ b/pkg/internal/ebpf/common/spanner.go @@ -2,7 +2,10 @@ package ebpfcommon import ( "bytes" + "debug/gosym" + "fmt" "log/slog" + "strings" "unsafe" trace2 "go.opentelemetry.io/otel/trace" @@ -13,7 +16,7 @@ import ( var log = slog.With("component", "goexec.spanner") -func HTTPRequestTraceToSpan(trace *HTTPRequestTrace) request.Span { +func HTTPRequestTraceToSpan(trace *HTTPRequestTrace, filter ServiceFilter) request.Span { // From C, assuming 0-ended strings methodLen := bytes.IndexByte(trace.Method[:], 0) if methodLen < 0 { @@ -24,6 +27,10 @@ func HTTPRequestTraceToSpan(trace *HTTPRequestTrace) request.Span { if pathLen < 0 { pathLen = len(trace.Path) } + errMsgLen := bytes.IndexByte(trace.Error.ErrMsg[:], 0) + if errMsgLen < 0 { + errMsgLen = len(trace.Error.ErrMsg) + } path := string(trace.Path[:pathLen]) peer := "" @@ -58,7 +65,25 @@ func HTTPRequestTraceToSpan(trace *HTTPRequestTrace) request.Span { UserPID: trace.Pid.UserPid, Namespace: trace.Pid.Ns, }, + ErrorMessage: string(trace.Error.ErrMsg[:errMsgLen]), + ErrorStacktrace: extractErrorStacktrace(trace, filter.GetSymTab(trace.Pid.UserPid)), + } +} + +func extractErrorStacktrace(trace *HTTPRequestTrace, symTab *gosym.Table) string { + var stacktrace strings.Builder + if symTab != nil && trace.Error.UstackSz > 0 { + for _, pc := range trace.Error.Ustack { + f := symTab.PCToFunc(pc) + if f == nil { + break + } + file, line, _ := symTab.PCToLine(pc) + stacktrace.WriteString(fmt.Sprintf("%s\n", f.Name)) + stacktrace.WriteString(fmt.Sprintf("\t%s:%d\n", file, line)) + } } + return stacktrace.String() } func SQLRequestTraceToSpan(trace *SQLRequestTrace) request.Span { diff --git a/pkg/internal/ebpf/common/spanner_test.go b/pkg/internal/ebpf/common/spanner_test.go index 00ff9538e..4062f5e3b 100644 --- a/pkg/internal/ebpf/common/spanner_test.go +++ b/pkg/internal/ebpf/common/spanner_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/grafana/beyla/pkg/internal/request" + "github.com/grafana/beyla/pkg/internal/svc" ) func tocstr(s string) []byte { @@ -54,38 +55,40 @@ func assertMatches(t *testing.T, span *request.Span, method, path string, status } func TestRequestTraceParsing(t *testing.T) { + fltr := TestPidsFilter{services: map[uint32]svc.ID{}} t.Run("Test basic parsing", func(t *testing.T) { tr := makeHTTPRequestTrace("POST", "/users", 200, 5) - s := HTTPRequestTraceToSpan(&tr) + s := HTTPRequestTraceToSpan(&tr, &fltr) assertMatches(t, &s, "POST", "/users", 200, 5) }) t.Run("Test with empty path and missing peer host", func(t *testing.T) { tr := makeHTTPRequestTrace("GET", "", 403, 6) - s := HTTPRequestTraceToSpan(&tr) + s := HTTPRequestTraceToSpan(&tr, &fltr) assertMatches(t, &s, "GET", "", 403, 6) }) t.Run("Test with missing peer port", func(t *testing.T) { tr := makeHTTPRequestTrace("GET", "/posts/1/1", 500, 1) - s := HTTPRequestTraceToSpan(&tr) + s := HTTPRequestTraceToSpan(&tr, &fltr) assertMatches(t, &s, "GET", "/posts/1/1", 500, 1) }) t.Run("Test with invalid peer port", func(t *testing.T) { tr := makeHTTPRequestTrace("GET", "/posts/1/1", 500, 1) - s := HTTPRequestTraceToSpan(&tr) + s := HTTPRequestTraceToSpan(&tr, &fltr) assertMatches(t, &s, "GET", "/posts/1/1", 500, 1) }) t.Run("Test with GRPC request", func(t *testing.T) { tr := makeGRPCRequestTrace("/posts/1/1", 2, 1) - s := HTTPRequestTraceToSpan(&tr) + s := HTTPRequestTraceToSpan(&tr, &fltr) assertMatches(t, &s, "", "/posts/1/1", 2, 1) }) } func makeSpanWithTimings(goStart, start, end uint64) request.Span { + fltr := TestPidsFilter{services: map[uint32]svc.ID{}} tr := HTTPRequestTrace{ Type: 1, Path: [100]uint8{}, @@ -95,7 +98,7 @@ func makeSpanWithTimings(goStart, start, end uint64) request.Span { EndMonotimeNs: end, } - return HTTPRequestTraceToSpan(&tr) + return HTTPRequestTraceToSpan(&tr, &fltr) } func TestSpanNesting(t *testing.T) { diff --git a/pkg/internal/ebpf/generictracer/bpf_arm64_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_arm64_bpfel.o index 0e866b83b..01f973c79 100644 --- a/pkg/internal/ebpf/generictracer/bpf_arm64_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:42ae803457a32667c58151079c5e52dffaca9399ee278d92049b9e48f068b8d7 -size 462544 +oid sha256:afe0b6e5da89e2a139a95701a4ea8a447f6d89e3d0270dc42a888b90c8c4cd1c +size 463808 diff --git a/pkg/internal/ebpf/generictracer/bpf_debug_arm64_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_debug_arm64_bpfel.o index 573e45e6f..90f8048e0 100644 --- a/pkg/internal/ebpf/generictracer/bpf_debug_arm64_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_debug_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2fe32652f06f776cf5dfd0eeb906bfd64473972a897ff6b0fa8edc98b9dfb264 -size 769376 +oid sha256:18eb1bed47d2f60e7c32f43519b7dff1717967a283066f2b1c11411371c2295d +size 766856 diff --git a/pkg/internal/ebpf/generictracer/bpf_debug_x86_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_debug_x86_bpfel.o index 4702b0898..026d26e01 100644 --- a/pkg/internal/ebpf/generictracer/bpf_debug_x86_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_debug_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:74d9cff43f30918bd407f2be7d53f32d4c856b36a6626c7e99485d9f18573f22 -size 768888 +oid sha256:49db164dc41b96f5d2ea6709585fe9bfd828b13146e1ff1f1e9fe994838db891 +size 766128 diff --git a/pkg/internal/ebpf/generictracer/bpf_tp_arm64_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_tp_arm64_bpfel.o index eec45258f..d2496e6b8 100644 --- a/pkg/internal/ebpf/generictracer/bpf_tp_arm64_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_tp_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:12e0c64938beeb0414a867b05d8882ff7adccc10037565105528358ee4202db4 -size 477232 +oid sha256:bad1b0a13db4be2f19d163ff26550ee2203019ef5e42acc6a8f60b56a8dcb508 +size 477336 diff --git a/pkg/internal/ebpf/generictracer/bpf_tp_debug_arm64_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_tp_debug_arm64_bpfel.o index dfe6e719e..2f590d18a 100644 --- a/pkg/internal/ebpf/generictracer/bpf_tp_debug_arm64_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_tp_debug_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:02ab01c95abf546beba3baf61961420647d90347789202c841bf5ac5fa3f7771 -size 787632 +oid sha256:c1f3e283a18498b3e0e5536f162667b4d996bacda3131fe2d4398fd3212800f9 +size 784160 diff --git a/pkg/internal/ebpf/generictracer/bpf_tp_debug_x86_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_tp_debug_x86_bpfel.o index 6f7b534a7..b16eade9b 100644 --- a/pkg/internal/ebpf/generictracer/bpf_tp_debug_x86_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_tp_debug_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d0d3ed640cfe8783ce63077ba1955b1a1246f22cfc913ce9d5808e7c3bf76ba4 -size 787152 +oid sha256:ade56940c86e7fc971e1d7b271ae0e0ef2d10881c71270cf3e6f210c5b87e0e2 +size 783432 diff --git a/pkg/internal/ebpf/generictracer/bpf_tp_x86_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_tp_x86_bpfel.o index 0f38c3c0e..b301e2c10 100644 --- a/pkg/internal/ebpf/generictracer/bpf_tp_x86_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_tp_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:64cc2193dc7ecd3e954a0fb1c9b00ebb8d187a38c89749bc0e39351b68b52d7a -size 476632 +oid sha256:e87054c5c65c57a705f9d836198fb3f9fa069d7ebe0894f86011ec19a9bba1e7 +size 476504 diff --git a/pkg/internal/ebpf/generictracer/bpf_x86_bpfel.o b/pkg/internal/ebpf/generictracer/bpf_x86_bpfel.o index ca950329c..9d34d1794 100644 --- a/pkg/internal/ebpf/generictracer/bpf_x86_bpfel.o +++ b/pkg/internal/ebpf/generictracer/bpf_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a364bb584d2ae638ec7afa90a74ef0b6ee6e09cec049f01c1eea8fae0477cdea -size 461952 +oid sha256:dee371856ed7f0e96bebded39aa52c7e2edbaf85cbdb651f573901cdad3fe502 +size 462968 diff --git a/pkg/internal/ebpf/generictracer/generictracer.go b/pkg/internal/ebpf/generictracer/generictracer.go index 0ae42a0aa..11fc7125e 100644 --- a/pkg/internal/ebpf/generictracer/generictracer.go +++ b/pkg/internal/ebpf/generictracer/generictracer.go @@ -4,6 +4,7 @@ package generictracer import ( "context" + "debug/gosym" "io" "log/slog" "sync" @@ -114,8 +115,8 @@ func (p *Tracer) rebuildValidPids() { } } -func (p *Tracer) AllowPID(pid, ns uint32, svc *svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeKProbes) +func (p *Tracer) AllowPID(pid, ns uint32, svc *svc.ID, _ *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeKProbes, nil) p.rebuildValidPids() } diff --git a/pkg/internal/ebpf/gotracer/bpf_arm64_bpfel.go b/pkg/internal/ebpf/gotracer/bpf_arm64_bpfel.go index 3b29b4439..f52df80a9 100644 --- a/pkg/internal/ebpf/gotracer/bpf_arm64_bpfel.go +++ b/pkg/internal/ebpf/gotracer/bpf_arm64_bpfel.go @@ -24,6 +24,16 @@ type bpfEgressKeyT struct { D_port uint16 } +type bpfErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpfGoAddrKeyT struct { Pid uint64 Addr uint64 @@ -224,6 +234,8 @@ type bpfProgramSpecs struct { UprobeClientRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -277,6 +289,7 @@ type bpfMapSpecs struct { GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` IncomingTraceMap *ebpf.MapSpec `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.MapSpec `ebpf:"kafka_requests"` + LastError *ebpf.MapSpec `ebpf:"last_error"` Newproc1 *ebpf.MapSpec `ebpf:"newproc1"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.MapSpec `ebpf:"ongoing_go_http"` @@ -330,6 +343,7 @@ type bpfMaps struct { GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` IncomingTraceMap *ebpf.Map `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.Map `ebpf:"kafka_requests"` + LastError *ebpf.Map `ebpf:"last_error"` Newproc1 *ebpf.Map `ebpf:"newproc1"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.Map `ebpf:"ongoing_go_http"` @@ -366,6 +380,7 @@ func (m *bpfMaps) Close() error { m.GolangMapbucketStorageMap, m.IncomingTraceMap, m.KafkaRequests, + m.LastError, m.Newproc1, m.OngoingClientConnections, m.OngoingGoHttp, @@ -409,6 +424,8 @@ type bpfPrograms struct { UprobeClientRoundTrip *ebpf.Program `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -464,6 +481,8 @@ func (p *bpfPrograms) Close() error { p.UprobeClientRoundTrip, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeGrpcFramerWriteHeaders, p.UprobeGrpcFramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/gotracer/bpf_arm64_bpfel.o b/pkg/internal/ebpf/gotracer/bpf_arm64_bpfel.o index 8b0f9524c..aacecd8bc 100644 --- a/pkg/internal/ebpf/gotracer/bpf_arm64_bpfel.o +++ b/pkg/internal/ebpf/gotracer/bpf_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:5867d68bb8ad50c575bb1aec00947a720efba13fe55ed68ccee5bc82d68108e2 -size 407528 +oid sha256:0384001c4e6a03d38211e2343cace298aab0f41480820b18aad25d1f87f88c51 +size 404168 diff --git a/pkg/internal/ebpf/gotracer/bpf_debug_arm64_bpfel.go b/pkg/internal/ebpf/gotracer/bpf_debug_arm64_bpfel.go index be7248e99..a948e1813 100644 --- a/pkg/internal/ebpf/gotracer/bpf_debug_arm64_bpfel.go +++ b/pkg/internal/ebpf/gotracer/bpf_debug_arm64_bpfel.go @@ -24,6 +24,16 @@ type bpf_debugEgressKeyT struct { D_port uint16 } +type bpf_debugErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_debugGoAddrKeyT struct { Pid uint64 Addr uint64 @@ -224,6 +234,8 @@ type bpf_debugProgramSpecs struct { UprobeClientRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -278,6 +290,7 @@ type bpf_debugMapSpecs struct { GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` IncomingTraceMap *ebpf.MapSpec `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.MapSpec `ebpf:"kafka_requests"` + LastError *ebpf.MapSpec `ebpf:"last_error"` Newproc1 *ebpf.MapSpec `ebpf:"newproc1"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.MapSpec `ebpf:"ongoing_go_http"` @@ -332,6 +345,7 @@ type bpf_debugMaps struct { GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` IncomingTraceMap *ebpf.Map `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.Map `ebpf:"kafka_requests"` + LastError *ebpf.Map `ebpf:"last_error"` Newproc1 *ebpf.Map `ebpf:"newproc1"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.Map `ebpf:"ongoing_go_http"` @@ -369,6 +383,7 @@ func (m *bpf_debugMaps) Close() error { m.GolangMapbucketStorageMap, m.IncomingTraceMap, m.KafkaRequests, + m.LastError, m.Newproc1, m.OngoingClientConnections, m.OngoingGoHttp, @@ -412,6 +427,8 @@ type bpf_debugPrograms struct { UprobeClientRoundTrip *ebpf.Program `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -467,6 +484,8 @@ func (p *bpf_debugPrograms) Close() error { p.UprobeClientRoundTrip, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeGrpcFramerWriteHeaders, p.UprobeGrpcFramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/gotracer/bpf_debug_arm64_bpfel.o b/pkg/internal/ebpf/gotracer/bpf_debug_arm64_bpfel.o index d5e0e5d76..37aa70395 100644 --- a/pkg/internal/ebpf/gotracer/bpf_debug_arm64_bpfel.o +++ b/pkg/internal/ebpf/gotracer/bpf_debug_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6bdcc06f351d441779c826b8d3e484c36dd0342fee1bb9dffb2b73fb5b14862d -size 899920 +oid sha256:d0dfb938ab30a290e052e1a23483038b5745e25db7c8810a785c5347f5c4b9d9 +size 903568 diff --git a/pkg/internal/ebpf/gotracer/bpf_debug_x86_bpfel.go b/pkg/internal/ebpf/gotracer/bpf_debug_x86_bpfel.go index f87332fa7..2d70642ba 100644 --- a/pkg/internal/ebpf/gotracer/bpf_debug_x86_bpfel.go +++ b/pkg/internal/ebpf/gotracer/bpf_debug_x86_bpfel.go @@ -24,6 +24,16 @@ type bpf_debugEgressKeyT struct { D_port uint16 } +type bpf_debugErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_debugGoAddrKeyT struct { Pid uint64 Addr uint64 @@ -224,6 +234,8 @@ type bpf_debugProgramSpecs struct { UprobeClientRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -278,6 +290,7 @@ type bpf_debugMapSpecs struct { GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` IncomingTraceMap *ebpf.MapSpec `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.MapSpec `ebpf:"kafka_requests"` + LastError *ebpf.MapSpec `ebpf:"last_error"` Newproc1 *ebpf.MapSpec `ebpf:"newproc1"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.MapSpec `ebpf:"ongoing_go_http"` @@ -332,6 +345,7 @@ type bpf_debugMaps struct { GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` IncomingTraceMap *ebpf.Map `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.Map `ebpf:"kafka_requests"` + LastError *ebpf.Map `ebpf:"last_error"` Newproc1 *ebpf.Map `ebpf:"newproc1"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.Map `ebpf:"ongoing_go_http"` @@ -369,6 +383,7 @@ func (m *bpf_debugMaps) Close() error { m.GolangMapbucketStorageMap, m.IncomingTraceMap, m.KafkaRequests, + m.LastError, m.Newproc1, m.OngoingClientConnections, m.OngoingGoHttp, @@ -412,6 +427,8 @@ type bpf_debugPrograms struct { UprobeClientRoundTrip *ebpf.Program `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -467,6 +484,8 @@ func (p *bpf_debugPrograms) Close() error { p.UprobeClientRoundTrip, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeGrpcFramerWriteHeaders, p.UprobeGrpcFramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/gotracer/bpf_debug_x86_bpfel.o b/pkg/internal/ebpf/gotracer/bpf_debug_x86_bpfel.o index 94c670e7e..03205ecb3 100644 --- a/pkg/internal/ebpf/gotracer/bpf_debug_x86_bpfel.o +++ b/pkg/internal/ebpf/gotracer/bpf_debug_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d6fe779702bd07cfeae83f447f141dde100d6d8886b58cef94a4e0abe303958 -size 901752 +oid sha256:0aac2bde080259f8a41d3a5868871cf2a04b5103c9c47e7f2cdf08cc246d2823 +size 905368 diff --git a/pkg/internal/ebpf/gotracer/bpf_tp_arm64_bpfel.go b/pkg/internal/ebpf/gotracer/bpf_tp_arm64_bpfel.go index 2f5dca811..14abf45bc 100644 --- a/pkg/internal/ebpf/gotracer/bpf_tp_arm64_bpfel.go +++ b/pkg/internal/ebpf/gotracer/bpf_tp_arm64_bpfel.go @@ -24,6 +24,16 @@ type bpf_tpEgressKeyT struct { D_port uint16 } +type bpf_tpErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_tpFramerFuncInvocationT struct { FramerPtr uint64 Tp bpf_tpTpInfoT @@ -236,6 +246,8 @@ type bpf_tpProgramSpecs struct { UprobeClientRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -293,6 +305,7 @@ type bpf_tpMapSpecs struct { Http2ReqMap *ebpf.MapSpec `ebpf:"http2_req_map"` IncomingTraceMap *ebpf.MapSpec `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.MapSpec `ebpf:"kafka_requests"` + LastError *ebpf.MapSpec `ebpf:"last_error"` Newproc1 *ebpf.MapSpec `ebpf:"newproc1"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.MapSpec `ebpf:"ongoing_go_http"` @@ -350,6 +363,7 @@ type bpf_tpMaps struct { Http2ReqMap *ebpf.Map `ebpf:"http2_req_map"` IncomingTraceMap *ebpf.Map `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.Map `ebpf:"kafka_requests"` + LastError *ebpf.Map `ebpf:"last_error"` Newproc1 *ebpf.Map `ebpf:"newproc1"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.Map `ebpf:"ongoing_go_http"` @@ -390,6 +404,7 @@ func (m *bpf_tpMaps) Close() error { m.Http2ReqMap, m.IncomingTraceMap, m.KafkaRequests, + m.LastError, m.Newproc1, m.OngoingClientConnections, m.OngoingGoHttp, @@ -433,6 +448,8 @@ type bpf_tpPrograms struct { UprobeClientRoundTrip *ebpf.Program `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -488,6 +505,8 @@ func (p *bpf_tpPrograms) Close() error { p.UprobeClientRoundTrip, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeGrpcFramerWriteHeaders, p.UprobeGrpcFramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/gotracer/bpf_tp_arm64_bpfel.o b/pkg/internal/ebpf/gotracer/bpf_tp_arm64_bpfel.o index 45c6abe3f..3079dac76 100644 --- a/pkg/internal/ebpf/gotracer/bpf_tp_arm64_bpfel.o +++ b/pkg/internal/ebpf/gotracer/bpf_tp_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:821f9adf0dc2e7e4eb9500e99f6ee3c9b3753308d2f90862a521ee8e64bfc464 -size 454152 +oid sha256:fe12671daa6f2bda3a3d3fa747f5d40bb600d84faa662a90c610777e397b443a +size 450496 diff --git a/pkg/internal/ebpf/gotracer/bpf_tp_debug_arm64_bpfel.go b/pkg/internal/ebpf/gotracer/bpf_tp_debug_arm64_bpfel.go index ebd6ce529..f4a432253 100644 --- a/pkg/internal/ebpf/gotracer/bpf_tp_debug_arm64_bpfel.go +++ b/pkg/internal/ebpf/gotracer/bpf_tp_debug_arm64_bpfel.go @@ -24,6 +24,16 @@ type bpf_tp_debugEgressKeyT struct { D_port uint16 } +type bpf_tp_debugErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_tp_debugFramerFuncInvocationT struct { FramerPtr uint64 Tp bpf_tp_debugTpInfoT @@ -236,6 +246,8 @@ type bpf_tp_debugProgramSpecs struct { UprobeClientRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -294,6 +306,7 @@ type bpf_tp_debugMapSpecs struct { Http2ReqMap *ebpf.MapSpec `ebpf:"http2_req_map"` IncomingTraceMap *ebpf.MapSpec `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.MapSpec `ebpf:"kafka_requests"` + LastError *ebpf.MapSpec `ebpf:"last_error"` Newproc1 *ebpf.MapSpec `ebpf:"newproc1"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.MapSpec `ebpf:"ongoing_go_http"` @@ -352,6 +365,7 @@ type bpf_tp_debugMaps struct { Http2ReqMap *ebpf.Map `ebpf:"http2_req_map"` IncomingTraceMap *ebpf.Map `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.Map `ebpf:"kafka_requests"` + LastError *ebpf.Map `ebpf:"last_error"` Newproc1 *ebpf.Map `ebpf:"newproc1"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.Map `ebpf:"ongoing_go_http"` @@ -393,6 +407,7 @@ func (m *bpf_tp_debugMaps) Close() error { m.Http2ReqMap, m.IncomingTraceMap, m.KafkaRequests, + m.LastError, m.Newproc1, m.OngoingClientConnections, m.OngoingGoHttp, @@ -436,6 +451,8 @@ type bpf_tp_debugPrograms struct { UprobeClientRoundTrip *ebpf.Program `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -491,6 +508,8 @@ func (p *bpf_tp_debugPrograms) Close() error { p.UprobeClientRoundTrip, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeGrpcFramerWriteHeaders, p.UprobeGrpcFramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/gotracer/bpf_tp_debug_arm64_bpfel.o b/pkg/internal/ebpf/gotracer/bpf_tp_debug_arm64_bpfel.o index e5a567cba..41cf97ca2 100644 --- a/pkg/internal/ebpf/gotracer/bpf_tp_debug_arm64_bpfel.o +++ b/pkg/internal/ebpf/gotracer/bpf_tp_debug_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6d3fe2321ea5a6ae0a2a32e9c34828efed363bd78a568dc2eeed5446d0a74fba -size 997832 +oid sha256:690f5446901a2c05c0c21f7a426b4016804bb5a8ee68351cf014c76da43453bd +size 1001072 diff --git a/pkg/internal/ebpf/gotracer/bpf_tp_debug_x86_bpfel.go b/pkg/internal/ebpf/gotracer/bpf_tp_debug_x86_bpfel.go index b234c3605..2f5261e85 100644 --- a/pkg/internal/ebpf/gotracer/bpf_tp_debug_x86_bpfel.go +++ b/pkg/internal/ebpf/gotracer/bpf_tp_debug_x86_bpfel.go @@ -24,6 +24,16 @@ type bpf_tp_debugEgressKeyT struct { D_port uint16 } +type bpf_tp_debugErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_tp_debugFramerFuncInvocationT struct { FramerPtr uint64 Tp bpf_tp_debugTpInfoT @@ -236,6 +246,8 @@ type bpf_tp_debugProgramSpecs struct { UprobeClientRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -294,6 +306,7 @@ type bpf_tp_debugMapSpecs struct { Http2ReqMap *ebpf.MapSpec `ebpf:"http2_req_map"` IncomingTraceMap *ebpf.MapSpec `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.MapSpec `ebpf:"kafka_requests"` + LastError *ebpf.MapSpec `ebpf:"last_error"` Newproc1 *ebpf.MapSpec `ebpf:"newproc1"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.MapSpec `ebpf:"ongoing_go_http"` @@ -352,6 +365,7 @@ type bpf_tp_debugMaps struct { Http2ReqMap *ebpf.Map `ebpf:"http2_req_map"` IncomingTraceMap *ebpf.Map `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.Map `ebpf:"kafka_requests"` + LastError *ebpf.Map `ebpf:"last_error"` Newproc1 *ebpf.Map `ebpf:"newproc1"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.Map `ebpf:"ongoing_go_http"` @@ -393,6 +407,7 @@ func (m *bpf_tp_debugMaps) Close() error { m.Http2ReqMap, m.IncomingTraceMap, m.KafkaRequests, + m.LastError, m.Newproc1, m.OngoingClientConnections, m.OngoingGoHttp, @@ -436,6 +451,8 @@ type bpf_tp_debugPrograms struct { UprobeClientRoundTrip *ebpf.Program `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -491,6 +508,8 @@ func (p *bpf_tp_debugPrograms) Close() error { p.UprobeClientRoundTrip, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeGrpcFramerWriteHeaders, p.UprobeGrpcFramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/gotracer/bpf_tp_debug_x86_bpfel.o b/pkg/internal/ebpf/gotracer/bpf_tp_debug_x86_bpfel.o index 1bfdb8816..226ad0810 100644 --- a/pkg/internal/ebpf/gotracer/bpf_tp_debug_x86_bpfel.o +++ b/pkg/internal/ebpf/gotracer/bpf_tp_debug_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:77fa58ed8bdfd4555c3806163f68ae7472673d0319115740aad5d7c753da9993 -size 999656 +oid sha256:e94d0d80646cdd20986ef50ece1ffe7e45c973a0f62405444bc3b08f442b1681 +size 1002872 diff --git a/pkg/internal/ebpf/gotracer/bpf_tp_x86_bpfel.go b/pkg/internal/ebpf/gotracer/bpf_tp_x86_bpfel.go index 7a76ff187..600b0b625 100644 --- a/pkg/internal/ebpf/gotracer/bpf_tp_x86_bpfel.go +++ b/pkg/internal/ebpf/gotracer/bpf_tp_x86_bpfel.go @@ -24,6 +24,16 @@ type bpf_tpEgressKeyT struct { D_port uint16 } +type bpf_tpErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpf_tpFramerFuncInvocationT struct { FramerPtr uint64 Tp bpf_tpTpInfoT @@ -236,6 +246,8 @@ type bpf_tpProgramSpecs struct { UprobeClientRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -293,6 +305,7 @@ type bpf_tpMapSpecs struct { Http2ReqMap *ebpf.MapSpec `ebpf:"http2_req_map"` IncomingTraceMap *ebpf.MapSpec `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.MapSpec `ebpf:"kafka_requests"` + LastError *ebpf.MapSpec `ebpf:"last_error"` Newproc1 *ebpf.MapSpec `ebpf:"newproc1"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.MapSpec `ebpf:"ongoing_go_http"` @@ -350,6 +363,7 @@ type bpf_tpMaps struct { Http2ReqMap *ebpf.Map `ebpf:"http2_req_map"` IncomingTraceMap *ebpf.Map `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.Map `ebpf:"kafka_requests"` + LastError *ebpf.Map `ebpf:"last_error"` Newproc1 *ebpf.Map `ebpf:"newproc1"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.Map `ebpf:"ongoing_go_http"` @@ -390,6 +404,7 @@ func (m *bpf_tpMaps) Close() error { m.Http2ReqMap, m.IncomingTraceMap, m.KafkaRequests, + m.LastError, m.Newproc1, m.OngoingClientConnections, m.OngoingGoHttp, @@ -433,6 +448,8 @@ type bpf_tpPrograms struct { UprobeClientRoundTrip *ebpf.Program `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -488,6 +505,8 @@ func (p *bpf_tpPrograms) Close() error { p.UprobeClientRoundTrip, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeGrpcFramerWriteHeaders, p.UprobeGrpcFramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/gotracer/bpf_tp_x86_bpfel.o b/pkg/internal/ebpf/gotracer/bpf_tp_x86_bpfel.o index 64cb25bd7..5cb7aa7c5 100644 --- a/pkg/internal/ebpf/gotracer/bpf_tp_x86_bpfel.o +++ b/pkg/internal/ebpf/gotracer/bpf_tp_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:44411f0e7f8a76cf0fd1977f9aec9778b7f8cbdf7acc0222d5d6512b90a9275e -size 455920 +oid sha256:72dd921144ae9884dc35bd6611cd1ff2dc971d135a00f50c374d8d461dbbd8ea +size 452240 diff --git a/pkg/internal/ebpf/gotracer/bpf_x86_bpfel.go b/pkg/internal/ebpf/gotracer/bpf_x86_bpfel.go index 26513c474..c9e4bd755 100644 --- a/pkg/internal/ebpf/gotracer/bpf_x86_bpfel.go +++ b/pkg/internal/ebpf/gotracer/bpf_x86_bpfel.go @@ -24,6 +24,16 @@ type bpfEgressKeyT struct { D_port uint16 } +type bpfErrorEvent struct { + Pid uint32 + CpuId uint32 + Comm [16]int8 + UstackSz int32 + _ [4]byte + Ustack [32]uint64 + ErrMsg [128]uint8 +} + type bpfGoAddrKeyT struct { Pid uint64 Addr uint64 @@ -224,6 +234,8 @@ type bpfProgramSpecs struct { UprobeClientRoundTrip *ebpf.ProgramSpec `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.ProgramSpec `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.ProgramSpec `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.ProgramSpec `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.ProgramSpec `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.ProgramSpec `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.ProgramSpec `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -277,6 +289,7 @@ type bpfMapSpecs struct { GolangMapbucketStorageMap *ebpf.MapSpec `ebpf:"golang_mapbucket_storage_map"` IncomingTraceMap *ebpf.MapSpec `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.MapSpec `ebpf:"kafka_requests"` + LastError *ebpf.MapSpec `ebpf:"last_error"` Newproc1 *ebpf.MapSpec `ebpf:"newproc1"` OngoingClientConnections *ebpf.MapSpec `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.MapSpec `ebpf:"ongoing_go_http"` @@ -330,6 +343,7 @@ type bpfMaps struct { GolangMapbucketStorageMap *ebpf.Map `ebpf:"golang_mapbucket_storage_map"` IncomingTraceMap *ebpf.Map `ebpf:"incoming_trace_map"` KafkaRequests *ebpf.Map `ebpf:"kafka_requests"` + LastError *ebpf.Map `ebpf:"last_error"` Newproc1 *ebpf.Map `ebpf:"newproc1"` OngoingClientConnections *ebpf.Map `ebpf:"ongoing_client_connections"` OngoingGoHttp *ebpf.Map `ebpf:"ongoing_go_http"` @@ -366,6 +380,7 @@ func (m *bpfMaps) Close() error { m.GolangMapbucketStorageMap, m.IncomingTraceMap, m.KafkaRequests, + m.LastError, m.Newproc1, m.OngoingClientConnections, m.OngoingGoHttp, @@ -409,6 +424,8 @@ type bpfPrograms struct { UprobeClientRoundTrip *ebpf.Program `ebpf:"uprobe_client_roundTrip"` UprobeConnServe *ebpf.Program `ebpf:"uprobe_connServe"` UprobeConnServeRet *ebpf.Program `ebpf:"uprobe_connServeRet"` + UprobeError *ebpf.Program `ebpf:"uprobe_error"` + UprobeErrorReturn *ebpf.Program `ebpf:"uprobe_errorReturn"` UprobeExecDC *ebpf.Program `ebpf:"uprobe_execDC"` UprobeGrpcFramerWriteHeaders *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders"` UprobeGrpcFramerWriteHeadersReturns *ebpf.Program `ebpf:"uprobe_grpcFramerWriteHeaders_returns"` @@ -464,6 +481,8 @@ func (p *bpfPrograms) Close() error { p.UprobeClientRoundTrip, p.UprobeConnServe, p.UprobeConnServeRet, + p.UprobeError, + p.UprobeErrorReturn, p.UprobeExecDC, p.UprobeGrpcFramerWriteHeaders, p.UprobeGrpcFramerWriteHeadersReturns, diff --git a/pkg/internal/ebpf/gotracer/bpf_x86_bpfel.o b/pkg/internal/ebpf/gotracer/bpf_x86_bpfel.o index 6a7437d14..820ab809f 100644 --- a/pkg/internal/ebpf/gotracer/bpf_x86_bpfel.o +++ b/pkg/internal/ebpf/gotracer/bpf_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:138e8984bd7eab126bae805788ef55475e59374c927c39c6cb722fd2281f416d -size 409360 +oid sha256:236e75e4bfb654d4c5bcfbf56aab2dce81a0f92f3de782972cb4ca7ff046ab29 +size 405976 diff --git a/pkg/internal/ebpf/gotracer/gotracer.go b/pkg/internal/ebpf/gotracer/gotracer.go index 9385cea4d..90be90e9b 100644 --- a/pkg/internal/ebpf/gotracer/gotracer.go +++ b/pkg/internal/ebpf/gotracer/gotracer.go @@ -16,6 +16,7 @@ package gotracer import ( "context" + "debug/gosym" "io" "log/slog" "unsafe" @@ -38,26 +39,28 @@ import ( //go:generate $BPF2GO -cc $BPF_CLANG -cflags $BPF_CFLAGS -target amd64,arm64 bpf_tp_debug ../../../../bpf/go_tracer.c -- -I../../../../bpf/headers -DBPF_DEBUG type Tracer struct { - log *slog.Logger - pidsFilter ebpfcommon.ServiceFilter - cfg *config.EPPFTracer - metrics imetrics.Reporter - bpfObjects bpfObjects - closers []io.Closer + log *slog.Logger + pidsFilter ebpfcommon.ServiceFilter + cfg *config.EPPFTracer + reportErrors bool + metrics imetrics.Reporter + bpfObjects bpfObjects + closers []io.Closer } func New(cfg *beyla.Config, metrics imetrics.Reporter) *Tracer { log := slog.With("component", "go.Tracer") return &Tracer{ - log: log, - pidsFilter: ebpfcommon.CommonPIDsFilter(&cfg.Discovery), - cfg: &cfg.EBPF, - metrics: metrics, + log: log, + pidsFilter: ebpfcommon.CommonPIDsFilter(&cfg.Discovery), + cfg: &cfg.EBPF, + reportErrors: cfg.Traces.ReportExceptionEvents, + metrics: metrics, } } -func (p *Tracer) AllowPID(pid, ns uint32, svc *svc.ID) { - p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo) +func (p *Tracer) AllowPID(pid, ns uint32, svc *svc.ID, symTab *gosym.Table) { + p.pidsFilter.AllowPID(pid, ns, svc, ebpfcommon.PIDTypeGo, symTab) } func (p *Tracer) BlockPID(pid, ns uint32) { @@ -359,6 +362,15 @@ func (p *Tracer) GoProbes() map[string][]ebpfcommon.FunctionPrograms { }} } + if p.reportErrors { + m["fmt.Errorf"] = []ebpfcommon.FunctionPrograms{{ + Start: p.bpfObjects.UprobeError, + }} + m["errors.(*errorString).Error"] = []ebpfcommon.FunctionPrograms{{ + End: p.bpfObjects.UprobeErrorReturn, + }} + } + return m } diff --git a/pkg/internal/ebpf/httptracer/bpf_arm64_bpfel.o b/pkg/internal/ebpf/httptracer/bpf_arm64_bpfel.o index 186ed744b..a2e0660df 100644 --- a/pkg/internal/ebpf/httptracer/bpf_arm64_bpfel.o +++ b/pkg/internal/ebpf/httptracer/bpf_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d091613317e96ee7b18e36a7c6cad8e4c14d58bf06ae244179ca472e3efd7ce1 -size 42512 +oid sha256:28b757f64601d5ffd074215806c8d2255b6db5d0c81ed4cd88d814d395287e22 +size 42880 diff --git a/pkg/internal/ebpf/httptracer/bpf_debug_arm64_bpfel.o b/pkg/internal/ebpf/httptracer/bpf_debug_arm64_bpfel.o index 97d515de4..e973624fc 100644 --- a/pkg/internal/ebpf/httptracer/bpf_debug_arm64_bpfel.o +++ b/pkg/internal/ebpf/httptracer/bpf_debug_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c3eb35f051ce41e980c49dd8e2ce62d373b184b79263cb3fd9cf0fa3aec7d016 -size 42752 +oid sha256:f388a75dcca8328bf812ac859cd51b46e722ed98b073ca4acb692de8b5429f01 +size 43120 diff --git a/pkg/internal/ebpf/httptracer/bpf_debug_x86_bpfel.o b/pkg/internal/ebpf/httptracer/bpf_debug_x86_bpfel.o index 610e9b50f..31fa6dd97 100644 --- a/pkg/internal/ebpf/httptracer/bpf_debug_x86_bpfel.o +++ b/pkg/internal/ebpf/httptracer/bpf_debug_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:70fc197c8ab1850376bde51e8c1f429a57286938c65a337262807ce409f6e406 -size 42640 +oid sha256:fc336226ef2d2b5c635500f678ade56da044f729d2e91bfc52c99956137fc260 +size 43008 diff --git a/pkg/internal/ebpf/httptracer/bpf_x86_bpfel.o b/pkg/internal/ebpf/httptracer/bpf_x86_bpfel.o index b715b233c..770d57ecc 100644 --- a/pkg/internal/ebpf/httptracer/bpf_x86_bpfel.o +++ b/pkg/internal/ebpf/httptracer/bpf_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9500aa3b559104eea0df776102a8c79f4d8eed5ec85017c14f52eb860be06670 -size 42400 +oid sha256:c161042872775089b7d9054216924f93e96c15cad086f749b61db6ce82d55bd4 +size 42768 diff --git a/pkg/internal/ebpf/httptracer/httptracer.go b/pkg/internal/ebpf/httptracer/httptracer.go index d8c9e76bc..95a7070a0 100644 --- a/pkg/internal/ebpf/httptracer/httptracer.go +++ b/pkg/internal/ebpf/httptracer/httptracer.go @@ -4,6 +4,7 @@ package httptracer import ( "context" + "debug/gosym" "io" "log/slog" @@ -43,7 +44,7 @@ func New(cfg *beyla.Config) *Tracer { } } -func (p *Tracer) AllowPID(uint32, uint32, *svc.ID) {} +func (p *Tracer) AllowPID(uint32, uint32, *svc.ID, *gosym.Table) {} func (p *Tracer) BlockPID(uint32, uint32) {} diff --git a/pkg/internal/ebpf/tctracer/bpf_arm64_bpfel.o b/pkg/internal/ebpf/tctracer/bpf_arm64_bpfel.o index 6a6a239ad..919b1fe91 100644 --- a/pkg/internal/ebpf/tctracer/bpf_arm64_bpfel.o +++ b/pkg/internal/ebpf/tctracer/bpf_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:021f8eea8cdbc5820ec1fc157da66359145394ea96fcad95484885dab73bd84a -size 36952 +oid sha256:f3cb18954fc0744c34f256ca23da5ec17ec8f700748a56eb90597e943d647b4f +size 37040 diff --git a/pkg/internal/ebpf/tctracer/bpf_debug_arm64_bpfel.o b/pkg/internal/ebpf/tctracer/bpf_debug_arm64_bpfel.o index 6a28492bf..e35b11ae0 100644 --- a/pkg/internal/ebpf/tctracer/bpf_debug_arm64_bpfel.o +++ b/pkg/internal/ebpf/tctracer/bpf_debug_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:97f7b0a0a3ee1a22e21840fee91dcfa2d3ae9664c0cb33922fa8930f08890cff -size 78760 +oid sha256:08799290bbc0b5cfaca7f65b90c50af0e3606f2593de41016e18a641840735ef +size 77848 diff --git a/pkg/internal/ebpf/tctracer/bpf_debug_x86_bpfel.o b/pkg/internal/ebpf/tctracer/bpf_debug_x86_bpfel.o index 87641db49..e0aa23d13 100644 --- a/pkg/internal/ebpf/tctracer/bpf_debug_x86_bpfel.o +++ b/pkg/internal/ebpf/tctracer/bpf_debug_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:16b80c465485dca1666e4d6fec3999e5c07329c07701587ef2d22145b399663e -size 78544 +oid sha256:26ff67087c19e80ea80937fc0193ebbdc25fd7bb63ce95d84b16ed2765aa4cad +size 77624 diff --git a/pkg/internal/ebpf/tctracer/bpf_x86_bpfel.o b/pkg/internal/ebpf/tctracer/bpf_x86_bpfel.o index 2253d66fb..7adf738ae 100644 --- a/pkg/internal/ebpf/tctracer/bpf_x86_bpfel.o +++ b/pkg/internal/ebpf/tctracer/bpf_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e782c2881f64ff39008afa683bbb0fc6cf5c4b78fd9d6f88d27cb74b638d3007 -size 36736 +oid sha256:1f1398a5e99a5ffd54062cb91e8323ac00e2de5b5cfc39fa955769c1a8fe0d14 +size 36824 diff --git a/pkg/internal/ebpf/tctracer/tctracer.go b/pkg/internal/ebpf/tctracer/tctracer.go index f199a760a..8eb5503f5 100644 --- a/pkg/internal/ebpf/tctracer/tctracer.go +++ b/pkg/internal/ebpf/tctracer/tctracer.go @@ -4,6 +4,7 @@ package tctracer import ( "context" + "debug/gosym" "io" "log/slog" @@ -43,7 +44,7 @@ func New(cfg *beyla.Config) *Tracer { } } -func (p *Tracer) AllowPID(uint32, uint32, *svc.ID) {} +func (p *Tracer) AllowPID(uint32, uint32, *svc.ID, *gosym.Table) {} func (p *Tracer) BlockPID(uint32, uint32) {} diff --git a/pkg/internal/ebpf/tracer.go b/pkg/internal/ebpf/tracer.go index e35c2df1b..0fdca06b7 100644 --- a/pkg/internal/ebpf/tracer.go +++ b/pkg/internal/ebpf/tracer.go @@ -2,6 +2,7 @@ package ebpf import ( "context" + "debug/gosym" "io" "log/slog" @@ -33,7 +34,7 @@ type PIDsAccounter interface { // traces from processes whose PID has not been allowed before // We must use a pointer for svc.ID so that all child processes share the same // object. This is important when we tag a service as exporting traces or metrics. - AllowPID(uint32, uint32, *svc.ID) + AllowPID(uint32, uint32, *svc.ID, *gosym.Table) // BlockPID notifies the tracer to stop accepting traces from the process // with the provided PID. After receiving them via ringbuffer, it should // discard them. @@ -117,9 +118,9 @@ type ProcessTracer struct { Instrumentables map[uint64]*instrumenter } -func (pt *ProcessTracer) AllowPID(pid, ns uint32, svc *svc.ID) { +func (pt *ProcessTracer) AllowPID(pid, ns uint32, svc *svc.ID, symTab *gosym.Table) { for i := range pt.Programs { - pt.Programs[i].AllowPID(pid, ns, svc) + pt.Programs[i].AllowPID(pid, ns, svc, symTab) } } diff --git a/pkg/internal/ebpf/watcher/bpf_arm64_bpfel.o b/pkg/internal/ebpf/watcher/bpf_arm64_bpfel.o index cd6d838b9..807ad9fe7 100644 --- a/pkg/internal/ebpf/watcher/bpf_arm64_bpfel.o +++ b/pkg/internal/ebpf/watcher/bpf_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e141ab06f68d4f191f0e3e505784bb6a61b78c30819966df7c7c301b34ed59c2 +oid sha256:f95505b6e9d3fc4fc131ab0d160f3abf41cab29dd50109a3f10aa2375dd38bae size 6912 diff --git a/pkg/internal/ebpf/watcher/bpf_debug_arm64_bpfel.o b/pkg/internal/ebpf/watcher/bpf_debug_arm64_bpfel.o index 570d08a84..9a7909b76 100644 --- a/pkg/internal/ebpf/watcher/bpf_debug_arm64_bpfel.o +++ b/pkg/internal/ebpf/watcher/bpf_debug_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c91beff8f0dca86a2b5156051ff0ed436390d582aec75f508ac38667daed8839 -size 8864 +oid sha256:068b0e73d3107b5523e53caf399ae3927d2e794133fcbc2847bac6f78894040e +size 8880 diff --git a/pkg/internal/ebpf/watcher/bpf_debug_x86_bpfel.o b/pkg/internal/ebpf/watcher/bpf_debug_x86_bpfel.o index e5b7fd9c6..6f2bf8bdb 100644 --- a/pkg/internal/ebpf/watcher/bpf_debug_x86_bpfel.o +++ b/pkg/internal/ebpf/watcher/bpf_debug_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ac5deb1999a2e133818760b3683d755e34ef0abc5a106dc904d4c3ed15f94f86 -size 8728 +oid sha256:1a88dc0fab324e0879b7f0766594b23274fab13aaf8a01fccb8d06bf6905c988 +size 8736 diff --git a/pkg/internal/ebpf/watcher/bpf_x86_bpfel.o b/pkg/internal/ebpf/watcher/bpf_x86_bpfel.o index 826e57682..a5a23fbee 100644 --- a/pkg/internal/ebpf/watcher/bpf_x86_bpfel.o +++ b/pkg/internal/ebpf/watcher/bpf_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:92eb613aa18fd1fb18a59297ede757f6f444f13014fadfb7262c2d2869e23e06 +oid sha256:1699bb5f44955548bbe22e52232713d2fc36469eba3b371887fbe0052b8d29b0 size 6768 diff --git a/pkg/internal/goexec/instructions.go b/pkg/internal/goexec/instructions.go index e96186170..4166888dd 100644 --- a/pkg/internal/goexec/instructions.go +++ b/pkg/internal/goexec/instructions.go @@ -12,9 +12,8 @@ import ( // instrumentationPoints loads the provided executable and looks for the addresses // where the start and return probes must be inserted. -// -//nolint:cyclop -func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncOffsets, error) { +// nolint:cyclop +func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncOffsets, *gosym.Table, error) { ilog := slog.With("component", "goexec.instructions") ilog.Debug("searching for instrumentation points", "functions", funcNames) functions := map[string]struct{}{} @@ -23,12 +22,12 @@ func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncO } symTab, err := findGoSymbolTable(elfF) if err != nil { - return nil, err + return nil, nil, err } goVersion, _, err := getGoDetails(elfF) if err == nil && !supportedGoVersion(goVersion) { - return nil, fmt.Errorf("unsupported Go version: %v. Minimum supported version is %v", goVersion, minGoVersion) + return nil, nil, fmt.Errorf("unsupported Go version: %v. Minimum supported version is %v", goVersion, minGoVersion) } gosyms := elfF.Section(".gosymtab") @@ -40,7 +39,7 @@ func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncO if gosyms == nil { allSyms, err = exec.FindExeSymbols(elfF, functions) if err != nil { - return nil, err + return nil, nil, err } } @@ -65,7 +64,7 @@ func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncO offs, ok, err := findFuncOffset(&f, elfF) if err != nil { - return nil, err + return nil, nil, err } if ok { ilog.Debug("found relevant function for instrumentation", "function", fName, "offsets", offs) @@ -74,7 +73,7 @@ func instrumentationPoints(elfF *elf.File, funcNames []string) (map[string]FuncO } } - return allOffsets, nil + return allOffsets, symTab, nil } func handleStaticSymbol(fName string, allOffsets map[string]FuncOffsets, allSyms map[string]exec.Sym, ilog *slog.Logger) { diff --git a/pkg/internal/goexec/offsets.go b/pkg/internal/goexec/offsets.go index 7221a85a4..cede09cd1 100644 --- a/pkg/internal/goexec/offsets.go +++ b/pkg/internal/goexec/offsets.go @@ -2,15 +2,18 @@ package goexec import ( + "debug/gosym" "fmt" + "github.com/grafana/beyla/pkg/export/otel" "github.com/grafana/beyla/pkg/internal/exec" ) type Offsets struct { // Funcs key: function name - Funcs map[string]FuncOffsets - Field FieldOffsets + Funcs map[string]FuncOffsets + Field FieldOffsets + SymTab *gosym.Table } type FuncOffsets struct { @@ -22,16 +25,22 @@ type FieldOffsets map[GoOffset]any // InspectOffsets gets the memory addresses/offsets of the instrumenting function, as well as the required // parameters fields to be read from the eBPF code -func InspectOffsets(execElf *exec.FileInfo, funcs []string) (*Offsets, error) { +func InspectOffsets(cfg *otel.TracesConfig, execElf *exec.FileInfo, funcs []string) (*Offsets, error) { if execElf == nil { return nil, fmt.Errorf("executable not found") } // Analyse executable ELF file and find instrumentation points - found, err := instrumentationPoints(execElf.ELF, funcs) + found, symTab, err := instrumentationPoints(execElf.ELF, funcs) if err != nil { return nil, fmt.Errorf("finding instrumentation points: %w", err) } + // symTab would be used to find the function name from the address when + // capturing Go errors. If the option is disabled, whe don't need to keep + // the symbol table in memory. + if !cfg.ReportExceptionEvents { + symTab = nil + } if len(found) == 0 { return nil, fmt.Errorf("couldn't find any instrumentation point in %s", execElf.CmdExePath) } @@ -43,7 +52,8 @@ func InspectOffsets(execElf *exec.FileInfo, funcs []string) (*Offsets, error) { } return &Offsets{ - Funcs: found, - Field: structFieldOffsets, + Funcs: found, + Field: structFieldOffsets, + SymTab: symTab, }, nil } diff --git a/pkg/internal/goexec/offsets_test.go b/pkg/internal/goexec/offsets_test.go index 5bfa88259..547572a11 100644 --- a/pkg/internal/goexec/offsets_test.go +++ b/pkg/internal/goexec/offsets_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/grafana/beyla/pkg/export/otel" "github.com/grafana/beyla/pkg/internal/testutil" ) @@ -18,7 +19,8 @@ func TestProcessNotFound(t *testing.T) { finish := make(chan struct{}) go func() { defer close(finish) - _, err := InspectOffsets(nil, nil) + cfg := &otel.TracesConfig{} + _, err := InspectOffsets(cfg, nil, nil) require.Error(t, err) }() testutil.ReadChannel(t, finish, 5*time.Second) diff --git a/pkg/internal/netolly/ebpf/net_arm64_bpfel.o b/pkg/internal/netolly/ebpf/net_arm64_bpfel.o index 29755eeae..e5b62d44d 100644 --- a/pkg/internal/netolly/ebpf/net_arm64_bpfel.o +++ b/pkg/internal/netolly/ebpf/net_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4f1926ab4ebb8e41fa9d2d5f9bc0bfc351cec56b6d48ba22be01beb4335d2c3e -size 31232 +oid sha256:f342b83229a3b3ab3d2213db8f98df776186589ec9962210af3cbdbb0091a7fb +size 31400 diff --git a/pkg/internal/netolly/ebpf/net_x86_bpfel.o b/pkg/internal/netolly/ebpf/net_x86_bpfel.o index d6a7147cc..4a6513b89 100644 --- a/pkg/internal/netolly/ebpf/net_x86_bpfel.o +++ b/pkg/internal/netolly/ebpf/net_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c6df5d77381c807c9770eda78ad45116fdf93bfcdbad9aaec7c1568d512693dd -size 31000 +oid sha256:58f97438ac4f934ddbb19f5fd142d7d20fd785ec279aaf4796117eb0f64e91d0 +size 31176 diff --git a/pkg/internal/netolly/ebpf/netsk_arm64_bpfel.o b/pkg/internal/netolly/ebpf/netsk_arm64_bpfel.o index 1b14dc48a..6dff9aa6f 100644 --- a/pkg/internal/netolly/ebpf/netsk_arm64_bpfel.o +++ b/pkg/internal/netolly/ebpf/netsk_arm64_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:804b4d262d43a1432af8027545e362ede6c4fe6fd274e521fd6084727103d268 -size 28584 +oid sha256:56c354ddfc8625aeba39f33f899d237e85c2109cfd641adc64c83137baafbd6d +size 28760 diff --git a/pkg/internal/netolly/ebpf/netsk_x86_bpfel.o b/pkg/internal/netolly/ebpf/netsk_x86_bpfel.o index 916be9db5..80f2d9fb2 100644 --- a/pkg/internal/netolly/ebpf/netsk_x86_bpfel.o +++ b/pkg/internal/netolly/ebpf/netsk_x86_bpfel.o @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b3746deaaa2b5df4c010273dbaee625518d85e857625d64ddc4327529ef9e773 -size 28368 +oid sha256:5ebb516660ed8a51b745049c455e8dc5006142def51214c93d45bb0fc5ac6010 +size 28544 diff --git a/pkg/internal/pipe/instrumenter.go b/pkg/internal/pipe/instrumenter.go index 23edc03e1..f0e7c7691 100644 --- a/pkg/internal/pipe/instrumenter.go +++ b/pkg/internal/pipe/instrumenter.go @@ -115,7 +115,7 @@ func newGraphBuilder(ctx context.Context, config *beyla.Config, ctxInfo *global. config.Traces.Grafana = &gb.config.Grafana.OTLP pipe.AddFinalProvider(gnb, otelTraces, otel.TracesReceiver(ctx, config.Traces, gb.ctxInfo, config.Attributes.Select)) pipe.AddFinalProvider(gnb, prometheus, prom.PrometheusEndpoint(ctx, gb.ctxInfo, &config.Prometheus, config.Attributes.Select)) - pipe.AddFinalProvider(gnb, alloyTraces, alloy.TracesReceiver(ctx, gb.ctxInfo, &config.TracesReceiver, config.Attributes.Select)) + pipe.AddFinalProvider(gnb, alloyTraces, alloy.TracesReceiver(ctx, gb.ctxInfo, config, config.Attributes.Select)) pipe.AddFinalProvider(gnb, printer, debug.PrinterNode(config.TracePrinter)) diff --git a/pkg/internal/request/span.go b/pkg/internal/request/span.go index 1b39df211..17ad42ede 100644 --- a/pkg/internal/request/span.go +++ b/pkg/internal/request/span.go @@ -124,30 +124,32 @@ type PidInfo struct { // REMINDER: any attribute here must be also added to the functions SpanOTELGetters, // SpanPromGetters and getDefinitions in pkg/export/attributes/attr_defs.go type Span struct { - Type EventType `json:"type"` - IgnoreSpan ignoreMode `json:"ignoreSpan"` - Method string `json:"-"` - Path string `json:"-"` - Route string `json:"-"` - Peer string `json:"peer"` - PeerPort int `json:"peerPort,string"` - Host string `json:"host"` - HostPort int `json:"hostPort,string"` - Status int `json:"-"` - ContentLength int64 `json:"-"` - RequestStart int64 `json:"-"` - Start int64 `json:"-"` - End int64 `json:"-"` - ServiceID svc.ID `json:"-"` // TODO: rename to Service or ResourceAttrs - TraceID trace2.TraceID `json:"traceID"` - SpanID trace2.SpanID `json:"spanID"` - ParentSpanID trace2.SpanID `json:"parentSpanID"` - Flags uint8 `json:"flags,string"` - Pid PidInfo `json:"-"` - PeerName string `json:"peerName"` - HostName string `json:"hostName"` - OtherNamespace string `json:"-"` - Statement string `json:"-"` + Type EventType `json:"type"` + IgnoreSpan ignoreMode `json:"ignoreSpan"` + Method string `json:"-"` + Path string `json:"-"` + Route string `json:"-"` + Peer string `json:"peer"` + PeerPort int `json:"peerPort,string"` + Host string `json:"host"` + HostPort int `json:"hostPort,string"` + Status int `json:"-"` + ContentLength int64 `json:"-"` + RequestStart int64 `json:"-"` + Start int64 `json:"-"` + End int64 `json:"-"` + ServiceID svc.ID `json:"-"` // TODO: rename to Service or ResourceAttrs + TraceID trace2.TraceID `json:"traceID"` + SpanID trace2.SpanID `json:"spanID"` + ParentSpanID trace2.SpanID `json:"parentSpanID"` + Flags uint8 `json:"flags,string"` + Pid PidInfo `json:"-"` + PeerName string `json:"peerName"` + HostName string `json:"hostName"` + OtherNamespace string `json:"-"` + Statement string `json:"-"` + ErrorMessage string `json:"-"` + ErrorStacktrace string `json:"-"` } func (s *Span) Inside(parent *Span) bool { diff --git a/test/integration/components/jaeger/jaeger.go b/test/integration/components/jaeger/jaeger.go index c3dfa19a1..3bf94d148 100644 --- a/test/integration/components/jaeger/jaeger.go +++ b/test/integration/components/jaeger/jaeger.go @@ -28,6 +28,7 @@ type Span struct { Duration int64 `json:"duration"` Tags []Tag `json:"tags"` ProcessID string `json:"processID"` + Logs []Log `json:"logs"` } type Tag struct { @@ -36,6 +37,11 @@ type Tag struct { Value interface{} `json:"value"` } +type Log struct { + Timestamp int64 `json:"timestamp"` + Fields []Tag `json:"fields"` +} + type Reference struct { RefType string `json:"refType"` TraceID string `json:"traceID"` diff --git a/test/integration/components/testserver/std/std.go b/test/integration/components/testserver/std/std.go index f26e05005..2719654c7 100644 --- a/test/integration/components/testserver/std/std.go +++ b/test/integration/components/testserver/std/std.go @@ -54,6 +54,10 @@ func HTTPHandler(log *slog.Logger, echoPort int) http.HandlerFunc { } else { status = s } + if status == 500 { + echoError(rw) + return + } case arg.Delay: if d, err := time.ParseDuration(v[0]); err != nil { log.Debug("wrong delay value. Ignoring", "error", err) @@ -180,6 +184,12 @@ func echoCall(rw http.ResponseWriter) { rw.WriteHeader(204) } +func echoError(rw http.ResponseWriter) { + _, err := httpClient.Get("htt://pytestserver:8083/error") + slog.Error("error making http request", "err", err) + rw.WriteHeader(500) +} + func Setup(port int) { log := slog.With("component", "std.Server") address := fmt.Sprintf(":%d", port) diff --git a/test/integration/docker-compose.yml b/test/integration/docker-compose.yml index eb47a2d35..edadd264a 100644 --- a/test/integration/docker-compose.yml +++ b/test/integration/docker-compose.yml @@ -51,6 +51,7 @@ services: BEYLA_INTERNAL_METRICS_PROMETHEUS_PORT: 8999 BEYLA_PROCESSES_INTERVAL: "100ms" BEYLA_HOSTNAME: "beyla" + BEYLA_TRACES_REPORT_EXCEPTION_EVENTS: "true" ports: - "8999:8999" # Prometheus scrape port, if enabled via config diff --git a/test/integration/http2go_test.go b/test/integration/http2go_test.go index 669e735b0..2672542ed 100644 --- a/test/integration/http2go_test.go +++ b/test/integration/http2go_test.go @@ -32,7 +32,7 @@ func testREDMetricsForHTTP2Library(t *testing.T, route, svcNs string) { `http_route="` + route + `",` + `url_path="` + route + `"}`) require.NoError(t, err) - // check duration_count has 3 calls and all the arguments + // check duration_count has 1 calls and all the arguments enoughPromResults(t, results) val := totalPromCount(t, results) assert.LessOrEqual(t, 1, val) diff --git a/test/integration/suites_test.go b/test/integration/suites_test.go index 27fe66156..165e16c62 100644 --- a/test/integration/suites_test.go +++ b/test/integration/suites_test.go @@ -124,7 +124,7 @@ func TestSuite_OldestGoVersion(t *testing.T) { require.NoError(t, err) require.NoError(t, compose.Up()) t.Run("RED metrics", testREDMetricsOldHTTP) - t.Run("HTTP traces", testHTTPTraces) + t.Run("HTTP traces", testHTTPTracesNoErrors) t.Run("GRPC traces", testGRPCTraces) t.Run("GRPC RED metrics", testREDMetricsGRPC) t.Run("Internal Prometheus metrics", testInternalPrometheusExport) diff --git a/test/integration/traces_test.go b/test/integration/traces_test.go index e4986bfe2..15f47bb65 100644 --- a/test/integration/traces_test.go +++ b/test/integration/traces_test.go @@ -19,14 +19,18 @@ import ( ) func testHTTPTracesNoTraceID(t *testing.T) { - testHTTPTracesCommon(t, false, 200) + testHTTPTracesCommon(t, false, 200, false) } func testHTTPTraces(t *testing.T) { - testHTTPTracesCommon(t, true, 500) + testHTTPTracesCommon(t, true, 500, true) } -func testHTTPTracesCommon(t *testing.T, doTraceID bool, httpCode int) { +func testHTTPTracesNoErrors(t *testing.T) { + testHTTPTracesCommon(t, true, 500, false) +} + +func testHTTPTracesCommon(t *testing.T, doTraceID bool, httpCode int, checkErrors bool) { var traceID string var parentID string @@ -58,7 +62,7 @@ func testHTTPTracesCommon(t *testing.T, doTraceID bool, httpCode int) { traces := tq.FindBySpan(jaeger.Tag{Key: "url.path", Type: "string", Value: "/" + slug}) require.Len(t, traces, 1) trace = traces[0] - require.Len(t, trace.Spans, 3) // parent - in queue - processing + require.GreaterOrEqual(t, len(trace.Spans), 3) // parent - in queue - processing }, test.Interval(100*time.Millisecond)) // Check the information of the parent span @@ -91,6 +95,20 @@ func testHTTPTracesCommon(t *testing.T, doTraceID bool, httpCode int) { jaeger.Tag{Key: "otel.status_code", Type: "string", Value: "ERROR"}, ) assert.Empty(t, sd, sd.String()) + if checkErrors { + assert.NotEmpty(t, parent.Logs) + eventAttr := parent.Logs[0].Fields + eventType, ok := jaeger.FindIn(eventAttr, "event") + require.True(t, ok) + assert.Equal(t, "exception", eventType.Value) + + errMessage, ok := jaeger.FindIn(eventAttr, "exception.message") + require.True(t, ok) + assert.Equal(t, "unsupported protocol scheme \"htt\"", errMessage.Value) + + _, ok = jaeger.FindIn(eventAttr, "exception.stacktrace") + require.True(t, ok) + } } // Check the information of the "in queue" span