Add probe loading tests across multiple kernel versions using vimto (#1005)

Author: RonFed

.github/workflows/probe_load.yaml

@@ -0,0 +1,47 @@
+name: probe_load
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+env:
+  go_version: '~1.22'
+  CGO_ENABLED: '0'
+
+jobs:
+    vm-test:
+        name: Run tests
+        runs-on: ubuntu-latest
+        timeout-minutes: 15
+        strategy:
+          fail-fast: false
+          matrix:
+            tag:
+              - "stable"
+              - "6.6"
+              - "5.15"
+              - "5.10"
+              - "5.4"
+        steps:
+          - uses: actions/checkout@v4
+          - name: Set up Go
+            uses: actions/setup-go@v5
+            with:
+              go-version: '${{ env.go_version }}'
+          - name: make docker-generate
+            run: make docker-generate
+          - name: verify output
+            run: make check-clean-work-tree
+          - name: Install vimto
+            run: go install lmb.io/vimto@latest
+          - name: Install qemu
+            run: |
+                sudo apt-get update && sudo apt-get install -y --no-install-recommends qemu-system-x86
+                sudo chmod 0666 /dev/kvm
+          - name: Test without verifier logs
+            id: no_verifier_logs_test
+            run: OTEL_GO_AUTO_SHOW_VERIFIER_LOG=false vimto -kernel :${{ matrix.tag }} -- go test -v -count=1 -tags=multi_kernel_test go.opentelemetry.io/auto/internal/pkg/instrumentation
+          - name: Test with verifier logs
+            run: OTEL_GO_AUTO_SHOW_VERIFIER_LOG=true vimto -kernel :${{ matrix.tag }} -- go test -v -count=1 -tags=multi_kernel_test go.opentelemetry.io/auto/internal/pkg/instrumentation
+            if: always() && steps.no_verifier_logs_test.outcome == 'failure'

.golangci.yml

@@ -74,6 +74,7 @@ linters-settings:
         files:
           - "!$test"
           - "!**/*test/*.go"
+          - "!**/testutils/*.go"
         deny:
           - pkg: "testing"
           - pkg: "github.com/stretchr/testify"

.vimto.toml

@@ -0,0 +1,12 @@
+kernel="ghcr.io/cilium/ci-kernels:stable"
+smp="cpus=4"
+memory="8G"
+user="root"
+setup=[
+   "mount -t cgroup2 -o nosuid,noexec,nodev cgroup2 /sys/fs/cgroup",
+   "/bin/sh -c 'modprobe bpf_testmod || true'",
+   "dmesg --clear",
+]
+teardown=[
+   "dmesg --read-clear",
+]

internal/include/otel_types.h

@@ -55,11 +55,6 @@ static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_
 {
 	u64 vtype = go_attr_value->vtype;
 
-	if (vtype == attr_type_invalid) {
-		bpf_printk("Invalid attribute value type\n");
-		return false;
-	}
-
 	// Constant size values
 	if (vtype == attr_type_bool ||
 		vtype == attr_type_int64 ||
@@ -74,7 +69,8 @@ static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_
 			bpf_printk("Aattribute string value is too long\n");
 			return false;
 		}
-		return get_go_string_from_user_ptr(&go_attr_value->string, attr->value, OTEL_ATTRIBUTE_VALUE_MAX_LEN);
+		long res = bpf_probe_read_user(attr->value, go_attr_value->string.len & (OTEL_ATTRIBUTE_VALUE_MAX_LEN -1), go_attr_value->string.str);
+ 		return res == 0;
 	}
 
 	// TODO (#525): handle slices
@@ -83,7 +79,7 @@ static __always_inline bool set_attr_value(otel_attirbute_t *attr, go_otel_attr_
 
 static __always_inline void convert_go_otel_attributes(void *attrs_buf, u64 slice_len, otel_attributes_t *enc_attrs)
 {
-	if (attrs_buf == NULL || enc_attrs == NULL){
+	if (attrs_buf == NULL){
 		return;
 	}
 
@@ -100,7 +96,10 @@ static __always_inline void convert_go_otel_attributes(void *attrs_buf, u64 slic
 		return;
 	}
 
-	for (u8 go_attr_index = 0; go_attr_index < num_attrs; go_attr_index++) {
+	for (u8 go_attr_index = 0; go_attr_index < OTEL_ATTRUBUTE_MAX_COUNT; go_attr_index++) {
+		if (go_attr_index >= slice_len) {
+ 			break;
+ 		}
 		__builtin_memset(&go_attr_value, 0, sizeof(go_otel_attr_value_t));
 		// Read the value struct
 		bpf_probe_read(&go_attr_value, sizeof(go_otel_attr_value_t), &go_attr[go_attr_index].value);
@@ -124,9 +123,7 @@ static __always_inline void convert_go_otel_attributes(void *attrs_buf, u64 slic
 			break;
 		}
 
-		if (!get_go_string_from_user_ptr(&go_str, enc_attrs->attrs[valid_attrs].key, OTEL_ATTRIBUTE_KEY_MAX_LEN)) {
-			continue;
-		}
+		bpf_probe_read_user(enc_attrs->attrs[valid_attrs].key, go_str.len & (OTEL_ATTRIBUTE_KEY_MAX_LEN -1), go_str.str);
 
 		if (!set_attr_value(&enc_attrs->attrs[valid_attrs], &go_attr_value)) {
 			continue;

internal/pkg/inject/consts.go

@@ -139,3 +139,7 @@ func WithOffset(key string, id structfield.ID, ver *version.Version) Option {
 	}
 	return WithKeyValue(key, off.Offset)
 }
+
+func GetLatestOffset(id structfield.ID) (structfield.OffsetKey, *version.Version) {
+	return offsets.GetLatestOffset(id)
+}

internal/pkg/instrumentation/manager.go

@@ -223,21 +223,28 @@ func (m *Manager) cleanup(target *process.TargetDetails) error {
 	return errors.Join(err, bpffsCleanup(target))
 }
 
-func (m *Manager) registerProbes() error {
+//nolint:revive // ignoring linter complaint about control flag
+func availableProbes(l logr.Logger, withTraceGlobal bool) []probe.Probe {
 	insts := []probe.Probe{
-		grpcClient.New(m.logger),
-		grpcServer.New(m.logger),
-		httpServer.New(m.logger),
-		httpClient.New(m.logger),
-		dbSql.New(m.logger),
-		kafkaProducer.New(m.logger),
-		kafkaConsumer.New(m.logger),
+		grpcClient.New(l),
+		grpcServer.New(l),
+		httpServer.New(l),
+		httpClient.New(l),
+		dbSql.New(l),
+		kafkaProducer.New(l),
+		kafkaConsumer.New(l),
 	}
 
-	if m.globalImpl {
-		insts = append(insts, otelTraceGlobal.New(m.logger))
+	if withTraceGlobal {
+		insts = append(insts, otelTraceGlobal.New(l))
 	}
 
+	return insts
+}
+
+func (m *Manager) registerProbes() error {
+	insts := availableProbes(m.logger, m.globalImpl)
+
 	for _, i := range insts {
 		err := m.registerProbe(i)
 		if err != nil {

internal/pkg/instrumentation/manager_load_test.go

@@ -0,0 +1,57 @@
+//go:build multi_kernel_test
+
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package instrumentation
+
+import (
+	"log"
+	"os"
+	"testing"
+
+	"github.com/go-logr/stdr"
+	"github.com/hashicorp/go-version"
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/auto/internal/pkg/inject"
+	"go.opentelemetry.io/auto/internal/pkg/instrumentation/testutils"
+	"go.opentelemetry.io/auto/internal/pkg/instrumentation/utils"
+)
+
+func TestLoadProbes(t *testing.T) {
+	ver, _ := utils.GetLinuxKernelVersion()
+	t.Logf("Running on kernel %s", ver.String())
+	m := fakeManager(t)
+
+	probes := availableProbes(m.logger, true)
+	assert.NotEmpty(t, probes)
+
+	for _, p := range probes {
+		manifest := p.Manifest()
+		fields := manifest.StructFields
+		offsets := map[string]*version.Version{}
+		for _, f := range fields {
+			_, ver := inject.GetLatestOffset(f)
+			if ver != nil {
+				offsets[f.PkgPath] = ver
+				offsets[f.ModPath] = ver
+			}
+		}
+		t.Run(p.Manifest().Id.String(), func(t *testing.T) {
+			testProbe, ok := p.(testutils.TestProbe)
+			assert.True(t, ok)
+			testutils.ProbesLoad(t, testProbe, offsets)
+		})
+	}
+}
+
+func fakeManager(t *testing.T) *Manager {
+	logger := stdr.New(log.New(os.Stderr, "", log.LstdFlags))
+	logger = logger.WithName("Instrumentation")
+
+	m, err := NewManager(logger, nil, true, nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, m)
+
+	return m
+} 

internal/pkg/instrumentation/manager_test.go

@@ -1,3 +1,5 @@
+//go:build !multi_kernel_test
+
 // Copyright The OpenTelemetry Authors
 // SPDX-License-Identifier: Apache-2.0
 

internal/pkg/instrumentation/probe/probe.go

@@ -92,14 +92,18 @@ func (i *Base[BPFObj, BPFEvent]) Manifest() Manifest {
 	return NewManifest(i.ID, structfields, symbols)
 }
 
+func (i *Base[BPFObj, BPFEvent]) Spec() (*ebpf.CollectionSpec, error) {
+	return i.SpecFn()
+}
+
 // Load loads all instrumentation offsets.
 func (i *Base[BPFObj, BPFEvent]) Load(exec *link.Executable, td *process.TargetDetails) error {
 	spec, err := i.SpecFn()
 	if err != nil {
 		return err
 	}
 
-	err = i.injectConsts(td, spec)
+	err = i.InjectConsts(td, spec)
 	if err != nil {
 		return err
 	}
@@ -123,7 +127,7 @@ func (i *Base[BPFObj, BPFEvent]) Load(exec *link.Executable, td *process.TargetD
 	return nil
 }
 
-func (i *Base[BPFObj, BPFEvent]) injectConsts(td *process.TargetDetails, spec *ebpf.CollectionSpec) error {
+func (i *Base[BPFObj, BPFEvent]) InjectConsts(td *process.TargetDetails, spec *ebpf.CollectionSpec) error {
 	opts, err := consts(i.Consts).injectOpts(td)
 	if err != nil {
 		return err

internal/pkg/instrumentation/testutils/testutils.go

@@ -0,0 +1,93 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package testutils
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/cilium/ebpf"
+	"github.com/cilium/ebpf/rlimit"
+	"github.com/hashicorp/go-version"
+	"github.com/stretchr/testify/assert"
+
+	"go.opentelemetry.io/auto/internal/pkg/instrumentation/bpffs"
+	"go.opentelemetry.io/auto/internal/pkg/instrumentation/utils"
+	"go.opentelemetry.io/auto/internal/pkg/process"
+)
+
+var testGoVersion = version.Must(version.NewVersion("1.22.1"))
+
+type TestProbe interface {
+	Spec() (*ebpf.CollectionSpec, error)
+	InjectConsts(td *process.TargetDetails, spec *ebpf.CollectionSpec) error
+}
+
+func ProbesLoad(t *testing.T, p TestProbe, libs map[string]*version.Version) {
+	err := rlimit.RemoveMemlock()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	td := &process.TargetDetails{
+		PID: 1,
+		AllocationDetails: &process.AllocationDetails{
+			StartAddr: 140434497441792,
+			EndAddr:   140434497507328,
+		},
+		Libraries: map[string]*version.Version{
+			"std": testGoVersion,
+		},
+		GoVersion: testGoVersion,
+	}
+	for k, v := range libs {
+		td.Libraries[k] = v
+	}
+
+	err = bpffs.Mount(td)
+	if !assert.NoError(t, err) {
+		return
+	}
+	defer func() {
+		_ = bpffs.Cleanup(td)
+	}()
+
+	spec, err := p.Spec()
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	// Inject the same constants as the BPF program.
+	// It is important to inject the same constants as those that will be used in the actual run,
+	// since From Linux 5.5 the verifier will use constants to eliminate dead code.
+	err = p.InjectConsts(td, spec)
+	if !assert.NoError(t, err) {
+		return
+	}
+
+	opts := ebpf.CollectionOptions{
+		Maps: ebpf.MapOptions{
+			PinPath: bpffs.PathForTargetApplication(td),
+		},
+	}
+
+	collectVerifierLogs := utils.ShouldShowVerifierLogs()
+	if collectVerifierLogs {
+		opts.Programs.LogLevel = ebpf.LogLevelStats | ebpf.LogLevelInstruction
+	}
+
+	c, err := ebpf.NewCollectionWithOptions(spec, opts)
+	if !assert.NoError(t, err) {
+		var ve *ebpf.VerifierError
+		if errors.As(err, &ve) && collectVerifierLogs {
+			t.Logf("Verifier log: %-100v\n", ve)
+		}
+	}
+
+	defer func() {
+		if c != nil {
+			c.Close()
+		}
+	}()
+}

internal/pkg/instrumentation/utils/ebpf.go

@@ -21,7 +21,7 @@ const (
 // If the environment variable OTEL_GO_AUTO_SHOW_VERIFIER_LOG is set to true, the verifier log will be printed.
 func InitializeEBPFCollection(spec *ebpf.CollectionSpec, opts *ebpf.CollectionOptions) (*ebpf.Collection, error) {
 	// Getting full verifier log is expensive, so we only do it if the user explicitly asks for it.
-	showVerifierLogs := shouldShowVerifierLogs()
+	showVerifierLogs := ShouldShowVerifierLogs()
 	if showVerifierLogs {
 		opts.Programs.LogLevel = ebpf.LogLevelInstruction | ebpf.LogLevelBranch | ebpf.LogLevelStats
 	}
@@ -37,8 +37,8 @@ func InitializeEBPFCollection(spec *ebpf.CollectionSpec, opts *ebpf.CollectionOp
 	return c, err
 }
 
-// shouldShowVerifierLogs returns if the user has configured verifier logs to be emitted.
-func shouldShowVerifierLogs() bool {
+// ShouldShowVerifierLogs returns if the user has configured verifier logs to be emitted.
+func ShouldShowVerifierLogs() bool {
 	val, exists := os.LookupEnv(showVerifierLogEnvVar)
 	if exists {
 		boolVal, err := strconv.ParseBool(val)

internal/pkg/structfield/structfield.go

@@ -49,6 +49,20 @@ func (i *Index) GetOffset(id ID, ver *version.Version) (OffsetKey, bool) {
 	return i.getOffset(id, ver)
 }
 
+// GetLatestOffset returns the latest known offset value and version for id
+// contained in the Index i.
+func (i *Index) GetLatestOffset(id ID) (OffsetKey, *version.Version) {
+	i.dataMu.RLock()
+	defer i.dataMu.RUnlock()
+
+	offs, ok := i.get(id)
+	if !ok {
+		return OffsetKey{}, nil
+	}
+	off, ver := offs.getLatest()
+	return off, ver.ToVersion()
+}
+
 func (i *Index) getOffset(id ID, ver *version.Version) (OffsetKey, bool) {
 	offs, ok := i.get(id)
 	if !ok {
@@ -259,6 +273,23 @@ func (o *Offsets) Get(ver *version.Version) (OffsetKey, bool) {
 	return v.offset, ok
 }
 
+// getLatest returns the latest known offset value and version.
+func (o *Offsets) getLatest() (OffsetKey, verKey) {
+	o.mu.RLock()
+	defer o.mu.RUnlock()
+
+	latestVersion := verKey{}
+	val := OffsetKey{}
+	for verKey, ov := range o.values {
+		if verKey.GreaterThan(latestVersion) && ov.offset.Valid {
+			latestVersion = verKey
+			val = ov.offset
+		}
+	}
+
+	return val, latestVersion
+}
+
 // Put sets the offset value for ver. If an offset for ver is already known
 // (i.e. ver.Equal(other) == true), this will overwrite that value.
 func (o *Offsets) Put(ver *version.Version, offset OffsetKey) {
@@ -281,6 +312,28 @@ func (o *Offsets) Put(ver *version.Version, offset OffsetKey) {
 	}
 }
 
+func (v verKey) GreaterThan(other verKey) bool {
+	if v.major != other.major {
+		return v.major > other.major
+	}
+	if v.minor != other.minor {
+		return v.minor > other.minor
+	}
+	if v.patch != other.patch {
+		return v.patch > other.patch
+	}
+	return false
+}
+
+func (v verKey) ToVersion() *version.Version {
+	vs := fmt.Sprintf("%d.%d.%d", v.major, v.minor, v.patch)
+	if v.prerelease != "" {
+		vs += "-" + v.prerelease
+	}
+	ver, _ := version.NewVersion(vs)
+	return ver
+}
+
 func (o *Offsets) index() map[OffsetKey][]*version.Version {
 	o.mu.RLock()
 	defer o.mu.RUnlock()

internal/pkg/structfield/structfield_test.go

@@ -53,6 +53,10 @@ func TestOffsets(t *testing.T) {
 	assert.True(t, ok, "did not get 1.2.1")
 	assert.Equal(t, OffsetKey{Offset: 2, Valid: true}, off, "invalid value for 1.2.1")
 
+	off, ver := o.getLatest()
+	assert.Equal(t, v121, ver.ToVersion(), "invalid version for latest")
+	assert.Equal(t, OffsetKey{Offset: 2, Valid: true}, off, "invalid value for latest")
+
 	o.Put(v120, OffsetKey{Offset: 1, Valid: true})
 	off, ok = o.Get(v120)
 	assert.True(t, ok, "did not get 1.2.0 after reset")
@@ -123,3 +127,21 @@ func TestIndexUnmarshalJSON(t *testing.T) {
 	require.NoError(t, json.NewDecoder(f).Decode(&got))
 	assert.Equal(t, index, &got)
 }
+
+func TestGetLatestOffsetFromIndex(t *testing.T) {
+	off, ver := index.GetLatestOffset(NewID("std", "net/http", "Request", "Method"))
+	assert.Equal(t, v130, ver, "invalid version for Request.Method")
+	assert.Equal(t, OffsetKey{Offset: 1, Valid: true}, off, "invalid value for Request.Method")
+
+	off, ver = index.GetLatestOffset(NewID("std", "net/http", "Request", "URL"))
+	assert.Equal(t, v130, ver, "invalid version for Request.URL")
+	assert.Equal(t, OffsetKey{Offset: 2, Valid: true}, off, "invalid value for Request.URL")
+
+	off, ver = index.GetLatestOffset(NewID("std", "net/http", "Response", "Status"))
+	assert.Equal(t, v120, ver, "invalid version for Response.Status")
+	assert.Equal(t, OffsetKey{Offset: 0, Valid: true}, off, "invalid value for Response.Status")
+
+	off, ver = index.GetLatestOffset(NewID("google.golang.org/grpc", "google.golang.org/grpc", "ClientConn", "target"))
+	assert.Equal(t, v120, ver, "invalid version for ClientConn.target")
+	assert.Equal(t, OffsetKey{Offset: 0, Valid: true}, off, "invalid value for ClientConn.target")
+}