Inter-zone traffic monitoring (#1593)

docs/sources/metrics.md

@@ -93,6 +93,7 @@ In order to configure which attributes to show or which attributes to hide, chec
 | `beyla.network.flow.bytes`     | `dst.cidr`                   | shown if the `cidrs` configuration section exists |
 | `beyla.network.flow.bytes`     | `dst.name`                   | hidden                                            |
 | `beyla.network.flow.bytes`     | `dst.port`                   | hidden                                            |
+| `beyla.network.flow.bytes`     | `dst.zone` (only Kubernetes) | hidden                                            |
 | `beyla.network.flow.bytes`     | `iface`                      | hidden                                            |
 | `beyla.network.flow.bytes`     | `k8s.cluster.name`           | shown if network metrics are enabled              |
 | `beyla.network.flow.bytes`     | `k8s.dst.name`               | hidden                                            |
@@ -113,6 +114,7 @@ In order to configure which attributes to show or which attributes to hide, chec
 | `beyla.network.flow.bytes`     | `src.cidr`                   | shown if the `cidrs` configuration section exists |
 | `beyla.network.flow.bytes`     | `src.name`                   | hidden                                            |
 | `beyla.network.flow.bytes`     | `src.port`                   | hidden                                            |
+| `beyla.network.flow.bytes`     | `src.zone` (only Kubernetes) | hidden                                            |
 | `beyla.network.flow.bytes`     | `transport`                  | hidden                                            |
 | Traces (SQL, Redis)            | `db.query.text`              | hidden                                            |
 

pkg/export/attributes/attr_defs.go

@@ -201,6 +201,8 @@ func getDefinitions(groups AttrGroups) map[Section]AttrReportGroup {
 				attr.DstName:        false,
 				attr.ServerPort:     false,
 				attr.ClientPort:     false,
+				attr.SrcZone:        false,
+				attr.DstZone:        false,
 				attr.IfaceDirection: Default(ifaceDirEnabled),
 				attr.Iface:          Default(ifaceDirEnabled),
 			},

pkg/export/attributes/attr_selector_test.go

@@ -27,7 +27,7 @@ func TestFor(t *testing.T) {
 	p, err := NewAttrSelector(GroupKubernetes, Selection{
 		"beyla_network_flow_bytes_total": InclusionLists{
 			Include: []string{"beyla_ip", "src.*", "k8s.*"},
-			Exclude: []string{"k8s_*_name", "k8s.*.type"},
+			Exclude: []string{"k8s_*_name", "k8s.*.type", "*zone"},
 		},
 	})
 	require.NoError(t, err)
@@ -54,7 +54,7 @@ func TestFor_GlobEntries(t *testing.T) {
 		},
 		"beyla_network_flow_bytes_total": InclusionLists{
 			Include: []string{"src.*", "k8s.*"},
-			Exclude: []string{"k8s.*.name"},
+			Exclude: []string{"k8s.*.name", "*zone"},
 		},
 	})
 	require.NoError(t, err)
@@ -81,7 +81,7 @@ func TestFor_GlobEntries_NoInclusion(t *testing.T) {
 			Exclude: []string{"*dst*"},
 		},
 		"beyla_network_flow_bytes_total": InclusionLists{
-			Exclude: []string{"k8s.*.namespace"},
+			Exclude: []string{"k8s.*.namespace", "*zone"},
 		},
 	})
 	require.NoError(t, err)
@@ -101,7 +101,7 @@ func TestFor_GlobEntries_Order(t *testing.T) {
 			Include: []string{"*"},
 		},
 		"beyla_network_*": InclusionLists{
-			Exclude: []string{"dst.*", "transport", "*direction", "iface"},
+			Exclude: []string{"dst.*", "transport", "*direction", "iface", "*zone"},
 		},
 		"beyla_network_flow_bytes_total": InclusionLists{
 			Include: []string{"dst.name"},
@@ -140,7 +140,7 @@ func TestFor_KubeDisabled(t *testing.T) {
 	p, err := NewAttrSelector(0, Selection{
 		"beyla_network_flow_bytes_total": InclusionLists{
 			Include: []string{"target.instance", "beyla_ip", "src.*", "k8s.*"},
-			Exclude: []string{"src.port"},
+			Exclude: []string{"src.port", "*zone"},
 		},
 	})
 	require.NoError(t, err)

pkg/export/attributes/names/attrs.go

@@ -80,6 +80,8 @@ const (
 	Iface      = Name("iface")
 	SrcCIDR    = Name("src.cidr")
 	DstCIDR    = Name("dst.cidr")
+	SrcZone    = Name("src.zone")
+	DstZone    = Name("dst.zone")
 
 	ClientPort = Name("client.port")
 

pkg/internal/netolly/ebpf/record.go

@@ -51,6 +51,10 @@ type RecordAttrs struct {
 	SrcName string
 	DstName string
 
+	// SrcZone and DstZone represent the Cloud availability zones of the source and destination
+	SrcZone string
+	DstZone string
+
 	Interface string
 	// BeylaIP provides information about the source of the flow (the Agent that traced it)
 	BeylaIP  string

pkg/internal/netolly/ebpf/record_getters.go

@@ -93,6 +93,10 @@ func RecordGetters(name attr.Name) (attributes.Getter[*Record, attribute.KeyValu
 			}
 			return attribute.Int(string(attr.ServerPort), int(serverPort))
 		}
+	case attr.SrcZone:
+		getter = func(r *Record) attribute.KeyValue { return attribute.String(string(attr.SrcZone), r.Attrs.SrcZone) }
+	case attr.DstZone:
+		getter = func(r *Record) attribute.KeyValue { return attribute.String(string(attr.DstZone), r.Attrs.DstZone) }
 	default:
 		getter = func(r *Record) attribute.KeyValue { return attribute.String(string(name), r.Attrs.Metadata[name]) }
 	}

pkg/internal/netolly/transform/k8s/kubernetes.go

@@ -29,6 +29,7 @@ import (
 	attr "github.com/grafana/beyla/pkg/export/attributes/names"
 	"github.com/grafana/beyla/pkg/internal/kube"
 	"github.com/grafana/beyla/pkg/internal/netolly/ebpf"
+	"github.com/grafana/beyla/pkg/kubecache/informer"
 	"github.com/grafana/beyla/pkg/transform"
 )
 
@@ -42,6 +43,8 @@ const (
 	attrSuffixOwnerType = ".owner.type"
 	attrSuffixHostIP    = ".node.ip"
 	attrSuffixHostName  = ".node.name"
+
+	cloudZoneLabel = "topology.kubernetes.io/zone"
 )
 
 const alreadyLoggedIPsCacheLen = 256
@@ -141,13 +144,9 @@ func (n *decorator) decorate(flow *ebpf.Record, prefix, ip string) bool {
 	flow.Attrs.Metadata[attr.Name(prefix+attrSuffixType)] = meta.Kind
 	flow.Attrs.Metadata[attr.Name(prefix+attrSuffixOwnerName)] = ownerName
 	flow.Attrs.Metadata[attr.Name(prefix+attrSuffixOwnerType)] = ownerKind
-	// add any other ownership label (they might be several, e.g. replicaset and deployment)¡
-	if meta.Pod != nil && meta.Pod.HostIp != "" {
-		flow.Attrs.Metadata[attr.Name(prefix+attrSuffixHostIP)] = meta.Pod.HostIp
-		if host := n.kube.ObjectMetaByIP(meta.Pod.HostIp); host != nil {
-			flow.Attrs.Metadata[attr.Name(prefix+attrSuffixHostName)] = host.Meta.Name
-		}
-	}
+
+	n.nodeLabels(flow, prefix, meta)
+
 	// decorate other names from metadata, if required
 	if prefix == attrPrefixDst {
 		if flow.Attrs.DstName == "" {
@@ -161,6 +160,31 @@ func (n *decorator) decorate(flow *ebpf.Record, prefix, ip string) bool {
 	return true
 }
 
+func (n *decorator) nodeLabels(flow *ebpf.Record, prefix string, meta *informer.ObjectMeta) {
+	var nodeLabels map[string]string
+	// add any other ownership label (they might be several, e.g. replicaset and deployment)
+	if meta.Pod != nil && meta.Pod.HostIp != "" {
+		flow.Attrs.Metadata[attr.Name(prefix+attrSuffixHostIP)] = meta.Pod.HostIp
+		if host := n.kube.ObjectMetaByIP(meta.Pod.HostIp); host != nil {
+			flow.Attrs.Metadata[attr.Name(prefix+attrSuffixHostName)] = host.Meta.Name
+			nodeLabels = host.Meta.Labels
+		}
+	} else if meta.Kind == "Node" {
+		nodeLabels = meta.Labels
+	}
+	if nodeLabels != nil {
+		// this isn't strictly a Kubernetes attribute, but in Kubernetes
+		// clusters this information is inferred from Node annotations
+		if zone, ok := nodeLabels[cloudZoneLabel]; ok {
+			if prefix == attrPrefixDst {
+				flow.Attrs.DstZone = zone
+			} else {
+				flow.Attrs.SrcZone = zone
+			}
+		}
+	}
+}
+
 // newDecorator create a new transform
 func newDecorator(ctx context.Context, cfg *transform.KubernetesDecorator, meta *kube.Store) (*decorator, error) {
 	nt := decorator{

test/integration/components/docker/build.go

@@ -44,7 +44,6 @@ func pullDockerfile(logger io.WriteCloser, ilog *slog.Logger, img ImageBuild) er
 		return err
 	}
 	return nil
-
 }
 
 func buildDockerfile(logger io.WriteCloser, rootPath string, ilog *slog.Logger, img ImageBuild) error {

test/integration/k8s/manifests/00-kind-multi-zone.yml

@@ -0,0 +1,50 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+name: beyla
+nodes:
+  - role: control-plane
+    labels:
+      # faking a multi-zone cluster
+      topology.kubernetes.io/zone: control-plane-zone
+    extraMounts:
+      # configuration files that need to be mounted in the host
+      - hostPath: ../../configs
+        containerPath: /configs
+      # testoutput folder to store coverage data
+      - hostPath: ../../../../testoutput
+        containerPath: /testoutput
+  - role: worker
+    labels:
+      # faking a multi-zone cluster
+      topology.kubernetes.io/zone: server-zone
+    extraMounts:
+      # configuration files that need to be mounted in the host
+      - hostPath: ../../configs
+        containerPath: /configs
+      # testoutput folder to store coverage data
+      - hostPath: ../../../../testoutput
+        containerPath: /testoutput
+  - role: worker
+    labels:
+      # faking a multi-zone cluster
+      topology.kubernetes.io/zone: client-zone
+      # otel collector needs to be placed here, due to port mappings
+      deployment/zone: otel
+    extraMounts:
+      # configuration files that need to be mounted in the host
+      - hostPath: ../../configs
+        containerPath: /configs
+      # testoutput folder to store coverage data
+      - hostPath: ../../../../testoutput
+        containerPath: /testoutput
+    extraPortMappings:
+      # to avoid having to do port-forwarding from the Go client (a bit cumbersome process),
+      # we just make visible some container ports through host ports
+      # hostPorts need to be in range 30000-32767
+      # containerPort must be the "hostPort" value that needs to be
+      # exposed on each Pod container
+      - containerPort: 8999
+        hostPort: 38999
+      - containerPort: 9090
+        hostPort: 39090
+

test/integration/k8s/manifests/05-uninstrumented-multizone-client-server.yml

@@ -0,0 +1,103 @@
+# this file depends in the annotations and 00-kind-multi-node.yml to deploy a testserver
+# and a client in different nodes.
+# Beyla will instrument both, but restricting the metadata only to the local node,
+# so network flows between client and testserver would be incomplete
+apiVersion: v1
+kind: Pod
+metadata:
+  name: httppinger
+  labels:
+    component: httppinger
+    # this label will trigger a deletion of beyla pods before tearing down
+    # kind, to force Beyla writing the coverage data
+    teardown: delete
+  annotations:
+    resource.opentelemetry.io/deployment.environment: integration-test
+    resource.opentelemetry.io/service.version: '3.2.1'
+spec:
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          # force multi-zone traffic: server is in us-west-1b
+          - matchExpressions:
+              - key: topology.kubernetes.io/zone
+                operator: In
+                values:
+                  - client-zone
+  containers:
+    - name: httppinger
+      image: httppinger:dev
+      env:
+        - name: TARGET_URL
+          value: "http://testserver:8080"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: testserver
+spec:
+  selector:
+    app: testserver
+  ports:
+    - port: 8080
+      name: http0
+      targetPort: http0
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: testserver
+  labels:
+    app: testserver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: testserver
+  template:
+    metadata:
+      name: testserver
+      labels:
+        app: testserver
+        # this label will trigger a deletion of beyla pods before tearing down
+        # kind, to force Beyla writing the coverage data
+        teardown: delete
+      annotations:
+        resource.opentelemetry.io/deployment.environment: integration-test
+        resource.opentelemetry.io/service.version: '3.2.1'
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  # force multi-zone traffic: client is in us-west-1a
+                  - key: topology.kubernetes.io/zone
+                    operator: In
+                    values:
+                      - server-zone
+      containers:
+        - name: testserver
+          image: testserver:dev
+          imagePullPolicy: Never # loaded into Kind from localhost
+          ports:
+            # exposing hostports to enable operation from tests
+            - containerPort: 8080
+              hostPort: 8080
+              name: http0
+            - containerPort: 8081
+              hostPort: 8081
+              name: http1
+            - containerPort: 8082
+              hostPort: 8082
+              name: http2
+            - containerPort: 8083
+              hostPort: 8083
+              name: http3
+            - containerPort: 5051
+              hostPort: 5051
+              name: grpc
+          env:
+            - name: LOG_LEVEL
+              value: "DEBUG"