ENG-52474: copy all hypertrace config to traceable

.github/workflows/pr.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - uses: bufbuild/[email protected]
 

.gitmodules

@@ -1,3 +1,6 @@
-[submodule "tools/hypertrace/agent-config"]
-	path = tools/hypertrace/agent-config
-	url = [email protected]:hypertrace/agent-config.git
+[submodule "tools/env-vars-generator/protobuf"]
+	path = tools/env-vars-generator/protobuf
+	url = https://github.com/protocolbuffers/protobuf.git
+[submodule "tools/go-generator/cmd/generator/_protobuf"]
+	path = tools/go-generator/cmd/generator/_protobuf
+	url = https://github.com/protocolbuffers/protobuf.git

ENV_VARS.md

@@ -53,3 +53,26 @@ Agents can be configured using environment variables:
 | TA_METRICS_CONFIG_LOGGING_ENABLED | Set this flag to print metrics in logs |
 | TA_METRICS_CONFIG_LOGGING_FREQUENCY | Set the frequency at which metrics should be printed. Examples are '1s', '2m', '3h'. Default value is 30m |
 | TA_ENVIRONMENT | Represents the environment name of agent |
+| TA_SERVICE_NAME | From HT config |
+| TA_REPORTING_ENDPOINT | Represents the endpoint for reporting the traces For ZIPKIN reporter type use http://api.traceable.ai:9411/api/v2/spans For OTLP reporter type use http://api.traceable.ai:4317 |
+| TA_REPORTING_SECURE | When `true`, connects to endpoints over TLS. |
+| TA_REPORTING_TOKEN | User specific token to access Traceable API |
+| TA_REPORTING_CERT_FILE | Certificate file containing the CA to verify the server's certificate. This is for private certificates. If this is set then `secure` above should also be set to `true`. |
+| TA_REPORTING_METRIC_ENDPOINT | Represents the endpoint for reporting the metrics. For OTLP metric reporter type use http://api.traceable.ai:4317 |
+| TA_REPORTING_ENABLE_GRPC_LOADBALANCING | When `true`, modifies grpc resolver to use dns instead of passthrough and configure round robin client side loadbalancing |
+| TA_DATA_CAPTURE_HTTP_HEADERS_REQUEST | When `false` it disables the capture for the request in a client/request operation |
+| TA_DATA_CAPTURE_HTTP_HEADERS_RESPONSE | When `false` it disables the capture for the response in a client/request operation |
+| TA_DATA_CAPTURE_HTTP_BODY_REQUEST | When `false` it disables the capture for the request in a client/request operation |
+| TA_DATA_CAPTURE_HTTP_BODY_RESPONSE | When `false` it disables the capture for the response in a client/request operation |
+| TA_DATA_CAPTURE_RPC_METADATA_REQUEST | When `false` it disables the capture for the request in a client/request operation |
+| TA_DATA_CAPTURE_RPC_METADATA_RESPONSE | When `false` it disables the capture for the response in a client/request operation |
+| TA_DATA_CAPTURE_RPC_BODY_REQUEST | When `false` it disables the capture for the request in a client/request operation |
+| TA_DATA_CAPTURE_RPC_BODY_RESPONSE | When `false` it disables the capture for the response in a client/request operation |
+| TA_DATA_CAPTURE_BODY_MAX_SIZE_BYTES | Is the maximum size of captured body in bytes. Default should be 131_072 (128 KiB). |
+| TA_DATA_CAPTURE_BODY_MAX_PROCESSING_SIZE_BYTES | Is maximum size of body being processed by filters in bytes. Default should be 1_048_576 (1MB).  For uncompressed bodies we capture all bytes up to `body_max_processing_size_bytes` in memory and pass that through the filter. For compressed and GRPC bodies, if the size of the body is larger than this, we ignore it entirely, otherwise we decompress/decode the body and then pass it to the filter. |
+| TA_DATA_CAPTURE_ALLOWED_CONTENT_TYPES | Array of allowed content type substrings to record default should be json, x-www-form-urlencoded ex: ["json"] will record any request bodies that have a content-type header that includes "json". The values should be separated by `,`. |
+| TA_PROPAGATION_FORMATS | List the supported propagation formats e.g. `TA_PROPAGATION_FORMATS="B3,TRACECONTEXT"`. |
+| TA_ENABLED | When `false`, disables the agent |
+| TA_TELEMETRY_STARTUP_SPAN_ENABLED | When `true`, an internal span is created and exported when the agent is initialized and started. It's useful to denote when the application the agent is in started. |
+| TA_TELEMETRY_METRICS_ENABLED | Whether to capture metrics or not. The metrics will be otel go metrics. See https://github.com/open-telemetry/opentelemetry-go/tree/main/metric |
+| TA_GOAGENT_USE_CUSTOM_BSP | Use the custom batch_span_processor adapted from the one in opentelemetry go and supports some additional metrics |

Makefile

@@ -15,21 +15,21 @@ generate-proto:
 	OUT_DIR="$(PWD)/gen/go" \
 	OPT_MODULE="github.com/Traceableai/agent-config/gen/go" \
 	ENV_PREFIX="TA_" \
-	$(MAKE) -C ./tools/hypertrace/agent-config/tools/go-generator
+	$(MAKE) -C ./tools/go-generator
 
 	@echo "Tidy generated modules."
-	@find $(PWD)/gen/go \( -name vendor -o -name '[._].*' -o -name node_modules \) -prune -o -name go.mod -print | sed 's:/go.mod::' | xargs -I {} bash -c 'cd {}; go mod tidy -go=1.19'
+	@find $(PWD)/gen/go \( -name vendor -o -name '[._].*' -o -name node_modules \) -prune -o -name go.mod -print | sed 's:/go.mod::' | xargs -I {} bash -c 'cd {}; go mod tidy -go=1.21'
 
 	@# Run gen/go load sanity tests
 	cd $(PWD)/gen/go && go test ./...
 
 generate-env-vars: ## Generates the ENV_VARS.md with all environment variables.
-	docker build -t hypertrace/agent-config/env-vars-generator tools/hypertrace/agent-config/tools/env-vars-generator
+	docker build -t tools/env-vars-generator tools/env-vars-generator
 	touch $(PWD)/ENV_VARS.md # makes sure this is created as a file and not as a directory
 	docker run \
 	-v $(PWD)/ENV_VARS.md:/usr/local/ENV_VARS.md \
 	-v $(PWD)/proto/ai/traceable/agent/config/v1/config.proto:/usr/local/config.proto \
-	hypertrace/agent-config/env-vars-generator \
+	tools/env-vars-generator \
 	-o "/usr/local/ENV_VARS.md" -p "TA_" \
 	/usr/local/config.proto
 

buf.yaml

@@ -5,6 +5,9 @@ build:
 lint:
   use:
     - DEFAULT
+  except:
+    - ENUM_ZERO_VALUE_SUFFIX
+    - ENUM_VALUE_PREFIX
 breaking:
   use:
     - PACKAGE

gen/go/go.mod

@@ -1,11 +1,11 @@
 module github.com/Traceableai/agent-config/gen/go
 
-go 1.19
+go 1.21
 
 require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/stretchr/testify v1.9.0
-	google.golang.org/protobuf v1.34.2
+	google.golang.org/protobuf v1.35.1
 )
 
 require (

gen/go/go.sum

@@ -3,13 +3,15 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
-google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
-google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

gen/go/proto/v1/config.proto

@@ -20,6 +20,64 @@ message AgentConfig {
   MetricsConfig metrics_config = 9;
   // represents the environment name of agent
   google.protobuf.StringValue environment = 10;
+
+  // From HT config
+  google.protobuf.StringValue service_name = 11;
+
+  // reporting holds the reporting settings for the agent
+  Reporting reporting = 12;
+
+  // data_capture describes the data being captured by instrumentation
+  DataCapture data_capture = 13;
+
+  // propagation_formats list the supported propagation formats
+  repeated PropagationFormat propagation_formats = 14;
+
+  // when `false`, disables the agent
+  google.protobuf.BoolValue enabled = 15;
+
+  // resource_attributes map define the static list of resources which is configured on the tracer
+  map<string, string> resource_attributes = 16;
+
+  // telemetry
+  Telemetry telemetry = 17;
+
+  // Goagent specific config
+  GoAgent goagent = 18;
+
+}
+
+// Reporting covers the options related to the mechanics for sending data to the
+// tracing server o collector.
+message Reporting {
+  // endpoint represents the endpoint for reporting the traces
+  // For ZIPKIN reporter type use http://api.traceable.ai:9411/api/v2/spans
+  // For OTLP reporter type use http://api.traceable.ai:4317
+  google.protobuf.StringValue endpoint = 1;
+
+  // when `true`, connects to endpoints over TLS.
+  google.protobuf.BoolValue secure = 2;
+
+  // user specific token to access Traceable API
+  google.protobuf.StringValue token = 3;
+
+  // reporter type for traces.
+  TraceReporterType trace_reporter_type = 5;
+
+  // Certificate file containing the CA to verify the server's certificate.
+  // This is for private certificates.
+  // If this is set then `secure` above should also be set to `true`.
+  google.protobuf.StringValue cert_file = 6;
+
+  // metric_endpoint represents the endpoint for reporting the metrics.
+  // For OTLP metric reporter type use http://api.traceable.ai:4317
+  google.protobuf.StringValue metric_endpoint = 7;
+
+  // reporter type for metrics.
+  MetricReporterType metric_reporter_type = 8;
+
+  // When `true`, modifies grpc resolver to use dns instead of passthrough and configure round robin client side loadbalancing
+  google.protobuf.BoolValue enable_grpc_loadbalancing = 9;
 }
 
 // Opa covers the options related to the mechanics for getting Open Policy Agent configuration file.
@@ -101,6 +159,7 @@ message SamplingConfig {
 message Javaagent {
   // set this flag to export certificates configured in JKS to libtraceable for making HTTPS connection to TPA.
   google.protobuf.BoolValue import_jks_certs = 1;
+  repeated google.protobuf.StringValue filter_jar_paths = 2;
 }
 
 enum LogMode {
@@ -182,3 +241,120 @@ message RateLimitConfig {
       // set the span type for rate limited spans
       SpanType span_type = 6;
 }
+
+// Message describes what message should be considered for certain DataCapture option
+message Message {
+  // when `false` it disables the capture for the request in a client/request operation
+  google.protobuf.BoolValue request = 1;
+
+  // when `false` it disables the capture for the response in a client/request operation
+  google.protobuf.BoolValue response = 2;
+}
+
+// DataCapture describes the elements to be captured by the agent instrumentation
+message DataCapture {
+  // http_headers enables/disables the capture of the request/response headers in HTTP
+  Message http_headers = 1;
+
+  // http_body enables/disables the capture of the request/response body in HTTP
+  Message http_body = 2;
+
+  // rpc_metadata enables/disables the capture of the request/response metadata in RPC
+  Message rpc_metadata = 3;
+
+  // rpc_body enables/disables the capture of the request/response body in RPC
+  Message rpc_body = 4;
+
+  // body_max_size_bytes is the maximum size of captured body in bytes.
+  // Default should be 131_072 (128 KiB).
+  google.protobuf.Int32Value body_max_size_bytes = 5;
+
+  // body_max_processing_size_bytes is maximum size of body being processed by filters in bytes.
+  // Default should be 1_048_576 (1MB).
+  //
+  // For uncompressed bodies we capture all bytes up to `body_max_processing_size_bytes`
+  // in memory and pass that through the filter.
+  // For compressed and GRPC bodies, if the size of the body is larger than this, we ignore
+  // it entirely, otherwise we decompress/decode the body and then pass it to the filter.
+  google.protobuf.Int32Value body_max_processing_size_bytes = 6;
+
+  // Array of allowed content type substrings to record
+  // default should be json, x-www-form-urlencoded
+  // ex: ["json"] will record any request bodies that have a content-type header that includes "json"
+  repeated google.protobuf.StringValue allowed_content_types = 10;
+}
+
+// PropagationFormat represents the propagation formats supported by agents
+enum PropagationFormat {
+  // B3 propagation format, agents should support both multi and single value formats
+  // see https://github.com/openzipkin/b3-propagation
+  B3 = 0;
+
+  // W3C Propagation format
+  // see https://www.w3.org/TR/trace-context/
+  TRACECONTEXT = 1;
+}
+
+// TraceReporterType represents the reporting format for trace data.
+enum TraceReporterType {
+
+  // Default to none. Agent will use it's default reporting type
+  UNSPECIFIED = 0;
+
+  // Zipkin protobuf reporting format.
+  // see https://github.com/openzipkin/zipkin-api
+  ZIPKIN = 1;
+
+  // OpenTelemetry protobuf reporting format.
+  // see https://github.com/open-telemetry/opentelemetry-proto
+  OTLP = 2;
+
+  // Logging reporting format
+  LOGGING = 3;
+
+  // Disable trace reporting
+  NONE = 4;
+
+  // OTLP over http
+  OTLP_HTTP = 5;
+}
+
+// MetricReporterType represents the reporting format for metric data.
+enum MetricReporterType {
+
+  // Default to none. Agent will use it's default reporting type
+  METRIC_REPORTER_TYPE_UNSPECIFIED = 0;
+
+  // OpenTelemetry protobuf reporting format.
+  // see https://github.com/open-telemetry/opentelemetry-proto
+  METRIC_REPORTER_TYPE_OTLP = 1;
+
+  // Prometheus exposition format.
+  // see https://github.com/prometheus/docs/blob/main/content/docs/instrumenting/exposition_formats.md
+  METRIC_REPORTER_TYPE_PROMETHEUS = 2;
+
+  // Logging reporting format
+  METRIC_REPORTER_TYPE_LOGGING = 3;
+
+  // Disable metric reporting
+  METRIC_REPORTER_TYPE_NONE = 4;
+}
+
+// GoAgent has the configs specific to goagent
+message GoAgent {
+  // use the custom batch_span_processor adapted from the one in opentelemetry go
+  // and supports some additional metrics
+  google.protobuf.BoolValue use_custom_bsp = 1;
+}
+
+// Telemetry has config for agent telemetry: traces and metrics on agent's
+// performance and events.
+message Telemetry {
+  // when `true`, an internal span is created and exported when the agent is initialized and started.
+  // It's useful to denote when the application the agent is in started.
+  google.protobuf.BoolValue startup_span_enabled = 1;
+
+  // Whether to capture metrics or not. The metrics will be otel go metrics.
+  // See https://github.com/open-telemetry/opentelemetry-go/tree/main/metric
+  google.protobuf.BoolValue metrics_enabled = 2;
+}