Skip to content

Commit

Permalink
Merge branch 'main' into warmup-ewmarate
Browse files Browse the repository at this point in the history
  • Loading branch information
slimjim777 committed Mar 7, 2025
2 parents 0e06465 + 769906a commit 559a6a7
Show file tree
Hide file tree
Showing 578 changed files with 10,701 additions and 4,355 deletions.
28 changes: 26 additions & 2 deletions .github/workflows/backport.yaml
Original file line number Diff line number Diff line change
@@ -1,14 +1,36 @@
name: Backport PR Creator

on:
pull_request_target:
pull_request:
types:
- closed
- labeled

permissions:
contents: read
id-token: write

jobs:
main:
runs-on: ubuntu-latest

steps:
- name: Retrieve GitHub App credentials from Vault
id: get-secrets
uses: grafana/shared-workflows/actions/[email protected]
with:
repo_secrets: |
APP_ID=mimir-github-bot:app_id
PRIVATE_KEY=mimir-github-bot:private_key
- name: Generate GitHub App token
id: app-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ env.APP_ID }}
private-key: ${{ env.PRIVATE_KEY }}
owner: ${{ github.repository_owner }}

- name: Checkout Actions
uses: actions/checkout@v4
with:
Expand All @@ -17,11 +39,13 @@ jobs:
# pin the version to before https://github.com/grafana/grafana-github-actions/pull/113 because
# we don't want to have the same strict rules for PR labels
ref: d284afd314ca3625c23595e9f62b52d215ead7ce

- name: Install Actions
run: npm install --production --prefix ./actions

- name: Run backport
uses: ./actions/backport
with:
token: ${{secrets.GH_BOT_ACCESS_TOKEN}}
token: ${{ steps.app-token.outputs.token }}
labelsToAdd: "backport"
title: "[{{base}}] {{originalTitle}}"
37 changes: 32 additions & 5 deletions .github/workflows/generate-docs-helm-tests-renovate-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,16 @@ on:
paths:
- 'operations/helm/**'

# These permissions are needed to assume roles from Github's OIDC.
permissions:
contents: read
id-token: write

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true

jobs:
jobs:
prepare:
if: github.event.pull_request.user.login == 'renovate[bot]'
runs-on: ubuntu-latest
Expand All @@ -33,28 +38,50 @@ jobs:
steps:
- name: Checkout Repository
uses: actions/checkout@v4

# Retrieve GitHub App Credentials from Vault
- name: Retrieve GitHub App Credentials from Vault
id: get-secrets
uses: grafana/shared-workflows/actions/[email protected]
with:
token: ${{ secrets.GH_BOT_ACCESS_TOKEN }}
repo_secrets: |
APP_ID=mimir-github-bot:app_id
PRIVATE_KEY=mimir-github-bot:private_key
# Generate GitHub App Token (Directly in the Job That Needs It)
- name: Generate GitHub App Token
id: token
uses: actions/create-github-app-token@v1
with:
# Variables generated by the previous step get-secrets
app-id: ${{ env.APP_ID }}
private-key: ${{ env.PRIVATE_KEY }}
owner: ${{ github.repository_owner }}

- name: Checkout Repository with App Token
uses: actions/checkout@v4
with:
token: ${{ steps.token.outputs.token }}

- name: Run Git Config
run: |
git config --global --add safe.directory '*'
git config --global user.email "${{ github.event.pull_request.user.login }}@users.noreply.github.com"
git config --global user.name "${{ github.event.pull_request.user.login }}"
- name: Install the gh cli
uses: ksivamuthu/actions-setup-gh-cli@v2

- name: Checkout Pull Request Branch
run: gh pr checkout ${{ github.event.pull_request.number }}
env:
GITHUB_TOKEN: ${{ secrets.GH_BOT_ACCESS_TOKEN }}
GITHUB_TOKEN: ${{ steps.token.outputs.token }}

- name: Run make targets
id: update
run: |
make BUILD_IN_CONTAINER=false doc build-helm-tests
- name: Check Updates
id: check_updates
run: |
Expand Down
25 changes: 21 additions & 4 deletions .github/workflows/grafanabot_reviewer.yml
Original file line number Diff line number Diff line change
@@ -1,20 +1,37 @@
name: Auto-review Grafanabot PRs
on: pull_request_target
on: pull_request

permissions:
pull-requests: write
contents: write
id-token: write

jobs:
dependabot-reviewer:
runs-on: ubuntu-latest

if: ${{ github.event.pull_request.user.login == 'grafanabot' }}

steps:
- name: Checkout Repository
uses: actions/checkout@v4

- name: Retrieve GitHub App Credentials from Vault
id: get-secrets
uses: grafana/shared-workflows/actions/[email protected]
with:
repo_secrets: |
APP_ID=mimir-github-bot:app_id
PRIVATE_KEY=mimir-github-bot:private_key
- name: Generate GitHub App Token
id: app-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ env.APP_ID }}
private-key: ${{ env.PRIVATE_KEY }}
owner: ${{ github.repository_owner }}

- name: Approve and auto-merge
id: auto-merge
if: contains(github.event.pull_request.head.ref, 'helm-chart-weekly-')
Expand All @@ -24,7 +41,7 @@ jobs:
--approve -b "**I'm approving** this pull request, since it is a helm release."
env:
PR_URL: ${{github.event.pull_request.html_url}}
GITHUB_TOKEN: ${{secrets.GH_BOT_ACCESS_TOKEN}}
GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}

- name: Manual review is required
if: steps.auto-merge.conclusion != 'success'
Expand All @@ -33,4 +50,4 @@ jobs:
env:
PR_URL: ${{github.event.pull_request.html_url}}
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
23 changes: 22 additions & 1 deletion .github/workflows/helm-weekly-release-pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,42 @@ on:

workflow_dispatch: # for manual testing

# These permissions are needed to assume roles from Github's OIDC.
permissions:
contents: read
id-token: write

jobs:
weekly-release-pr:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: imjasonh/[email protected]

- name: Retrieve GitHub App credentials from Vault
id: get-secrets
uses: grafana/shared-workflows/actions/[email protected]
with:
repo_secrets: |
APP_ID=mimir-github-bot:app_id
PRIVATE_KEY=mimir-github-bot:private_key
- name: Generate GitHub App token
id: app-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ env.APP_ID }}
private-key: ${{ env.PRIVATE_KEY }}
owner: ${{ github.repository_owner }}

- name: Update/regenerate files
id: update
run: bash .github/workflows/scripts/helm-weekly-release.sh

- name: Create Pull Request
uses: peter-evans/create-pull-request@v5
with:
token: ${{ secrets.GH_BOT_ACCESS_TOKEN }}
token: ${{ steps.app-token.outputs.token }}
title: Release mimir-distributed Helm chart ${{ steps.update.outputs.new_chart_version }}
body: Automated PR created by [helm-weekly-release-pr.yaml](https://github.com/grafana/mimir/blob/main/.github/workflows/helm-weekly-release-pr.yaml)
commit-message: Update mimir-distributed chart to ${{ steps.update.outputs.new_chart_version }}
Expand Down
15 changes: 12 additions & 3 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

* [FEATURE] Ingester/Distributor: Add support for exporting cost attribution metrics (`cortex_ingester_attributed_active_series`, `cortex_distributor_received_attributed_samples_total`, and `cortex_discarded_attributed_samples_total`) with labels specified by customers to a custom Prometheus registry. This feature enables more flexible billing data tracking. #10269 #10702
* [FEATURE] Ruler: Added `/ruler/tenants` endpoints to list the discovered tenants with rule groups. #10738
* [FEATURE] Distributor: Add experimental Influx handler. #10153
* [CHANGE] Querier: pass context to queryable `IsApplicable` hook. #10451
* [CHANGE] Distributor: OTLP and push handler replace all non-UTF8 characters with the unicode replacement character `\uFFFD` in error messages before propagating them. #10236
* [CHANGE] Querier: pass query matchers to queryable `IsApplicable` hook. #10256
Expand All @@ -17,7 +18,7 @@
* [CHANGE] Ruler: Add `user` and `reason` labels to `cortex_ruler_write_requests_failed_total` and `cortex_ruler_queries_failed_total`; add `user` to
`cortex_ruler_write_requests_total` and `cortex_ruler_queries_total` metrics. #10536
* [CHANGE] Querier / Query-frontend: Remove experimental `-querier.promql-experimental-functions-enabled` and `-query-frontend.block-promql-experimental-functions` CLI flags and respective YAML configuration options to enable experimental PromQL functions. Instead access to experimental PromQL functions is always blocked. You can enable them using the per-tenant setting `enabled_promql_experimental_functions`. #10660 #10712
* [FEATURE] Distributor: Add experimental Influx handler. #10153
* [CHANGE] Store-gateway: Include posting sampling rate in sparse index headers. When the sampling rate isn't set in a sparse index header, store gateway will rebuild the sparse header with the configured `blocks-storage.bucket-store.posting-offsets-in-mem-sampling` value. If the sparse header's sampling rate is set, but doesn't match the configured rate, store gateway will either rebuild the sparse header or downsample to the configured sampling rate. #10684
* [ENHANCEMENT] Compactor: Expose `cortex_bucket_index_last_successful_update_timestamp_seconds` for all tenants assigned to the compactor before starting the block cleanup job. #10569
* [ENHANCEMENT] Query Frontend: Return server-side `samples_processed` statistics. #10103
* [ENHANCEMENT] Distributor: OTLP receiver now converts also metric metadata. See also https://github.com/prometheus/prometheus/pull/15416. #10168
Expand Down Expand Up @@ -46,9 +47,8 @@
* `cortex_ingester_tsdb_block_postings_for_matchers_cache_evictions_total`
* [ENHANCEMENT] Compactor: Shuffle users' order in `BlocksCleaner`. Prevents bucket indexes from going an extended period without cleanup during compactor restarts. #10513
* [ENHANCEMENT] Distributor, querier, ingester and store-gateway: Add support for `limit` parameter for label names and values requests. #10410
* [ENHANCEMENT] Query-frontend: Allow adjustment of queries looking into the future to a maximum duration with experimental `-query-frontend.max-future-query-window` flag. #10547
* [ENHANCEMENT] Ruler: Adds support for filtering results from rule status endpoint by `file[]`, `rule_group[]` and `rule_name[]`. #10589
* [ENHANCEMENT] Query-frontend: Add option to "spin off" subqueries as actual range queries, so that they benefit from query acceleration techniques such as sharding, splitting, and caching. To enable this feature, set the `-query-frontend.instant-queries-with-subquery-spin-off=<comma separated list>` option on the frontend or the `instant_queries_with_subquery_spin_off` per-tenant override with regular expressions matching the queries to enable. #10460 #10603 #10621 #10742
* [ENHANCEMENT] Query-frontend: Add option to "spin off" subqueries as actual range queries, so that they benefit from query acceleration techniques such as sharding, splitting, and caching. To enable this feature, set the `-query-frontend.instant-queries-with-subquery-spin-off=<comma separated list>` option on the frontend or the `instant_queries_with_subquery_spin_off` per-tenant override with regular expressions matching the queries to enable. #10460 #10603 #10621 #10742 #10796
* [ENHANCEMENT] Querier, ingester: The series API respects passed `limit` parameter. #10620 #10652
* [ENHANCEMENT] Store-gateway: Add experimental settings under `-store-gateway.dynamic-replication` to allow more than the default of 3 store-gateways to own recent blocks. #10382 #10637
* [ENHANCEMENT] Ingester: Add reactive concurrency limiters to protect push and read operations from overload. #10574
Expand All @@ -58,6 +58,14 @@
* `go_cpu_classes_gc_total_cpu_seconds_total`
* `go_cpu_classes_total_cpu_seconds_total`
* `go_cpu_classes_idle_cpu_seconds_total`
* [ENHANCEMENT] All: Add support for cluster validation in gRCP calls. When it is enabled, gRPC server verifies if a request coming from a gRPC client comes from an expected cluster. This validation can be configured by the following experimental configuration options: #10767
* `-server.cluster-validation.label`
* `-server.cluster-validation.grpc.enabled`
* `-server.cluster-validation.grpc.soft-validation`
* [ENHANCEMENT] All: Add `cortex_client_request_invalid_cluster_validation_labels_total` metrics, that is used by Mimir's gRPC clients to track invalid cluster validations. #10767
* [ENHANCEMENT] Ingester client: Add support to configure cluster validation for ingester clients. Failed cluster validations are tracked by `cortex_client_request_invalid_cluster_validation_labels_total` with label `client=ingester`. #10767
* [ENHANCEMENT] Add experimental metric `cortex_distributor_dropped_native_histograms_total` to measure native histograms silently dropped when native histograms are disabled for a tenant. #10760
* [ENCHACEMENT] Compactor: Add experimental `-compactor.upload-sparse-index-headers` option. When enabled, the compactor will attempt to upload sparse index headers to object storage. This prevents latency spikes after adding store-gateway replicas. #10684
* [BUGFIX] Distributor: Use a boolean to track changes while merging the ReplicaDesc components, rather than comparing the objects directly. #10185
* [BUGFIX] Querier: fix timeout responding to query-frontend when response size is very close to `-querier.frontend-client.grpc-max-send-msg-size`. #10154
* [BUGFIX] Query-frontend and querier: show warning/info annotations in some cases where they were missing (if a lazy querier was used). #10277
Expand All @@ -78,6 +86,7 @@
* [BUGFIX] Distributor: Report partially converted OTLP requests with status 400 Bad Request. #10588
* [BUGFIX] Ruler: fix issue where rule evaluations could be missed while shutting down a ruler instance if that instance owns many rule groups. prometheus/prometheus#15804 #10762
* [BUGFIX] Ingester: Add additional check on reactive limiter queue sizes. #10722
* [BUGFIX] TSDB: fix unknown series errors and possible lost data during WAL replay when series are removed from the head due to inactivity and reappear before the next WAL checkpoint. https://github.com/prometheus/prometheus/pull/16060 #10824

### Mixin

Expand Down
12 changes: 10 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,8 @@ images: ## Print all image names.
PROTO_DEFS := $(shell find . $(DONT_FIND) -type f -name '*.proto' -print)
PROTO_GOS := $(patsubst %.proto,%.pb.go,$(PROTO_DEFS))

PROMQL_TESTS := $(shell find pkg/streamingpromql/testdata/ours pkg/streamingpromql/testdata/ours-only $(DONT_FIND) -type f -path '*.test' -print)

# Generating OTLP translation code is automated.
OTLP_GOS := $(shell find ./pkg/distributor/otlp/ -type f -name '*_generated.go' -print)

Expand All @@ -239,7 +241,7 @@ all: $(UPTODATE_FILES)
test: protos
test-with-race: protos
mod-check: protos
lint: lint-gh-action lint-packaging-scripts protos
lint: lint-gh-action lint-packaging-scripts protos check-promql-tests
mimir-build-image/$(UPTODATE): mimir-build-image/*

# All the boiler plate for building golang follows:
Expand Down Expand Up @@ -270,7 +272,7 @@ GOVOLUMES= -v mimir-go-cache:/go/cache \
# Mount local ssh credentials to be able to clone private repos when doing `mod-check`
SSHVOLUME= -v ~/.ssh/:/root/.ssh:$(CONTAINER_MOUNT_OPTIONS)

exes $(EXES) $(EXES_RACE) protos $(PROTO_GOS) lint lint-gh-action lint-packaging-scripts test test-with-race cover shell mod-check check-protos doc format dist build-mixin format-mixin check-mixin-tests license check-license conftest-fmt check-conftest-fmt helm-conftest-test helm-conftest-quick-test conftest-verify check-helm-tests build-helm-tests print-go-version: fetch-build-image
exes $(EXES) $(EXES_RACE) protos $(PROTO_GOS) lint lint-gh-action lint-packaging-scripts test test-with-race cover shell mod-check check-protos doc format dist build-mixin format-mixin check-mixin-tests license check-license conftest-fmt check-conftest-fmt helm-conftest-test helm-conftest-quick-test conftest-verify check-helm-tests build-helm-tests print-go-version format-promql-tests check-promql-tests: fetch-build-image
@echo ">>>> Entering build container: $@"
$(SUDO) time docker run --rm $(TTY) -i $(SSHVOLUME) $(GOVOLUMES) $(BUILD_IMAGE) GOOS=$(GOOS) GOARCH=$(GOARCH) BINARY_SUFFIX=$(BINARY_SUFFIX) $@;

Expand Down Expand Up @@ -496,6 +498,12 @@ check-protos: ## Check the protobuf files are up to date.
check-protos: clean-protos protos
@./tools/find-diff-or-untracked.sh $(PROTO_GOS) || (echo "Please rebuild protobuf code by running 'check-protos'" && false)

format-promql-tests:
@./tools/format-promql-test.sh $(PROMQL_TESTS)

check-promql-tests: format-promql-tests
@./tools/find-diff-or-untracked.sh $(PROMQL_TESTS) || (echo "Please format PromQL test files by running 'format-promql-tests'" && false)

%.md : %.template
go run ./tools/doc-generator $< > $@

Expand Down
Loading

0 comments on commit 559a6a7

Please sign in to comment.