From b30ee4e2309cbe7e33ce15e77e9f67668954ebe1 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 5 Nov 2026 08:41:50 +0000 Subject: [PATCH 1/5] Make the compaction group graceful period configurable per block range The partitioning grouper holds a compaction group back until its time range ended more than twice the cleanup interval ago. That graceful period is the same for every block range, so the group for the last sub-range of a larger range expires at the same moment as the larger range's own group. A larger range can therefore start compacting while its last sub-range is still collecting blocks, which splits the result into a group compacted early and a second group for the blocks that arrived afterwards. Add -compactor.block-ranges-compaction-delay, a list matched to -compactor.block-ranges by position, so a larger range can be made to wait for its sub-ranges to settle. Leaving it empty keeps the existing behaviour: every range falls back to twice the cleanup interval, which is also what any range without a configured entry uses. The delay is a lower bound. Groups only become eligible on a compaction run, so the effective wait can be up to the compaction interval longer. Signed-off-by: Ben Ye --- docs/blocks-storage/compactor.md | 11 + docs/configuration/config-file-reference.md | 11 + pkg/compactor/compactor.go | 21 +- pkg/compactor/compactor_test.go | 47 ++++ pkg/compactor/partition_compaction_grouper.go | 28 ++- .../partition_compaction_grouper_test.go | 234 ++++++++++++++++++ schemas/cortex-config-schema.json | 8 + 7 files changed, 357 insertions(+), 3 deletions(-) diff --git a/docs/blocks-storage/compactor.md b/docs/blocks-storage/compactor.md index e9148823955..10abc9c8195 100644 --- a/docs/blocks-storage/compactor.md +++ b/docs/blocks-storage/compactor.md @@ -92,6 +92,17 @@ compactor: # CLI flag: -compactor.block-ranges [block_ranges: | default = 2h0m0s,12h0m0s,24h0m0s] + # Per compaction time range, the minimum age the end of a compaction group's + # time range must reach before the group becomes eligible for compaction. + # Positionally matched against -compactor.block-ranges: it must either be + # empty or have the same number of entries, and no entry may be negative. When + # empty, every range defaults to twice -compactor.cleanup-interval. This is a + # lower bound: a group only becomes eligible on a compaction run, so the + # effective wait can be up to -compactor.compaction-interval longer. Only used + # by the partitioning compaction strategy. + # CLI flag: -compactor.block-ranges-compaction-delay + [block_ranges_compaction_delay: | default = ] + # Number of Go routines to use when syncing block index and chunks files from # the long term storage. # CLI flag: -compactor.block-sync-concurrency diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 805fe51dee6..18e00bb3db4 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3103,6 +3103,17 @@ The `compactor_config` configures the compactor for the blocks storage. # CLI flag: -compactor.block-ranges [block_ranges: | default = 2h0m0s,12h0m0s,24h0m0s] +# Per compaction time range, the minimum age the end of a compaction group's +# time range must reach before the group becomes eligible for compaction. +# Positionally matched against -compactor.block-ranges: it must either be empty +# or have the same number of entries, and no entry may be negative. When empty, +# every range defaults to twice -compactor.cleanup-interval. This is a lower +# bound: a group only becomes eligible on a compaction run, so the effective +# wait can be up to -compactor.compaction-interval longer. Only used by the +# partitioning compaction strategy. +# CLI flag: -compactor.block-ranges-compaction-delay +[block_ranges_compaction_delay: | default = ] + # Number of Go routines to use when syncing block index and chunks files from # the long term storage. # CLI flag: -compactor.block-sync-concurrency diff --git a/pkg/compactor/compactor.go b/pkg/compactor/compactor.go index dc008f9908b..692b8bb3b7f 100644 --- a/pkg/compactor/compactor.go +++ b/pkg/compactor/compactor.go @@ -52,8 +52,10 @@ const ( ) var ( - errInvalidBlockRanges = "compactor block range periods should be divisible by the previous one, but %s is not divisible by %s" - RingOp = ring.NewOp([]ring.InstanceState{ring.ACTIVE}, nil) + errInvalidBlockRanges = "compactor block range periods should be divisible by the previous one, but %s is not divisible by %s" + errInvalidBlockRangesCompactionDelayLen = "compactor block ranges compaction delay must have the same number of entries as compactor block ranges, but got %d and %d" + errNegativeBlockRangesCompactionDelay = "compactor block ranges compaction delay cannot be negative, but got %s at position %d" + RingOp = ring.NewOp([]ring.InstanceState{ring.ACTIVE}, nil) supportedShardingStrategies = []string{util.ShardingStrategyDefault, util.ShardingStrategyShuffle} errInvalidShardingStrategy = errors.New("invalid sharding strategy") @@ -255,6 +257,7 @@ type Limits interface { // Config holds the Compactor config. type Config struct { BlockRanges cortex_tsdb.DurationList `yaml:"block_ranges"` + BlockRangesCompactionDelay cortex_tsdb.DurationList `yaml:"block_ranges_compaction_delay"` BlockSyncConcurrency int `yaml:"block_sync_concurrency"` MetaSyncConcurrency int `yaml:"meta_sync_concurrency"` ConsistencyDelay time.Duration `yaml:"consistency_delay"` @@ -317,6 +320,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.retryMaxBackoff = time.Minute f.Var(&cfg.BlockRanges, "compactor.block-ranges", "List of compaction time ranges.") + f.Var(&cfg.BlockRangesCompactionDelay, "compactor.block-ranges-compaction-delay", "Per compaction time range, the minimum age the end of a compaction group's time range must reach before the group becomes eligible for compaction. Positionally matched against -compactor.block-ranges: it must either be empty or have the same number of entries, and no entry may be negative. When empty, every range defaults to twice -compactor.cleanup-interval. This is a lower bound: a group only becomes eligible on a compaction run, so the effective wait can be up to -compactor.compaction-interval longer. Only used by the partitioning compaction strategy.") f.DurationVar(&cfg.ConsistencyDelay, "compactor.consistency-delay", 0, fmt.Sprintf("Minimum age of fresh (non-compacted) blocks before they are being processed. Malformed blocks older than the maximum of consistency-delay and %s will be removed.", compact.PartialUploadThresholdAge)) f.IntVar(&cfg.BlockSyncConcurrency, "compactor.block-sync-concurrency", 20, "Number of Go routines to use when syncing block index and chunks files from the long term storage.") f.IntVar(&cfg.MetaSyncConcurrency, "compactor.meta-sync-concurrency", 20, "Number of Go routines to use when syncing block meta files from the long term storage.") @@ -365,6 +369,19 @@ func (cfg *Config) Validate(limits validation.Limits) error { } } + // The compaction delay is matched to block ranges by position, so it must either be + // left empty (every range falls back to the default delay) or cover every range. + if len(cfg.BlockRangesCompactionDelay) > 0 { + if len(cfg.BlockRangesCompactionDelay) != len(cfg.BlockRanges) { + return errors.Errorf(errInvalidBlockRangesCompactionDelayLen, len(cfg.BlockRangesCompactionDelay), len(cfg.BlockRanges)) + } + for i, delay := range cfg.BlockRangesCompactionDelay { + if delay < 0 { + return errors.Errorf(errNegativeBlockRangesCompactionDelay, delay.String(), i) + } + } + } + // Make sure a valid sharding strategy is being used if !slices.Contains(supportedShardingStrategies, cfg.ShardingStrategy) { return errInvalidShardingStrategy diff --git a/pkg/compactor/compactor_test.go b/pkg/compactor/compactor_test.go index 2fbe01b1192..abb9268ea19 100644 --- a/pkg/compactor/compactor_test.go +++ b/pkg/compactor/compactor_test.go @@ -145,6 +145,53 @@ func TestConfig_Validate(t *testing.T) { initLimits: func(_ *validation.Limits) {}, expected: errInvalidTenantShardSize.Error(), }, + "should pass with an empty block ranges compaction delay": { + setup: func(cfg *Config) { + cfg.BlockRangesCompactionDelay = cortex_tsdb.DurationList{} + }, + initLimits: func(_ *validation.Limits) {}, + expected: "", + }, + "should pass with a block ranges compaction delay entry per block range": { + setup: func(cfg *Config) { + cfg.BlockRanges = cortex_tsdb.DurationList{2 * time.Hour, 12 * time.Hour, 24 * time.Hour} + cfg.BlockRangesCompactionDelay = cortex_tsdb.DurationList{2 * time.Hour, 150 * time.Minute, 30 * time.Minute} + }, + initLimits: func(_ *validation.Limits) {}, + expected: "", + }, + "should pass with a zero block ranges compaction delay entry": { + setup: func(cfg *Config) { + cfg.BlockRanges = cortex_tsdb.DurationList{2 * time.Hour, 12 * time.Hour, 24 * time.Hour} + cfg.BlockRangesCompactionDelay = cortex_tsdb.DurationList{0, 0, 0} + }, + initLimits: func(_ *validation.Limits) {}, + expected: "", + }, + "should fail with fewer block ranges compaction delay entries than block ranges": { + setup: func(cfg *Config) { + cfg.BlockRanges = cortex_tsdb.DurationList{2 * time.Hour, 12 * time.Hour, 24 * time.Hour} + cfg.BlockRangesCompactionDelay = cortex_tsdb.DurationList{2 * time.Hour} + }, + initLimits: func(_ *validation.Limits) {}, + expected: errors.Errorf(errInvalidBlockRangesCompactionDelayLen, 1, 3).Error(), + }, + "should fail with more block ranges compaction delay entries than block ranges": { + setup: func(cfg *Config) { + cfg.BlockRanges = cortex_tsdb.DurationList{2 * time.Hour, 12 * time.Hour} + cfg.BlockRangesCompactionDelay = cortex_tsdb.DurationList{2 * time.Hour, 150 * time.Minute, 30 * time.Minute} + }, + initLimits: func(_ *validation.Limits) {}, + expected: errors.Errorf(errInvalidBlockRangesCompactionDelayLen, 3, 2).Error(), + }, + "should fail with a negative block ranges compaction delay entry": { + setup: func(cfg *Config) { + cfg.BlockRanges = cortex_tsdb.DurationList{2 * time.Hour, 12 * time.Hour, 24 * time.Hour} + cfg.BlockRangesCompactionDelay = cortex_tsdb.DurationList{2 * time.Hour, -1 * time.Minute, 30 * time.Minute} + }, + initLimits: func(_ *validation.Limits) {}, + expected: errors.Errorf(errNegativeBlockRangesCompactionDelay, (-1 * time.Minute).String(), 1).Error(), + }, } for testName, testData := range tests { diff --git a/pkg/compactor/partition_compaction_grouper.go b/pkg/compactor/partition_compaction_grouper.go index b3bd0ce2c20..71e73d02f70 100644 --- a/pkg/compactor/partition_compaction_grouper.go +++ b/pkg/compactor/partition_compaction_grouper.go @@ -29,6 +29,14 @@ var ( DUMMY_BLOCK_ID = ulid.ULID{} ) +// defaultCompactionDelayCleanupIntervalMultiplier is how many cleanup intervals a +// compaction group's time range end must be in the past before the group becomes +// eligible for compaction, when -compactor.block-ranges-compaction-delay does not +// configure a delay for that range. The cleanup interval also drives how often the +// bucket index is refreshed, so waiting for two of them keeps grouping from acting on +// a bucket index that is missing recently uploaded blocks. +const defaultCompactionDelayCleanupIntervalMultiplier = 2 + type PartitionCompactionGrouper struct { ctx context.Context logger log.Logger @@ -283,7 +291,7 @@ func (g *PartitionCompactionGrouper) groupBlocksByRange(blocks []*metadata.Meta, } if len(group.blocks) > 1 { - if time.UnixMilli(group.rangeEnd).Before(time.Now().Add(-2 * g.compactorCfg.CleanupInterval)) { + if time.UnixMilli(group.rangeEnd).Before(time.Now().Add(-g.compactionDelayForRange(tr))) { ret = append(ret, group) } } @@ -292,6 +300,24 @@ func (g *PartitionCompactionGrouper) groupBlocksByRange(blocks []*metadata.Meta, return ret } +// compactionDelayForRange returns the minimum age the end of a compaction group's time +// range must reach before the group becomes eligible for compaction. The configured +// delays are matched to the configured block ranges by position; any range without a +// configured delay falls back to defaultCompactionDelayCleanupIntervalMultiplier times +// the cleanup interval, which keeps the grouping decision at least two bucket index +// refreshes behind the blocks it is grouping. +func (g *PartitionCompactionGrouper) compactionDelayForRange(tr int64) time.Duration { + if len(g.compactorCfg.BlockRangesCompactionDelay) == len(g.compactorCfg.BlockRanges) { + for i, blockRange := range g.compactorCfg.BlockRanges { + if blockRange.Milliseconds() == tr { + return g.compactorCfg.BlockRangesCompactionDelay[i] + } + } + } + + return defaultCompactionDelayCleanupIntervalMultiplier * g.compactorCfg.CleanupInterval +} + func (g *PartitionCompactionGrouper) sortBlockGroups(groups []blocksGroupWithPartition) { // Ensure groups are sorted by smallest range, oldest min time first. The rationale // is that we wanna favor smaller ranges first (ie. to deduplicate samples sooner diff --git a/pkg/compactor/partition_compaction_grouper_test.go b/pkg/compactor/partition_compaction_grouper_test.go index f5e2d70ac54..68170810cf5 100644 --- a/pkg/compactor/partition_compaction_grouper_test.go +++ b/pkg/compactor/partition_compaction_grouper_test.go @@ -2294,3 +2294,237 @@ type expectedCompactionJob struct { rangeStart int64 rangeEnd int64 } + +// testBlockRanges mirrors the default -compactor.block-ranges. +var testBlockRanges = cortextsdb.DurationList{2 * time.Hour, 12 * time.Hour, 24 * time.Hour} + +// elapsedRange returns an aligned time range of size r that ended between r and 2r ago, +// along with how long ago it ended. Stepping one full range back from the current window +// keeps the range's age comfortably above the margins the delay tests rely on, whatever +// time of day the test runs at. +func elapsedRange(r time.Duration) (rangeStart, rangeEnd int64, age time.Duration) { + rangeEnd = getRangeStartBasedOnNow(r) - r.Milliseconds() + rangeStart = rangeEnd - r.Milliseconds() + age = time.Duration(time.Now().UnixMilli()-rangeEnd) * time.Millisecond + return rangeStart, rangeEnd, age +} + +func TestPartitionCompactionGrouper_CompactionDelayForRange(t *testing.T) { + const cleanupInterval = 15 * time.Minute + defaultDelay := defaultCompactionDelayCleanupIntervalMultiplier * cleanupInterval + configuredDelays := cortextsdb.DurationList{2 * time.Hour, 150 * time.Minute, 30 * time.Minute} + + tests := map[string]struct { + blockRangesCompactionDelay cortextsdb.DurationList + timeRange time.Duration + expected time.Duration + }{ + "falls back to the default delay for the first range when no delay is configured": { + timeRange: 2 * time.Hour, + expected: defaultDelay, + }, + "falls back to the default delay for the middle range when no delay is configured": { + timeRange: 12 * time.Hour, + expected: defaultDelay, + }, + "uses the configured delay of the first range": { + blockRangesCompactionDelay: configuredDelays, + timeRange: 2 * time.Hour, + expected: 2 * time.Hour, + }, + "uses the configured delay of the middle range": { + blockRangesCompactionDelay: configuredDelays, + timeRange: 12 * time.Hour, + expected: 150 * time.Minute, + }, + "uses the configured delay of the last range": { + blockRangesCompactionDelay: configuredDelays, + timeRange: 24 * time.Hour, + expected: 30 * time.Minute, + }, + "honours a configured zero delay instead of falling back to the default": { + blockRangesCompactionDelay: cortextsdb.DurationList{0, 150 * time.Minute, 30 * time.Minute}, + timeRange: 2 * time.Hour, + expected: 0, + }, + "falls back to the default delay for a time range that is not a configured block range": { + blockRangesCompactionDelay: configuredDelays, + timeRange: 8 * time.Hour, + expected: defaultDelay, + }, + "falls back to the default delay when the configured delay does not cover every range": { + blockRangesCompactionDelay: cortextsdb.DurationList{2 * time.Hour}, + timeRange: 2 * time.Hour, + expected: defaultDelay, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + g := &PartitionCompactionGrouper{ + compactorCfg: Config{ + BlockRanges: testBlockRanges, + BlockRangesCompactionDelay: testData.blockRangesCompactionDelay, + CleanupInterval: cleanupInterval, + }, + } + + require.Equal(t, testData.expected, g.compactionDelayForRange(testData.timeRange.Milliseconds())) + }) + } +} + +func TestPartitionCompactionGrouper_GroupBlocksByRange_CompactionDelay(t *testing.T) { + // Two blocks filling an elapsed 2h range, so the only thing deciding whether they are + // grouped is the delay applied to the 2h range. + rangeStart, rangeEnd, age := elapsedRange(2 * time.Hour) + blocks := []*metadata.Meta{ + {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil), MinTime: rangeStart, MaxTime: rangeEnd}}, + {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(2, nil), MinTime: rangeStart, MaxTime: rangeEnd}}, + } + + // The gate is rangeEnd < now-delay, so a delay shorter than the range age lets the group + // through and a longer one holds it back. The clock only advances while the test runs, + // which makes the range older, so the eligible cases can sit 1ms from the boundary while + // the ineligible ones need a margin wider than the test's own runtime. + tests := map[string]struct { + blockRangesCompactionDelay cortextsdb.DurationList + expectGroup bool + }{ + "eligible with a zero delay": { + blockRangesCompactionDelay: cortextsdb.DurationList{0, 0, 0}, + expectGroup: true, + }, + "eligible when the delay is 1ms shorter than the range age": { + blockRangesCompactionDelay: cortextsdb.DurationList{age - time.Millisecond, 0, 0}, + expectGroup: true, + }, + "not eligible when the delay exceeds the range age": { + blockRangesCompactionDelay: cortextsdb.DurationList{age + time.Minute, 0, 0}, + expectGroup: false, + }, + // Proves the lookup is positional: the 2h range must ignore the entries belonging to + // the 12h and 24h ranges, even when those are long enough to block everything. + "reads only the entry of its own range": { + blockRangesCompactionDelay: cortextsdb.DurationList{0, 365 * 24 * time.Hour, 365 * 24 * time.Hour}, + expectGroup: true, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + g := &PartitionCompactionGrouper{ + compactorCfg: Config{ + BlockRanges: testBlockRanges, + BlockRangesCompactionDelay: testData.blockRangesCompactionDelay, + CleanupInterval: 15 * time.Minute, + }, + } + + groups := g.groupBlocksByRange(blocks, (2 * time.Hour).Milliseconds()) + + if !testData.expectGroup { + require.Empty(t, groups) + return + } + + require.Len(t, groups, 1) + require.Equal(t, rangeEnd, groups[0].rangeEnd) + require.Len(t, groups[0].blocks, len(blocks)) + }) + } +} + +func TestPartitionCompactionGrouper_GroupBlocksByRange_CompactionDelayOfNonFirstRange(t *testing.T) { + // Two blocks inside an elapsed 12h range, so grouping at the 12h range depends on the + // second entry of the delay list. The first entry is set long enough to block everything, + // proving the 12h range does not read it. + rangeStart, rangeEnd, age := elapsedRange(12 * time.Hour) + blocks := []*metadata.Meta{ + {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil), MinTime: rangeStart, MaxTime: rangeStart + 2*H}}, + {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(2, nil), MinTime: rangeStart + 2*H, MaxTime: rangeStart + 4*H}}, + } + blockingDelay := 365 * 24 * time.Hour + + tests := map[string]struct { + blockRangesCompactionDelay cortextsdb.DurationList + expectGroup bool + }{ + "eligible when the 12h delay has elapsed": { + blockRangesCompactionDelay: cortextsdb.DurationList{blockingDelay, 0, blockingDelay}, + expectGroup: true, + }, + "not eligible when the 12h delay exceeds the range age": { + blockRangesCompactionDelay: cortextsdb.DurationList{0, age + time.Minute, 0}, + expectGroup: false, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + g := &PartitionCompactionGrouper{ + compactorCfg: Config{ + BlockRanges: testBlockRanges, + BlockRangesCompactionDelay: testData.blockRangesCompactionDelay, + CleanupInterval: 15 * time.Minute, + }, + } + + groups := g.groupBlocksByRange(blocks, (12 * time.Hour).Milliseconds()) + + if !testData.expectGroup { + require.Empty(t, groups) + return + } + + require.Len(t, groups, 1) + require.Equal(t, rangeEnd, groups[0].rangeEnd) + require.Len(t, groups[0].blocks, len(blocks)) + }) + } +} + +func TestPartitionCompactionGrouper_GroupBlocksByRange_DefaultDelayFollowsCleanupInterval(t *testing.T) { + rangeStart, rangeEnd, age := elapsedRange(2 * time.Hour) + blocks := []*metadata.Meta{ + {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil), MinTime: rangeStart, MaxTime: rangeEnd}}, + {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(2, nil), MinTime: rangeStart, MaxTime: rangeEnd}}, + } + + // With no delay configured the gate must still be twice the cleanup interval, so a + // cleanup interval either side of age/2 decides whether the group is eligible. + tests := map[string]struct { + cleanupInterval time.Duration + expectGroup bool + }{ + "eligible once twice the cleanup interval has elapsed": { + cleanupInterval: (age - time.Minute) / defaultCompactionDelayCleanupIntervalMultiplier, + expectGroup: true, + }, + "not eligible before twice the cleanup interval has elapsed": { + cleanupInterval: (age + time.Minute) / defaultCompactionDelayCleanupIntervalMultiplier, + expectGroup: false, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + g := &PartitionCompactionGrouper{ + compactorCfg: Config{ + BlockRanges: testBlockRanges, + CleanupInterval: testData.cleanupInterval, + }, + } + + groups := g.groupBlocksByRange(blocks, (2 * time.Hour).Milliseconds()) + + if !testData.expectGroup { + require.Empty(t, groups) + return + } + + require.Len(t, groups, 1) + require.Equal(t, rangeEnd, groups[0].rangeEnd) + }) + } +} diff --git a/schemas/cortex-config-schema.json b/schemas/cortex-config-schema.json index 363f6362d0b..7bffd6ae002 100644 --- a/schemas/cortex-config-schema.json +++ b/schemas/cortex-config-schema.json @@ -3756,6 +3756,14 @@ "type": "array", "x-cli-flag": "compactor.block-ranges" }, + "block_ranges_compaction_delay": { + "description": "Per compaction time range, the minimum age the end of a compaction group's time range must reach before the group becomes eligible for compaction. Positionally matched against -compactor.block-ranges: it must either be empty or have the same number of entries, and no entry may be negative. When empty, every range defaults to twice -compactor.cleanup-interval. This is a lower bound: a group only becomes eligible on a compaction run, so the effective wait can be up to -compactor.compaction-interval longer. Only used by the partitioning compaction strategy.", + "items": { + "type": "string" + }, + "type": "array", + "x-cli-flag": "compactor.block-ranges-compaction-delay" + }, "block_sync_concurrency": { "default": 20, "description": "Number of Go routines to use when syncing block index and chunks files from the long term storage.", From 7d13a4f0ac0cc600eb78a71528f2c5011a9ec1ee Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 5 Nov 2026 08:42:31 +0000 Subject: [PATCH 2/5] Add CHANGELOG entry for -compactor.block-ranges-compaction-delay Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bfc9ac8532..6042a0b9aa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## master / unreleased +* [FEATURE] Compactor: Add `-compactor.block-ranges-compaction-delay` to configure the compaction group graceful period per `-compactor.block-ranges` entry, matched by position. A larger block range can now be held back until its sub-ranges have settled, instead of every range sharing one period derived from `-compactor.cleanup-interval`. Leaving it empty, and any range without an entry, keeps the current behaviour of twice `-compactor.cleanup-interval`. The delay is a lower bound: groups only become eligible on a compaction run, so the effective wait can be up to `-compactor.compaction-interval` longer. Only used by the partitioning compaction strategy. #7916 * [FEATURE] Ruler: Add experimental support for federated rule groups. A rule group listing tenants in its `source_tenants` field is evaluated against those tenants while the resulting series and alerts are written to the tenant owning the rule group. Enabled with `-ruler.enable-federated-rules` (requires `-tenant-federation.enabled`), and restricted to selected tenants with `-ruler.allowed-federated-tenants` and `-ruler.disallowed-federated-tenants`. #7828 * [FEATURE] Distributor: Add experimental OTLP metrics ingestion over gRPC (`opentelemetry.proto.collector.metrics.v1.MetricsService/Export`) on the gRPC server port. Enable it with `-distributor.otlp.grpc-enabled`. The tenant is read from the `X-Scope-OrgID` gRPC metadata, and the request size is limited by `-server.grpc-max-recv-msg-size-bytes`. #7873 * [FEATURE] Storage: Add support for Alibaba Cloud OSS as a backend for blocks, ruler, and alertmanager storage. Configured via `-.oss.*` flags with `backend: oss`. #7907 From 18c970dbdb462a02ff78119ef61460df5abc33c9 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 5 Nov 2026 08:54:24 +0000 Subject: [PATCH 3/5] Build the test block metas with a helper to satisfy the modernize linter Signed-off-by: Ben Ye --- .../partition_compaction_grouper_test.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/compactor/partition_compaction_grouper_test.go b/pkg/compactor/partition_compaction_grouper_test.go index 68170810cf5..e10c396ce88 100644 --- a/pkg/compactor/partition_compaction_grouper_test.go +++ b/pkg/compactor/partition_compaction_grouper_test.go @@ -2309,6 +2309,13 @@ func elapsedRange(r time.Duration) (rangeStart, rangeEnd int64, age time.Duratio return rangeStart, rangeEnd, age } +// blockMetaForRange returns the meta of a block spanning [minTime, maxTime]. +func blockMetaForRange(id uint64, minTime, maxTime int64) *metadata.Meta { + return &metadata.Meta{ + BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(id, nil), MinTime: minTime, MaxTime: maxTime}, + } +} + func TestPartitionCompactionGrouper_CompactionDelayForRange(t *testing.T) { const cleanupInterval = 15 * time.Minute defaultDelay := defaultCompactionDelayCleanupIntervalMultiplier * cleanupInterval @@ -2379,8 +2386,8 @@ func TestPartitionCompactionGrouper_GroupBlocksByRange_CompactionDelay(t *testin // grouped is the delay applied to the 2h range. rangeStart, rangeEnd, age := elapsedRange(2 * time.Hour) blocks := []*metadata.Meta{ - {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil), MinTime: rangeStart, MaxTime: rangeEnd}}, - {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(2, nil), MinTime: rangeStart, MaxTime: rangeEnd}}, + blockMetaForRange(1, rangeStart, rangeEnd), + blockMetaForRange(2, rangeStart, rangeEnd), } // The gate is rangeEnd < now-delay, so a delay shorter than the range age lets the group @@ -2441,8 +2448,8 @@ func TestPartitionCompactionGrouper_GroupBlocksByRange_CompactionDelayOfNonFirst // proving the 12h range does not read it. rangeStart, rangeEnd, age := elapsedRange(12 * time.Hour) blocks := []*metadata.Meta{ - {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil), MinTime: rangeStart, MaxTime: rangeStart + 2*H}}, - {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(2, nil), MinTime: rangeStart + 2*H, MaxTime: rangeStart + 4*H}}, + blockMetaForRange(1, rangeStart, rangeStart+2*H), + blockMetaForRange(2, rangeStart+2*H, rangeStart+4*H), } blockingDelay := 365 * 24 * time.Hour @@ -2487,8 +2494,8 @@ func TestPartitionCompactionGrouper_GroupBlocksByRange_CompactionDelayOfNonFirst func TestPartitionCompactionGrouper_GroupBlocksByRange_DefaultDelayFollowsCleanupInterval(t *testing.T) { rangeStart, rangeEnd, age := elapsedRange(2 * time.Hour) blocks := []*metadata.Meta{ - {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(1, nil), MinTime: rangeStart, MaxTime: rangeEnd}}, - {BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(2, nil), MinTime: rangeStart, MaxTime: rangeEnd}}, + blockMetaForRange(1, rangeStart, rangeEnd), + blockMetaForRange(2, rangeStart, rangeEnd), } // With no delay configured the gate must still be twice the cleanup interval, so a From 9fddc00038f407caee8da64e6396414de4976c29 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 5 Nov 2026 09:03:25 +0000 Subject: [PATCH 4/5] Set the promoted block meta fields directly in the test helper Signed-off-by: Ben Ye --- pkg/compactor/partition_compaction_grouper_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/compactor/partition_compaction_grouper_test.go b/pkg/compactor/partition_compaction_grouper_test.go index e10c396ce88..6c260cf6e3f 100644 --- a/pkg/compactor/partition_compaction_grouper_test.go +++ b/pkg/compactor/partition_compaction_grouper_test.go @@ -2311,9 +2311,7 @@ func elapsedRange(r time.Duration) (rangeStart, rangeEnd int64, age time.Duratio // blockMetaForRange returns the meta of a block spanning [minTime, maxTime]. func blockMetaForRange(id uint64, minTime, maxTime int64) *metadata.Meta { - return &metadata.Meta{ - BlockMeta: tsdb.BlockMeta{ULID: ulid.MustNew(id, nil), MinTime: minTime, MaxTime: maxTime}, - } + return &metadata.Meta{ULID: ulid.MustNew(id, nil), MinTime: minTime, MaxTime: maxTime} } func TestPartitionCompactionGrouper_CompactionDelayForRange(t *testing.T) { From 2fc504e35c28cf7aefb7aca7a72c83cf648bab0b Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Thu, 5 Nov 2026 19:49:24 +0000 Subject: [PATCH 5/5] Reclassify the CHANGELOG entry as an enhancement Signed-off-by: Ben Ye --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6042a0b9aa5..7beb525e1d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,10 @@ ## master / unreleased -* [FEATURE] Compactor: Add `-compactor.block-ranges-compaction-delay` to configure the compaction group graceful period per `-compactor.block-ranges` entry, matched by position. A larger block range can now be held back until its sub-ranges have settled, instead of every range sharing one period derived from `-compactor.cleanup-interval`. Leaving it empty, and any range without an entry, keeps the current behaviour of twice `-compactor.cleanup-interval`. The delay is a lower bound: groups only become eligible on a compaction run, so the effective wait can be up to `-compactor.compaction-interval` longer. Only used by the partitioning compaction strategy. #7916 * [FEATURE] Ruler: Add experimental support for federated rule groups. A rule group listing tenants in its `source_tenants` field is evaluated against those tenants while the resulting series and alerts are written to the tenant owning the rule group. Enabled with `-ruler.enable-federated-rules` (requires `-tenant-federation.enabled`), and restricted to selected tenants with `-ruler.allowed-federated-tenants` and `-ruler.disallowed-federated-tenants`. #7828 * [FEATURE] Distributor: Add experimental OTLP metrics ingestion over gRPC (`opentelemetry.proto.collector.metrics.v1.MetricsService/Export`) on the gRPC server port. Enable it with `-distributor.otlp.grpc-enabled`. The tenant is read from the `X-Scope-OrgID` gRPC metadata, and the request size is limited by `-server.grpc-max-recv-msg-size-bytes`. #7873 * [FEATURE] Storage: Add support for Alibaba Cloud OSS as a backend for blocks, ruler, and alertmanager storage. Configured via `-.oss.*` flags with `backend: oss`. #7907 +* [ENHANCEMENT] Compactor: Add `-compactor.block-ranges-compaction-delay` to configure the compaction group graceful period per `-compactor.block-ranges` entry, matched by position. A larger block range can now be held back until its sub-ranges have settled, instead of every range sharing one period derived from `-compactor.cleanup-interval`. Leaving it empty, and any range without an entry, keeps the current behaviour of twice `-compactor.cleanup-interval`. The delay is a lower bound: groups only become eligible on a compaction run, so the effective wait can be up to `-compactor.compaction-interval` longer. Only used by the partitioning compaction strategy. #7916 * [ENHANCEMENT] Ring: Add the `cortex_ring_member_zonal_ownership_percent{name, member, zone}` metric, reporting the fraction of its own zone's token space owned by each ring member. With zone-aware replication a write is routed within a single zone, so the existing `cortex_ring_member_ownership_percent`, which is computed over the merged token list of all zones, does not reflect the hash range a member actually serves. Reported when ring detailed metrics and zone awareness are both enabled. #7910 * [ENHANCEMENT] Ruler: Add rule information to query parameters sent to the Query Frontend when `-ruler.frontend-address` is configured, so the Query Frontend query stats logs (`-frontend.enabled-ruler-query-stats`) can identify the evaluated rule. In this mode, the Ruler query stats log (`-ruler.query-stats-enabled`) omits the fetched statistics fields, which are always zero. #6539 * [ENHANCEMENT] gRPC: When `-auth.enabled=true`, a gRPC request without the `X-Scope-OrgID` metadata is now rejected with the `UNAUTHENTICATED` status code instead of `UNKNOWN`. #7873