Skip to content

Commit

Permalink
fix: querier integration flaky test
Browse files Browse the repository at this point in the history 
Saw this test fail: https://github.com/cortexproject/cortex/actions/runs/10761255507/job/29840284232

Adding this test option to make the test wait until the metric is available.

```
    querier_test.go:231: 

        	Error Trace:	/home/runner/work/cortex/cortex/integration/querier_test.go:231

        	Error:      	Received unexpected error:

        	            	metric not found

        	            	github.com/cortexproject/cortex/integration/e2e.init

        	            		<autogenerated>:1

        	            	runtime.doInit1

        	            		/opt/hostedtoolcache/go/1.22.5/x64/src/runtime/proc.go:7176

        	            	runtime.doInit

        	            		/opt/hostedtoolcache/go/1.22.5/x64/src/runtime/proc.go:7143

        	            	runtime.main

        	            		/opt/hostedtoolcache/go/1.22.5/x64/src/runtime/proc.go:253

        	            	runtime.goexit

        	            		/opt/hostedtoolcache/go/1.22.5/x64/src/runtime/asm_amd64.s:1695

        	            	metric=cortex_bucket_store_blocks_loaded service=store-gateway-1

        	            	github.com/cortexproject/cortex/integration/e2e.(*HTTPService).SumMetrics

        	            		/home/runner/work/cortex/cortex/integration/e2e/service.go:619

        	            	github.com/cortexproject/cortex/integration/e2e.(*CompositeHTTPService).SumMetrics

        	            		/home/runner/work/cortex/cortex/integration/e2e/composite_service.go:78

        	            	github.com/cortexproject/cortex/integration/e2e.(*CompositeHTTPService).WaitSumMetricsWithOptions

        	            		/home/runner/work/cortex/cortex/integration/e2e/composite_service.go:55

        	            	github.com/cortexproject/cortex/integration/e2e.(*CompositeHTTPService).WaitSumMetrics

        	            		/home/runner/work/cortex/cortex/integration/e2e/composite_service.go:44

        	            	github.com/cortexproject/cortex/integration.TestQuerierWithBlocksStorageRunningInMicroservicesMode.func1

        	            		/home/runner/work/cortex/cortex/integration/querier_test.go:231

        	            	testing.tRunner

        	            		/opt/hostedtoolcache/go/1.22.5/x64/src/testing/testing.go:1689

        	            	runtime.goexit

        	            		/opt/hostedtoolcache/go/1.22.5/x64/src/runtime/asm_amd64.s:1695

        	Test:       	TestQuerierWithBlocksStorageRunningInMicroservicesMode/blocks_sharding_disabled,_memcached_index_cache,thanosEngine=false
```

Signed-off-by: Charlie Le <[email protected]>
  • Loading branch information
@CharlieTLe
CharlieTLe authored Sep 8, 2024
1 parent bc69e73 commit eb915f8
Showing 1 changed file with 1 addition and 1 deletion.
2 changes: 1 addition & 1 deletion integration/querier_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
// then the metric only appears in one store gateway instance.
require.NoError(t, storeGateways.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"cortex_bucket_store_blocks_loaded"}, e2e.SkipMissingMetrics))
} else {
require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(float64(2*storeGateways.NumInstances())), "cortex_bucket_store_blocks_loaded"))
require.NoError(t, storeGateways.WaitSumMetricsWithOptions(e2e.Equals(float64(2*storeGateways.NumInstances())), []string{"cortex_bucket_store_blocks_loaded"}, e2e.WaitMissingMetrics))
}

// Check how many tenants have been discovered and synced by store-gateways.
Expand Down

0 comments on commit eb915f8

Please sign in to comment.